summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2013-09-07 14:31:18 -0700
committerLinus Torvalds <torvalds@linux-foundation.org>2013-09-07 14:31:18 -0700
commit6be48f2940af9ea8d93c23a0dd8e322672c92efd (patch)
tree1bdc85a9d3fd0c19e108ea27a29a83ef2b44f5d0
parent0ffb01d9def22f1954e99529b7e4ded497b2e88b (diff)
parent68411521cc6055edc6274e03ab3210a5893533ba (diff)
downloadlinux-6be48f2940af9ea8d93c23a0dd8e322672c92efd.tar.bz2
Merge git://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6
Pull crypto update from Herbert Xu: "Here is the crypto update for 3.12: - Added MODULE_SOFTDEP to allow pre-loading of modules. - Reinstated crct10dif driver using the module softdep feature. - Allow via rng driver to be auto-loaded. - Split large input data when necessary in nx. - Handle zero length messages correctly for GCM/XCBC in nx. - Handle SHA-2 chunks bigger than block size properly in nx. - Handle unaligned lengths in omap-aes. - Added SHA384/SHA512 to omap-sham. - Added OMAP5/AM43XX SHAM support. - Added OMAP4 TRNG support. - Misc fixes" * git://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6: (66 commits) Reinstate "crypto: crct10dif - Wrap crc_t10dif function all to use crypto transform framework" hwrng: via - Add MODULE_DEVICE_TABLE crypto: fcrypt - Fix bitoperation for compilation with clang crypto: nx - fix SHA-2 for chunks bigger than block size crypto: nx - fix GCM for zero length messages crypto: nx - fix XCBC for zero length messages crypto: nx - fix limits to sg lists for AES-CCM crypto: nx - fix limits to sg lists for AES-XCBC crypto: nx - fix limits to sg lists for AES-GCM crypto: nx - fix limits to sg lists for AES-CTR crypto: nx - fix limits to sg lists for AES-CBC crypto: nx - fix limits to sg lists for AES-ECB crypto: nx - add offset to nx_build_sg_lists() padata - Register hotcpu notifier after initialization padata - share code between CPU_ONLINE and CPU_DOWN_FAILED, same to CPU_DOWN_PREPARE and CPU_UP_CANCELED hwrng: omap - reorder OMAP TRNG driver code crypto: omap-sham - correct dma burst size crypto: omap-sham - Enable Polling mode if DMA fails crypto: tegra-aes - bitwise vs logical and crypto: sahara - checking the wrong variable ...
-rw-r--r--arch/arm/mach-omap2/devices.c2
-rw-r--r--arch/x86/crypto/Makefile2
-rw-r--r--arch/x86/crypto/camellia_glue.c64
-rw-r--r--arch/x86/crypto/crct10dif-pcl-asm_64.S643
-rw-r--r--arch/x86/crypto/crct10dif-pclmul_glue.c151
-rw-r--r--arch/x86/include/asm/xor_avx.h4
-rw-r--r--crypto/Kconfig19
-rw-r--r--crypto/Makefile1
-rw-r--r--crypto/aes_generic.c8
-rw-r--r--crypto/camellia_generic.c48
-rw-r--r--crypto/cast_common.c8
-rw-r--r--crypto/crct10dif.c178
-rw-r--r--crypto/fcrypt.c2
-rw-r--r--crypto/scatterwalk.c22
-rw-r--r--crypto/tcrypt.c8
-rw-r--r--crypto/testmgr.c12
-rw-r--r--crypto/testmgr.h33
-rw-r--r--drivers/char/hw_random/Kconfig6
-rw-r--r--drivers/char/hw_random/mxc-rnga.c4
-rw-r--r--drivers/char/hw_random/omap-rng.c386
-rw-r--r--drivers/char/hw_random/picoxcell-rng.c2
-rw-r--r--drivers/char/hw_random/tx4939-rng.c4
-rw-r--r--drivers/char/hw_random/via-rng.c7
-rw-r--r--drivers/crypto/Kconfig13
-rw-r--r--drivers/crypto/amcc/crypto4xx_alg.c15
-rw-r--r--drivers/crypto/caam/Kconfig8
-rw-r--r--drivers/crypto/caam/Makefile3
-rw-r--r--drivers/crypto/caam/caamalg.c80
-rw-r--r--drivers/crypto/caam/caamhash.c70
-rw-r--r--drivers/crypto/caam/ctrl.c77
-rw-r--r--drivers/crypto/caam/desc_constr.h1
-rw-r--r--drivers/crypto/caam/intern.h5
-rw-r--r--drivers/crypto/caam/jr.c67
-rw-r--r--drivers/crypto/caam/jr.h2
-rw-r--r--drivers/crypto/caam/key_gen.c6
-rw-r--r--drivers/crypto/caam/regs.h12
-rw-r--r--drivers/crypto/nx/nx-aes-cbc.c57
-rw-r--r--drivers/crypto/nx/nx-aes-ccm.c283
-rw-r--r--drivers/crypto/nx/nx-aes-ctr.c52
-rw-r--r--drivers/crypto/nx/nx-aes-ecb.c50
-rw-r--r--drivers/crypto/nx/nx-aes-gcm.c296
-rw-r--r--drivers/crypto/nx/nx-aes-xcbc.c205
-rw-r--r--drivers/crypto/nx/nx-sha256.c124
-rw-r--r--drivers/crypto/nx/nx-sha512.c131
-rw-r--r--drivers/crypto/nx/nx.c35
-rw-r--r--drivers/crypto/nx/nx.h3
-rw-r--r--drivers/crypto/omap-aes.c468
-rw-r--r--drivers/crypto/omap-sham.c382
-rw-r--r--drivers/crypto/sahara.c4
-rw-r--r--drivers/crypto/tegra-aes.c6
-rw-r--r--drivers/crypto/ux500/hash/hash_core.c586
-rw-r--r--include/crypto/scatterwalk.h2
-rw-r--r--include/linux/crc-t10dif.h4
-rw-r--r--kernel/padata.c32
-rw-r--r--lib/Kconfig2
-rw-r--r--lib/crc-t10dif.c74
56 files changed, 3405 insertions, 1364 deletions
diff --git a/arch/arm/mach-omap2/devices.c b/arch/arm/mach-omap2/devices.c
index 73ae7536a32b..5c5315ba129b 100644
--- a/arch/arm/mach-omap2/devices.c
+++ b/arch/arm/mach-omap2/devices.c
@@ -530,12 +530,12 @@ static int __init omap2_init_devices(void)
omap_init_mcspi();
omap_init_sham();
omap_init_aes();
+ omap_init_rng();
} else {
/* These can be removed when bindings are done */
omap_init_wl12xx_of();
}
omap_init_sti();
- omap_init_rng();
omap_init_vout();
return 0;
diff --git a/arch/x86/crypto/Makefile b/arch/x86/crypto/Makefile
index 6c63c358a7e6..7d6ba9db1be9 100644
--- a/arch/x86/crypto/Makefile
+++ b/arch/x86/crypto/Makefile
@@ -27,6 +27,7 @@ obj-$(CONFIG_CRYPTO_SHA1_SSSE3) += sha1-ssse3.o
obj-$(CONFIG_CRYPTO_CRC32_PCLMUL) += crc32-pclmul.o
obj-$(CONFIG_CRYPTO_SHA256_SSSE3) += sha256-ssse3.o
obj-$(CONFIG_CRYPTO_SHA512_SSSE3) += sha512-ssse3.o
+obj-$(CONFIG_CRYPTO_CRCT10DIF_PCLMUL) += crct10dif-pclmul.o
# These modules require assembler to support AVX.
ifeq ($(avx_supported),yes)
@@ -81,3 +82,4 @@ crc32c-intel-$(CONFIG_64BIT) += crc32c-pcl-intel-asm_64.o
crc32-pclmul-y := crc32-pclmul_asm.o crc32-pclmul_glue.o
sha256-ssse3-y := sha256-ssse3-asm.o sha256-avx-asm.o sha256-avx2-asm.o sha256_ssse3_glue.o
sha512-ssse3-y := sha512-ssse3-asm.o sha512-avx-asm.o sha512-avx2-asm.o sha512_ssse3_glue.o
+crct10dif-pclmul-y := crct10dif-pcl-asm_64.o crct10dif-pclmul_glue.o
diff --git a/arch/x86/crypto/camellia_glue.c b/arch/x86/crypto/camellia_glue.c
index 5cb86ccd4acb..c171dcbf192d 100644
--- a/arch/x86/crypto/camellia_glue.c
+++ b/arch/x86/crypto/camellia_glue.c
@@ -62,7 +62,7 @@ static void camellia_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
}
/* camellia sboxes */
-const u64 camellia_sp10011110[256] = {
+__visible const u64 camellia_sp10011110[256] = {
0x7000007070707000ULL, 0x8200008282828200ULL, 0x2c00002c2c2c2c00ULL,
0xec0000ecececec00ULL, 0xb30000b3b3b3b300ULL, 0x2700002727272700ULL,
0xc00000c0c0c0c000ULL, 0xe50000e5e5e5e500ULL, 0xe40000e4e4e4e400ULL,
@@ -151,7 +151,7 @@ const u64 camellia_sp10011110[256] = {
0x9e00009e9e9e9e00ULL,
};
-const u64 camellia_sp22000222[256] = {
+__visible const u64 camellia_sp22000222[256] = {
0xe0e0000000e0e0e0ULL, 0x0505000000050505ULL, 0x5858000000585858ULL,
0xd9d9000000d9d9d9ULL, 0x6767000000676767ULL, 0x4e4e0000004e4e4eULL,
0x8181000000818181ULL, 0xcbcb000000cbcbcbULL, 0xc9c9000000c9c9c9ULL,
@@ -240,7 +240,7 @@ const u64 camellia_sp22000222[256] = {
0x3d3d0000003d3d3dULL,
};
-const u64 camellia_sp03303033[256] = {
+__visible const u64 camellia_sp03303033[256] = {
0x0038380038003838ULL, 0x0041410041004141ULL, 0x0016160016001616ULL,
0x0076760076007676ULL, 0x00d9d900d900d9d9ULL, 0x0093930093009393ULL,
0x0060600060006060ULL, 0x00f2f200f200f2f2ULL, 0x0072720072007272ULL,
@@ -329,7 +329,7 @@ const u64 camellia_sp03303033[256] = {
0x004f4f004f004f4fULL,
};
-const u64 camellia_sp00444404[256] = {
+__visible const u64 camellia_sp00444404[256] = {
0x0000707070700070ULL, 0x00002c2c2c2c002cULL, 0x0000b3b3b3b300b3ULL,
0x0000c0c0c0c000c0ULL, 0x0000e4e4e4e400e4ULL, 0x0000575757570057ULL,
0x0000eaeaeaea00eaULL, 0x0000aeaeaeae00aeULL, 0x0000232323230023ULL,
@@ -418,7 +418,7 @@ const u64 camellia_sp00444404[256] = {
0x00009e9e9e9e009eULL,
};
-const u64 camellia_sp02220222[256] = {
+__visible const u64 camellia_sp02220222[256] = {
0x00e0e0e000e0e0e0ULL, 0x0005050500050505ULL, 0x0058585800585858ULL,
0x00d9d9d900d9d9d9ULL, 0x0067676700676767ULL, 0x004e4e4e004e4e4eULL,
0x0081818100818181ULL, 0x00cbcbcb00cbcbcbULL, 0x00c9c9c900c9c9c9ULL,
@@ -507,7 +507,7 @@ const u64 camellia_sp02220222[256] = {
0x003d3d3d003d3d3dULL,
};
-const u64 camellia_sp30333033[256] = {
+__visible const u64 camellia_sp30333033[256] = {
0x3800383838003838ULL, 0x4100414141004141ULL, 0x1600161616001616ULL,
0x7600767676007676ULL, 0xd900d9d9d900d9d9ULL, 0x9300939393009393ULL,
0x6000606060006060ULL, 0xf200f2f2f200f2f2ULL, 0x7200727272007272ULL,
@@ -596,7 +596,7 @@ const u64 camellia_sp30333033[256] = {
0x4f004f4f4f004f4fULL,
};
-const u64 camellia_sp44044404[256] = {
+__visible const u64 camellia_sp44044404[256] = {
0x7070007070700070ULL, 0x2c2c002c2c2c002cULL, 0xb3b300b3b3b300b3ULL,
0xc0c000c0c0c000c0ULL, 0xe4e400e4e4e400e4ULL, 0x5757005757570057ULL,
0xeaea00eaeaea00eaULL, 0xaeae00aeaeae00aeULL, 0x2323002323230023ULL,
@@ -685,7 +685,7 @@ const u64 camellia_sp44044404[256] = {
0x9e9e009e9e9e009eULL,
};
-const u64 camellia_sp11101110[256] = {
+__visible const u64 camellia_sp11101110[256] = {
0x7070700070707000ULL, 0x8282820082828200ULL, 0x2c2c2c002c2c2c00ULL,
0xececec00ececec00ULL, 0xb3b3b300b3b3b300ULL, 0x2727270027272700ULL,
0xc0c0c000c0c0c000ULL, 0xe5e5e500e5e5e500ULL, 0xe4e4e400e4e4e400ULL,
@@ -828,8 +828,8 @@ static void camellia_setup_tail(u64 *subkey, u64 *subRL, int max)
subRL[1] ^= (subRL[1] & ~subRL[9]) << 32;
/* modified for FLinv(kl2) */
- dw = (subRL[1] & subRL[9]) >> 32,
- subRL[1] ^= rol32(dw, 1);
+ dw = (subRL[1] & subRL[9]) >> 32;
+ subRL[1] ^= rol32(dw, 1);
/* round 8 */
subRL[11] ^= subRL[1];
@@ -840,8 +840,8 @@ static void camellia_setup_tail(u64 *subkey, u64 *subRL, int max)
subRL[1] ^= (subRL[1] & ~subRL[17]) << 32;
/* modified for FLinv(kl4) */
- dw = (subRL[1] & subRL[17]) >> 32,
- subRL[1] ^= rol32(dw, 1);
+ dw = (subRL[1] & subRL[17]) >> 32;
+ subRL[1] ^= rol32(dw, 1);
/* round 14 */
subRL[19] ^= subRL[1];
@@ -859,8 +859,8 @@ static void camellia_setup_tail(u64 *subkey, u64 *subRL, int max)
} else {
subRL[1] ^= (subRL[1] & ~subRL[25]) << 32;
/* modified for FLinv(kl6) */
- dw = (subRL[1] & subRL[25]) >> 32,
- subRL[1] ^= rol32(dw, 1);
+ dw = (subRL[1] & subRL[25]) >> 32;
+ subRL[1] ^= rol32(dw, 1);
/* round 20 */
subRL[27] ^= subRL[1];
@@ -882,8 +882,8 @@ static void camellia_setup_tail(u64 *subkey, u64 *subRL, int max)
kw4 ^= (kw4 & ~subRL[24]) << 32;
/* modified for FL(kl5) */
- dw = (kw4 & subRL[24]) >> 32,
- kw4 ^= rol32(dw, 1);
+ dw = (kw4 & subRL[24]) >> 32;
+ kw4 ^= rol32(dw, 1);
}
/* round 17 */
@@ -895,8 +895,8 @@ static void camellia_setup_tail(u64 *subkey, u64 *subRL, int max)
kw4 ^= (kw4 & ~subRL[16]) << 32;
/* modified for FL(kl3) */
- dw = (kw4 & subRL[16]) >> 32,
- kw4 ^= rol32(dw, 1);
+ dw = (kw4 & subRL[16]) >> 32;
+ kw4 ^= rol32(dw, 1);
/* round 11 */
subRL[14] ^= kw4;
@@ -907,8 +907,8 @@ static void camellia_setup_tail(u64 *subkey, u64 *subRL, int max)
kw4 ^= (kw4 & ~subRL[8]) << 32;
/* modified for FL(kl1) */
- dw = (kw4 & subRL[8]) >> 32,
- kw4 ^= rol32(dw, 1);
+ dw = (kw4 & subRL[8]) >> 32;
+ kw4 ^= rol32(dw, 1);
/* round 5 */
subRL[6] ^= kw4;
@@ -928,8 +928,8 @@ static void camellia_setup_tail(u64 *subkey, u64 *subRL, int max)
SET_SUBKEY_LR(6, subRL[5] ^ subRL[7]); /* round 5 */
tl = (subRL[10] >> 32) ^ (subRL[10] & ~subRL[8]);
- dw = tl & (subRL[8] >> 32), /* FL(kl1) */
- tr = subRL[10] ^ rol32(dw, 1);
+ dw = tl & (subRL[8] >> 32); /* FL(kl1) */
+ tr = subRL[10] ^ rol32(dw, 1);
tt = (tr | ((u64)tl << 32));
SET_SUBKEY_LR(7, subRL[6] ^ tt); /* round 6 */
@@ -937,8 +937,8 @@ static void camellia_setup_tail(u64 *subkey, u64 *subRL, int max)
SET_SUBKEY_LR(9, subRL[9]); /* FLinv(kl2) */
tl = (subRL[7] >> 32) ^ (subRL[7] & ~subRL[9]);
- dw = tl & (subRL[9] >> 32), /* FLinv(kl2) */
- tr = subRL[7] ^ rol32(dw, 1);
+ dw = tl & (subRL[9] >> 32); /* FLinv(kl2) */
+ tr = subRL[7] ^ rol32(dw, 1);
tt = (tr | ((u64)tl << 32));
SET_SUBKEY_LR(10, subRL[11] ^ tt); /* round 7 */
@@ -948,8 +948,8 @@ static void camellia_setup_tail(u64 *subkey, u64 *subRL, int max)
SET_SUBKEY_LR(14, subRL[13] ^ subRL[15]); /* round 11 */
tl = (subRL[18] >> 32) ^ (subRL[18] & ~subRL[16]);
- dw = tl & (subRL[16] >> 32), /* FL(kl3) */
- tr = subRL[18] ^ rol32(dw, 1);
+ dw = tl & (subRL[16] >> 32); /* FL(kl3) */
+ tr = subRL[18] ^ rol32(dw, 1);
tt = (tr | ((u64)tl << 32));
SET_SUBKEY_LR(15, subRL[14] ^ tt); /* round 12 */
@@ -957,8 +957,8 @@ static void camellia_setup_tail(u64 *subkey, u64 *subRL, int max)
SET_SUBKEY_LR(17, subRL[17]); /* FLinv(kl4) */
tl = (subRL[15] >> 32) ^ (subRL[15] & ~subRL[17]);
- dw = tl & (subRL[17] >> 32), /* FLinv(kl4) */
- tr = subRL[15] ^ rol32(dw, 1);
+ dw = tl & (subRL[17] >> 32); /* FLinv(kl4) */
+ tr = subRL[15] ^ rol32(dw, 1);
tt = (tr | ((u64)tl << 32));
SET_SUBKEY_LR(18, subRL[19] ^ tt); /* round 13 */
@@ -972,8 +972,8 @@ static void camellia_setup_tail(u64 *subkey, u64 *subRL, int max)
SET_SUBKEY_LR(24, subRL[24] ^ subRL[23]); /* kw3 */
} else {
tl = (subRL[26] >> 32) ^ (subRL[26] & ~subRL[24]);
- dw = tl & (subRL[24] >> 32), /* FL(kl5) */
- tr = subRL[26] ^ rol32(dw, 1);
+ dw = tl & (subRL[24] >> 32); /* FL(kl5) */
+ tr = subRL[26] ^ rol32(dw, 1);
tt = (tr | ((u64)tl << 32));
SET_SUBKEY_LR(23, subRL[22] ^ tt); /* round 18 */
@@ -981,8 +981,8 @@ static void camellia_setup_tail(u64 *subkey, u64 *subRL, int max)
SET_SUBKEY_LR(25, subRL[25]); /* FLinv(kl6) */
tl = (subRL[23] >> 32) ^ (subRL[23] & ~subRL[25]);
- dw = tl & (subRL[25] >> 32), /* FLinv(kl6) */
- tr = subRL[23] ^ rol32(dw, 1);
+ dw = tl & (subRL[25] >> 32); /* FLinv(kl6) */
+ tr = subRL[23] ^ rol32(dw, 1);
tt = (tr | ((u64)tl << 32));
SET_SUBKEY_LR(26, subRL[27] ^ tt); /* round 19 */
diff --git a/arch/x86/crypto/crct10dif-pcl-asm_64.S b/arch/x86/crypto/crct10dif-pcl-asm_64.S
new file mode 100644
index 000000000000..35e97569d05f
--- /dev/null
+++ b/arch/x86/crypto/crct10dif-pcl-asm_64.S
@@ -0,0 +1,643 @@
+########################################################################
+# Implement fast CRC-T10DIF computation with SSE and PCLMULQDQ instructions
+#
+# Copyright (c) 2013, Intel Corporation
+#
+# Authors:
+# Erdinc Ozturk <erdinc.ozturk@intel.com>
+# Vinodh Gopal <vinodh.gopal@intel.com>
+# James Guilford <james.guilford@intel.com>
+# Tim Chen <tim.c.chen@linux.intel.com>
+#
+# This software is available to you under a choice of one of two
+# licenses. You may choose to be licensed under the terms of the GNU
+# General Public License (GPL) Version 2, available from the file
+# COPYING in the main directory of this source tree, or the
+# OpenIB.org BSD license below:
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#
+# * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the
+# distribution.
+#
+# * Neither the name of the Intel Corporation nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+#
+# THIS SOFTWARE IS PROVIDED BY INTEL CORPORATION ""AS IS"" AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL INTEL CORPORATION OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+########################################################################
+# Function API:
+# UINT16 crc_t10dif_pcl(
+# UINT16 init_crc, //initial CRC value, 16 bits
+# const unsigned char *buf, //buffer pointer to calculate CRC on
+# UINT64 len //buffer length in bytes (64-bit data)
+# );
+#
+# Reference paper titled "Fast CRC Computation for Generic
+# Polynomials Using PCLMULQDQ Instruction"
+# URL: http://www.intel.com/content/dam/www/public/us/en/documents
+# /white-papers/fast-crc-computation-generic-polynomials-pclmulqdq-paper.pdf
+#
+#
+
+#include <linux/linkage.h>
+
+.text
+
+#define arg1 %rdi
+#define arg2 %rsi
+#define arg3 %rdx
+
+#define arg1_low32 %edi
+
+ENTRY(crc_t10dif_pcl)
+.align 16
+
+ # adjust the 16-bit initial_crc value, scale it to 32 bits
+ shl $16, arg1_low32
+
+ # Allocate Stack Space
+ mov %rsp, %rcx
+ sub $16*2, %rsp
+ # align stack to 16 byte boundary
+ and $~(0x10 - 1), %rsp
+
+ # check if smaller than 256
+ cmp $256, arg3
+
+ # for sizes less than 128, we can't fold 64B at a time...
+ jl _less_than_128
+
+
+ # load the initial crc value
+ movd arg1_low32, %xmm10 # initial crc
+
+ # crc value does not need to be byte-reflected, but it needs
+ # to be moved to the high part of the register.
+ # because data will be byte-reflected and will align with
+ # initial crc at correct place.
+ pslldq $12, %xmm10
+
+ movdqa SHUF_MASK(%rip), %xmm11
+ # receive the initial 64B data, xor the initial crc value
+ movdqu 16*0(arg2), %xmm0
+ movdqu 16*1(arg2), %xmm1
+ movdqu 16*2(arg2), %xmm2
+ movdqu 16*3(arg2), %xmm3
+ movdqu 16*4(arg2), %xmm4
+ movdqu 16*5(arg2), %xmm5
+ movdqu 16*6(arg2), %xmm6
+ movdqu 16*7(arg2), %xmm7
+
+ pshufb %xmm11, %xmm0
+ # XOR the initial_crc value
+ pxor %xmm10, %xmm0
+ pshufb %xmm11, %xmm1
+ pshufb %xmm11, %xmm2
+ pshufb %xmm11, %xmm3
+ pshufb %xmm11, %xmm4
+ pshufb %xmm11, %xmm5
+ pshufb %xmm11, %xmm6
+ pshufb %xmm11, %xmm7
+
+ movdqa rk3(%rip), %xmm10 #xmm10 has rk3 and rk4
+ #imm value of pclmulqdq instruction
+ #will determine which constant to use
+
+ #################################################################
+ # we subtract 256 instead of 128 to save one instruction from the loop
+ sub $256, arg3
+
+ # at this section of the code, there is 64*x+y (0<=y<64) bytes of
+ # buffer. The _fold_64_B_loop will fold 64B at a time
+ # until we have 64+y Bytes of buffer
+
+
+ # fold 64B at a time. This section of the code folds 4 xmm
+ # registers in parallel
+_fold_64_B_loop:
+
+ # update the buffer pointer
+ add $128, arg2 # buf += 64#
+
+ movdqu 16*0(arg2), %xmm9
+ movdqu 16*1(arg2), %xmm12
+ pshufb %xmm11, %xmm9
+ pshufb %xmm11, %xmm12
+ movdqa %xmm0, %xmm8
+ movdqa %xmm1, %xmm13
+ pclmulqdq $0x0 , %xmm10, %xmm0
+ pclmulqdq $0x11, %xmm10, %xmm8
+ pclmulqdq $0x0 , %xmm10, %xmm1
+ pclmulqdq $0x11, %xmm10, %xmm13
+ pxor %xmm9 , %xmm0
+ xorps %xmm8 , %xmm0
+ pxor %xmm12, %xmm1
+ xorps %xmm13, %xmm1
+
+ movdqu 16*2(arg2), %xmm9
+ movdqu 16*3(arg2), %xmm12
+ pshufb %xmm11, %xmm9
+ pshufb %xmm11, %xmm12
+ movdqa %xmm2, %xmm8
+ movdqa %xmm3, %xmm13
+ pclmulqdq $0x0, %xmm10, %xmm2
+ pclmulqdq $0x11, %xmm10, %xmm8
+ pclmulqdq $0x0, %xmm10, %xmm3
+ pclmulqdq $0x11, %xmm10, %xmm13
+ pxor %xmm9 , %xmm2
+ xorps %xmm8 , %xmm2
+ pxor %xmm12, %xmm3
+ xorps %xmm13, %xmm3
+
+ movdqu 16*4(arg2), %xmm9
+ movdqu 16*5(arg2), %xmm12
+ pshufb %xmm11, %xmm9
+ pshufb %xmm11, %xmm12
+ movdqa %xmm4, %xmm8
+ movdqa %xmm5, %xmm13
+ pclmulqdq $0x0, %xmm10, %xmm4
+ pclmulqdq $0x11, %xmm10, %xmm8
+ pclmulqdq $0x0, %xmm10, %xmm5
+ pclmulqdq $0x11, %xmm10, %xmm13
+ pxor %xmm9 , %xmm4
+ xorps %xmm8 , %xmm4
+ pxor %xmm12, %xmm5
+ xorps %xmm13, %xmm5
+
+ movdqu 16*6(arg2), %xmm9
+ movdqu 16*7(arg2), %xmm12
+ pshufb %xmm11, %xmm9
+ pshufb %xmm11, %xmm12
+ movdqa %xmm6 , %xmm8
+ movdqa %xmm7 , %xmm13
+ pclmulqdq $0x0 , %xmm10, %xmm6
+ pclmulqdq $0x11, %xmm10, %xmm8
+ pclmulqdq $0x0 , %xmm10, %xmm7
+ pclmulqdq $0x11, %xmm10, %xmm13
+ pxor %xmm9 , %xmm6
+ xorps %xmm8 , %xmm6
+ pxor %xmm12, %xmm7
+ xorps %xmm13, %xmm7
+
+ sub $128, arg3
+
+ # check if there is another 64B in the buffer to be able to fold
+ jge _fold_64_B_loop
+ ##################################################################
+
+
+ add $128, arg2
+ # at this point, the buffer pointer is pointing at the last y Bytes
+ # of the buffer the 64B of folded data is in 4 of the xmm
+ # registers: xmm0, xmm1, xmm2, xmm3
+
+
+ # fold the 8 xmm registers to 1 xmm register with different constants
+
+ movdqa rk9(%rip), %xmm10
+ movdqa %xmm0, %xmm8
+ pclmulqdq $0x11, %xmm10, %xmm0
+ pclmulqdq $0x0 , %xmm10, %xmm8
+ pxor %xmm8, %xmm7
+ xorps %xmm0, %xmm7
+
+ movdqa rk11(%rip), %xmm10
+ movdqa %xmm1, %xmm8
+ pclmulqdq $0x11, %xmm10, %xmm1
+ pclmulqdq $0x0 , %xmm10, %xmm8
+ pxor %xmm8, %xmm7
+ xorps %xmm1, %xmm7
+
+ movdqa rk13(%rip), %xmm10
+ movdqa %xmm2, %xmm8
+ pclmulqdq $0x11, %xmm10, %xmm2
+ pclmulqdq $0x0 , %xmm10, %xmm8
+ pxor %xmm8, %xmm7
+ pxor %xmm2, %xmm7
+
+ movdqa rk15(%rip), %xmm10
+ movdqa %xmm3, %xmm8
+ pclmulqdq $0x11, %xmm10, %xmm3
+ pclmulqdq $0x0 , %xmm10, %xmm8
+ pxor %xmm8, %xmm7
+ xorps %xmm3, %xmm7
+
+ movdqa rk17(%rip), %xmm10
+ movdqa %xmm4, %xmm8
+ pclmulqdq $0x11, %xmm10, %xmm4
+ pclmulqdq $0x0 , %xmm10, %xmm8
+ pxor %xmm8, %xmm7
+ pxor %xmm4, %xmm7
+
+ movdqa rk19(%rip), %xmm10
+ movdqa %xmm5, %xmm8
+ pclmulqdq $0x11, %xmm10, %xmm5
+ pclmulqdq $0x0 , %xmm10, %xmm8
+ pxor %xmm8, %xmm7
+ xorps %xmm5, %xmm7
+
+ movdqa rk1(%rip), %xmm10 #xmm10 has rk1 and rk2
+ #imm value of pclmulqdq instruction
+ #will determine which constant to use
+ movdqa %xmm6, %xmm8
+ pclmulqdq $0x11, %xmm10, %xmm6
+ pclmulqdq $0x0 , %xmm10, %xmm8
+ pxor %xmm8, %xmm7
+ pxor %xmm6, %xmm7
+
+
+ # instead of 64, we add 48 to the loop counter to save 1 instruction
+ # from the loop instead of a cmp instruction, we use the negative
+ # flag with the jl instruction
+ add $128-16, arg3
+ jl _final_reduction_for_128
+
+ # now we have 16+y bytes left to reduce. 16 Bytes is in register xmm7
+ # and the rest is in memory. We can fold 16 bytes at a time if y>=16
+ # continue folding 16B at a time
+
+_16B_reduction_loop:
+ movdqa %xmm7, %xmm8
+ pclmulqdq $0x11, %xmm10, %xmm7
+ pclmulqdq $0x0 , %xmm10, %xmm8
+ pxor %xmm8, %xmm7
+ movdqu (arg2), %xmm0
+ pshufb %xmm11, %xmm0
+ pxor %xmm0 , %xmm7
+ add $16, arg2
+ sub $16, arg3
+ # instead of a cmp instruction, we utilize the flags with the
+ # jge instruction equivalent of: cmp arg3, 16-16
+ # check if there is any more 16B in the buffer to be able to fold
+ jge _16B_reduction_loop
+
+ #now we have 16+z bytes left to reduce, where 0<= z < 16.
+ #first, we reduce the data in the xmm7 register
+
+
+_final_reduction_for_128:
+ # check if any more data to fold. If not, compute the CRC of
+ # the final 128 bits
+ add $16, arg3
+ je _128_done
+
+ # here we are getting data that is less than 16 bytes.
+ # since we know that there was data before the pointer, we can
+ # offset the input pointer before the actual point, to receive
+ # exactly 16 bytes. after that the registers need to be adjusted.
+_get_last_two_xmms:
+ movdqa %xmm7, %xmm2
+
+ movdqu -16(arg2, arg3), %xmm1
+ pshufb %xmm11, %xmm1
+
+ # get rid of the extra data that was loaded before
+ # load the shift constant
+ lea pshufb_shf_table+16(%rip), %rax
+ sub arg3, %rax
+ movdqu (%rax), %xmm0
+
+ # shift xmm2 to the left by arg3 bytes
+ pshufb %xmm0, %xmm2
+
+ # shift xmm7 to the right by 16-arg3 bytes
+ pxor mask1(%rip), %xmm0
+ pshufb %xmm0, %xmm7
+ pblendvb %xmm2, %xmm1 #xmm0 is implicit
+
+ # fold 16 Bytes
+ movdqa %xmm1, %xmm2
+ movdqa %xmm7, %xmm8
+ pclmulqdq $0x11, %xmm10, %xmm7
+ pclmulqdq $0x0 , %xmm10, %xmm8
+ pxor %xmm8, %xmm7
+ pxor %xmm2, %xmm7
+
+_128_done:
+ # compute crc of a 128-bit value
+ movdqa rk5(%rip), %xmm10 # rk5 and rk6 in xmm10
+ movdqa %xmm7, %xmm0
+
+ #64b fold
+ pclmulqdq $0x1, %xmm10, %xmm7
+ pslldq $8 , %xmm0
+ pxor %xmm0, %xmm7
+
+ #32b fold
+ movdqa %xmm7, %xmm0
+
+ pand mask2(%rip), %xmm0
+
+ psrldq $12, %xmm7
+ pclmulqdq $0x10, %xmm10, %xmm7
+ pxor %xmm0, %xmm7
+
+ #barrett reduction
+_barrett:
+ movdqa rk7(%rip), %xmm10 # rk7 and rk8 in xmm10
+ movdqa %xmm7, %xmm0
+ pclmulqdq $0x01, %xmm10, %xmm7
+ pslldq $4, %xmm7
+ pclmulqdq $0x11, %xmm10, %xmm7
+
+ pslldq $4, %xmm7
+ pxor %xmm0, %xmm7
+ pextrd $1, %xmm7, %eax
+
+_cleanup:
+ # scale the result back to 16 bits
+ shr $16, %eax
+ mov %rcx, %rsp
+ ret
+
+########################################################################
+
+.align 16
+_less_than_128:
+
+ # check if there is enough buffer to be able to fold 16B at a time
+ cmp $32, arg3
+ jl _less_than_32
+ movdqa SHUF_MASK(%rip), %xmm11
+
+ # now if there is, load the constants
+ movdqa rk1(%rip), %xmm10 # rk1 and rk2 in xmm10
+
+ movd arg1_low32, %xmm0 # get the initial crc value
+ pslldq $12, %xmm0 # align it to its correct place
+ movdqu (arg2), %xmm7 # load the plaintext
+ pshufb %xmm11, %xmm7 # byte-reflect the plaintext
+ pxor %xmm0, %xmm7
+
+
+ # update the buffer pointer
+ add $16, arg2
+
+ # update the counter. subtract 32 instead of 16 to save one
+ # instruction from the loop
+ sub $32, arg3
+
+ jmp _16B_reduction_loop
+
+
+.align 16
+_less_than_32:
+ # mov initial crc to the return value. this is necessary for
+ # zero-length buffers.
+ mov arg1_low32, %eax
+ test arg3, arg3
+ je _cleanup
+
+ movdqa SHUF_MASK(%rip), %xmm11
+
+ movd arg1_low32, %xmm0 # get the initial crc value
+ pslldq $12, %xmm0 # align it to its correct place
+
+ cmp $16, arg3
+ je _exact_16_left
+ jl _less_than_16_left
+
+ movdqu (arg2), %xmm7 # load the plaintext
+ pshufb %xmm11, %xmm7 # byte-reflect the plaintext
+ pxor %xmm0 , %xmm7 # xor the initial crc value
+ add $16, arg2
+ sub $16, arg3
+ movdqa rk1(%rip), %xmm10 # rk1 and rk2 in xmm10
+ jmp _get_last_two_xmms
+
+
+.align 16
+_less_than_16_left:
+ # use stack space to load data less than 16 bytes, zero-out
+ # the 16B in memory first.
+
+ pxor %xmm1, %xmm1
+ mov %rsp, %r11
+ movdqa %xmm1, (%r11)
+
+ cmp $4, arg3
+ jl _only_less_than_4
+
+ # backup the counter value
+ mov arg3, %r9
+ cmp $8, arg3
+ jl _less_than_8_left
+
+ # load 8 Bytes
+ mov (arg2), %rax
+ mov %rax, (%r11)
+ add $8, %r11
+ sub $8, arg3
+ add $8, arg2
+_less_than_8_left:
+
+ cmp $4, arg3
+ jl _less_than_4_left
+
+ # load 4 Bytes
+ mov (arg2), %eax
+ mov %eax, (%r11)
+ add $4, %r11
+ sub $4, arg3
+ add $4, arg2
+_less_than_4_left:
+
+ cmp $2, arg3
+ jl _less_than_2_left
+
+ # load 2 Bytes
+ mov (arg2), %ax
+ mov %ax, (%r11)
+ add $2, %r11
+ sub $2, arg3
+ add $2, arg2
+_less_than_2_left:
+ cmp $1, arg3
+ jl _zero_left
+
+ # load 1 Byte
+ mov (arg2), %al
+ mov %al, (%r11)
+_zero_left:
+ movdqa (%rsp), %xmm7
+ pshufb %xmm11, %xmm7
+ pxor %xmm0 , %xmm7 # xor the initial crc value
+
+ # shl r9, 4
+ lea pshufb_shf_table+16(%rip), %rax
+ sub %r9, %rax
+ movdqu (%rax), %xmm0
+ pxor mask1(%rip), %xmm0
+
+ pshufb %xmm0, %xmm7
+ jmp _128_done
+
+.align 16
+_exact_16_left:
+ movdqu (arg2), %xmm7
+ pshufb %xmm11, %xmm7
+ pxor %xmm0 , %xmm7 # xor the initial crc value
+
+ jmp _128_done
+
+_only_less_than_4:
+ cmp $3, arg3
+ jl _only_less_than_3
+
+ # load 3 Bytes
+ mov (arg2), %al
+ mov %al, (%r11)
+
+ mov 1(arg2), %al
+ mov %al, 1(%r11)
+
+ mov 2(arg2), %al
+ mov %al, 2(%r11)
+
+ movdqa (%rsp), %xmm7
+ pshufb %xmm11, %xmm7
+ pxor %xmm0 , %xmm7 # xor the initial crc value
+
+ psrldq $5, %xmm7
+
+ jmp _barrett
+_only_less_than_3:
+ cmp $2, arg3
+ jl _only_less_than_2
+
+ # load 2 Bytes
+ mov (arg2), %al
+ mov %al, (%r11)
+
+ mov 1(arg2), %al
+ mov %al, 1(%r11)
+
+ movdqa (%rsp), %xmm7
+ pshufb %xmm11, %xmm7
+ pxor %xmm0 , %xmm7 # xor the initial crc value
+
+ psrldq $6, %xmm7
+
+ jmp _barrett
+_only_less_than_2:
+
+ # load 1 Byte
+ mov (arg2), %al
+ mov %al, (%r11)
+
+ movdqa (%rsp), %xmm7
+ pshufb %xmm11, %xmm7
+ pxor %xmm0 , %xmm7 # xor the initial crc value
+
+ psrldq $7, %xmm7
+
+ jmp _barrett
+
+ENDPROC(crc_t10dif_pcl)
+
+.data
+
+# precomputed constants
+# these constants are precomputed from the poly:
+# 0x8bb70000 (0x8bb7 scaled to 32 bits)
+.align 16
+# Q = 0x18BB70000
+# rk1 = 2^(32*3) mod Q << 32
+# rk2 = 2^(32*5) mod Q << 32
+# rk3 = 2^(32*15) mod Q << 32
+# rk4 = 2^(32*17) mod Q << 32
+# rk5 = 2^(32*3) mod Q << 32
+# rk6 = 2^(32*2) mod Q << 32
+# rk7 = floor(2^64/Q)
+# rk8 = Q
+rk1:
+.quad 0x2d56000000000000
+rk2:
+.quad 0x06df000000000000
+rk3:
+.quad 0x9d9d000000000000
+rk4:
+.quad 0x7cf5000000000000
+rk5:
+.quad 0x2d56000000000000
+rk6:
+.quad 0x1368000000000000
+rk7:
+.quad 0x00000001f65a57f8
+rk8:
+.quad 0x000000018bb70000
+
+rk9:
+.quad 0xceae000000000000
+rk10:
+.quad 0xbfd6000000000000
+rk11:
+.quad 0x1e16000000000000
+rk12:
+.quad 0x713c000000000000
+rk13:
+.quad 0xf7f9000000000000
+rk14:
+.quad 0x80a6000000000000
+rk15:
+.quad 0x044c000000000000
+rk16:
+.quad 0xe658000000000000
+rk17:
+.quad 0xad18000000000000
+rk18:
+.quad 0xa497000000000000
+rk19:
+.quad 0x6ee3000000000000
+rk20:
+.quad 0xe7b5000000000000
+
+
+
+mask1:
+.octa 0x80808080808080808080808080808080
+mask2:
+.octa 0x00000000FFFFFFFFFFFFFFFFFFFFFFFF
+
+SHUF_MASK:
+.octa 0x000102030405060708090A0B0C0D0E0F
+
+pshufb_shf_table:
+# use these values for shift constants for the pshufb instruction
+# different alignments result in values as shown:
+# DDQ 0x008f8e8d8c8b8a898887868584838281 # shl 15 (16-1) / shr1
+# DDQ 0x01008f8e8d8c8b8a8988878685848382 # shl 14 (16-3) / shr2
+# DDQ 0x0201008f8e8d8c8b8a89888786858483 # shl 13 (16-4) / shr3
+# DDQ 0x030201008f8e8d8c8b8a898887868584 # shl 12 (16-4) / shr4
+# DDQ 0x04030201008f8e8d8c8b8a8988878685 # shl 11 (16-5) / shr5
+# DDQ 0x0504030201008f8e8d8c8b8a89888786 # shl 10 (16-6) / shr6
+# DDQ 0x060504030201008f8e8d8c8b8a898887 # shl 9 (16-7) / shr7
+# DDQ 0x07060504030201008f8e8d8c8b8a8988 # shl 8 (16-8) / shr8
+# DDQ 0x0807060504030201008f8e8d8c8b8a89 # shl 7 (16-9) / shr9
+# DDQ 0x090807060504030201008f8e8d8c8b8a # shl 6 (16-10) / shr10
+# DDQ 0x0a090807060504030201008f8e8d8c8b # shl 5 (16-11) / shr11
+# DDQ 0x0b0a090807060504030201008f8e8d8c # shl 4 (16-12) / shr12
+# DDQ 0x0c0b0a090807060504030201008f8e8d # shl 3 (16-13) / shr13
+# DDQ 0x0d0c0b0a090807060504030201008f8e # shl 2 (16-14) / shr14
+# DDQ 0x0e0d0c0b0a090807060504030201008f # shl 1 (16-15) / shr15
+.octa 0x8f8e8d8c8b8a89888786858483828100
+.octa 0x000e0d0c0b0a09080706050403020100
diff --git a/arch/x86/crypto/crct10dif-pclmul_glue.c b/arch/x86/crypto/crct10dif-pclmul_glue.c
new file mode 100644
index 000000000000..7845d7fd54c0
--- /dev/null
+++ b/arch/x86/crypto/crct10dif-pclmul_glue.c
@@ -0,0 +1,151 @@
+/*
+ * Cryptographic API.
+ *
+ * T10 Data Integrity Field CRC16 Crypto Transform using PCLMULQDQ Instructions
+ *
+ * Copyright (C) 2013 Intel Corporation
+ * Author: Tim Chen <tim.c.chen@linux.intel.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; either version 2 of the License, or (at your option)
+ * any later version.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ */
+
+#include <linux/types.h>
+#include <linux/module.h>
+#include <linux/crc-t10dif.h>
+#include <crypto/internal/hash.h>
+#include <linux/init.h>
+#include <linux/string.h>
+#include <linux/kernel.h>
+#include <asm/i387.h>
+#include <asm/cpufeature.h>
+#include <asm/cpu_device_id.h>
+
+asmlinkage __u16 crc_t10dif_pcl(__u16 crc, const unsigned char *buf,
+ size_t len);
+
+struct chksum_desc_ctx {
+ __u16 crc;
+};
+
+/*
+ * Steps through buffer one byte at at time, calculates reflected
+ * crc using table.
+ */
+
+static int chksum_init(struct shash_desc *desc)
+{
+ struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
+
+ ctx->crc = 0;
+
+ return 0;
+}
+
+static int chksum_update(struct shash_desc *desc, const u8 *data,
+ unsigned int length)
+{
+ struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
+
+ if (irq_fpu_usable()) {
+ kernel_fpu_begin();
+ ctx->crc = crc_t10dif_pcl(ctx->crc, data, length);
+ kernel_fpu_end();
+ } else
+ ctx->crc = crc_t10dif_generic(ctx->crc, data, length);
+ return 0;
+}
+
+static int chksum_final(struct shash_desc *desc, u8 *out)
+{
+ struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
+
+ *(__u16 *)out = ctx->crc;
+ return 0;
+}
+
+static int __chksum_finup(__u16 *crcp, const u8 *data, unsigned int len,
+ u8 *out)
+{
+ if (irq_fpu_usable()) {
+ kernel_fpu_begin();
+ *(__u16 *)out = crc_t10dif_pcl(*crcp, data, len);
+ kernel_fpu_end();
+ } else
+ *(__u16 *)out = crc_t10dif_generic(*crcp, data, len);
+ return 0;
+}
+
+static int chksum_finup(struct shash_desc *desc, const u8 *data,
+ unsigned int len, u8 *out)
+{
+ struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
+
+ return __chksum_finup(&ctx->crc, data, len, out);
+}
+
+static int chksum_digest(struct shash_desc *desc, const u8 *data,
+ unsigned int length, u8 *out)
+{
+ struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
+
+ return __chksum_finup(&ctx->crc, data, length, out);
+}
+
+static struct shash_alg alg = {
+ .digestsize = CRC_T10DIF_DIGEST_SIZE,
+ .init = chksum_init,
+ .update = chksum_update,
+ .final = chksum_final,
+ .finup = chksum_finup,
+ .digest = chksum_digest,
+ .descsize = sizeof(struct chksum_desc_ctx),
+ .base = {
+ .cra_name = "crct10dif",
+ .cra_driver_name = "crct10dif-pclmul",
+ .cra_priority = 200,
+ .cra_blocksize = CRC_T10DIF_BLOCK_SIZE,
+ .cra_module = THIS_MODULE,
+ }
+};
+
+static const struct x86_cpu_id crct10dif_cpu_id[] = {
+ X86_FEATURE_MATCH(X86_FEATURE_PCLMULQDQ),
+ {}
+};
+MODULE_DEVICE_TABLE(x86cpu, crct10dif_cpu_id);
+
+static int __init crct10dif_intel_mod_init(void)
+{
+ if (!x86_match_cpu(crct10dif_cpu_id))
+ return -ENODEV;
+
+ return crypto_register_shash(&alg);
+}
+
+static void __exit crct10dif_intel_mod_fini(void)
+{
+ crypto_unregister_shash(&alg);
+}
+
+module_init(crct10dif_intel_mod_init);
+module_exit(crct10dif_intel_mod_fini);
+
+MODULE_AUTHOR("Tim Chen <tim.c.chen@linux.intel.com>");
+MODULE_DESCRIPTION("T10 DIF CRC calculation accelerated with PCLMULQDQ.");
+MODULE_LICENSE("GPL");
+
+MODULE_ALIAS("crct10dif");
+MODULE_ALIAS("crct10dif-pclmul");
diff --git a/arch/x86/include/asm/xor_avx.h b/arch/x86/include/asm/xor_avx.h
index 7ea79c5fa1f2..492b29802f57 100644
--- a/arch/x86/include/asm/xor_avx.h
+++ b/arch/x86/include/asm/xor_avx.h
@@ -167,12 +167,12 @@ static struct xor_block_template xor_block_avx = {
#define AVX_XOR_SPEED \
do { \
- if (cpu_has_avx) \
+ if (cpu_has_avx && cpu_has_osxsave) \
xor_speed(&xor_block_avx); \
} while (0)
#define AVX_SELECT(FASTEST) \
- (cpu_has_avx ? &xor_block_avx : FASTEST)
+ (cpu_has_avx && cpu_has_osxsave ? &xor_block_avx : FASTEST)
#else
diff --git a/crypto/Kconfig b/crypto/Kconfig
index aca01164f002..69ce573f1224 100644
--- a/crypto/Kconfig
+++ b/crypto/Kconfig
@@ -376,6 +376,25 @@ config CRYPTO_CRC32_PCLMUL
which will enable any routine to use the CRC-32-IEEE 802.3 checksum
and gain better performance as compared with the table implementation.
+config CRYPTO_CRCT10DIF
+ tristate "CRCT10DIF algorithm"
+ select CRYPTO_HASH
+ help
+ CRC T10 Data Integrity Field computation is being cast as
+ a crypto transform. This allows for faster crc t10 diff
+ transforms to be used if they are available.
+
+config CRYPTO_CRCT10DIF_PCLMUL
+ tristate "CRCT10DIF PCLMULQDQ hardware acceleration"
+ depends on X86 && 64BIT && CRC_T10DIF
+ select CRYPTO_HASH
+ help
+ For x86_64 processors with SSE4.2 and PCLMULQDQ supported,
+ CRC T10 DIF PCLMULQDQ computation can be hardware
+ accelerated PCLMULQDQ instruction. This option will create
+ 'crct10dif-plcmul' module, which is faster when computing the
+ crct10dif checksum as compared with the generic table implementation.
+
config CRYPTO_GHASH
tristate "GHASH digest algorithm"
select CRYPTO_GF128MUL
diff --git a/crypto/Makefile b/crypto/Makefile
index 2ba0df2f908f..2d5ed08a239f 100644
--- a/crypto/Makefile
+++ b/crypto/Makefile
@@ -83,6 +83,7 @@ obj-$(CONFIG_CRYPTO_ZLIB) += zlib.o
obj-$(CONFIG_CRYPTO_MICHAEL_MIC) += michael_mic.o
obj-$(CONFIG_CRYPTO_CRC32C) += crc32c.o
obj-$(CONFIG_CRYPTO_CRC32) += crc32.o
+obj-$(CONFIG_CRYPTO_CRCT10DIF) += crct10dif.o
obj-$(CONFIG_CRYPTO_AUTHENC) += authenc.o authencesn.o
obj-$(CONFIG_CRYPTO_LZO) += lzo.o
obj-$(CONFIG_CRYPTO_LZ4) += lz4.o
diff --git a/crypto/aes_generic.c b/crypto/aes_generic.c
index 47f2e5c71759..fd0d6b454975 100644
--- a/crypto/aes_generic.c
+++ b/crypto/aes_generic.c
@@ -62,7 +62,7 @@ static inline u8 byte(const u32 x, const unsigned n)
static const u32 rco_tab[10] = { 1, 2, 4, 8, 16, 32, 64, 128, 27, 54 };
-const u32 crypto_ft_tab[4][256] = {
+__visible const u32 crypto_ft_tab[4][256] = {
{
0xa56363c6, 0x847c7cf8, 0x997777ee, 0x8d7b7bf6,
0x0df2f2ff, 0xbd6b6bd6, 0xb16f6fde, 0x54c5c591,
@@ -326,7 +326,7 @@ const u32 crypto_ft_tab[4][256] = {
}
};
-const u32 crypto_fl_tab[4][256] = {
+__visible const u32 crypto_fl_tab[4][256] = {
{
0x00000063, 0x0000007c, 0x00000077, 0x0000007b,
0x000000f2, 0x0000006b, 0x0000006f, 0x000000c5,
@@ -590,7 +590,7 @@ const u32 crypto_fl_tab[4][256] = {
}
};
-const u32 crypto_it_tab[4][256] = {
+__visible const u32 crypto_it_tab[4][256] = {
{
0x50a7f451, 0x5365417e, 0xc3a4171a, 0x965e273a,
0xcb6bab3b, 0xf1459d1f, 0xab58faac, 0x9303e34b,
@@ -854,7 +854,7 @@ const u32 crypto_it_tab[4][256] = {
}
};
-const u32 crypto_il_tab[4][256] = {
+__visible const u32 crypto_il_tab[4][256] = {
{
0x00000052, 0x00000009, 0x0000006a, 0x000000d5,
0x00000030, 0x00000036, 0x000000a5, 0x00000038,
diff --git a/crypto/camellia_generic.c b/crypto/camellia_generic.c
index 75efa2052305..26bcd7a2d6b4 100644
--- a/crypto/camellia_generic.c
+++ b/crypto/camellia_generic.c
@@ -388,8 +388,8 @@ static void camellia_setup_tail(u32 *subkey, u32 *subL, u32 *subR, int max)
/* round 6 */
subL[7] ^= subL[1]; subR[7] ^= subR[1];
subL[1] ^= subR[1] & ~subR[9];
- dw = subL[1] & subL[9],
- subR[1] ^= rol32(dw, 1); /* modified for FLinv(kl2) */
+ dw = subL[1] & subL[9];
+ subR[1] ^= rol32(dw, 1); /* modified for FLinv(kl2) */
/* round 8 */
subL[11] ^= subL[1]; subR[11] ^= subR[1];
/* round 10 */
@@ -397,8 +397,8 @@ static void camellia_setup_tail(u32 *subkey, u32 *subL, u32 *subR, int max)
/* round 12 */
subL[15] ^= subL[1]; subR[15] ^= subR[1];
subL[1] ^= subR[1] & ~subR[17];
- dw = subL[1] & subL[17],
- subR[1] ^= rol32(dw, 1); /* modified for FLinv(kl4) */
+ dw = subL[1] & subL[17];
+ subR[1] ^= rol32(dw, 1); /* modified for FLinv(kl4) */
/* round 14 */
subL[19] ^= subL[1]; subR[19] ^= subR[1];
/* round 16 */
@@ -413,8 +413,8 @@ static void camellia_setup_tail(u32 *subkey, u32 *subL, u32 *subR, int max)
kw4l = subL[25]; kw4r = subR[25];
} else {
subL[1] ^= subR[1] & ~subR[25];
- dw = subL[1] & subL[25],
- subR[1] ^= rol32(dw, 1); /* modified for FLinv(kl6) */
+ dw = subL[1] & subL[25];
+ subR[1] ^= rol32(dw, 1); /* modified for FLinv(kl6) */
/* round 20 */
subL[27] ^= subL[1]; subR[27] ^= subR[1];
/* round 22 */
@@ -433,8 +433,8 @@ static void camellia_setup_tail(u32 *subkey, u32 *subL, u32 *subR, int max)
/* round 19 */
subL[26] ^= kw4l; subR[26] ^= kw4r;
kw4l ^= kw4r & ~subR[24];
- dw = kw4l & subL[24],
- kw4r ^= rol32(dw, 1); /* modified for FL(kl5) */
+ dw = kw4l & subL[24];
+ kw4r ^= rol32(dw, 1); /* modified for FL(kl5) */
}
/* round 17 */
subL[22] ^= kw4l; subR[22] ^= kw4r;
@@ -443,8 +443,8 @@ static void camellia_setup_tail(u32 *subkey, u32 *subL, u32 *subR, int max)
/* round 13 */
subL[18] ^= kw4l; subR[18] ^= kw4r;
kw4l ^= kw4r & ~subR[16];
- dw = kw4l & subL[16],
- kw4r ^= rol32(dw, 1); /* modified for FL(kl3) */
+ dw = kw4l & subL[16];
+ kw4r ^= rol32(dw, 1); /* modified for FL(kl3) */
/* round 11 */
subL[14] ^= kw4l; subR[14] ^= kw4r;
/* round 9 */
@@ -452,8 +452,8 @@ static void camellia_setup_tail(u32 *subkey, u32 *subL, u32 *subR, int max)
/* round 7 */
subL[10] ^= kw4l; subR[10] ^= kw4r;
kw4l ^= kw4r & ~subR[8];
- dw = kw4l & subL[8],
- kw4r ^= rol32(dw, 1); /* modified for FL(kl1) */
+ dw = kw4l & subL[8];
+ kw4r ^= rol32(dw, 1); /* modified for FL(kl1) */
/* round 5 */
subL[6] ^= kw4l; subR[6] ^= kw4r;
/* round 3 */
@@ -477,8 +477,8 @@ static void camellia_setup_tail(u32 *subkey, u32 *subL, u32 *subR, int max)
SUBKEY_L(6) = subL[5] ^ subL[7]; /* round 5 */
SUBKEY_R(6) = subR[5] ^ subR[7];
tl = subL[10] ^ (subR[10] & ~subR[8]);
- dw = tl & subL[8], /* FL(kl1) */
- tr = subR[10] ^ rol32(dw, 1);
+ dw = tl & subL[8]; /* FL(kl1) */
+ tr = subR[10] ^ rol32(dw, 1);
SUBKEY_L(7) = subL[6] ^ tl; /* round 6 */
SUBKEY_R(7) = subR[6] ^ tr;
SUBKEY_L(8) = subL[8]; /* FL(kl1) */
@@ -486,8 +486,8 @@ static void camellia_setup_tail(u32 *subkey, u32 *subL, u32 *subR, int max)
SUBKEY_L(9) = subL[9]; /* FLinv(kl2) */
SUBKEY_R(9) = subR[9];
tl = subL[7] ^ (subR[7] & ~subR[9]);
- dw = tl & subL[9], /* FLinv(kl2) */
- tr = subR[7] ^ rol32(dw, 1);
+ dw = tl & subL[9]; /* FLinv(kl2) */
+ tr = subR[7] ^ rol32(dw, 1);
SUBKEY_L(10) = tl ^ subL[11]; /* round 7 */
SUBKEY_R(10) = tr ^ subR[11];
SUBKEY_L(11) = subL[10] ^ subL[12]; /* round 8 */
@@ -499,8 +499,8 @@ static void camellia_setup_tail(u32 *subkey, u32 *subL, u32 *subR, int max)
SUBKEY_L(14) = subL[13] ^ subL[15]; /* round 11 */
SUBKEY_R(14) = subR[13] ^ subR[15];
tl = subL[18] ^ (subR[18] & ~subR[16]);
- dw = tl & subL[16], /* FL(kl3) */
- tr = subR[18] ^ rol32(dw, 1);
+ dw = tl & subL[16]; /* FL(kl3) */
+ tr = subR[18] ^ rol32(dw, 1);
SUBKEY_L(15) = subL[14] ^ tl; /* round 12 */
SUBKEY_R(15) = subR[14] ^ tr;
SUBKEY_L(16) = subL[16]; /* FL(kl3) */
@@ -508,8 +508,8 @@ static void camellia_setup_tail(u32 *subkey, u32 *subL, u32 *subR, int max)
SUBKEY_L(17) = subL[17]; /* FLinv(kl4) */
SUBKEY_R(17) = subR[17];
tl = subL[15] ^ (subR[15] & ~subR[17]);
- dw = tl & subL[17], /* FLinv(kl4) */
- tr = subR[15] ^ rol32(dw, 1);
+ dw = tl & subL[17]; /* FLinv(kl4) */
+ tr = subR[15] ^ rol32(dw, 1);
SUBKEY_L(18) = tl ^ subL[19]; /* round 13 */
SUBKEY_R(18) = tr ^ subR[19];
SUBKEY_L(19) = subL[18] ^ subL[20]; /* round 14 */
@@ -527,8 +527,8 @@ static void camellia_setup_tail(u32 *subkey, u32 *subL, u32 *subR, int max)
SUBKEY_R(24) = subR[24] ^ subR[23];
} else {
tl = subL[26] ^ (subR[26] & ~subR[24]);
- dw = tl & subL[24], /* FL(kl5) */
- tr = subR[26] ^ rol32(dw, 1);
+ dw = tl & subL[24]; /* FL(kl5) */
+ tr = subR[26] ^ rol32(dw, 1);
SUBKEY_L(23) = subL[22] ^ tl; /* round 18 */
SUBKEY_R(23) = subR[22] ^ tr;
SUBKEY_L(24) = subL[24]; /* FL(kl5) */
@@ -536,8 +536,8 @@ static void camellia_setup_tail(u32 *subkey, u32 *subL, u32 *subR, int max)
SUBKEY_L(25) = subL[25]; /* FLinv(kl6) */
SUBKEY_R(25) = subR[25];
tl = subL[23] ^ (subR[23] & ~subR[25]);
- dw = tl & subL[25], /* FLinv(kl6) */
- tr = subR[23] ^ rol32(dw, 1);
+ dw = tl & subL[25]; /* FLinv(kl6) */
+ tr = subR[23] ^ rol32(dw, 1);
SUBKEY_L(26) = tl ^ subL[27]; /* round 19 */
SUBKEY_R(26) = tr ^ subR[27];
SUBKEY_L(27) = subL[26] ^ subL[28]; /* round 20 */
diff --git a/crypto/cast_common.c b/crypto/cast_common.c
index a15f523d5f56..117dd8250f27 100644
--- a/crypto/cast_common.c
+++ b/crypto/cast_common.c
@@ -15,7 +15,7 @@
#include <linux/module.h>
#include <crypto/cast_common.h>
-const u32 cast_s1[256] = {
+__visible const u32 cast_s1[256] = {
0x30fb40d4, 0x9fa0ff0b, 0x6beccd2f, 0x3f258c7a, 0x1e213f2f,
0x9c004dd3, 0x6003e540, 0xcf9fc949,
0xbfd4af27, 0x88bbbdb5, 0xe2034090, 0x98d09675, 0x6e63a0e0,
@@ -83,7 +83,7 @@ const u32 cast_s1[256] = {
};
EXPORT_SYMBOL_GPL(cast_s1);
-const u32 cast_s2[256] = {
+__visible const u32 cast_s2[256] = {
0x1f201094, 0xef0ba75b, 0x69e3cf7e, 0x393f4380, 0xfe61cf7a,
0xeec5207a, 0x55889c94, 0x72fc0651,
0xada7ef79, 0x4e1d7235, 0xd55a63ce, 0xde0436ba, 0x99c430ef,
@@ -151,7 +151,7 @@ const u32 cast_s2[256] = {
};
EXPORT_SYMBOL_GPL(cast_s2);
-const u32 cast_s3[256] = {
+__visible const u32 cast_s3[256] = {
0x8defc240, 0x25fa5d9f, 0xeb903dbf, 0xe810c907, 0x47607fff,
0x369fe44b, 0x8c1fc644, 0xaececa90,
0xbeb1f9bf, 0xeefbcaea, 0xe8cf1950, 0x51df07ae, 0x920e8806,
@@ -219,7 +219,7 @@ const u32 cast_s3[256] = {
};
EXPORT_SYMBOL_GPL(cast_s3);
-const u32 cast_s4[256] = {
+__visible const u32 cast_s4[256] = {
0x9db30420, 0x1fb6e9de, 0xa7be7bef, 0xd273a298, 0x4a4f7bdb,
0x64ad8c57, 0x85510443, 0xfa020ed1,
0x7e287aff, 0xe60fb663, 0x095f35a1, 0x79ebf120, 0xfd059d43,
diff --git a/crypto/crct10dif.c b/crypto/crct10dif.c
new file mode 100644
index 000000000000..92aca96d6b98
--- /dev/null
+++ b/crypto/crct10dif.c
@@ -0,0 +1,178 @@
+/*
+ * Cryptographic API.
+ *
+ * T10 Data Integrity Field CRC16 Crypto Transform
+ *
+ * Copyright (c) 2007 Oracle Corporation. All rights reserved.
+ * Written by Martin K. Petersen <martin.petersen@oracle.com>
+ * Copyright (C) 2013 Intel Corporation
+ * Author: Tim Chen <tim.c.chen@linux.intel.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; either version 2 of the License, or (at your option)
+ * any later version.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ */
+
+#include <linux/types.h>
+#include <linux/module.h>
+#include <linux/crc-t10dif.h>
+#include <crypto/internal/hash.h>
+#include <linux/init.h>
+#include <linux/string.h>
+#include <linux/kernel.h>
+
+struct chksum_desc_ctx {
+ __u16 crc;
+};
+
+/* Table generated using the following polynomium:
+ * x^16 + x^15 + x^11 + x^9 + x^8 + x^7 + x^5 + x^4 + x^2 + x + 1
+ * gt: 0x8bb7
+ */
+static const __u16 t10_dif_crc_table[256] = {
+ 0x0000, 0x8BB7, 0x9CD9, 0x176E, 0xB205, 0x39B2, 0x2EDC, 0xA56B,
+ 0xEFBD, 0x640A, 0x7364, 0xF8D3, 0x5DB8, 0xD60F, 0xC161, 0x4AD6,
+ 0x54CD, 0xDF7A, 0xC814, 0x43A3, 0xE6C8, 0x6D7F, 0x7A11, 0xF1A6,
+ 0xBB70, 0x30C7, 0x27A9, 0xAC1E, 0x0975, 0x82C2, 0x95AC, 0x1E1B,
+ 0xA99A, 0x222D, 0x3543, 0xBEF4, 0x1B9F, 0x9028, 0x8746, 0x0CF1,
+ 0x4627, 0xCD90, 0xDAFE, 0x5149, 0xF422, 0x7F95, 0x68FB, 0xE34C,
+ 0xFD57, 0x76E0, 0x618E, 0xEA39, 0x4F52, 0xC4E5, 0xD38B, 0x583C,
+ 0x12EA, 0x995D, 0x8E33, 0x0584, 0xA0EF, 0x2B58, 0x3C36, 0xB781,
+ 0xD883, 0x5334, 0x445A, 0xCFED, 0x6A86, 0xE131, 0xF65F, 0x7DE8,
+ 0x373E, 0xBC89, 0xABE7, 0x2050, 0x853B, 0x0E8C, 0x19E2, 0x9255,
+ 0x8C4E, 0x07F9, 0x1097, 0x9B20, 0x3E4B, 0xB5FC, 0xA292, 0x2925,
+ 0x63F3, 0xE844, 0xFF2A, 0x749D, 0xD1F6, 0x5A41, 0x4D2F, 0xC698,
+ 0x7119, 0xFAAE, 0xEDC0, 0x6677, 0xC31C, 0x48AB, 0x5FC5, 0xD472,
+ 0x9EA4, 0x1513, 0x027D, 0x89CA, 0x2CA1, 0xA716, 0xB078, 0x3BCF,
+ 0x25D4, 0xAE63, 0xB90D, 0x32BA, 0x97D1, 0x1C66, 0x0B08, 0x80BF,
+ 0xCA69, 0x41DE, 0x56B0, 0xDD07, 0x786C, 0xF3DB, 0xE4B5, 0x6F02,
+ 0x3AB1, 0xB106, 0xA668, 0x2DDF, 0x88B4, 0x0303, 0x146D, 0x9FDA,
+ 0xD50C, 0x5EBB, 0x49D5, 0xC262, 0x6709, 0xECBE, 0xFBD0, 0x7067,
+ 0x6E7C, 0xE5CB, 0xF2A5, 0x7912, 0xDC79, 0x57CE, 0x40A0, 0xCB17,
+ 0x81C1, 0x0A76, 0x1D18, 0x96AF, 0x33C4, 0xB873, 0xAF1D, 0x24AA,
+ 0x932B, 0x189C, 0x0FF2, 0x8445, 0x212E, 0xAA99, 0xBDF7, 0x3640,
+ 0x7C96, 0xF721, 0xE04F, 0x6BF8, 0xCE93, 0x4524, 0x524A, 0xD9FD,
+ 0xC7E6, 0x4C51, 0x5B3F, 0xD088, 0x75E3, 0xFE54, 0xE93A, 0x628D,
+ 0x285B, 0xA3EC, 0xB482, 0x3F35, 0x9A5E, 0x11E9, 0x0687, 0x8D30,
+ 0xE232, 0x6985, 0x7EEB, 0xF55C, 0x5037, 0xDB80, 0xCCEE, 0x4759,
+ 0x0D8F, 0x8638, 0x9156, 0x1AE1, 0xBF8A, 0x343D, 0x2353, 0xA8E4,
+ 0xB6FF, 0x3D48, 0x2A26, 0xA191, 0x04FA, 0x8F4D, 0x9823, 0x1394,
+ 0x5942, 0xD2F5, 0xC59B, 0x4E2C, 0xEB47, 0x60F0, 0x779E, 0xFC29,
+ 0x4BA8, 0xC01F, 0xD771, 0x5CC6, 0xF9AD, 0x721A, 0x6574, 0xEEC3,
+ 0xA415, 0x2FA2, 0x38CC, 0xB37B, 0x1610, 0x9DA7, 0x8AC9, 0x017E,
+ 0x1F65, 0x94D2, 0x83BC, 0x080B, 0xAD60, 0x26D7, 0x31B9, 0xBA0E,
+ 0xF0D8, 0x7B6F, 0x6C01, 0xE7B6, 0x42DD, 0xC96A, 0xDE04, 0x55B3
+};
+
+__u16 crc_t10dif_generic(__u16 crc, const unsigned char *buffer, size_t len)
+{
+ unsigned int i;
+
+ for (i = 0 ; i < len ; i++)
+ crc = (crc << 8) ^ t10_dif_crc_table[((crc >> 8) ^ buffer[i]) & 0xff];
+
+ return crc;
+}
+EXPORT_SYMBOL(crc_t10dif_generic);
+
+/*
+ * Steps through buffer one byte at at time, calculates reflected
+ * crc using table.
+ */
+
+static int chksum_init(struct shash_desc *desc)
+{
+ struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
+
+ ctx->crc = 0;
+
+ return 0;
+}
+
+static int chksum_update(struct shash_desc *desc, const u8 *data,
+ unsigned int length)
+{
+ struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
+
+ ctx->crc = crc_t10dif_generic(ctx->crc, data, length);
+ return 0;
+}
+
+static int chksum_final(struct shash_desc *desc, u8 *out)
+{
+ struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
+
+ *(__u16 *)out = ctx->crc;
+ return 0;
+}
+
+static int __chksum_finup(__u16 *crcp, const u8 *data, unsigned int len,
+ u8 *out)
+{
+ *(__u16 *)out = crc_t10dif_generic(*crcp, data, len);
+ return 0;
+}
+
+static int chksum_finup(struct shash_desc *desc, const u8 *data,
+ unsigned int len, u8 *out)
+{
+ struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
+
+ return __chksum_finup(&ctx->crc, data, len, out);
+}
+
+static int chksum_digest(struct shash_desc *desc, const u8 *data,
+ unsigned int length, u8 *out)
+{
+ struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
+
+ return __chksum_finup(&ctx->crc, data, length, out);
+}
+
+static struct shash_alg alg = {
+ .digestsize = CRC_T10DIF_DIGEST_SIZE,
+ .init = chksum_init,
+ .update = chksum_update,
+ .final = chksum_final,
+ .finup = chksum_finup,
+ .digest = chksum_digest,
+ .descsize = sizeof(struct chksum_desc_ctx),
+ .base = {
+ .cra_name = "crct10dif",
+ .cra_driver_name = "crct10dif-generic",
+ .cra_priority = 100,
+ .cra_blocksize = CRC_T10DIF_BLOCK_SIZE,
+ .cra_module = THIS_MODULE,
+ }
+};
+
+static int __init crct10dif_mod_init(void)
+{
+ int ret;
+
+ ret = crypto_register_shash(&alg);
+ return ret;
+}
+
+static void __exit crct10dif_mod_fini(void)
+{
+ crypto_unregister_shash(&alg);
+}
+
+module_init(crct10dif_mod_init);
+module_exit(crct10dif_mod_fini);
+
+MODULE_AUTHOR("Tim Chen <tim.c.chen@linux.intel.com>");
+MODULE_DESCRIPTION("T10 DIF CRC calculation.");
+MODULE_LICENSE("GPL");
diff --git a/crypto/fcrypt.c b/crypto/fcrypt.c
index 3b2cf569c684..021d7fec6bc8 100644
--- a/crypto/fcrypt.c
+++ b/crypto/fcrypt.c
@@ -110,7 +110,7 @@ static const __be32 sbox0[256] = {
};
#undef Z
-#define Z(x) cpu_to_be32((x << 27) | (x >> 5))
+#define Z(x) cpu_to_be32(((x & 0x1f) << 27) | (x >> 5))
static const __be32 sbox1[256] = {
Z(0x77), Z(0x14), Z(0xa6), Z(0xfe), Z(0xb2), Z(0x5e), Z(0x8c), Z(0x3e),
Z(0x67), Z(0x6c), Z(0xa1), Z(0x0d), Z(0xc2), Z(0xa2), Z(0xc1), Z(0x85),
diff --git a/crypto/scatterwalk.c b/crypto/scatterwalk.c
index 7281b8a93ad3..79ca2278c2a3 100644
--- a/crypto/scatterwalk.c
+++ b/crypto/scatterwalk.c
@@ -124,3 +124,25 @@ void scatterwalk_map_and_copy(void *buf, struct scatterlist *sg,
scatterwalk_done(&walk, out, 0);
}
EXPORT_SYMBOL_GPL(scatterwalk_map_and_copy);
+
+int scatterwalk_bytes_sglen(struct scatterlist *sg, int num_bytes)
+{
+ int offset = 0, n = 0;
+
+ /* num_bytes is too small */
+ if (num_bytes < sg->length)
+ return -1;
+
+ do {
+ offset += sg->length;
+ n++;
+ sg = scatterwalk_sg_next(sg);
+
+ /* num_bytes is too large */
+ if (unlikely(!sg && (num_bytes < offset)))
+ return -1;
+ } while (sg && (num_bytes > offset));
+
+ return n;
+}
+EXPORT_SYMBOL_GPL(scatterwalk_bytes_sglen);
diff --git a/crypto/tcrypt.c b/crypto/tcrypt.c
index 66d254ce0d11..25a5934f0e50 100644
--- a/crypto/tcrypt.c
+++ b/crypto/tcrypt.c
@@ -1174,6 +1174,10 @@ static int do_test(int m)
ret += tcrypt_test("ghash");
break;
+ case 47:
+ ret += tcrypt_test("crct10dif");
+ break;
+
case 100:
ret += tcrypt_test("hmac(md5)");
break;
@@ -1498,6 +1502,10 @@ static int do_test(int m)
test_hash_speed("crc32c", sec, generic_hash_speed_template);
if (mode > 300 && mode < 400) break;
+ case 320:
+ test_hash_speed("crct10dif", sec, generic_hash_speed_template);
+ if (mode > 300 && mode < 400) break;
+
case 399:
break;
diff --git a/crypto/testmgr.c b/crypto/testmgr.c
index ecddf921a9db..e091ef6e1791 100644
--- a/crypto/testmgr.c
+++ b/crypto/testmgr.c
@@ -2046,6 +2046,16 @@ static const struct alg_test_desc alg_test_descs[] = {
}
}
}, {
+ .alg = "crct10dif",
+ .test = alg_test_hash,
+ .fips_allowed = 1,
+ .suite = {
+ .hash = {
+ .vecs = crct10dif_tv_template,
+ .count = CRCT10DIF_TEST_VECTORS
+ }
+ }
+ }, {
.alg = "cryptd(__driver-cbc-aes-aesni)",
.test = alg_test_null,
.fips_allowed = 1,
@@ -3224,7 +3234,7 @@ int alg_test(const char *driver, const char *alg, u32 type, u32 mask)
if (i >= 0)
rc |= alg_test_descs[i].test(alg_test_descs + i, driver,
type, mask);
- if (j >= 0)
+ if (j >= 0 && j != i)
rc |= alg_test_descs[j].test(alg_test_descs + j, driver,
type, mask);
diff --git a/crypto/testmgr.h b/crypto/testmgr.h
index 1e701bc075b9..7d44aa3d6b44 100644
--- a/crypto/testmgr.h
+++ b/crypto/testmgr.h
@@ -450,6 +450,39 @@ static struct hash_testvec rmd320_tv_template[] = {
}
};
+#define CRCT10DIF_TEST_VECTORS 3
+static struct hash_testvec crct10dif_tv_template[] = {
+ {
+ .plaintext = "abc",
+ .psize = 3,
+#ifdef __LITTLE_ENDIAN
+ .digest = "\x3b\x44",
+#else
+ .digest = "\x44\x3b",
+#endif
+ }, {
+ .plaintext = "1234567890123456789012345678901234567890"
+ "123456789012345678901234567890123456789",
+ .psize = 79,
+#ifdef __LITTLE_ENDIAN
+ .digest = "\x70\x4b",
+#else
+ .digest = "\x4b\x70",
+#endif
+ }, {
+ .plaintext =
+ "abcddddddddddddddddddddddddddddddddddddddddddddddddddddd",
+ .psize = 56,
+#ifdef __LITTLE_ENDIAN
+ .digest = "\xe3\x9c",
+#else
+ .digest = "\x9c\xe3",
+#endif
+ .np = 2,
+ .tap = { 28, 28 }
+ }
+};
+
/*
* SHA1 test vectors from from FIPS PUB 180-1
* Long vector from CAVS 5.0
diff --git a/drivers/char/hw_random/Kconfig b/drivers/char/hw_random/Kconfig
index 40a865449f35..0aa9d91daef5 100644
--- a/drivers/char/hw_random/Kconfig
+++ b/drivers/char/hw_random/Kconfig
@@ -153,12 +153,12 @@ config HW_RANDOM_IXP4XX
config HW_RANDOM_OMAP
tristate "OMAP Random Number Generator support"
- depends on HW_RANDOM && (ARCH_OMAP16XX || ARCH_OMAP2)
+ depends on HW_RANDOM && (ARCH_OMAP16XX || ARCH_OMAP2PLUS)
default HW_RANDOM
---help---
This driver provides kernel-side support for the Random Number
- Generator hardware found on OMAP16xx and OMAP24xx multimedia
- processors.
+ Generator hardware found on OMAP16xx, OMAP2/3/4/5 and AM33xx/AM43xx
+ multimedia processors.
To compile this driver as a module, choose M here: the
module will be called omap-rng.
diff --git a/drivers/char/hw_random/mxc-rnga.c b/drivers/char/hw_random/mxc-rnga.c
index 19a12ac64a9e..6a86b6f56af2 100644
--- a/drivers/char/hw_random/mxc-rnga.c
+++ b/drivers/char/hw_random/mxc-rnga.c
@@ -164,7 +164,9 @@ static int __init mxc_rnga_probe(struct platform_device *pdev)
goto out;
}
- clk_prepare_enable(mxc_rng->clk);
+ err = clk_prepare_enable(mxc_rng->clk);
+ if (err)
+ goto out;
res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
mxc_rng->mem = devm_ioremap_resource(&pdev->dev, res);
diff --git a/drivers/char/hw_random/omap-rng.c b/drivers/char/hw_random/omap-rng.c
index 6843ec87b98b..9b89ff4881de 100644
--- a/drivers/char/hw_random/omap-rng.c
+++ b/drivers/char/hw_random/omap-rng.c
@@ -24,57 +24,131 @@
#include <linux/delay.h>
#include <linux/slab.h>
#include <linux/pm_runtime.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
+#include <linux/of_address.h>
+#include <linux/interrupt.h>
#include <asm/io.h>
-#define RNG_OUT_REG 0x00 /* Output register */
-#define RNG_STAT_REG 0x04 /* Status register
- [0] = STAT_BUSY */
-#define RNG_ALARM_REG 0x24 /* Alarm register
- [7:0] = ALARM_COUNTER */
-#define RNG_CONFIG_REG 0x28 /* Configuration register
- [11:6] = RESET_COUNT
- [5:3] = RING2_DELAY
- [2:0] = RING1_DELAY */
-#define RNG_REV_REG 0x3c /* Revision register
- [7:0] = REV_NB */
-#define RNG_MASK_REG 0x40 /* Mask and reset register
- [2] = IT_EN
- [1] = SOFTRESET
- [0] = AUTOIDLE */
-#define RNG_SYSSTATUS 0x44 /* System status
- [0] = RESETDONE */
+#define RNG_REG_STATUS_RDY (1 << 0)
+
+#define RNG_REG_INTACK_RDY_MASK (1 << 0)
+#define RNG_REG_INTACK_SHUTDOWN_OFLO_MASK (1 << 1)
+#define RNG_SHUTDOWN_OFLO_MASK (1 << 1)
+
+#define RNG_CONTROL_STARTUP_CYCLES_SHIFT 16
+#define RNG_CONTROL_STARTUP_CYCLES_MASK (0xffff << 16)
+#define RNG_CONTROL_ENABLE_TRNG_SHIFT 10
+#define RNG_CONTROL_ENABLE_TRNG_MASK (1 << 10)
+
+#define RNG_CONFIG_MAX_REFIL_CYCLES_SHIFT 16
+#define RNG_CONFIG_MAX_REFIL_CYCLES_MASK (0xffff << 16)
+#define RNG_CONFIG_MIN_REFIL_CYCLES_SHIFT 0
+#define RNG_CONFIG_MIN_REFIL_CYCLES_MASK (0xff << 0)
+
+#define RNG_CONTROL_STARTUP_CYCLES 0xff
+#define RNG_CONFIG_MIN_REFIL_CYCLES 0x21
+#define RNG_CONFIG_MAX_REFIL_CYCLES 0x22
+
+#define RNG_ALARMCNT_ALARM_TH_SHIFT 0x0
+#define RNG_ALARMCNT_ALARM_TH_MASK (0xff << 0)
+#define RNG_ALARMCNT_SHUTDOWN_TH_SHIFT 16
+#define RNG_ALARMCNT_SHUTDOWN_TH_MASK (0x1f << 16)
+#define RNG_ALARM_THRESHOLD 0xff
+#define RNG_SHUTDOWN_THRESHOLD 0x4
+
+#define RNG_REG_FROENABLE_MASK 0xffffff
+#define RNG_REG_FRODETUNE_MASK 0xffffff
+
+#define OMAP2_RNG_OUTPUT_SIZE 0x4
+#define OMAP4_RNG_OUTPUT_SIZE 0x8
+
+enum {
+ RNG_OUTPUT_L_REG = 0,
+ RNG_OUTPUT_H_REG,
+ RNG_STATUS_REG,
+ RNG_INTMASK_REG,
+ RNG_INTACK_REG,
+ RNG_CONTROL_REG,
+ RNG_CONFIG_REG,
+ RNG_ALARMCNT_REG,
+ RNG_FROENABLE_REG,
+ RNG_FRODETUNE_REG,
+ RNG_ALARMMASK_REG,
+ RNG_ALARMSTOP_REG,
+ RNG_REV_REG,
+ RNG_SYSCONFIG_REG,
+};
+
+static const u16 reg_map_omap2[] = {
+ [RNG_OUTPUT_L_REG] = 0x0,
+ [RNG_STATUS_REG] = 0x4,
+ [RNG_CONFIG_REG] = 0x28,
+ [RNG_REV_REG] = 0x3c,
+ [RNG_SYSCONFIG_REG] = 0x40,
+};
+static const u16 reg_map_omap4[] = {
+ [RNG_OUTPUT_L_REG] = 0x0,
+ [RNG_OUTPUT_H_REG] = 0x4,
+ [RNG_STATUS_REG] = 0x8,
+ [RNG_INTMASK_REG] = 0xc,
+ [RNG_INTACK_REG] = 0x10,
+ [RNG_CONTROL_REG] = 0x14,
+ [RNG_CONFIG_REG] = 0x18,
+ [RNG_ALARMCNT_REG] = 0x1c,
+ [RNG_FROENABLE_REG] = 0x20,
+ [RNG_FRODETUNE_REG] = 0x24,
+ [RNG_ALARMMASK_REG] = 0x28,
+ [RNG_ALARMSTOP_REG] = 0x2c,
+ [RNG_REV_REG] = 0x1FE0,
+ [RNG_SYSCONFIG_REG] = 0x1FE4,
+};
+
+struct omap_rng_dev;
/**
- * struct omap_rng_private_data - RNG IP block-specific data
- * @base: virtual address of the beginning of the RNG IP block registers
- * @mem_res: struct resource * for the IP block registers physical memory
+ * struct omap_rng_pdata - RNG IP block-specific data
+ * @regs: Pointer to the register offsets structure.
+ * @data_size: No. of bytes in RNG output.
+ * @data_present: Callback to determine if data is available.
+ * @init: Callback for IP specific initialization sequence.
+ * @cleanup: Callback for IP specific cleanup sequence.
*/
-struct omap_rng_private_data {
- void __iomem *base;
- struct resource *mem_res;
+struct omap_rng_pdata {
+ u16 *regs;
+ u32 data_size;
+ u32 (*data_present)(struct omap_rng_dev *priv);
+ int (*init)(struct omap_rng_dev *priv);
+ void (*cleanup)(struct omap_rng_dev *priv);
};
-static inline u32 omap_rng_read_reg(struct omap_rng_private_data *priv, int reg)
+struct omap_rng_dev {
+ void __iomem *base;
+ struct device *dev;
+ const struct omap_rng_pdata *pdata;
+};
+
+static inline u32 omap_rng_read(struct omap_rng_dev *priv, u16 reg)
{
- return __raw_readl(priv->base + reg);
+ return __raw_readl(priv->base + priv->pdata->regs[reg]);
}
-static inline void omap_rng_write_reg(struct omap_rng_private_data *priv,
- int reg, u32 val)
+static inline void omap_rng_write(struct omap_rng_dev *priv, u16 reg,
+ u32 val)
{
- __raw_writel(val, priv->base + reg);
+ __raw_writel(val, priv->base + priv->pdata->regs[reg]);
}
static int omap_rng_data_present(struct hwrng *rng, int wait)
{
- struct omap_rng_private_data *priv;
+ struct omap_rng_dev *priv;
int data, i;
- priv = (struct omap_rng_private_data *)rng->priv;
+ priv = (struct omap_rng_dev *)rng->priv;
for (i = 0; i < 20; i++) {
- data = omap_rng_read_reg(priv, RNG_STAT_REG) ? 0 : 1;
+ data = priv->pdata->data_present(priv);
if (data || !wait)
break;
/* RNG produces data fast enough (2+ MBit/sec, even
@@ -89,27 +163,212 @@ static int omap_rng_data_present(struct hwrng *rng, int wait)
static int omap_rng_data_read(struct hwrng *rng, u32 *data)
{
- struct omap_rng_private_data *priv;
+ struct omap_rng_dev *priv;
+ u32 data_size, i;
+
+ priv = (struct omap_rng_dev *)rng->priv;
+ data_size = priv->pdata->data_size;
+
+ for (i = 0; i < data_size / sizeof(u32); i++)
+ data[i] = omap_rng_read(priv, RNG_OUTPUT_L_REG + i);
+
+ if (priv->pdata->regs[RNG_INTACK_REG])
+ omap_rng_write(priv, RNG_INTACK_REG, RNG_REG_INTACK_RDY_MASK);
+ return data_size;
+}
+
+static int omap_rng_init(struct hwrng *rng)
+{
+ struct omap_rng_dev *priv;
- priv = (struct omap_rng_private_data *)rng->priv;
+ priv = (struct omap_rng_dev *)rng->priv;
+ return priv->pdata->init(priv);
+}
- *data = omap_rng_read_reg(priv, RNG_OUT_REG);
+static void omap_rng_cleanup(struct hwrng *rng)
+{
+ struct omap_rng_dev *priv;
- return sizeof(u32);
+ priv = (struct omap_rng_dev *)rng->priv;
+ priv->pdata->cleanup(priv);
}
static struct hwrng omap_rng_ops = {
.name = "omap",
.data_present = omap_rng_data_present,
.data_read = omap_rng_data_read,
+ .init = omap_rng_init,
+ .cleanup = omap_rng_cleanup,
+};
+
+static inline u32 omap2_rng_data_present(struct omap_rng_dev *priv)
+{
+ return omap_rng_read(priv, RNG_STATUS_REG) ? 0 : 1;
+}
+
+static int omap2_rng_init(struct omap_rng_dev *priv)
+{
+ omap_rng_write(priv, RNG_SYSCONFIG_REG, 0x1);
+ return 0;
+}
+
+static void omap2_rng_cleanup(struct omap_rng_dev *priv)
+{
+ omap_rng_write(priv, RNG_SYSCONFIG_REG, 0x0);
+}
+
+static struct omap_rng_pdata omap2_rng_pdata = {
+ .regs = (u16 *)reg_map_omap2,
+ .data_size = OMAP2_RNG_OUTPUT_SIZE,
+ .data_present = omap2_rng_data_present,
+ .init = omap2_rng_init,
+ .cleanup = omap2_rng_cleanup,
};
+#if defined(CONFIG_OF)
+static inline u32 omap4_rng_data_present(struct omap_rng_dev *priv)
+{
+ return omap_rng_read(priv, RNG_STATUS_REG) & RNG_REG_STATUS_RDY;
+}
+
+static int omap4_rng_init(struct omap_rng_dev *priv)
+{
+ u32 val;
+
+ /* Return if RNG is already running. */
+ if (omap_rng_read(priv, RNG_CONFIG_REG) & RNG_CONTROL_ENABLE_TRNG_MASK)
+ return 0;
+
+ val = RNG_CONFIG_MIN_REFIL_CYCLES << RNG_CONFIG_MIN_REFIL_CYCLES_SHIFT;
+ val |= RNG_CONFIG_MAX_REFIL_CYCLES << RNG_CONFIG_MAX_REFIL_CYCLES_SHIFT;
+ omap_rng_write(priv, RNG_CONFIG_REG, val);
+
+ omap_rng_write(priv, RNG_FRODETUNE_REG, 0x0);
+ omap_rng_write(priv, RNG_FROENABLE_REG, RNG_REG_FROENABLE_MASK);
+ val = RNG_ALARM_THRESHOLD << RNG_ALARMCNT_ALARM_TH_SHIFT;
+ val |= RNG_SHUTDOWN_THRESHOLD << RNG_ALARMCNT_SHUTDOWN_TH_SHIFT;
+ omap_rng_write(priv, RNG_ALARMCNT_REG, val);
+
+ val = RNG_CONTROL_STARTUP_CYCLES << RNG_CONTROL_STARTUP_CYCLES_SHIFT;
+ val |= RNG_CONTROL_ENABLE_TRNG_MASK;
+ omap_rng_write(priv, RNG_CONTROL_REG, val);
+
+ return 0;
+}
+
+static void omap4_rng_cleanup(struct omap_rng_dev *priv)
+{
+ int val;
+
+ val = omap_rng_read(priv, RNG_CONTROL_REG);
+ val &= ~RNG_CONTROL_ENABLE_TRNG_MASK;
+ omap_rng_write(priv, RNG_CONFIG_REG, val);
+}
+
+static irqreturn_t omap4_rng_irq(int irq, void *dev_id)
+{
+ struct omap_rng_dev *priv = dev_id;
+ u32 fro_detune, fro_enable;
+
+ /*
+ * Interrupt raised by a fro shutdown threshold, do the following:
+ * 1. Clear the alarm events.
+ * 2. De tune the FROs which are shutdown.
+ * 3. Re enable the shutdown FROs.
+ */
+ omap_rng_write(priv, RNG_ALARMMASK_REG, 0x0);
+ omap_rng_write(priv, RNG_ALARMSTOP_REG, 0x0);
+
+ fro_enable = omap_rng_read(priv, RNG_FROENABLE_REG);
+ fro_detune = ~fro_enable & RNG_REG_FRODETUNE_MASK;
+ fro_detune = fro_detune | omap_rng_read(priv, RNG_FRODETUNE_REG);
+ fro_enable = RNG_REG_FROENABLE_MASK;
+
+ omap_rng_write(priv, RNG_FRODETUNE_REG, fro_detune);
+ omap_rng_write(priv, RNG_FROENABLE_REG, fro_enable);
+
+ omap_rng_write(priv, RNG_INTACK_REG, RNG_REG_INTACK_SHUTDOWN_OFLO_MASK);
+
+ return IRQ_HANDLED;
+}
+
+static struct omap_rng_pdata omap4_rng_pdata = {
+ .regs = (u16 *)reg_map_omap4,
+ .data_size = OMAP4_RNG_OUTPUT_SIZE,
+ .data_present = omap4_rng_data_present,
+ .init = omap4_rng_init,
+ .cleanup = omap4_rng_cleanup,
+};
+
+static const struct of_device_id omap_rng_of_match[] = {
+ {
+ .compatible = "ti,omap2-rng",
+ .data = &omap2_rng_pdata,
+ },
+ {
+ .compatible = "ti,omap4-rng",
+ .data = &omap4_rng_pdata,
+ },
+ {},
+};
+MODULE_DEVICE_TABLE(of, omap_rng_of_match);
+
+static int of_get_omap_rng_device_details(struct omap_rng_dev *priv,
+ struct platform_device *pdev)
+{
+ const struct of_device_id *match;
+ struct device *dev = &pdev->dev;
+ int irq, err;
+
+ match = of_match_device(of_match_ptr(omap_rng_of_match), dev);
+ if (!match) {
+ dev_err(dev, "no compatible OF match\n");
+ return -EINVAL;
+ }
+ priv->pdata = match->data;
+
+ if (of_device_is_compatible(dev->of_node, "ti,omap4-rng")) {
+ irq = platform_get_irq(pdev, 0);
+ if (irq < 0) {
+ dev_err(dev, "%s: error getting IRQ resource - %d\n",
+ __func__, irq);
+ return irq;
+ }
+
+ err = devm_request_irq(dev, irq, omap4_rng_irq,
+ IRQF_TRIGGER_NONE, dev_name(dev), priv);
+ if (err) {
+ dev_err(dev, "unable to request irq %d, err = %d\n",
+ irq, err);
+ return err;
+ }
+ omap_rng_write(priv, RNG_INTMASK_REG, RNG_SHUTDOWN_OFLO_MASK);
+ }
+ return 0;
+}
+#else
+static int of_get_omap_rng_device_details(struct omap_rng_dev *omap_rng,
+ struct platform_device *pdev)
+{
+ return -EINVAL;
+}
+#endif
+
+static int get_omap_rng_device_details(struct omap_rng_dev *omap_rng)
+{
+ /* Only OMAP2/3 can be non-DT */
+ omap_rng->pdata = &omap2_rng_pdata;
+ return 0;
+}
+
static int omap_rng_probe(struct platform_device *pdev)
{
- struct omap_rng_private_data *priv;
+ struct omap_rng_dev *priv;
+ struct resource *res;
+ struct device *dev = &pdev->dev;
int ret;
- priv = kzalloc(sizeof(struct omap_rng_private_data), GFP_KERNEL);
+ priv = devm_kzalloc(dev, sizeof(struct omap_rng_dev), GFP_KERNEL);
if (!priv) {
dev_err(&pdev->dev, "could not allocate memory\n");
return -ENOMEM;
@@ -117,26 +376,29 @@ static int omap_rng_probe(struct platform_device *pdev)
omap_rng_ops.priv = (unsigned long)priv;
platform_set_drvdata(pdev, priv);
+ priv->dev = dev;
- priv->mem_res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
- priv->base = devm_ioremap_resource(&pdev->dev, priv->mem_res);
+ res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+ priv->base = devm_ioremap_resource(dev, res);
if (IS_ERR(priv->base)) {
ret = PTR_ERR(priv->base);
goto err_ioremap;
}
- platform_set_drvdata(pdev, priv);
pm_runtime_enable(&pdev->dev);
pm_runtime_get_sync(&pdev->dev);
+ ret = (dev->of_node) ? of_get_omap_rng_device_details(priv, pdev) :
+ get_omap_rng_device_details(priv);
+ if (ret)
+ goto err_ioremap;
+
ret = hwrng_register(&omap_rng_ops);
if (ret)
goto err_register;
dev_info(&pdev->dev, "OMAP Random Number Generator ver. %02x\n",
- omap_rng_read_reg(priv, RNG_REV_REG));
-
- omap_rng_write_reg(priv, RNG_MASK_REG, 0x1);
+ omap_rng_read(priv, RNG_REV_REG));
return 0;
@@ -144,26 +406,21 @@ err_register:
priv->base = NULL;
pm_runtime_disable(&pdev->dev);
err_ioremap:
- kfree(priv);
-
+ dev_err(dev, "initialization failed.\n");
return ret;
}
static int __exit omap_rng_remove(struct platform_device *pdev)
{
- struct omap_rng_private_data *priv = platform_get_drvdata(pdev);
+ struct omap_rng_dev *priv = platform_get_drvdata(pdev);
hwrng_unregister(&omap_rng_ops);
- omap_rng_write_reg(priv, RNG_MASK_REG, 0x0);
+ priv->pdata->cleanup(priv);
pm_runtime_put_sync(&pdev->dev);
pm_runtime_disable(&pdev->dev);
- release_mem_region(priv->mem_res->start, resource_size(priv->mem_res));
-
- kfree(priv);
-
return 0;
}
@@ -171,9 +428,9 @@ static int __exit omap_rng_remove(struct platform_device *pdev)
static int omap_rng_suspend(struct device *dev)
{
- struct omap_rng_private_data *priv = dev_get_drvdata(dev);
+ struct omap_rng_dev *priv = dev_get_drvdata(dev);
- omap_rng_write_reg(priv, RNG_MASK_REG, 0x0);
+ priv->pdata->cleanup(priv);
pm_runtime_put_sync(dev);
return 0;
@@ -181,10 +438,10 @@ static int omap_rng_suspend(struct device *dev)
static int omap_rng_resume(struct device *dev)
{
- struct omap_rng_private_data *priv = dev_get_drvdata(dev);
+ struct omap_rng_dev *priv = dev_get_drvdata(dev);
pm_runtime_get_sync(dev);
- omap_rng_write_reg(priv, RNG_MASK_REG, 0x1);
+ priv->pdata->init(priv);
return 0;
}
@@ -198,31 +455,18 @@ static SIMPLE_DEV_PM_OPS(omap_rng_pm, omap_rng_suspend, omap_rng_resume);
#endif
-/* work with hotplug and coldplug */
-MODULE_ALIAS("platform:omap_rng");
-
static struct platform_driver omap_rng_driver = {
.driver = {
.name = "omap_rng",
.owner = THIS_MODULE,
.pm = OMAP_RNG_PM,
+ .of_match_table = of_match_ptr(omap_rng_of_match),
},
.probe = omap_rng_probe,
.remove = __exit_p(omap_rng_remove),
};
-static int __init omap_rng_init(void)
-{
- return platform_driver_register(&omap_rng_driver);
-}
-
-static void __exit omap_rng_exit(void)
-{
- platform_driver_unregister(&omap_rng_driver);
-}
-
-module_init(omap_rng_init);
-module_exit(omap_rng_exit);
-
+module_platform_driver(omap_rng_driver);
+MODULE_ALIAS("platform:omap_rng");
MODULE_AUTHOR("Deepak Saxena (and others)");
MODULE_LICENSE("GPL");
diff --git a/drivers/char/hw_random/picoxcell-rng.c b/drivers/char/hw_random/picoxcell-rng.c
index 973b95113edf..3d4c2293c6f5 100644
--- a/drivers/char/hw_random/picoxcell-rng.c
+++ b/drivers/char/hw_random/picoxcell-rng.c
@@ -33,7 +33,7 @@
static void __iomem *rng_base;
static struct clk *rng_clk;
-struct device *rng_dev;
+static struct device *rng_dev;
static inline u32 picoxcell_trng_read_csr(void)
{
diff --git a/drivers/char/hw_random/tx4939-rng.c b/drivers/char/hw_random/tx4939-rng.c
index 00593c847cf0..09c5fbea2b93 100644
--- a/drivers/char/hw_random/tx4939-rng.c
+++ b/drivers/char/hw_random/tx4939-rng.c
@@ -110,12 +110,10 @@ static int __init tx4939_rng_probe(struct platform_device *dev)
struct resource *r;
int i;
- r = platform_get_resource(dev, IORESOURCE_MEM, 0);
- if (!r)
- return -EBUSY;
rngdev = devm_kzalloc(&dev->dev, sizeof(*rngdev), GFP_KERNEL);
if (!rngdev)
return -ENOMEM;
+ r = platform_get_resource(dev, IORESOURCE_MEM, 0);
rngdev->base = devm_ioremap_resource(&dev->dev, r);
if (IS_ERR(rngdev->base))
return PTR_ERR(rngdev->base);
diff --git a/drivers/char/hw_random/via-rng.c b/drivers/char/hw_random/via-rng.c
index d0387a84eec1..e737772ad69a 100644
--- a/drivers/char/hw_random/via-rng.c
+++ b/drivers/char/hw_random/via-rng.c
@@ -29,6 +29,7 @@
#include <linux/kernel.h>
#include <linux/hw_random.h>
#include <linux/delay.h>
+#include <asm/cpu_device_id.h>
#include <asm/io.h>
#include <asm/msr.h>
#include <asm/cpufeature.h>
@@ -220,5 +221,11 @@ static void __exit mod_exit(void)
module_init(mod_init);
module_exit(mod_exit);
+static struct x86_cpu_id via_rng_cpu_id[] = {
+ X86_FEATURE_MATCH(X86_FEATURE_XSTORE),
+ {}
+};
+
MODULE_DESCRIPTION("H/W RNG driver for VIA CPU with PadLock");
MODULE_LICENSE("GPL");
+MODULE_DEVICE_TABLE(x86cpu, via_rng_cpu_id);
diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig
index 8ff7c230d82e..f4fd837bcb82 100644
--- a/drivers/crypto/Kconfig
+++ b/drivers/crypto/Kconfig
@@ -242,17 +242,20 @@ config CRYPTO_DEV_PPC4XX
This option allows you to have support for AMCC crypto acceleration.
config CRYPTO_DEV_OMAP_SHAM
- tristate "Support for OMAP SHA1/MD5 hw accelerator"
- depends on ARCH_OMAP2 || ARCH_OMAP3
+ tristate "Support for OMAP MD5/SHA1/SHA2 hw accelerator"
+ depends on ARCH_OMAP2PLUS
select CRYPTO_SHA1
select CRYPTO_MD5
+ select CRYPTO_SHA256
+ select CRYPTO_SHA512
+ select CRYPTO_HMAC
help
- OMAP processors have SHA1/MD5 hw accelerator. Select this if you
- want to use the OMAP module for SHA1/MD5 algorithms.
+ OMAP processors have MD5/SHA1/SHA2 hw accelerator. Select this if you
+ want to use the OMAP module for MD5/SHA1/SHA2 algorithms.
config CRYPTO_DEV_OMAP_AES
tristate "Support for OMAP AES hw engine"
- depends on ARCH_OMAP2 || ARCH_OMAP3
+ depends on ARCH_OMAP2 || ARCH_OMAP3 || ARCH_OMAP2PLUS
select CRYPTO_AES
select CRYPTO_BLKCIPHER2
help
diff --git a/drivers/crypto/amcc/crypto4xx_alg.c b/drivers/crypto/amcc/crypto4xx_alg.c
index a33243c17b00..4afca3968773 100644
--- a/drivers/crypto/amcc/crypto4xx_alg.c
+++ b/drivers/crypto/amcc/crypto4xx_alg.c
@@ -32,10 +32,10 @@
#include "crypto4xx_sa.h"
#include "crypto4xx_core.h"
-void set_dynamic_sa_command_0(struct dynamic_sa_ctl *sa, u32 save_h,
- u32 save_iv, u32 ld_h, u32 ld_iv, u32 hdr_proc,
- u32 h, u32 c, u32 pad_type, u32 op_grp, u32 op,
- u32 dir)
+static void set_dynamic_sa_command_0(struct dynamic_sa_ctl *sa, u32 save_h,
+ u32 save_iv, u32 ld_h, u32 ld_iv,
+ u32 hdr_proc, u32 h, u32 c, u32 pad_type,
+ u32 op_grp, u32 op, u32 dir)
{
sa->sa_command_0.w = 0;
sa->sa_command_0.bf.save_hash_state = save_h;
@@ -52,9 +52,10 @@ void set_dynamic_sa_command_0(struct dynamic_sa_ctl *sa, u32 save_h,
sa->sa_command_0.bf.dir = dir;
}
-void set_dynamic_sa_command_1(struct dynamic_sa_ctl *sa, u32 cm, u32 hmac_mc,
- u32 cfb, u32 esn, u32 sn_mask, u32 mute,
- u32 cp_pad, u32 cp_pay, u32 cp_hdr)
+static void set_dynamic_sa_command_1(struct dynamic_sa_ctl *sa, u32 cm,
+ u32 hmac_mc, u32 cfb, u32 esn,
+ u32 sn_mask, u32 mute, u32 cp_pad,
+ u32 cp_pay, u32 cp_hdr)
{
sa->sa_command_1.w = 0;
sa->sa_command_1.bf.crypto_mode31 = (cm & 4) >> 2;
diff --git a/drivers/crypto/caam/Kconfig b/drivers/crypto/caam/Kconfig
index b44091c47f75..ca89f6b84b06 100644
--- a/drivers/crypto/caam/Kconfig
+++ b/drivers/crypto/caam/Kconfig
@@ -98,3 +98,11 @@ config CRYPTO_DEV_FSL_CAAM_RNG_API
To compile this as a module, choose M here: the module
will be called caamrng.
+
+config CRYPTO_DEV_FSL_CAAM_DEBUG
+ bool "Enable debug output in CAAM driver"
+ depends on CRYPTO_DEV_FSL_CAAM
+ default n
+ help
+ Selecting this will enable printing of various debug
+ information in the CAAM driver.
diff --git a/drivers/crypto/caam/Makefile b/drivers/crypto/caam/Makefile
index b1eb44838db5..d56bd0ec65d8 100644
--- a/drivers/crypto/caam/Makefile
+++ b/drivers/crypto/caam/Makefile
@@ -1,6 +1,9 @@
#
# Makefile for the CAAM backend and dependent components
#
+ifeq ($(CONFIG_CRYPTO_DEV_FSL_CAAM_DEBUG), y)
+ EXTRA_CFLAGS := -DDEBUG
+endif
obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM) += caam.o
obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API) += caamalg.o
diff --git a/drivers/crypto/caam/caamalg.c b/drivers/crypto/caam/caamalg.c
index bf416a8391a7..7c63b72ecd75 100644
--- a/drivers/crypto/caam/caamalg.c
+++ b/drivers/crypto/caam/caamalg.c
@@ -65,8 +65,6 @@
#define CAAM_MAX_IV_LENGTH 16
/* length of descriptors text */
-#define DESC_JOB_IO_LEN (CAAM_CMD_SZ * 5 + CAAM_PTR_SZ * 3)
-
#define DESC_AEAD_BASE (4 * CAAM_CMD_SZ)
#define DESC_AEAD_ENC_LEN (DESC_AEAD_BASE + 16 * CAAM_CMD_SZ)
#define DESC_AEAD_DEC_LEN (DESC_AEAD_BASE + 21 * CAAM_CMD_SZ)
@@ -84,8 +82,6 @@
#ifdef DEBUG
/* for print_hex_dumps with line references */
-#define xstr(s) str(s)
-#define str(s) #s
#define debug(format, arg...) printk(format, arg)
#else
#define debug(format, arg...)
@@ -285,7 +281,7 @@ static int aead_set_sh_desc(struct crypto_aead *aead)
return -ENOMEM;
}
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "aead enc shdesc@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "aead enc shdesc@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc,
desc_bytes(desc), 1);
#endif
@@ -353,7 +349,7 @@ static int aead_set_sh_desc(struct crypto_aead *aead)
return -ENOMEM;
}
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "aead dec shdesc@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "aead dec shdesc@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc,
desc_bytes(desc), 1);
#endif
@@ -436,7 +432,7 @@ static int aead_set_sh_desc(struct crypto_aead *aead)
return -ENOMEM;
}
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "aead givenc shdesc@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "aead givenc shdesc@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc,
desc_bytes(desc), 1);
#endif
@@ -500,7 +496,7 @@ static int aead_setkey(struct crypto_aead *aead,
keylen, enckeylen, authkeylen);
printk(KERN_ERR "split_key_len %d split_key_pad_len %d\n",
ctx->split_key_len, ctx->split_key_pad_len);
- print_hex_dump(KERN_ERR, "key in @"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
#endif
@@ -519,7 +515,7 @@ static int aead_setkey(struct crypto_aead *aead,
return -ENOMEM;
}
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "ctx.key@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "ctx.key@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, ctx->key,
ctx->split_key_pad_len + enckeylen, 1);
#endif
@@ -549,7 +545,7 @@ static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher,
u32 *desc;
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "key in @"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
#endif
@@ -598,7 +594,8 @@ static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher,
return -ENOMEM;
}
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "ablkcipher enc shdesc@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR,
+ "ablkcipher enc shdesc@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc,
desc_bytes(desc), 1);
#endif
@@ -643,7 +640,8 @@ static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher,
}
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "ablkcipher dec shdesc@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR,
+ "ablkcipher dec shdesc@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc,
desc_bytes(desc), 1);
#endif
@@ -780,13 +778,13 @@ static void aead_encrypt_done(struct device *jrdev, u32 *desc, u32 err,
aead_unmap(jrdev, edesc, req);
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "assoc @"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "assoc @"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->assoc),
req->assoclen , 1);
- print_hex_dump(KERN_ERR, "dstiv @"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "dstiv @"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src) - ivsize,
edesc->src_nents ? 100 : ivsize, 1);
- print_hex_dump(KERN_ERR, "dst @"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "dst @"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
edesc->src_nents ? 100 : req->cryptlen +
ctx->authsize + 4, 1);
@@ -814,10 +812,10 @@ static void aead_decrypt_done(struct device *jrdev, u32 *desc, u32 err,
offsetof(struct aead_edesc, hw_desc));
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "dstiv @"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "dstiv @"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, req->iv,
ivsize, 1);
- print_hex_dump(KERN_ERR, "dst @"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "dst @"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->dst),
req->cryptlen, 1);
#endif
@@ -837,7 +835,7 @@ static void aead_decrypt_done(struct device *jrdev, u32 *desc, u32 err,
err = -EBADMSG;
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "iphdrout@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "iphdrout@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4,
((char *)sg_virt(req->assoc) - sizeof(struct iphdr)),
sizeof(struct iphdr) + req->assoclen +
@@ -845,7 +843,7 @@ static void aead_decrypt_done(struct device *jrdev, u32 *desc, u32 err,
ctx->authsize + 36, 1);
if (!err && edesc->sec4_sg_bytes) {
struct scatterlist *sg = sg_last(req->src, edesc->src_nents);
- print_hex_dump(KERN_ERR, "sglastout@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "sglastout@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(sg),
sg->length + ctx->authsize + 16, 1);
}
@@ -878,10 +876,10 @@ static void ablkcipher_encrypt_done(struct device *jrdev, u32 *desc, u32 err,
}
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "dstiv @"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "dstiv @"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, req->info,
edesc->src_nents > 1 ? 100 : ivsize, 1);
- print_hex_dump(KERN_ERR, "dst @"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "dst @"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
edesc->dst_nents > 1 ? 100 : req->nbytes, 1);
#endif
@@ -913,10 +911,10 @@ static void ablkcipher_decrypt_done(struct device *jrdev, u32 *desc, u32 err,
}
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "dstiv @"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "dstiv @"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, req->info,
ivsize, 1);
- print_hex_dump(KERN_ERR, "dst @"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "dst @"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
edesc->dst_nents > 1 ? 100 : req->nbytes, 1);
#endif
@@ -947,16 +945,16 @@ static void init_aead_job(u32 *sh_desc, dma_addr_t ptr,
#ifdef DEBUG
debug("assoclen %d cryptlen %d authsize %d\n",
req->assoclen, req->cryptlen, authsize);
- print_hex_dump(KERN_ERR, "assoc @"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "assoc @"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->assoc),
req->assoclen , 1);
- print_hex_dump(KERN_ERR, "presciv@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "presciv@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, req->iv,
edesc->src_nents ? 100 : ivsize, 1);
- print_hex_dump(KERN_ERR, "src @"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "src @"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
edesc->src_nents ? 100 : req->cryptlen, 1);
- print_hex_dump(KERN_ERR, "shrdesc@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "shrdesc@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, sh_desc,
desc_bytes(sh_desc), 1);
#endif
@@ -1025,15 +1023,15 @@ static void init_aead_giv_job(u32 *sh_desc, dma_addr_t ptr,
#ifdef DEBUG
debug("assoclen %d cryptlen %d authsize %d\n",
req->assoclen, req->cryptlen, authsize);
- print_hex_dump(KERN_ERR, "assoc @"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "assoc @"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->assoc),
req->assoclen , 1);
- print_hex_dump(KERN_ERR, "presciv@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "presciv@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, req->iv, ivsize, 1);
- print_hex_dump(KERN_ERR, "src @"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "src @"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
edesc->src_nents > 1 ? 100 : req->cryptlen, 1);
- print_hex_dump(KERN_ERR, "shrdesc@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "shrdesc@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, sh_desc,
desc_bytes(sh_desc), 1);
#endif
@@ -1086,10 +1084,10 @@ static void init_ablkcipher_job(u32 *sh_desc, dma_addr_t ptr,
int len, sec4_sg_index = 0;
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "presciv@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "presciv@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, req->info,
ivsize, 1);
- print_hex_dump(KERN_ERR, "src @"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "src @"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
edesc->src_nents ? 100 : req->nbytes, 1);
#endif
@@ -1247,7 +1245,7 @@ static int aead_encrypt(struct aead_request *req)
init_aead_job(ctx->sh_desc_enc, ctx->sh_desc_enc_dma, edesc, req,
all_contig, true);
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "aead jobdesc@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
desc_bytes(edesc->hw_desc), 1);
#endif
@@ -1281,7 +1279,7 @@ static int aead_decrypt(struct aead_request *req)
return PTR_ERR(edesc);
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "dec src@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "dec src@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
req->cryptlen, 1);
#endif
@@ -1290,7 +1288,7 @@ static int aead_decrypt(struct aead_request *req)
init_aead_job(ctx->sh_desc_dec,
ctx->sh_desc_dec_dma, edesc, req, all_contig, false);
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "aead jobdesc@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
desc_bytes(edesc->hw_desc), 1);
#endif
@@ -1437,7 +1435,7 @@ static int aead_givencrypt(struct aead_givcrypt_request *areq)
return PTR_ERR(edesc);
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "giv src@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "giv src@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
req->cryptlen, 1);
#endif
@@ -1446,7 +1444,7 @@ static int aead_givencrypt(struct aead_givcrypt_request *areq)
init_aead_giv_job(ctx->sh_desc_givenc,
ctx->sh_desc_givenc_dma, edesc, req, contig);
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "aead jobdesc@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
desc_bytes(edesc->hw_desc), 1);
#endif
@@ -1546,7 +1544,7 @@ static struct ablkcipher_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request
edesc->iv_dma = iv_dma;
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "ablkcipher sec4_sg@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "ablkcipher sec4_sg@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, edesc->sec4_sg,
sec4_sg_bytes, 1);
#endif
@@ -1575,7 +1573,7 @@ static int ablkcipher_encrypt(struct ablkcipher_request *req)
init_ablkcipher_job(ctx->sh_desc_enc,
ctx->sh_desc_enc_dma, edesc, req, iv_contig);
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "ablkcipher jobdesc@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "ablkcipher jobdesc@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
desc_bytes(edesc->hw_desc), 1);
#endif
@@ -1613,7 +1611,7 @@ static int ablkcipher_decrypt(struct ablkcipher_request *req)
ctx->sh_desc_dec_dma, edesc, req, iv_contig);
desc = edesc->hw_desc;
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "ablkcipher jobdesc@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "ablkcipher jobdesc@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
desc_bytes(edesc->hw_desc), 1);
#endif
diff --git a/drivers/crypto/caam/caamhash.c b/drivers/crypto/caam/caamhash.c
index 84573b4d6f92..e732bd962e98 100644
--- a/drivers/crypto/caam/caamhash.c
+++ b/drivers/crypto/caam/caamhash.c
@@ -72,8 +72,6 @@
#define CAAM_MAX_HASH_DIGEST_SIZE SHA512_DIGEST_SIZE
/* length of descriptors text */
-#define DESC_JOB_IO_LEN (CAAM_CMD_SZ * 5 + CAAM_PTR_SZ * 3)
-
#define DESC_AHASH_BASE (4 * CAAM_CMD_SZ)
#define DESC_AHASH_UPDATE_LEN (6 * CAAM_CMD_SZ)
#define DESC_AHASH_UPDATE_FIRST_LEN (DESC_AHASH_BASE + 4 * CAAM_CMD_SZ)
@@ -91,8 +89,6 @@
#ifdef DEBUG
/* for print_hex_dumps with line references */
-#define xstr(s) str(s)
-#define str(s) #s
#define debug(format, arg...) printk(format, arg)
#else
#define debug(format, arg...)
@@ -331,7 +327,8 @@ static int ahash_set_sh_desc(struct crypto_ahash *ahash)
return -ENOMEM;
}
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "ahash update shdesc@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR,
+ "ahash update shdesc@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
#endif
@@ -349,7 +346,8 @@ static int ahash_set_sh_desc(struct crypto_ahash *ahash)
return -ENOMEM;
}
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "ahash update first shdesc@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR,
+ "ahash update first shdesc@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
#endif
@@ -366,7 +364,7 @@ static int ahash_set_sh_desc(struct crypto_ahash *ahash)
return -ENOMEM;
}
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "ahash final shdesc@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "ahash final shdesc@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc,
desc_bytes(desc), 1);
#endif
@@ -384,7 +382,7 @@ static int ahash_set_sh_desc(struct crypto_ahash *ahash)
return -ENOMEM;
}
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "ahash finup shdesc@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "ahash finup shdesc@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc,
desc_bytes(desc), 1);
#endif
@@ -403,7 +401,8 @@ static int ahash_set_sh_desc(struct crypto_ahash *ahash)
return -ENOMEM;
}
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "ahash digest shdesc@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR,
+ "ahash digest shdesc@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc,
desc_bytes(desc), 1);
#endif
@@ -464,9 +463,9 @@ static int hash_digest_key(struct caam_hash_ctx *ctx, const u8 *key_in,
LDST_SRCDST_BYTE_CONTEXT);
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "key_in@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "key_in@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, key_in, *keylen, 1);
- print_hex_dump(KERN_ERR, "jobdesc@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
#endif
@@ -479,7 +478,8 @@ static int hash_digest_key(struct caam_hash_ctx *ctx, const u8 *key_in,
wait_for_completion_interruptible(&result.completion);
ret = result.err;
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "digested key@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR,
+ "digested key@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, key_in,
digestsize, 1);
#endif
@@ -530,7 +530,7 @@ static int ahash_setkey(struct crypto_ahash *ahash,
#ifdef DEBUG
printk(KERN_ERR "split_key_len %d split_key_pad_len %d\n",
ctx->split_key_len, ctx->split_key_pad_len);
- print_hex_dump(KERN_ERR, "key in @"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
#endif
@@ -545,7 +545,7 @@ static int ahash_setkey(struct crypto_ahash *ahash,
return -ENOMEM;
}
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "ctx.key@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "ctx.key@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, ctx->key,
ctx->split_key_pad_len, 1);
#endif
@@ -638,11 +638,11 @@ static void ahash_done(struct device *jrdev, u32 *desc, u32 err,
kfree(edesc);
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "ctx@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "ctx@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
ctx->ctx_len, 1);
if (req->result)
- print_hex_dump(KERN_ERR, "result@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "result@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, req->result,
digestsize, 1);
#endif
@@ -676,11 +676,11 @@ static void ahash_done_bi(struct device *jrdev, u32 *desc, u32 err,
kfree(edesc);
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "ctx@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "ctx@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
ctx->ctx_len, 1);
if (req->result)
- print_hex_dump(KERN_ERR, "result@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "result@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, req->result,
digestsize, 1);
#endif
@@ -714,11 +714,11 @@ static void ahash_done_ctx_src(struct device *jrdev, u32 *desc, u32 err,
kfree(edesc);
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "ctx@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "ctx@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
ctx->ctx_len, 1);
if (req->result)
- print_hex_dump(KERN_ERR, "result@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "result@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, req->result,
digestsize, 1);
#endif
@@ -752,11 +752,11 @@ static void ahash_done_ctx_dst(struct device *jrdev, u32 *desc, u32 err,
kfree(edesc);
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "ctx@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "ctx@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
ctx->ctx_len, 1);
if (req->result)
- print_hex_dump(KERN_ERR, "result@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "result@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, req->result,
digestsize, 1);
#endif
@@ -852,7 +852,7 @@ static int ahash_update_ctx(struct ahash_request *req)
append_seq_out_ptr(desc, state->ctx_dma, ctx->ctx_len, 0);
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "jobdesc@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc,
desc_bytes(desc), 1);
#endif
@@ -871,9 +871,9 @@ static int ahash_update_ctx(struct ahash_request *req)
*next_buflen = last_buflen;
}
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "buf@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "buf@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, buf, *buflen, 1);
- print_hex_dump(KERN_ERR, "next buf@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "next buf@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, next_buf,
*next_buflen, 1);
#endif
@@ -937,7 +937,7 @@ static int ahash_final_ctx(struct ahash_request *req)
digestsize);
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "jobdesc@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
#endif
@@ -1016,7 +1016,7 @@ static int ahash_finup_ctx(struct ahash_request *req)
digestsize);
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "jobdesc@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
#endif
@@ -1086,7 +1086,7 @@ static int ahash_digest(struct ahash_request *req)
digestsize);
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "jobdesc@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
#endif
@@ -1140,7 +1140,7 @@ static int ahash_final_no_ctx(struct ahash_request *req)
edesc->src_nents = 0;
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "jobdesc@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
#endif
@@ -1228,7 +1228,7 @@ static int ahash_update_no_ctx(struct ahash_request *req)
map_seq_out_ptr_ctx(desc, jrdev, state, ctx->ctx_len);
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "jobdesc@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc,
desc_bytes(desc), 1);
#endif
@@ -1250,9 +1250,9 @@ static int ahash_update_no_ctx(struct ahash_request *req)
*next_buflen = 0;
}
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "buf@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "buf@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, buf, *buflen, 1);
- print_hex_dump(KERN_ERR, "next buf@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "next buf@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, next_buf,
*next_buflen, 1);
#endif
@@ -1321,7 +1321,7 @@ static int ahash_finup_no_ctx(struct ahash_request *req)
digestsize);
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "jobdesc@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
#endif
@@ -1414,7 +1414,7 @@ static int ahash_update_first(struct ahash_request *req)
map_seq_out_ptr_ctx(desc, jrdev, state, ctx->ctx_len);
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "jobdesc@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc,
desc_bytes(desc), 1);
#endif
@@ -1438,7 +1438,7 @@ static int ahash_update_first(struct ahash_request *req)
sg_copy(next_buf, req->src, req->nbytes);
}
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "next buf@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "next buf@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, next_buf,
*next_buflen, 1);
#endif
diff --git a/drivers/crypto/caam/ctrl.c b/drivers/crypto/caam/ctrl.c
index f5d6deced1cb..b010d42a1803 100644
--- a/drivers/crypto/caam/ctrl.c
+++ b/drivers/crypto/caam/ctrl.c
@@ -75,55 +75,53 @@ static void build_instantiation_desc(u32 *desc)
OP_ALG_RNG4_SK);
}
-struct instantiate_result {
- struct completion completion;
- int err;
-};
-
-static void rng4_init_done(struct device *dev, u32 *desc, u32 err,
- void *context)
-{
- struct instantiate_result *instantiation = context;
-
- if (err) {
- char tmp[CAAM_ERROR_STR_MAX];
-
- dev_err(dev, "%08x: %s\n", err, caam_jr_strstatus(tmp, err));
- }
-
- instantiation->err = err;
- complete(&instantiation->completion);
-}
-
-static int instantiate_rng(struct device *jrdev)
+static int instantiate_rng(struct device *ctrldev)
{
- struct instantiate_result instantiation;
-
- dma_addr_t desc_dma;
+ struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctrldev);
+ struct caam_full __iomem *topregs;
+ unsigned int timeout = 100000;
u32 *desc;
- int ret;
+ int i, ret = 0;
desc = kmalloc(CAAM_CMD_SZ * 6, GFP_KERNEL | GFP_DMA);
if (!desc) {
- dev_err(jrdev, "cannot allocate RNG init descriptor memory\n");
+ dev_err(ctrldev, "can't allocate RNG init descriptor memory\n");
return -ENOMEM;
}
-
build_instantiation_desc(desc);
- desc_dma = dma_map_single(jrdev, desc, desc_bytes(desc), DMA_TO_DEVICE);
- init_completion(&instantiation.completion);
- ret = caam_jr_enqueue(jrdev, desc, rng4_init_done, &instantiation);
- if (!ret) {
- wait_for_completion_interruptible(&instantiation.completion);
- ret = instantiation.err;
- if (ret)
- dev_err(jrdev, "unable to instantiate RNG\n");
+
+ /* Set the bit to request direct access to DECO0 */
+ topregs = (struct caam_full __iomem *)ctrlpriv->ctrl;
+ setbits32(&topregs->ctrl.deco_rq, DECORR_RQD0ENABLE);
+
+ while (!(rd_reg32(&topregs->ctrl.deco_rq) & DECORR_DEN0) &&
+ --timeout)
+ cpu_relax();
+
+ if (!timeout) {
+ dev_err(ctrldev, "failed to acquire DECO 0\n");
+ ret = -EIO;
+ goto out;
}
- dma_unmap_single(jrdev, desc_dma, desc_bytes(desc), DMA_TO_DEVICE);
+ for (i = 0; i < desc_len(desc); i++)
+ topregs->deco.descbuf[i] = *(desc + i);
- kfree(desc);
+ wr_reg32(&topregs->deco.jr_ctl_hi, DECO_JQCR_WHL | DECO_JQCR_FOUR);
+ timeout = 10000000;
+ while ((rd_reg32(&topregs->deco.desc_dbg) & DECO_DBG_VALID) &&
+ --timeout)
+ cpu_relax();
+
+ if (!timeout) {
+ dev_err(ctrldev, "failed to instantiate RNG\n");
+ ret = -EIO;
+ }
+
+ clrbits32(&topregs->ctrl.deco_rq, DECORR_RQD0ENABLE);
+out:
+ kfree(desc);
return ret;
}
@@ -303,7 +301,7 @@ static int caam_probe(struct platform_device *pdev)
if ((cha_vid & CHA_ID_RNG_MASK) >> CHA_ID_RNG_SHIFT >= 4 &&
!(rd_reg32(&topregs->ctrl.r4tst[0].rdsta) & RDSTA_IF0)) {
kick_trng(pdev);
- ret = instantiate_rng(ctrlpriv->jrdev[0]);
+ ret = instantiate_rng(dev);
if (ret) {
caam_remove(pdev);
return ret;
@@ -315,9 +313,6 @@ static int caam_probe(struct platform_device *pdev)
/* NOTE: RTIC detection ought to go here, around Si time */
- /* Initialize queue allocator lock */
- spin_lock_init(&ctrlpriv->jr_alloc_lock);
-
caam_id = rd_reg64(&topregs->ctrl.perfmon.caam_id);
/* Report "alive" for developer to see */
diff --git a/drivers/crypto/caam/desc_constr.h b/drivers/crypto/caam/desc_constr.h
index fe3bfd1b08ca..cd5f678847ce 100644
--- a/drivers/crypto/caam/desc_constr.h
+++ b/drivers/crypto/caam/desc_constr.h
@@ -10,6 +10,7 @@
#define CAAM_CMD_SZ sizeof(u32)
#define CAAM_PTR_SZ sizeof(dma_addr_t)
#define CAAM_DESC_BYTES_MAX (CAAM_CMD_SZ * MAX_CAAM_DESCSIZE)
+#define DESC_JOB_IO_LEN (CAAM_CMD_SZ * 5 + CAAM_PTR_SZ * 3)
#ifdef DEBUG
#define PRINT_POS do { printk(KERN_DEBUG "%02d: %s\n", desc_len(desc),\
diff --git a/drivers/crypto/caam/intern.h b/drivers/crypto/caam/intern.h
index e4a16b741371..34c4b9f7fbfa 100644
--- a/drivers/crypto/caam/intern.h
+++ b/drivers/crypto/caam/intern.h
@@ -9,9 +9,6 @@
#ifndef INTERN_H
#define INTERN_H
-#define JOBR_UNASSIGNED 0
-#define JOBR_ASSIGNED 1
-
/* Currently comes from Kconfig param as a ^2 (driver-required) */
#define JOBR_DEPTH (1 << CONFIG_CRYPTO_DEV_FSL_CAAM_RINGSIZE)
@@ -46,7 +43,6 @@ struct caam_drv_private_jr {
struct caam_job_ring __iomem *rregs; /* JobR's register space */
struct tasklet_struct irqtask;
int irq; /* One per queue */
- int assign; /* busy/free */
/* Job ring info */
int ringsize; /* Size of rings (assume input = output) */
@@ -68,7 +64,6 @@ struct caam_drv_private {
struct device *dev;
struct device **jrdev; /* Alloc'ed array per sub-device */
- spinlock_t jr_alloc_lock;
struct platform_device *pdev;
/* Physical-presence section */
diff --git a/drivers/crypto/caam/jr.c b/drivers/crypto/caam/jr.c
index b4aa773ecbc8..105ba4da6180 100644
--- a/drivers/crypto/caam/jr.c
+++ b/drivers/crypto/caam/jr.c
@@ -126,72 +126,6 @@ static void caam_jr_dequeue(unsigned long devarg)
}
/**
- * caam_jr_register() - Alloc a ring for someone to use as needed. Returns
- * an ordinal of the rings allocated, else returns -ENODEV if no rings
- * are available.
- * @ctrldev: points to the controller level dev (parent) that
- * owns rings available for use.
- * @dev: points to where a pointer to the newly allocated queue's
- * dev can be written to if successful.
- **/
-int caam_jr_register(struct device *ctrldev, struct device **rdev)
-{
- struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctrldev);
- struct caam_drv_private_jr *jrpriv = NULL;
- int ring;
-
- /* Lock, if free ring - assign, unlock */
- spin_lock(&ctrlpriv->jr_alloc_lock);
- for (ring = 0; ring < ctrlpriv->total_jobrs; ring++) {
- jrpriv = dev_get_drvdata(ctrlpriv->jrdev[ring]);
- if (jrpriv->assign == JOBR_UNASSIGNED) {
- jrpriv->assign = JOBR_ASSIGNED;
- *rdev = ctrlpriv->jrdev[ring];
- spin_unlock(&ctrlpriv->jr_alloc_lock);
- return ring;
- }
- }
-
- /* If assigned, write dev where caller needs it */
- spin_unlock(&ctrlpriv->jr_alloc_lock);
- *rdev = NULL;
-
- return -ENODEV;
-}
-EXPORT_SYMBOL(caam_jr_register);
-
-/**
- * caam_jr_deregister() - Deregister an API and release the queue.
- * Returns 0 if OK, -EBUSY if queue still contains pending entries
- * or unprocessed results at the time of the call
- * @dev - points to the dev that identifies the queue to
- * be released.
- **/
-int caam_jr_deregister(struct device *rdev)
-{
- struct caam_drv_private_jr *jrpriv = dev_get_drvdata(rdev);
- struct caam_drv_private *ctrlpriv;
-
- /* Get the owning controller's private space */
- ctrlpriv = dev_get_drvdata(jrpriv->parentdev);
-
- /*
- * Make sure ring empty before release
- */
- if (rd_reg32(&jrpriv->rregs->outring_used) ||
- (rd_reg32(&jrpriv->rregs->inpring_avail) != JOBR_DEPTH))
- return -EBUSY;
-
- /* Release ring */
- spin_lock(&ctrlpriv->jr_alloc_lock);
- jrpriv->assign = JOBR_UNASSIGNED;
- spin_unlock(&ctrlpriv->jr_alloc_lock);
-
- return 0;
-}
-EXPORT_SYMBOL(caam_jr_deregister);
-
-/**
* caam_jr_enqueue() - Enqueue a job descriptor head. Returns 0 if OK,
* -EBUSY if the queue is full, -EIO if it cannot map the caller's
* descriptor.
@@ -379,7 +313,6 @@ static int caam_jr_init(struct device *dev)
(JOBR_INTC_COUNT_THLD << JRCFG_ICDCT_SHIFT) |
(JOBR_INTC_TIME_THLD << JRCFG_ICTT_SHIFT));
- jrp->assign = JOBR_UNASSIGNED;
return 0;
}
diff --git a/drivers/crypto/caam/jr.h b/drivers/crypto/caam/jr.h
index c23df395b622..9d8741a59037 100644
--- a/drivers/crypto/caam/jr.h
+++ b/drivers/crypto/caam/jr.h
@@ -8,8 +8,6 @@
#define JR_H
/* Prototypes for backend-level services exposed to APIs */
-int caam_jr_register(struct device *ctrldev, struct device **rdev);
-int caam_jr_deregister(struct device *rdev);
int caam_jr_enqueue(struct device *dev, u32 *desc,
void (*cbk)(struct device *dev, u32 *desc, u32 status,
void *areq),
diff --git a/drivers/crypto/caam/key_gen.c b/drivers/crypto/caam/key_gen.c
index 87138d2adb5f..ea2e406610eb 100644
--- a/drivers/crypto/caam/key_gen.c
+++ b/drivers/crypto/caam/key_gen.c
@@ -95,9 +95,9 @@ int gen_split_key(struct device *jrdev, u8 *key_out, int split_key_len,
LDST_CLASS_2_CCB | FIFOST_TYPE_SPLIT_KEK);
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "ctx.key@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "ctx.key@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, key_in, keylen, 1);
- print_hex_dump(KERN_ERR, "jobdesc@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
#endif
@@ -110,7 +110,7 @@ int gen_split_key(struct device *jrdev, u8 *key_out, int split_key_len,
wait_for_completion_interruptible(&result.completion);
ret = result.err;
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "ctx.key@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "ctx.key@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, key_out,
split_key_pad_len, 1);
#endif
diff --git a/drivers/crypto/caam/regs.h b/drivers/crypto/caam/regs.h
index c09142fc13e3..4455396918de 100644
--- a/drivers/crypto/caam/regs.h
+++ b/drivers/crypto/caam/regs.h
@@ -341,6 +341,8 @@ struct caam_ctrl {
#define MCFGR_DMA_RESET 0x10000000
#define MCFGR_LONG_PTR 0x00010000 /* Use >32-bit desc addressing */
#define SCFGR_RDBENABLE 0x00000400
+#define DECORR_RQD0ENABLE 0x00000001 /* Enable DECO0 for direct access */
+#define DECORR_DEN0 0x00010000 /* DECO0 available for access*/
/* AXI read cache control */
#define MCFGR_ARCACHE_SHIFT 12
@@ -703,9 +705,16 @@ struct caam_deco {
struct deco_sg_table sctr_tbl[4]; /* DxSTR - Scatter Tables */
u32 rsvd29[48];
u32 descbuf[64]; /* DxDESB - Descriptor buffer */
- u32 rsvd30[320];
+ u32 rscvd30[193];
+ u32 desc_dbg; /* DxDDR - DECO Debug Register */
+ u32 rsvd31[126];
};
+/* DECO DBG Register Valid Bit*/
+#define DECO_DBG_VALID 0x80000000
+#define DECO_JQCR_WHL 0x20000000
+#define DECO_JQCR_FOUR 0x10000000
+
/*
* Current top-level view of memory map is:
*
@@ -733,6 +742,7 @@ struct caam_full {
u64 rsvd[512];
struct caam_assurance assure;
struct caam_queue_if qi;
+ struct caam_deco deco;
};
#endif /* REGS_H */
diff --git a/drivers/crypto/nx/nx-aes-cbc.c b/drivers/crypto/nx/nx-aes-cbc.c
index 35d483f8db66..cc00b52306ba 100644
--- a/drivers/crypto/nx/nx-aes-cbc.c
+++ b/drivers/crypto/nx/nx-aes-cbc.c
@@ -70,35 +70,52 @@ static int cbc_aes_nx_crypt(struct blkcipher_desc *desc,
{
struct nx_crypto_ctx *nx_ctx = crypto_blkcipher_ctx(desc->tfm);
struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
+ unsigned long irq_flags;
+ unsigned int processed = 0, to_process;
+ u32 max_sg_len;
int rc;
- if (nbytes > nx_ctx->ap->databytelen)
- return -EINVAL;
+ spin_lock_irqsave(&nx_ctx->lock, irq_flags);
+
+ max_sg_len = min_t(u32, nx_driver.of.max_sg_len/sizeof(struct nx_sg),
+ nx_ctx->ap->sglen);
if (enc)
NX_CPB_FDM(csbcpb) |= NX_FDM_ENDE_ENCRYPT;
else
NX_CPB_FDM(csbcpb) &= ~NX_FDM_ENDE_ENCRYPT;
- rc = nx_build_sg_lists(nx_ctx, desc, dst, src, nbytes,
- csbcpb->cpb.aes_cbc.iv);
- if (rc)
- goto out;
-
- if (!nx_ctx->op.inlen || !nx_ctx->op.outlen) {
- rc = -EINVAL;
- goto out;
- }
-
- rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
- desc->flags & CRYPTO_TFM_REQ_MAY_SLEEP);
- if (rc)
- goto out;
-
- atomic_inc(&(nx_ctx->stats->aes_ops));
- atomic64_add(csbcpb->csb.processed_byte_count,
- &(nx_ctx->stats->aes_bytes));
+ do {
+ to_process = min_t(u64, nbytes - processed,
+ nx_ctx->ap->databytelen);
+ to_process = min_t(u64, to_process,
+ NX_PAGE_SIZE * (max_sg_len - 1));
+ to_process = to_process & ~(AES_BLOCK_SIZE - 1);
+
+ rc = nx_build_sg_lists(nx_ctx, desc, dst, src, to_process,
+ processed, csbcpb->cpb.aes_cbc.iv);
+ if (rc)
+ goto out;
+
+ if (!nx_ctx->op.inlen || !nx_ctx->op.outlen) {
+ rc = -EINVAL;
+ goto out;
+ }
+
+ rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
+ desc->flags & CRYPTO_TFM_REQ_MAY_SLEEP);
+ if (rc)
+ goto out;
+
+ memcpy(desc->info, csbcpb->cpb.aes_cbc.cv, AES_BLOCK_SIZE);
+ atomic_inc(&(nx_ctx->stats->aes_ops));
+ atomic64_add(csbcpb->csb.processed_byte_count,
+ &(nx_ctx->stats->aes_bytes));
+
+ processed += to_process;
+ } while (processed < nbytes);
out:
+ spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
return rc;
}
diff --git a/drivers/crypto/nx/nx-aes-ccm.c b/drivers/crypto/nx/nx-aes-ccm.c
index ef5eae6d1400..5ecd4c2414aa 100644
--- a/drivers/crypto/nx/nx-aes-ccm.c
+++ b/drivers/crypto/nx/nx-aes-ccm.c
@@ -179,13 +179,26 @@ static int generate_pat(u8 *iv,
struct nx_sg *nx_insg = nx_ctx->in_sg;
struct nx_sg *nx_outsg = nx_ctx->out_sg;
unsigned int iauth_len = 0;
- struct vio_pfo_op *op = NULL;
u8 tmp[16], *b1 = NULL, *b0 = NULL, *result = NULL;
int rc;
/* zero the ctr value */
memset(iv + 15 - iv[0], 0, iv[0] + 1);
+ /* page 78 of nx_wb.pdf has,
+ * Note: RFC3610 allows the AAD data to be up to 2^64 -1 bytes
+ * in length. If a full message is used, the AES CCA implementation
+ * restricts the maximum AAD length to 2^32 -1 bytes.
+ * If partial messages are used, the implementation supports
+ * 2^64 -1 bytes maximum AAD length.
+ *
+ * However, in the cryptoapi's aead_request structure,
+ * assoclen is an unsigned int, thus it cannot hold a length
+ * value greater than 2^32 - 1.
+ * Thus the AAD is further constrained by this and is never
+ * greater than 2^32.
+ */
+
if (!req->assoclen) {
b0 = nx_ctx->csbcpb->cpb.aes_ccm.in_pat_or_b0;
} else if (req->assoclen <= 14) {
@@ -195,7 +208,46 @@ static int generate_pat(u8 *iv,
b0 = nx_ctx->csbcpb->cpb.aes_ccm.in_pat_or_b0;
b1 = nx_ctx->priv.ccm.iauth_tag;
iauth_len = req->assoclen;
+ } else if (req->assoclen <= 65280) {
+ /* if associated data is less than (2^16 - 2^8), we construct
+ * B1 differently and feed in the associated data to a CCA
+ * operation */
+ b0 = nx_ctx->csbcpb_aead->cpb.aes_cca.b0;
+ b1 = nx_ctx->csbcpb_aead->cpb.aes_cca.b1;
+ iauth_len = 14;
+ } else {
+ b0 = nx_ctx->csbcpb_aead->cpb.aes_cca.b0;
+ b1 = nx_ctx->csbcpb_aead->cpb.aes_cca.b1;
+ iauth_len = 10;
+ }
+
+ /* generate B0 */
+ rc = generate_b0(iv, req->assoclen, authsize, nbytes, b0);
+ if (rc)
+ return rc;
+
+ /* generate B1:
+ * add control info for associated data
+ * RFC 3610 and NIST Special Publication 800-38C
+ */
+ if (b1) {
+ memset(b1, 0, 16);
+ if (req->assoclen <= 65280) {
+ *(u16 *)b1 = (u16)req->assoclen;
+ scatterwalk_map_and_copy(b1 + 2, req->assoc, 0,
+ iauth_len, SCATTERWALK_FROM_SG);
+ } else {
+ *(u16 *)b1 = (u16)(0xfffe);
+ *(u32 *)&b1[2] = (u32)req->assoclen;
+ scatterwalk_map_and_copy(b1 + 6, req->assoc, 0,
+ iauth_len, SCATTERWALK_FROM_SG);
+ }
+ }
+ /* now copy any remaining AAD to scatterlist and call nx... */
+ if (!req->assoclen) {
+ return rc;
+ } else if (req->assoclen <= 14) {
nx_insg = nx_build_sg_list(nx_insg, b1, 16, nx_ctx->ap->sglen);
nx_outsg = nx_build_sg_list(nx_outsg, tmp, 16,
nx_ctx->ap->sglen);
@@ -210,56 +262,74 @@ static int generate_pat(u8 *iv,
NX_CPB_FDM(nx_ctx->csbcpb) |= NX_FDM_ENDE_ENCRYPT;
NX_CPB_FDM(nx_ctx->csbcpb) |= NX_FDM_INTERMEDIATE;
- op = &nx_ctx->op;
result = nx_ctx->csbcpb->cpb.aes_ccm.out_pat_or_mac;
- } else if (req->assoclen <= 65280) {
- /* if associated data is less than (2^16 - 2^8), we construct
- * B1 differently and feed in the associated data to a CCA
- * operation */
- b0 = nx_ctx->csbcpb_aead->cpb.aes_cca.b0;
- b1 = nx_ctx->csbcpb_aead->cpb.aes_cca.b1;
- iauth_len = 14;
- /* remaining assoc data must have scatterlist built for it */
- nx_insg = nx_walk_and_build(nx_insg, nx_ctx->ap->sglen,
- req->assoc, iauth_len,
- req->assoclen - iauth_len);
- nx_ctx->op_aead.inlen = (nx_ctx->in_sg - nx_insg) *
- sizeof(struct nx_sg);
+ rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
+ req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP);
+ if (rc)
+ return rc;
+
+ atomic_inc(&(nx_ctx->stats->aes_ops));
+ atomic64_add(req->assoclen, &(nx_ctx->stats->aes_bytes));
- op = &nx_ctx->op_aead;
- result = nx_ctx->csbcpb_aead->cpb.aes_cca.out_pat_or_b0;
} else {
- /* if associated data is less than (2^32), we construct B1
- * differently yet again and feed in the associated data to a
- * CCA operation */
- pr_err("associated data len is %u bytes (returning -EINVAL)\n",
- req->assoclen);
- rc = -EINVAL;
- }
+ u32 max_sg_len;
+ unsigned int processed = 0, to_process;
+
+ /* page_limit: number of sg entries that fit on one page */
+ max_sg_len = min_t(u32,
+ nx_driver.of.max_sg_len/sizeof(struct nx_sg),
+ nx_ctx->ap->sglen);
+
+ processed += iauth_len;
+
+ do {
+ to_process = min_t(u32, req->assoclen - processed,
+ nx_ctx->ap->databytelen);
+ to_process = min_t(u64, to_process,
+ NX_PAGE_SIZE * (max_sg_len - 1));
+
+ if ((to_process + processed) < req->assoclen) {
+ NX_CPB_FDM(nx_ctx->csbcpb_aead) |=
+ NX_FDM_INTERMEDIATE;
+ } else {
+ NX_CPB_FDM(nx_ctx->csbcpb_aead) &=
+ ~NX_FDM_INTERMEDIATE;
+ }
+
+ nx_insg = nx_walk_and_build(nx_ctx->in_sg,
+ nx_ctx->ap->sglen,
+ req->assoc, processed,
+ to_process);
+
+ nx_ctx->op_aead.inlen = (nx_ctx->in_sg - nx_insg) *
+ sizeof(struct nx_sg);
- rc = generate_b0(iv, req->assoclen, authsize, nbytes, b0);
- if (rc)
- goto done;
+ result = nx_ctx->csbcpb_aead->cpb.aes_cca.out_pat_or_b0;
- if (b1) {
- memset(b1, 0, 16);
- *(u16 *)b1 = (u16)req->assoclen;
+ rc = nx_hcall_sync(nx_ctx, &nx_ctx->op_aead,
+ req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP);
+ if (rc)
+ return rc;
- scatterwalk_map_and_copy(b1 + 2, req->assoc, 0,
- iauth_len, SCATTERWALK_FROM_SG);
+ memcpy(nx_ctx->csbcpb_aead->cpb.aes_cca.b0,
+ nx_ctx->csbcpb_aead->cpb.aes_cca.out_pat_or_b0,
+ AES_BLOCK_SIZE);
- rc = nx_hcall_sync(nx_ctx, op,
- req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP);
- if (rc)
- goto done;
+ NX_CPB_FDM(nx_ctx->csbcpb_aead) |= NX_FDM_CONTINUATION;
- atomic_inc(&(nx_ctx->stats->aes_ops));
- atomic64_add(req->assoclen, &(nx_ctx->stats->aes_bytes));
+ atomic_inc(&(nx_ctx->stats->aes_ops));
+ atomic64_add(req->assoclen,
+ &(nx_ctx->stats->aes_bytes));
- memcpy(out, result, AES_BLOCK_SIZE);
+ processed += to_process;
+ } while (processed < req->assoclen);
+
+ result = nx_ctx->csbcpb_aead->cpb.aes_cca.out_pat_or_b0;
}
-done:
+
+ memcpy(out, result, AES_BLOCK_SIZE);
+
return rc;
}
@@ -271,10 +341,12 @@ static int ccm_nx_decrypt(struct aead_request *req,
unsigned int nbytes = req->cryptlen;
unsigned int authsize = crypto_aead_authsize(crypto_aead_reqtfm(req));
struct nx_ccm_priv *priv = &nx_ctx->priv.ccm;
+ unsigned long irq_flags;
+ unsigned int processed = 0, to_process;
+ u32 max_sg_len;
int rc = -1;
- if (nbytes > nx_ctx->ap->databytelen)
- return -EINVAL;
+ spin_lock_irqsave(&nx_ctx->lock, irq_flags);
nbytes -= authsize;
@@ -288,26 +360,61 @@ static int ccm_nx_decrypt(struct aead_request *req,
if (rc)
goto out;
- rc = nx_build_sg_lists(nx_ctx, desc, req->dst, req->src, nbytes,
- csbcpb->cpb.aes_ccm.iv_or_ctr);
- if (rc)
- goto out;
+ /* page_limit: number of sg entries that fit on one page */
+ max_sg_len = min_t(u32, nx_driver.of.max_sg_len/sizeof(struct nx_sg),
+ nx_ctx->ap->sglen);
- NX_CPB_FDM(nx_ctx->csbcpb) &= ~NX_FDM_ENDE_ENCRYPT;
- NX_CPB_FDM(nx_ctx->csbcpb) &= ~NX_FDM_INTERMEDIATE;
+ do {
+
+ /* to_process: the AES_BLOCK_SIZE data chunk to process in this
+ * update. This value is bound by sg list limits.
+ */
+ to_process = min_t(u64, nbytes - processed,
+ nx_ctx->ap->databytelen);
+ to_process = min_t(u64, to_process,
+ NX_PAGE_SIZE * (max_sg_len - 1));
+
+ if ((to_process + processed) < nbytes)
+ NX_CPB_FDM(csbcpb) |= NX_FDM_INTERMEDIATE;
+ else
+ NX_CPB_FDM(csbcpb) &= ~NX_FDM_INTERMEDIATE;
+
+ NX_CPB_FDM(nx_ctx->csbcpb) &= ~NX_FDM_ENDE_ENCRYPT;
+
+ rc = nx_build_sg_lists(nx_ctx, desc, req->dst, req->src,
+ to_process, processed,
+ csbcpb->cpb.aes_ccm.iv_or_ctr);
+ if (rc)
+ goto out;
- rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
+ rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP);
- if (rc)
- goto out;
+ if (rc)
+ goto out;
- atomic_inc(&(nx_ctx->stats->aes_ops));
- atomic64_add(csbcpb->csb.processed_byte_count,
- &(nx_ctx->stats->aes_bytes));
+ /* for partial completion, copy following for next
+ * entry into loop...
+ */
+ memcpy(desc->info, csbcpb->cpb.aes_ccm.out_ctr, AES_BLOCK_SIZE);
+ memcpy(csbcpb->cpb.aes_ccm.in_pat_or_b0,
+ csbcpb->cpb.aes_ccm.out_pat_or_mac, AES_BLOCK_SIZE);
+ memcpy(csbcpb->cpb.aes_ccm.in_s0,
+ csbcpb->cpb.aes_ccm.out_s0, AES_BLOCK_SIZE);
+
+ NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION;
+
+ /* update stats */
+ atomic_inc(&(nx_ctx->stats->aes_ops));
+ atomic64_add(csbcpb->csb.processed_byte_count,
+ &(nx_ctx->stats->aes_bytes));
+
+ processed += to_process;
+ } while (processed < nbytes);
rc = memcmp(csbcpb->cpb.aes_ccm.out_pat_or_mac, priv->oauth_tag,
authsize) ? -EBADMSG : 0;
out:
+ spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
return rc;
}
@@ -318,38 +425,76 @@ static int ccm_nx_encrypt(struct aead_request *req,
struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
unsigned int nbytes = req->cryptlen;
unsigned int authsize = crypto_aead_authsize(crypto_aead_reqtfm(req));
+ unsigned long irq_flags;
+ unsigned int processed = 0, to_process;
+ u32 max_sg_len;
int rc = -1;
- if (nbytes > nx_ctx->ap->databytelen)
- return -EINVAL;
+ spin_lock_irqsave(&nx_ctx->lock, irq_flags);
rc = generate_pat(desc->info, req, nx_ctx, authsize, nbytes,
csbcpb->cpb.aes_ccm.in_pat_or_b0);
if (rc)
goto out;
- rc = nx_build_sg_lists(nx_ctx, desc, req->dst, req->src, nbytes,
- csbcpb->cpb.aes_ccm.iv_or_ctr);
- if (rc)
- goto out;
+ /* page_limit: number of sg entries that fit on one page */
+ max_sg_len = min_t(u32, nx_driver.of.max_sg_len/sizeof(struct nx_sg),
+ nx_ctx->ap->sglen);
+
+ do {
+ /* to process: the AES_BLOCK_SIZE data chunk to process in this
+ * update. This value is bound by sg list limits.
+ */
+ to_process = min_t(u64, nbytes - processed,
+ nx_ctx->ap->databytelen);
+ to_process = min_t(u64, to_process,
+ NX_PAGE_SIZE * (max_sg_len - 1));
+
+ if ((to_process + processed) < nbytes)
+ NX_CPB_FDM(csbcpb) |= NX_FDM_INTERMEDIATE;
+ else
+ NX_CPB_FDM(csbcpb) &= ~NX_FDM_INTERMEDIATE;
+
+ NX_CPB_FDM(csbcpb) |= NX_FDM_ENDE_ENCRYPT;
+
+ rc = nx_build_sg_lists(nx_ctx, desc, req->dst, req->src,
+ to_process, processed,
+ csbcpb->cpb.aes_ccm.iv_or_ctr);
+ if (rc)
+ goto out;
- NX_CPB_FDM(csbcpb) |= NX_FDM_ENDE_ENCRYPT;
- NX_CPB_FDM(csbcpb) &= ~NX_FDM_INTERMEDIATE;
+ rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
+ req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP);
+ if (rc)
+ goto out;
- rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
- req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP);
- if (rc)
- goto out;
+ /* for partial completion, copy following for next
+ * entry into loop...
+ */
+ memcpy(desc->info, csbcpb->cpb.aes_ccm.out_ctr, AES_BLOCK_SIZE);
+ memcpy(csbcpb->cpb.aes_ccm.in_pat_or_b0,
+ csbcpb->cpb.aes_ccm.out_pat_or_mac, AES_BLOCK_SIZE);
+ memcpy(csbcpb->cpb.aes_ccm.in_s0,
+ csbcpb->cpb.aes_ccm.out_s0, AES_BLOCK_SIZE);
- atomic_inc(&(nx_ctx->stats->aes_ops));
- atomic64_add(csbcpb->csb.processed_byte_count,
- &(nx_ctx->stats->aes_bytes));
+ NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION;
+
+ /* update stats */
+ atomic_inc(&(nx_ctx->stats->aes_ops));
+ atomic64_add(csbcpb->csb.processed_byte_count,
+ &(nx_ctx->stats->aes_bytes));
+
+ processed += to_process;
+
+ } while (processed < nbytes);
/* copy out the auth tag */
scatterwalk_map_and_copy(csbcpb->cpb.aes_ccm.out_pat_or_mac,
req->dst, nbytes, authsize,
SCATTERWALK_TO_SG);
+
out:
+ spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
return rc;
}
diff --git a/drivers/crypto/nx/nx-aes-ctr.c b/drivers/crypto/nx/nx-aes-ctr.c
index b6286f14680b..a37d009dc75c 100644
--- a/drivers/crypto/nx/nx-aes-ctr.c
+++ b/drivers/crypto/nx/nx-aes-ctr.c
@@ -88,30 +88,48 @@ static int ctr_aes_nx_crypt(struct blkcipher_desc *desc,
{
struct nx_crypto_ctx *nx_ctx = crypto_blkcipher_ctx(desc->tfm);
struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
+ unsigned long irq_flags;
+ unsigned int processed = 0, to_process;
+ u32 max_sg_len;
int rc;
- if (nbytes > nx_ctx->ap->databytelen)
- return -EINVAL;
+ spin_lock_irqsave(&nx_ctx->lock, irq_flags);
- rc = nx_build_sg_lists(nx_ctx, desc, dst, src, nbytes,
- csbcpb->cpb.aes_ctr.iv);
- if (rc)
- goto out;
+ max_sg_len = min_t(u32, nx_driver.of.max_sg_len/sizeof(struct nx_sg),
+ nx_ctx->ap->sglen);
- if (!nx_ctx->op.inlen || !nx_ctx->op.outlen) {
- rc = -EINVAL;
- goto out;
- }
+ do {
+ to_process = min_t(u64, nbytes - processed,
+ nx_ctx->ap->databytelen);
+ to_process = min_t(u64, to_process,
+ NX_PAGE_SIZE * (max_sg_len - 1));
+ to_process = to_process & ~(AES_BLOCK_SIZE - 1);
+
+ rc = nx_build_sg_lists(nx_ctx, desc, dst, src, to_process,
+ processed, csbcpb->cpb.aes_ctr.iv);
+ if (rc)
+ goto out;
+
+ if (!nx_ctx->op.inlen || !nx_ctx->op.outlen) {
+ rc = -EINVAL;
+ goto out;
+ }
+
+ rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
+ desc->flags & CRYPTO_TFM_REQ_MAY_SLEEP);
+ if (rc)
+ goto out;
+
+ memcpy(desc->info, csbcpb->cpb.aes_cbc.cv, AES_BLOCK_SIZE);
- rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
- desc->flags & CRYPTO_TFM_REQ_MAY_SLEEP);
- if (rc)
- goto out;
+ atomic_inc(&(nx_ctx->stats->aes_ops));
+ atomic64_add(csbcpb->csb.processed_byte_count,
+ &(nx_ctx->stats->aes_bytes));
- atomic_inc(&(nx_ctx->stats->aes_ops));
- atomic64_add(csbcpb->csb.processed_byte_count,
- &(nx_ctx->stats->aes_bytes));
+ processed += to_process;
+ } while (processed < nbytes);
out:
+ spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
return rc;
}
diff --git a/drivers/crypto/nx/nx-aes-ecb.c b/drivers/crypto/nx/nx-aes-ecb.c
index 7bbc9a81da21..85a8d23cf29d 100644
--- a/drivers/crypto/nx/nx-aes-ecb.c
+++ b/drivers/crypto/nx/nx-aes-ecb.c
@@ -70,34 +70,52 @@ static int ecb_aes_nx_crypt(struct blkcipher_desc *desc,
{
struct nx_crypto_ctx *nx_ctx = crypto_blkcipher_ctx(desc->tfm);
struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
+ unsigned long irq_flags;
+ unsigned int processed = 0, to_process;
+ u32 max_sg_len;
int rc;
- if (nbytes > nx_ctx->ap->databytelen)
- return -EINVAL;
+ spin_lock_irqsave(&nx_ctx->lock, irq_flags);
+
+ max_sg_len = min_t(u32, nx_driver.of.max_sg_len/sizeof(struct nx_sg),
+ nx_ctx->ap->sglen);
if (enc)
NX_CPB_FDM(csbcpb) |= NX_FDM_ENDE_ENCRYPT;
else
NX_CPB_FDM(csbcpb) &= ~NX_FDM_ENDE_ENCRYPT;
- rc = nx_build_sg_lists(nx_ctx, desc, dst, src, nbytes, NULL);
- if (rc)
- goto out;
+ do {
+ to_process = min_t(u64, nbytes - processed,
+ nx_ctx->ap->databytelen);
+ to_process = min_t(u64, to_process,
+ NX_PAGE_SIZE * (max_sg_len - 1));
+ to_process = to_process & ~(AES_BLOCK_SIZE - 1);
- if (!nx_ctx->op.inlen || !nx_ctx->op.outlen) {
- rc = -EINVAL;
- goto out;
- }
+ rc = nx_build_sg_lists(nx_ctx, desc, dst, src, to_process,
+ processed, NULL);
+ if (rc)
+ goto out;
+
+ if (!nx_ctx->op.inlen || !nx_ctx->op.outlen) {
+ rc = -EINVAL;
+ goto out;
+ }
+
+ rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
+ desc->flags & CRYPTO_TFM_REQ_MAY_SLEEP);
+ if (rc)
+ goto out;
+
+ atomic_inc(&(nx_ctx->stats->aes_ops));
+ atomic64_add(csbcpb->csb.processed_byte_count,
+ &(nx_ctx->stats->aes_bytes));
- rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
- desc->flags & CRYPTO_TFM_REQ_MAY_SLEEP);
- if (rc)
- goto out;
+ processed += to_process;
+ } while (processed < nbytes);
- atomic_inc(&(nx_ctx->stats->aes_ops));
- atomic64_add(csbcpb->csb.processed_byte_count,
- &(nx_ctx->stats->aes_bytes));
out:
+ spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
return rc;
}
diff --git a/drivers/crypto/nx/nx-aes-gcm.c b/drivers/crypto/nx/nx-aes-gcm.c
index 6cca6c392b00..025d9a8d5b19 100644
--- a/drivers/crypto/nx/nx-aes-gcm.c
+++ b/drivers/crypto/nx/nx-aes-gcm.c
@@ -125,38 +125,187 @@ static int nx_gca(struct nx_crypto_ctx *nx_ctx,
struct aead_request *req,
u8 *out)
{
+ int rc;
struct nx_csbcpb *csbcpb_aead = nx_ctx->csbcpb_aead;
- int rc = -EINVAL;
struct scatter_walk walk;
struct nx_sg *nx_sg = nx_ctx->in_sg;
+ unsigned int nbytes = req->assoclen;
+ unsigned int processed = 0, to_process;
+ u32 max_sg_len;
- if (req->assoclen > nx_ctx->ap->databytelen)
- goto out;
-
- if (req->assoclen <= AES_BLOCK_SIZE) {
+ if (nbytes <= AES_BLOCK_SIZE) {
scatterwalk_start(&walk, req->assoc);
- scatterwalk_copychunks(out, &walk, req->assoclen,
- SCATTERWALK_FROM_SG);
+ scatterwalk_copychunks(out, &walk, nbytes, SCATTERWALK_FROM_SG);
scatterwalk_done(&walk, SCATTERWALK_FROM_SG, 0);
-
- rc = 0;
- goto out;
+ return 0;
}
- nx_sg = nx_walk_and_build(nx_sg, nx_ctx->ap->sglen, req->assoc, 0,
- req->assoclen);
- nx_ctx->op_aead.inlen = (nx_ctx->in_sg - nx_sg) * sizeof(struct nx_sg);
+ NX_CPB_FDM(csbcpb_aead) &= ~NX_FDM_CONTINUATION;
+
+ /* page_limit: number of sg entries that fit on one page */
+ max_sg_len = min_t(u32, nx_driver.of.max_sg_len/sizeof(struct nx_sg),
+ nx_ctx->ap->sglen);
+
+ do {
+ /*
+ * to_process: the data chunk to process in this update.
+ * This value is bound by sg list limits.
+ */
+ to_process = min_t(u64, nbytes - processed,
+ nx_ctx->ap->databytelen);
+ to_process = min_t(u64, to_process,
+ NX_PAGE_SIZE * (max_sg_len - 1));
+
+ if ((to_process + processed) < nbytes)
+ NX_CPB_FDM(csbcpb_aead) |= NX_FDM_INTERMEDIATE;
+ else
+ NX_CPB_FDM(csbcpb_aead) &= ~NX_FDM_INTERMEDIATE;
+
+ nx_sg = nx_walk_and_build(nx_ctx->in_sg, nx_ctx->ap->sglen,
+ req->assoc, processed, to_process);
+ nx_ctx->op_aead.inlen = (nx_ctx->in_sg - nx_sg)
+ * sizeof(struct nx_sg);
+
+ rc = nx_hcall_sync(nx_ctx, &nx_ctx->op_aead,
+ req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP);
+ if (rc)
+ return rc;
+
+ memcpy(csbcpb_aead->cpb.aes_gca.in_pat,
+ csbcpb_aead->cpb.aes_gca.out_pat,
+ AES_BLOCK_SIZE);
+ NX_CPB_FDM(csbcpb_aead) |= NX_FDM_CONTINUATION;
+
+ atomic_inc(&(nx_ctx->stats->aes_ops));
+ atomic64_add(req->assoclen, &(nx_ctx->stats->aes_bytes));
+
+ processed += to_process;
+ } while (processed < nbytes);
+
+ memcpy(out, csbcpb_aead->cpb.aes_gca.out_pat, AES_BLOCK_SIZE);
+
+ return rc;
+}
+
+static int gmac(struct aead_request *req, struct blkcipher_desc *desc)
+{
+ int rc;
+ struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(req->base.tfm);
+ struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
+ struct nx_sg *nx_sg;
+ unsigned int nbytes = req->assoclen;
+ unsigned int processed = 0, to_process;
+ u32 max_sg_len;
+
+ /* Set GMAC mode */
+ csbcpb->cpb.hdr.mode = NX_MODE_AES_GMAC;
+
+ NX_CPB_FDM(csbcpb) &= ~NX_FDM_CONTINUATION;
+
+ /* page_limit: number of sg entries that fit on one page */
+ max_sg_len = min_t(u32, nx_driver.of.max_sg_len/sizeof(struct nx_sg),
+ nx_ctx->ap->sglen);
+
+ /* Copy IV */
+ memcpy(csbcpb->cpb.aes_gcm.iv_or_cnt, desc->info, AES_BLOCK_SIZE);
+
+ do {
+ /*
+ * to_process: the data chunk to process in this update.
+ * This value is bound by sg list limits.
+ */
+ to_process = min_t(u64, nbytes - processed,
+ nx_ctx->ap->databytelen);
+ to_process = min_t(u64, to_process,
+ NX_PAGE_SIZE * (max_sg_len - 1));
+
+ if ((to_process + processed) < nbytes)
+ NX_CPB_FDM(csbcpb) |= NX_FDM_INTERMEDIATE;
+ else
+ NX_CPB_FDM(csbcpb) &= ~NX_FDM_INTERMEDIATE;
+
+ nx_sg = nx_walk_and_build(nx_ctx->in_sg, nx_ctx->ap->sglen,
+ req->assoc, processed, to_process);
+ nx_ctx->op.inlen = (nx_ctx->in_sg - nx_sg)
+ * sizeof(struct nx_sg);
+
+ csbcpb->cpb.aes_gcm.bit_length_data = 0;
+ csbcpb->cpb.aes_gcm.bit_length_aad = 8 * nbytes;
+
+ rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
+ req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP);
+ if (rc)
+ goto out;
+
+ memcpy(csbcpb->cpb.aes_gcm.in_pat_or_aad,
+ csbcpb->cpb.aes_gcm.out_pat_or_mac, AES_BLOCK_SIZE);
+ memcpy(csbcpb->cpb.aes_gcm.in_s0,
+ csbcpb->cpb.aes_gcm.out_s0, AES_BLOCK_SIZE);
+
+ NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION;
+
+ atomic_inc(&(nx_ctx->stats->aes_ops));
+ atomic64_add(req->assoclen, &(nx_ctx->stats->aes_bytes));
+
+ processed += to_process;
+ } while (processed < nbytes);
+
+out:
+ /* Restore GCM mode */
+ csbcpb->cpb.hdr.mode = NX_MODE_AES_GCM;
+ return rc;
+}
+
+static int gcm_empty(struct aead_request *req, struct blkcipher_desc *desc,
+ int enc)
+{
+ int rc;
+ struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(req->base.tfm);
+ struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
+ char out[AES_BLOCK_SIZE];
+ struct nx_sg *in_sg, *out_sg;
+
+ /* For scenarios where the input message is zero length, AES CTR mode
+ * may be used. Set the source data to be a single block (16B) of all
+ * zeros, and set the input IV value to be the same as the GMAC IV
+ * value. - nx_wb 4.8.1.3 */
+
+ /* Change to ECB mode */
+ csbcpb->cpb.hdr.mode = NX_MODE_AES_ECB;
+ memcpy(csbcpb->cpb.aes_ecb.key, csbcpb->cpb.aes_gcm.key,
+ sizeof(csbcpb->cpb.aes_ecb.key));
+ if (enc)
+ NX_CPB_FDM(csbcpb) |= NX_FDM_ENDE_ENCRYPT;
+ else
+ NX_CPB_FDM(csbcpb) &= ~NX_FDM_ENDE_ENCRYPT;
+
+ /* Encrypt the counter/IV */
+ in_sg = nx_build_sg_list(nx_ctx->in_sg, (u8 *) desc->info,
+ AES_BLOCK_SIZE, nx_ctx->ap->sglen);
+ out_sg = nx_build_sg_list(nx_ctx->out_sg, (u8 *) out, sizeof(out),
+ nx_ctx->ap->sglen);
+ nx_ctx->op.inlen = (nx_ctx->in_sg - in_sg) * sizeof(struct nx_sg);
+ nx_ctx->op.outlen = (nx_ctx->out_sg - out_sg) * sizeof(struct nx_sg);
- rc = nx_hcall_sync(nx_ctx, &nx_ctx->op_aead,
- req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP);
+ rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
+ desc->flags & CRYPTO_TFM_REQ_MAY_SLEEP);
if (rc)
goto out;
-
atomic_inc(&(nx_ctx->stats->aes_ops));
- atomic64_add(req->assoclen, &(nx_ctx->stats->aes_bytes));
- memcpy(out, csbcpb_aead->cpb.aes_gca.out_pat, AES_BLOCK_SIZE);
+ /* Copy out the auth tag */
+ memcpy(csbcpb->cpb.aes_gcm.out_pat_or_mac, out,
+ crypto_aead_authsize(crypto_aead_reqtfm(req)));
out:
+ /* Restore XCBC mode */
+ csbcpb->cpb.hdr.mode = NX_MODE_AES_GCM;
+
+ /*
+ * ECB key uses the same region that GCM AAD and counter, so it's safe
+ * to just fill it with zeroes.
+ */
+ memset(csbcpb->cpb.aes_ecb.key, 0, sizeof(csbcpb->cpb.aes_ecb.key));
+
return rc;
}
@@ -166,88 +315,104 @@ static int gcm_aes_nx_crypt(struct aead_request *req, int enc)
struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
struct blkcipher_desc desc;
unsigned int nbytes = req->cryptlen;
+ unsigned int processed = 0, to_process;
+ unsigned long irq_flags;
+ u32 max_sg_len;
int rc = -EINVAL;
- if (nbytes > nx_ctx->ap->databytelen)
- goto out;
+ spin_lock_irqsave(&nx_ctx->lock, irq_flags);
desc.info = nx_ctx->priv.gcm.iv;
/* initialize the counter */
*(u32 *)(desc.info + NX_GCM_CTR_OFFSET) = 1;
- /* For scenarios where the input message is zero length, AES CTR mode
- * may be used. Set the source data to be a single block (16B) of all
- * zeros, and set the input IV value to be the same as the GMAC IV
- * value. - nx_wb 4.8.1.3 */
if (nbytes == 0) {
- char src[AES_BLOCK_SIZE] = {};
- struct scatterlist sg;
-
- desc.tfm = crypto_alloc_blkcipher("ctr(aes)", 0, 0);
- if (IS_ERR(desc.tfm)) {
- rc = -ENOMEM;
+ if (req->assoclen == 0)
+ rc = gcm_empty(req, &desc, enc);
+ else
+ rc = gmac(req, &desc);
+ if (rc)
goto out;
- }
-
- crypto_blkcipher_setkey(desc.tfm, csbcpb->cpb.aes_gcm.key,
- NX_CPB_KEY_SIZE(csbcpb) == NX_KS_AES_128 ? 16 :
- NX_CPB_KEY_SIZE(csbcpb) == NX_KS_AES_192 ? 24 : 32);
-
- sg_init_one(&sg, src, AES_BLOCK_SIZE);
- if (enc)
- crypto_blkcipher_encrypt_iv(&desc, req->dst, &sg,
- AES_BLOCK_SIZE);
else
- crypto_blkcipher_decrypt_iv(&desc, req->dst, &sg,
- AES_BLOCK_SIZE);
- crypto_free_blkcipher(desc.tfm);
-
- rc = 0;
- goto out;
+ goto mac;
}
- desc.tfm = (struct crypto_blkcipher *)req->base.tfm;
-
+ /* Process associated data */
csbcpb->cpb.aes_gcm.bit_length_aad = req->assoclen * 8;
-
if (req->assoclen) {
rc = nx_gca(nx_ctx, req, csbcpb->cpb.aes_gcm.in_pat_or_aad);
if (rc)
goto out;
}
- if (enc)
+ /* Set flags for encryption */
+ NX_CPB_FDM(csbcpb) &= ~NX_FDM_CONTINUATION;
+ if (enc) {
NX_CPB_FDM(csbcpb) |= NX_FDM_ENDE_ENCRYPT;
- else
+ } else {
+ NX_CPB_FDM(csbcpb) &= ~NX_FDM_ENDE_ENCRYPT;
nbytes -= crypto_aead_authsize(crypto_aead_reqtfm(req));
+ }
- csbcpb->cpb.aes_gcm.bit_length_data = nbytes * 8;
+ /* page_limit: number of sg entries that fit on one page */
+ max_sg_len = min_t(u32, nx_driver.of.max_sg_len/sizeof(struct nx_sg),
+ nx_ctx->ap->sglen);
+
+ do {
+ /*
+ * to_process: the data chunk to process in this update.
+ * This value is bound by sg list limits.
+ */
+ to_process = min_t(u64, nbytes - processed,
+ nx_ctx->ap->databytelen);
+ to_process = min_t(u64, to_process,
+ NX_PAGE_SIZE * (max_sg_len - 1));
+
+ if ((to_process + processed) < nbytes)
+ NX_CPB_FDM(csbcpb) |= NX_FDM_INTERMEDIATE;
+ else
+ NX_CPB_FDM(csbcpb) &= ~NX_FDM_INTERMEDIATE;
- rc = nx_build_sg_lists(nx_ctx, &desc, req->dst, req->src, nbytes,
- csbcpb->cpb.aes_gcm.iv_or_cnt);
- if (rc)
- goto out;
+ csbcpb->cpb.aes_gcm.bit_length_data = nbytes * 8;
+ desc.tfm = (struct crypto_blkcipher *) req->base.tfm;
+ rc = nx_build_sg_lists(nx_ctx, &desc, req->dst,
+ req->src, to_process, processed,
+ csbcpb->cpb.aes_gcm.iv_or_cnt);
+ if (rc)
+ goto out;
- rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
- req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP);
- if (rc)
- goto out;
+ rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
+ req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP);
+ if (rc)
+ goto out;
- atomic_inc(&(nx_ctx->stats->aes_ops));
- atomic64_add(csbcpb->csb.processed_byte_count,
- &(nx_ctx->stats->aes_bytes));
+ memcpy(desc.info, csbcpb->cpb.aes_gcm.out_cnt, AES_BLOCK_SIZE);
+ memcpy(csbcpb->cpb.aes_gcm.in_pat_or_aad,
+ csbcpb->cpb.aes_gcm.out_pat_or_mac, AES_BLOCK_SIZE);
+ memcpy(csbcpb->cpb.aes_gcm.in_s0,
+ csbcpb->cpb.aes_gcm.out_s0, AES_BLOCK_SIZE);
+
+ NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION;
+
+ atomic_inc(&(nx_ctx->stats->aes_ops));
+ atomic64_add(csbcpb->csb.processed_byte_count,
+ &(nx_ctx->stats->aes_bytes));
+
+ processed += to_process;
+ } while (processed < nbytes);
+mac:
if (enc) {
/* copy out the auth tag */
scatterwalk_map_and_copy(csbcpb->cpb.aes_gcm.out_pat_or_mac,
req->dst, nbytes,
crypto_aead_authsize(crypto_aead_reqtfm(req)),
SCATTERWALK_TO_SG);
- } else if (req->assoclen) {
+ } else {
u8 *itag = nx_ctx->priv.gcm.iauth_tag;
u8 *otag = csbcpb->cpb.aes_gcm.out_pat_or_mac;
- scatterwalk_map_and_copy(itag, req->dst, nbytes,
+ scatterwalk_map_and_copy(itag, req->src, nbytes,
crypto_aead_authsize(crypto_aead_reqtfm(req)),
SCATTERWALK_FROM_SG);
rc = memcmp(itag, otag,
@@ -255,6 +420,7 @@ static int gcm_aes_nx_crypt(struct aead_request *req, int enc)
-EBADMSG : 0;
}
out:
+ spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
return rc;
}
diff --git a/drivers/crypto/nx/nx-aes-xcbc.c b/drivers/crypto/nx/nx-aes-xcbc.c
index 93923e4628c0..03c4bf57d066 100644
--- a/drivers/crypto/nx/nx-aes-xcbc.c
+++ b/drivers/crypto/nx/nx-aes-xcbc.c
@@ -56,6 +56,77 @@ static int nx_xcbc_set_key(struct crypto_shash *desc,
return 0;
}
+/*
+ * Based on RFC 3566, for a zero-length message:
+ *
+ * n = 1
+ * K1 = E(K, 0x01010101010101010101010101010101)
+ * K3 = E(K, 0x03030303030303030303030303030303)
+ * E[0] = 0x00000000000000000000000000000000
+ * M[1] = 0x80000000000000000000000000000000 (0 length message with padding)
+ * E[1] = (K1, M[1] ^ E[0] ^ K3)
+ * Tag = M[1]
+ */
+static int nx_xcbc_empty(struct shash_desc *desc, u8 *out)
+{
+ struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base);
+ struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
+ struct nx_sg *in_sg, *out_sg;
+ u8 keys[2][AES_BLOCK_SIZE];
+ u8 key[32];
+ int rc = 0;
+
+ /* Change to ECB mode */
+ csbcpb->cpb.hdr.mode = NX_MODE_AES_ECB;
+ memcpy(key, csbcpb->cpb.aes_xcbc.key, AES_BLOCK_SIZE);
+ memcpy(csbcpb->cpb.aes_ecb.key, key, AES_BLOCK_SIZE);
+ NX_CPB_FDM(csbcpb) |= NX_FDM_ENDE_ENCRYPT;
+
+ /* K1 and K3 base patterns */
+ memset(keys[0], 0x01, sizeof(keys[0]));
+ memset(keys[1], 0x03, sizeof(keys[1]));
+
+ /* Generate K1 and K3 encrypting the patterns */
+ in_sg = nx_build_sg_list(nx_ctx->in_sg, (u8 *) keys, sizeof(keys),
+ nx_ctx->ap->sglen);
+ out_sg = nx_build_sg_list(nx_ctx->out_sg, (u8 *) keys, sizeof(keys),
+ nx_ctx->ap->sglen);
+ nx_ctx->op.inlen = (nx_ctx->in_sg - in_sg) * sizeof(struct nx_sg);
+ nx_ctx->op.outlen = (nx_ctx->out_sg - out_sg) * sizeof(struct nx_sg);
+
+ rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
+ desc->flags & CRYPTO_TFM_REQ_MAY_SLEEP);
+ if (rc)
+ goto out;
+ atomic_inc(&(nx_ctx->stats->aes_ops));
+
+ /* XOr K3 with the padding for a 0 length message */
+ keys[1][0] ^= 0x80;
+
+ /* Encrypt the final result */
+ memcpy(csbcpb->cpb.aes_ecb.key, keys[0], AES_BLOCK_SIZE);
+ in_sg = nx_build_sg_list(nx_ctx->in_sg, (u8 *) keys[1], sizeof(keys[1]),
+ nx_ctx->ap->sglen);
+ out_sg = nx_build_sg_list(nx_ctx->out_sg, out, AES_BLOCK_SIZE,
+ nx_ctx->ap->sglen);
+ nx_ctx->op.inlen = (nx_ctx->in_sg - in_sg) * sizeof(struct nx_sg);
+ nx_ctx->op.outlen = (nx_ctx->out_sg - out_sg) * sizeof(struct nx_sg);
+
+ rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
+ desc->flags & CRYPTO_TFM_REQ_MAY_SLEEP);
+ if (rc)
+ goto out;
+ atomic_inc(&(nx_ctx->stats->aes_ops));
+
+out:
+ /* Restore XCBC mode */
+ csbcpb->cpb.hdr.mode = NX_MODE_AES_XCBC_MAC;
+ memcpy(csbcpb->cpb.aes_xcbc.key, key, AES_BLOCK_SIZE);
+ NX_CPB_FDM(csbcpb) &= ~NX_FDM_ENDE_ENCRYPT;
+
+ return rc;
+}
+
static int nx_xcbc_init(struct shash_desc *desc)
{
struct xcbc_state *sctx = shash_desc_ctx(desc);
@@ -88,76 +159,99 @@ static int nx_xcbc_update(struct shash_desc *desc,
struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base);
struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
struct nx_sg *in_sg;
- u32 to_process, leftover;
+ u32 to_process, leftover, total;
+ u32 max_sg_len;
+ unsigned long irq_flags;
int rc = 0;
- if (NX_CPB_FDM(csbcpb) & NX_FDM_CONTINUATION) {
- /* we've hit the nx chip previously and we're updating again,
- * so copy over the partial digest */
- memcpy(csbcpb->cpb.aes_xcbc.cv,
- csbcpb->cpb.aes_xcbc.out_cv_mac, AES_BLOCK_SIZE);
- }
+ spin_lock_irqsave(&nx_ctx->lock, irq_flags);
+
+
+ total = sctx->count + len;
/* 2 cases for total data len:
* 1: <= AES_BLOCK_SIZE: copy into state, return 0
* 2: > AES_BLOCK_SIZE: process X blocks, copy in leftover
*/
- if (len + sctx->count <= AES_BLOCK_SIZE) {
+ if (total <= AES_BLOCK_SIZE) {
memcpy(sctx->buffer + sctx->count, data, len);
sctx->count += len;
goto out;
}
- /* to_process: the AES_BLOCK_SIZE data chunk to process in this
- * update */
- to_process = (sctx->count + len) & ~(AES_BLOCK_SIZE - 1);
- leftover = (sctx->count + len) & (AES_BLOCK_SIZE - 1);
-
- /* the hardware will not accept a 0 byte operation for this algorithm
- * and the operation MUST be finalized to be correct. So if we happen
- * to get an update that falls on a block sized boundary, we must
- * save off the last block to finalize with later. */
- if (!leftover) {
- to_process -= AES_BLOCK_SIZE;
- leftover = AES_BLOCK_SIZE;
- }
-
- if (sctx->count) {
- in_sg = nx_build_sg_list(nx_ctx->in_sg, sctx->buffer,
- sctx->count, nx_ctx->ap->sglen);
- in_sg = nx_build_sg_list(in_sg, (u8 *)data,
- to_process - sctx->count,
- nx_ctx->ap->sglen);
+ in_sg = nx_ctx->in_sg;
+ max_sg_len = min_t(u32, nx_driver.of.max_sg_len/sizeof(struct nx_sg),
+ nx_ctx->ap->sglen);
+
+ do {
+
+ /* to_process: the AES_BLOCK_SIZE data chunk to process in this
+ * update */
+ to_process = min_t(u64, total, nx_ctx->ap->databytelen);
+ to_process = min_t(u64, to_process,
+ NX_PAGE_SIZE * (max_sg_len - 1));
+ to_process = to_process & ~(AES_BLOCK_SIZE - 1);
+ leftover = total - to_process;
+
+ /* the hardware will not accept a 0 byte operation for this
+ * algorithm and the operation MUST be finalized to be correct.
+ * So if we happen to get an update that falls on a block sized
+ * boundary, we must save off the last block to finalize with
+ * later. */
+ if (!leftover) {
+ to_process -= AES_BLOCK_SIZE;
+ leftover = AES_BLOCK_SIZE;
+ }
+
+ if (sctx->count) {
+ in_sg = nx_build_sg_list(nx_ctx->in_sg,
+ (u8 *) sctx->buffer,
+ sctx->count,
+ max_sg_len);
+ }
+ in_sg = nx_build_sg_list(in_sg,
+ (u8 *) data,
+ to_process - sctx->count,
+ max_sg_len);
nx_ctx->op.inlen = (nx_ctx->in_sg - in_sg) *
sizeof(struct nx_sg);
- } else {
- in_sg = nx_build_sg_list(nx_ctx->in_sg, (u8 *)data, to_process,
- nx_ctx->ap->sglen);
- nx_ctx->op.inlen = (nx_ctx->in_sg - in_sg) *
- sizeof(struct nx_sg);
- }
- NX_CPB_FDM(csbcpb) |= NX_FDM_INTERMEDIATE;
+ /* we've hit the nx chip previously and we're updating again,
+ * so copy over the partial digest */
+ if (NX_CPB_FDM(csbcpb) & NX_FDM_CONTINUATION) {
+ memcpy(csbcpb->cpb.aes_xcbc.cv,
+ csbcpb->cpb.aes_xcbc.out_cv_mac,
+ AES_BLOCK_SIZE);
+ }
+
+ NX_CPB_FDM(csbcpb) |= NX_FDM_INTERMEDIATE;
+ if (!nx_ctx->op.inlen || !nx_ctx->op.outlen) {
+ rc = -EINVAL;
+ goto out;
+ }
+
+ rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
+ desc->flags & CRYPTO_TFM_REQ_MAY_SLEEP);
+ if (rc)
+ goto out;
- if (!nx_ctx->op.inlen || !nx_ctx->op.outlen) {
- rc = -EINVAL;
- goto out;
- }
+ atomic_inc(&(nx_ctx->stats->aes_ops));
- rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
- desc->flags & CRYPTO_TFM_REQ_MAY_SLEEP);
- if (rc)
- goto out;
+ /* everything after the first update is continuation */
+ NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION;
- atomic_inc(&(nx_ctx->stats->aes_ops));
+ total -= to_process;
+ data += to_process - sctx->count;
+ sctx->count = 0;
+ in_sg = nx_ctx->in_sg;
+ } while (leftover > AES_BLOCK_SIZE);
/* copy the leftover back into the state struct */
- memcpy(sctx->buffer, data + len - leftover, leftover);
+ memcpy(sctx->buffer, data, leftover);
sctx->count = leftover;
- /* everything after the first update is continuation */
- NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION;
out:
+ spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
return rc;
}
@@ -167,21 +261,23 @@ static int nx_xcbc_final(struct shash_desc *desc, u8 *out)
struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base);
struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
struct nx_sg *in_sg, *out_sg;
+ unsigned long irq_flags;
int rc = 0;
+ spin_lock_irqsave(&nx_ctx->lock, irq_flags);
+
if (NX_CPB_FDM(csbcpb) & NX_FDM_CONTINUATION) {
/* we've hit the nx chip previously, now we're finalizing,
* so copy over the partial digest */
memcpy(csbcpb->cpb.aes_xcbc.cv,
csbcpb->cpb.aes_xcbc.out_cv_mac, AES_BLOCK_SIZE);
} else if (sctx->count == 0) {
- /* we've never seen an update, so this is a 0 byte op. The
- * hardware cannot handle a 0 byte op, so just copy out the
- * known 0 byte result. This is cheaper than allocating a
- * software context to do a 0 byte op */
- u8 data[] = { 0x75, 0xf0, 0x25, 0x1d, 0x52, 0x8a, 0xc0, 0x1c,
- 0x45, 0x73, 0xdf, 0xd5, 0x84, 0xd7, 0x9f, 0x29 };
- memcpy(out, data, sizeof(data));
+ /*
+ * we've never seen an update, so this is a 0 byte op. The
+ * hardware cannot handle a 0 byte op, so just ECB to
+ * generate the hash.
+ */
+ rc = nx_xcbc_empty(desc, out);
goto out;
}
@@ -211,6 +307,7 @@ static int nx_xcbc_final(struct shash_desc *desc, u8 *out)
memcpy(out, csbcpb->cpb.aes_xcbc.out_cv_mac, AES_BLOCK_SIZE);
out:
+ spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
return rc;
}
diff --git a/drivers/crypto/nx/nx-sha256.c b/drivers/crypto/nx/nx-sha256.c
index 67024f2f0b78..da0b24a7633f 100644
--- a/drivers/crypto/nx/nx-sha256.c
+++ b/drivers/crypto/nx/nx-sha256.c
@@ -55,71 +55,91 @@ static int nx_sha256_update(struct shash_desc *desc, const u8 *data,
struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base);
struct nx_csbcpb *csbcpb = (struct nx_csbcpb *)nx_ctx->csbcpb;
struct nx_sg *in_sg;
- u64 to_process, leftover;
+ u64 to_process, leftover, total;
+ u32 max_sg_len;
+ unsigned long irq_flags;
int rc = 0;
- if (NX_CPB_FDM(csbcpb) & NX_FDM_CONTINUATION) {
- /* we've hit the nx chip previously and we're updating again,
- * so copy over the partial digest */
- memcpy(csbcpb->cpb.sha256.input_partial_digest,
- csbcpb->cpb.sha256.message_digest, SHA256_DIGEST_SIZE);
- }
+ spin_lock_irqsave(&nx_ctx->lock, irq_flags);
/* 2 cases for total data len:
- * 1: <= SHA256_BLOCK_SIZE: copy into state, return 0
- * 2: > SHA256_BLOCK_SIZE: process X blocks, copy in leftover
+ * 1: < SHA256_BLOCK_SIZE: copy into state, return 0
+ * 2: >= SHA256_BLOCK_SIZE: process X blocks, copy in leftover
*/
- if (len + sctx->count < SHA256_BLOCK_SIZE) {
+ total = sctx->count + len;
+ if (total < SHA256_BLOCK_SIZE) {
memcpy(sctx->buf + sctx->count, data, len);
sctx->count += len;
goto out;
}
- /* to_process: the SHA256_BLOCK_SIZE data chunk to process in this
- * update */
- to_process = (sctx->count + len) & ~(SHA256_BLOCK_SIZE - 1);
- leftover = (sctx->count + len) & (SHA256_BLOCK_SIZE - 1);
-
- if (sctx->count) {
- in_sg = nx_build_sg_list(nx_ctx->in_sg, (u8 *)sctx->buf,
- sctx->count, nx_ctx->ap->sglen);
- in_sg = nx_build_sg_list(in_sg, (u8 *)data,
+ in_sg = nx_ctx->in_sg;
+ max_sg_len = min_t(u32, nx_driver.of.max_sg_len/sizeof(struct nx_sg),
+ nx_ctx->ap->sglen);
+
+ do {
+ /*
+ * to_process: the SHA256_BLOCK_SIZE data chunk to process in
+ * this update. This value is also restricted by the sg list
+ * limits.
+ */
+ to_process = min_t(u64, total, nx_ctx->ap->databytelen);
+ to_process = min_t(u64, to_process,
+ NX_PAGE_SIZE * (max_sg_len - 1));
+ to_process = to_process & ~(SHA256_BLOCK_SIZE - 1);
+ leftover = total - to_process;
+
+ if (sctx->count) {
+ in_sg = nx_build_sg_list(nx_ctx->in_sg,
+ (u8 *) sctx->buf,
+ sctx->count, max_sg_len);
+ }
+ in_sg = nx_build_sg_list(in_sg, (u8 *) data,
to_process - sctx->count,
- nx_ctx->ap->sglen);
+ max_sg_len);
nx_ctx->op.inlen = (nx_ctx->in_sg - in_sg) *
sizeof(struct nx_sg);
- } else {
- in_sg = nx_build_sg_list(nx_ctx->in_sg, (u8 *)data,
- to_process, nx_ctx->ap->sglen);
- nx_ctx->op.inlen = (nx_ctx->in_sg - in_sg) *
- sizeof(struct nx_sg);
- }
- NX_CPB_FDM(csbcpb) |= NX_FDM_INTERMEDIATE;
+ if (NX_CPB_FDM(csbcpb) & NX_FDM_CONTINUATION) {
+ /*
+ * we've hit the nx chip previously and we're updating
+ * again, so copy over the partial digest.
+ */
+ memcpy(csbcpb->cpb.sha256.input_partial_digest,
+ csbcpb->cpb.sha256.message_digest,
+ SHA256_DIGEST_SIZE);
+ }
- if (!nx_ctx->op.inlen || !nx_ctx->op.outlen) {
- rc = -EINVAL;
- goto out;
- }
+ NX_CPB_FDM(csbcpb) |= NX_FDM_INTERMEDIATE;
+ if (!nx_ctx->op.inlen || !nx_ctx->op.outlen) {
+ rc = -EINVAL;
+ goto out;
+ }
- rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
- desc->flags & CRYPTO_TFM_REQ_MAY_SLEEP);
- if (rc)
- goto out;
+ rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
+ desc->flags & CRYPTO_TFM_REQ_MAY_SLEEP);
+ if (rc)
+ goto out;
- atomic_inc(&(nx_ctx->stats->sha256_ops));
+ atomic_inc(&(nx_ctx->stats->sha256_ops));
+ csbcpb->cpb.sha256.message_bit_length += (u64)
+ (csbcpb->cpb.sha256.spbc * 8);
+
+ /* everything after the first update is continuation */
+ NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION;
+
+ total -= to_process;
+ data += to_process - sctx->count;
+ sctx->count = 0;
+ in_sg = nx_ctx->in_sg;
+ } while (leftover >= SHA256_BLOCK_SIZE);
/* copy the leftover back into the state struct */
if (leftover)
- memcpy(sctx->buf, data + len - leftover, leftover);
+ memcpy(sctx->buf, data, leftover);
sctx->count = leftover;
-
- csbcpb->cpb.sha256.message_bit_length += (u64)
- (csbcpb->cpb.sha256.spbc * 8);
-
- /* everything after the first update is continuation */
- NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION;
out:
+ spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
return rc;
}
@@ -129,8 +149,13 @@ static int nx_sha256_final(struct shash_desc *desc, u8 *out)
struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base);
struct nx_csbcpb *csbcpb = (struct nx_csbcpb *)nx_ctx->csbcpb;
struct nx_sg *in_sg, *out_sg;
+ u32 max_sg_len;
+ unsigned long irq_flags;
int rc;
+ spin_lock_irqsave(&nx_ctx->lock, irq_flags);
+
+ max_sg_len = min_t(u32, nx_driver.of.max_sg_len, nx_ctx->ap->sglen);
if (NX_CPB_FDM(csbcpb) & NX_FDM_CONTINUATION) {
/* we've hit the nx chip previously, now we're finalizing,
@@ -146,9 +171,9 @@ static int nx_sha256_final(struct shash_desc *desc, u8 *out)
csbcpb->cpb.sha256.message_bit_length += (u64)(sctx->count * 8);
in_sg = nx_build_sg_list(nx_ctx->in_sg, (u8 *)sctx->buf,
- sctx->count, nx_ctx->ap->sglen);
+ sctx->count, max_sg_len);
out_sg = nx_build_sg_list(nx_ctx->out_sg, out, SHA256_DIGEST_SIZE,
- nx_ctx->ap->sglen);
+ max_sg_len);
nx_ctx->op.inlen = (nx_ctx->in_sg - in_sg) * sizeof(struct nx_sg);
nx_ctx->op.outlen = (nx_ctx->out_sg - out_sg) * sizeof(struct nx_sg);
@@ -168,6 +193,7 @@ static int nx_sha256_final(struct shash_desc *desc, u8 *out)
&(nx_ctx->stats->sha256_bytes));
memcpy(out, csbcpb->cpb.sha256.message_digest, SHA256_DIGEST_SIZE);
out:
+ spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
return rc;
}
@@ -177,6 +203,9 @@ static int nx_sha256_export(struct shash_desc *desc, void *out)
struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base);
struct nx_csbcpb *csbcpb = (struct nx_csbcpb *)nx_ctx->csbcpb;
struct sha256_state *octx = out;
+ unsigned long irq_flags;
+
+ spin_lock_irqsave(&nx_ctx->lock, irq_flags);
octx->count = sctx->count +
(csbcpb->cpb.sha256.message_bit_length / 8);
@@ -199,6 +228,7 @@ static int nx_sha256_export(struct shash_desc *desc, void *out)
octx->state[7] = SHA256_H7;
}
+ spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
return 0;
}
@@ -208,6 +238,9 @@ static int nx_sha256_import(struct shash_desc *desc, const void *in)
struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base);
struct nx_csbcpb *csbcpb = (struct nx_csbcpb *)nx_ctx->csbcpb;
const struct sha256_state *ictx = in;
+ unsigned long irq_flags;
+
+ spin_lock_irqsave(&nx_ctx->lock, irq_flags);
memcpy(sctx->buf, ictx->buf, sizeof(ictx->buf));
@@ -222,6 +255,7 @@ static int nx_sha256_import(struct shash_desc *desc, const void *in)
NX_CPB_FDM(csbcpb) |= NX_FDM_INTERMEDIATE;
}
+ spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
return 0;
}
diff --git a/drivers/crypto/nx/nx-sha512.c b/drivers/crypto/nx/nx-sha512.c
index 08eee1122349..4ae5b0f221d5 100644
--- a/drivers/crypto/nx/nx-sha512.c
+++ b/drivers/crypto/nx/nx-sha512.c
@@ -55,73 +55,93 @@ static int nx_sha512_update(struct shash_desc *desc, const u8 *data,
struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base);
struct nx_csbcpb *csbcpb = (struct nx_csbcpb *)nx_ctx->csbcpb;
struct nx_sg *in_sg;
- u64 to_process, leftover, spbc_bits;
+ u64 to_process, leftover, total, spbc_bits;
+ u32 max_sg_len;
+ unsigned long irq_flags;
int rc = 0;
- if (NX_CPB_FDM(csbcpb) & NX_FDM_CONTINUATION) {
- /* we've hit the nx chip previously and we're updating again,
- * so copy over the partial digest */
- memcpy(csbcpb->cpb.sha512.input_partial_digest,
- csbcpb->cpb.sha512.message_digest, SHA512_DIGEST_SIZE);
- }
+ spin_lock_irqsave(&nx_ctx->lock, irq_flags);
/* 2 cases for total data len:
- * 1: <= SHA512_BLOCK_SIZE: copy into state, return 0
- * 2: > SHA512_BLOCK_SIZE: process X blocks, copy in leftover
+ * 1: < SHA512_BLOCK_SIZE: copy into state, return 0
+ * 2: >= SHA512_BLOCK_SIZE: process X blocks, copy in leftover
*/
- if ((u64)len + sctx->count[0] < SHA512_BLOCK_SIZE) {
+ total = sctx->count[0] + len;
+ if (total < SHA512_BLOCK_SIZE) {
memcpy(sctx->buf + sctx->count[0], data, len);
sctx->count[0] += len;
goto out;
}
- /* to_process: the SHA512_BLOCK_SIZE data chunk to process in this
- * update */
- to_process = (sctx->count[0] + len) & ~(SHA512_BLOCK_SIZE - 1);
- leftover = (sctx->count[0] + len) & (SHA512_BLOCK_SIZE - 1);
-
- if (sctx->count[0]) {
- in_sg = nx_build_sg_list(nx_ctx->in_sg, (u8 *)sctx->buf,
- sctx->count[0], nx_ctx->ap->sglen);
- in_sg = nx_build_sg_list(in_sg, (u8 *)data,
+ in_sg = nx_ctx->in_sg;
+ max_sg_len = min_t(u32, nx_driver.of.max_sg_len/sizeof(struct nx_sg),
+ nx_ctx->ap->sglen);
+
+ do {
+ /*
+ * to_process: the SHA512_BLOCK_SIZE data chunk to process in
+ * this update. This value is also restricted by the sg list
+ * limits.
+ */
+ to_process = min_t(u64, total, nx_ctx->ap->databytelen);
+ to_process = min_t(u64, to_process,
+ NX_PAGE_SIZE * (max_sg_len - 1));
+ to_process = to_process & ~(SHA512_BLOCK_SIZE - 1);
+ leftover = total - to_process;
+
+ if (sctx->count[0]) {
+ in_sg = nx_build_sg_list(nx_ctx->in_sg,
+ (u8 *) sctx->buf,
+ sctx->count[0], max_sg_len);
+ }
+ in_sg = nx_build_sg_list(in_sg, (u8 *) data,
to_process - sctx->count[0],
- nx_ctx->ap->sglen);
- nx_ctx->op.inlen = (nx_ctx->in_sg - in_sg) *
- sizeof(struct nx_sg);
- } else {
- in_sg = nx_build_sg_list(nx_ctx->in_sg, (u8 *)data,
- to_process, nx_ctx->ap->sglen);
+ max_sg_len);
nx_ctx->op.inlen = (nx_ctx->in_sg - in_sg) *
sizeof(struct nx_sg);
- }
- NX_CPB_FDM(csbcpb) |= NX_FDM_INTERMEDIATE;
+ if (NX_CPB_FDM(csbcpb) & NX_FDM_CONTINUATION) {
+ /*
+ * we've hit the nx chip previously and we're updating
+ * again, so copy over the partial digest.
+ */
+ memcpy(csbcpb->cpb.sha512.input_partial_digest,
+ csbcpb->cpb.sha512.message_digest,
+ SHA512_DIGEST_SIZE);
+ }
- if (!nx_ctx->op.inlen || !nx_ctx->op.outlen) {
- rc = -EINVAL;
- goto out;
- }
-
- rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
- desc->flags & CRYPTO_TFM_REQ_MAY_SLEEP);
- if (rc)
- goto out;
+ NX_CPB_FDM(csbcpb) |= NX_FDM_INTERMEDIATE;
+ if (!nx_ctx->op.inlen || !nx_ctx->op.outlen) {
+ rc = -EINVAL;
+ goto out;
+ }
+
+ rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
+ desc->flags & CRYPTO_TFM_REQ_MAY_SLEEP);
+ if (rc)
+ goto out;
+
+ atomic_inc(&(nx_ctx->stats->sha512_ops));
+ spbc_bits = csbcpb->cpb.sha512.spbc * 8;
+ csbcpb->cpb.sha512.message_bit_length_lo += spbc_bits;
+ if (csbcpb->cpb.sha512.message_bit_length_lo < spbc_bits)
+ csbcpb->cpb.sha512.message_bit_length_hi++;
+
+ /* everything after the first update is continuation */
+ NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION;
- atomic_inc(&(nx_ctx->stats->sha512_ops));
+ total -= to_process;
+ data += to_process - sctx->count[0];
+ sctx->count[0] = 0;
+ in_sg = nx_ctx->in_sg;
+ } while (leftover >= SHA512_BLOCK_SIZE);
/* copy the leftover back into the state struct */
if (leftover)
- memcpy(sctx->buf, data + len - leftover, leftover);
+ memcpy(sctx->buf, data, leftover);
sctx->count[0] = leftover;
-
- spbc_bits = csbcpb->cpb.sha512.spbc * 8;
- csbcpb->cpb.sha512.message_bit_length_lo += spbc_bits;
- if (csbcpb->cpb.sha512.message_bit_length_lo < spbc_bits)
- csbcpb->cpb.sha512.message_bit_length_hi++;
-
- /* everything after the first update is continuation */
- NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION;
out:
+ spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
return rc;
}
@@ -131,9 +151,15 @@ static int nx_sha512_final(struct shash_desc *desc, u8 *out)
struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base);
struct nx_csbcpb *csbcpb = (struct nx_csbcpb *)nx_ctx->csbcpb;
struct nx_sg *in_sg, *out_sg;
+ u32 max_sg_len;
u64 count0;
+ unsigned long irq_flags;
int rc;
+ spin_lock_irqsave(&nx_ctx->lock, irq_flags);
+
+ max_sg_len = min_t(u32, nx_driver.of.max_sg_len, nx_ctx->ap->sglen);
+
if (NX_CPB_FDM(csbcpb) & NX_FDM_CONTINUATION) {
/* we've hit the nx chip previously, now we're finalizing,
* so copy over the partial digest */
@@ -152,9 +178,9 @@ static int nx_sha512_final(struct shash_desc *desc, u8 *out)
csbcpb->cpb.sha512.message_bit_length_hi++;
in_sg = nx_build_sg_list(nx_ctx->in_sg, sctx->buf, sctx->count[0],
- nx_ctx->ap->sglen);
+ max_sg_len);
out_sg = nx_build_sg_list(nx_ctx->out_sg, out, SHA512_DIGEST_SIZE,
- nx_ctx->ap->sglen);
+ max_sg_len);
nx_ctx->op.inlen = (nx_ctx->in_sg - in_sg) * sizeof(struct nx_sg);
nx_ctx->op.outlen = (nx_ctx->out_sg - out_sg) * sizeof(struct nx_sg);
@@ -174,6 +200,7 @@ static int nx_sha512_final(struct shash_desc *desc, u8 *out)
memcpy(out, csbcpb->cpb.sha512.message_digest, SHA512_DIGEST_SIZE);
out:
+ spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
return rc;
}
@@ -183,6 +210,9 @@ static int nx_sha512_export(struct shash_desc *desc, void *out)
struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base);
struct nx_csbcpb *csbcpb = (struct nx_csbcpb *)nx_ctx->csbcpb;
struct sha512_state *octx = out;
+ unsigned long irq_flags;
+
+ spin_lock_irqsave(&nx_ctx->lock, irq_flags);
/* move message_bit_length (128 bits) into count and convert its value
* to bytes */
@@ -214,6 +244,7 @@ static int nx_sha512_export(struct shash_desc *desc, void *out)
octx->state[7] = SHA512_H7;
}
+ spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
return 0;
}
@@ -223,6 +254,9 @@ static int nx_sha512_import(struct shash_desc *desc, const void *in)
struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base);
struct nx_csbcpb *csbcpb = (struct nx_csbcpb *)nx_ctx->csbcpb;
const struct sha512_state *ictx = in;
+ unsigned long irq_flags;
+
+ spin_lock_irqsave(&nx_ctx->lock, irq_flags);
memcpy(sctx->buf, ictx->buf, sizeof(ictx->buf));
sctx->count[0] = ictx->count[0] & 0x3f;
@@ -240,6 +274,7 @@ static int nx_sha512_import(struct shash_desc *desc, const void *in)
NX_CPB_FDM(csbcpb) |= NX_FDM_INTERMEDIATE;
}
+ spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
return 0;
}
diff --git a/drivers/crypto/nx/nx.c b/drivers/crypto/nx/nx.c
index bbdab6e5ccf0..5533fe31c90d 100644
--- a/drivers/crypto/nx/nx.c
+++ b/drivers/crypto/nx/nx.c
@@ -61,8 +61,7 @@ int nx_hcall_sync(struct nx_crypto_ctx *nx_ctx,
do {
rc = vio_h_cop_sync(viodev, op);
- } while ((rc == -EBUSY && !may_sleep && retries--) ||
- (rc == -EBUSY && may_sleep && cond_resched()));
+ } while (rc == -EBUSY && !may_sleep && retries--);
if (rc) {
dev_dbg(&viodev->dev, "vio_h_cop_sync failed: rc: %d "
@@ -114,13 +113,29 @@ struct nx_sg *nx_build_sg_list(struct nx_sg *sg_head,
* have been described (or @sgmax elements have been written), the
* loop ends. min_t is used to ensure @end_addr falls on the same page
* as sg_addr, if not, we need to create another nx_sg element for the
- * data on the next page */
+ * data on the next page.
+ *
+ * Also when using vmalloc'ed data, every time that a system page
+ * boundary is crossed the physical address needs to be re-calculated.
+ */
for (sg = sg_head; sg_len < len; sg++) {
+ u64 next_page;
+
sg->addr = sg_addr;
- sg_addr = min_t(u64, NX_PAGE_NUM(sg_addr + NX_PAGE_SIZE), end_addr);
- sg->len = sg_addr - sg->addr;
+ sg_addr = min_t(u64, NX_PAGE_NUM(sg_addr + NX_PAGE_SIZE),
+ end_addr);
+
+ next_page = (sg->addr & PAGE_MASK) + PAGE_SIZE;
+ sg->len = min_t(u64, sg_addr, next_page) - sg->addr;
sg_len += sg->len;
+ if (sg_addr >= next_page &&
+ is_vmalloc_addr(start_addr + sg_len)) {
+ sg_addr = page_to_phys(vmalloc_to_page(
+ start_addr + sg_len));
+ end_addr = sg_addr + len - sg_len;
+ }
+
if ((sg - sg_head) == sgmax) {
pr_err("nx: scatter/gather list overflow, pid: %d\n",
current->pid);
@@ -196,6 +211,8 @@ struct nx_sg *nx_walk_and_build(struct nx_sg *nx_dst,
* @dst: destination scatterlist
* @src: source scatterlist
* @nbytes: length of data described in the scatterlists
+ * @offset: number of bytes to fast-forward past at the beginning of
+ * scatterlists.
* @iv: destination for the iv data, if the algorithm requires it
*
* This is common code shared by all the AES algorithms. It uses the block
@@ -207,6 +224,7 @@ int nx_build_sg_lists(struct nx_crypto_ctx *nx_ctx,
struct scatterlist *dst,
struct scatterlist *src,
unsigned int nbytes,
+ unsigned int offset,
u8 *iv)
{
struct nx_sg *nx_insg = nx_ctx->in_sg;
@@ -215,8 +233,10 @@ int nx_build_sg_lists(struct nx_crypto_ctx *nx_ctx,
if (iv)
memcpy(iv, desc->info, AES_BLOCK_SIZE);
- nx_insg = nx_walk_and_build(nx_insg, nx_ctx->ap->sglen, src, 0, nbytes);
- nx_outsg = nx_walk_and_build(nx_outsg, nx_ctx->ap->sglen, dst, 0, nbytes);
+ nx_insg = nx_walk_and_build(nx_insg, nx_ctx->ap->sglen, src,
+ offset, nbytes);
+ nx_outsg = nx_walk_and_build(nx_outsg, nx_ctx->ap->sglen, dst,
+ offset, nbytes);
/* these lengths should be negative, which will indicate to phyp that
* the input and output parameters are scatterlists, not linear
@@ -235,6 +255,7 @@ int nx_build_sg_lists(struct nx_crypto_ctx *nx_ctx,
*/
void nx_ctx_init(struct nx_crypto_ctx *nx_ctx, unsigned int function)
{
+ spin_lock_init(&nx_ctx->lock);
memset(nx_ctx->kmem, 0, nx_ctx->kmem_len);
nx_ctx->csbcpb->csb.valid |= NX_CSB_VALID_BIT;
diff --git a/drivers/crypto/nx/nx.h b/drivers/crypto/nx/nx.h
index 3232b182dd28..befda07ca1da 100644
--- a/drivers/crypto/nx/nx.h
+++ b/drivers/crypto/nx/nx.h
@@ -117,6 +117,7 @@ struct nx_ctr_priv {
};
struct nx_crypto_ctx {
+ spinlock_t lock; /* synchronize access to the context */
void *kmem; /* unaligned, kmalloc'd buffer */
size_t kmem_len; /* length of kmem */
struct nx_csbcpb *csbcpb; /* aligned page given to phyp @ hcall time */
@@ -155,7 +156,7 @@ int nx_hcall_sync(struct nx_crypto_ctx *ctx, struct vio_pfo_op *op,
struct nx_sg *nx_build_sg_list(struct nx_sg *, u8 *, unsigned int, u32);
int nx_build_sg_lists(struct nx_crypto_ctx *, struct blkcipher_desc *,
struct scatterlist *, struct scatterlist *, unsigned int,
- u8 *);
+ unsigned int, u8 *);
struct nx_sg *nx_walk_and_build(struct nx_sg *, unsigned int,
struct scatterlist *, unsigned int,
unsigned int);
diff --git a/drivers/crypto/omap-aes.c b/drivers/crypto/omap-aes.c
index 5f7980586850..ce791c2f81f7 100644
--- a/drivers/crypto/omap-aes.c
+++ b/drivers/crypto/omap-aes.c
@@ -13,7 +13,9 @@
*
*/
-#define pr_fmt(fmt) "%s: " fmt, __func__
+#define pr_fmt(fmt) "%20s: " fmt, __func__
+#define prn(num) pr_debug(#num "=%d\n", num)
+#define prx(num) pr_debug(#num "=%x\n", num)
#include <linux/err.h>
#include <linux/module.h>
@@ -38,6 +40,8 @@
#define DST_MAXBURST 4
#define DMA_MIN (DST_MAXBURST * sizeof(u32))
+#define _calc_walked(inout) (dd->inout##_walk.offset - dd->inout##_sg->offset)
+
/* OMAP TRM gives bitfields as start:end, where start is the higher bit
number. For example 7:0 */
#define FLD_MASK(start, end) (((1 << ((start) - (end) + 1)) - 1) << (end))
@@ -74,6 +78,10 @@
#define AES_REG_LENGTH_N(x) (0x54 + ((x) * 0x04))
+#define AES_REG_IRQ_STATUS(dd) ((dd)->pdata->irq_status_ofs)
+#define AES_REG_IRQ_ENABLE(dd) ((dd)->pdata->irq_enable_ofs)
+#define AES_REG_IRQ_DATA_IN BIT(1)
+#define AES_REG_IRQ_DATA_OUT BIT(2)
#define DEFAULT_TIMEOUT (5*HZ)
#define FLAGS_MODE_MASK 0x000f
@@ -86,6 +94,8 @@
#define FLAGS_FAST BIT(5)
#define FLAGS_BUSY BIT(6)
+#define AES_BLOCK_WORDS (AES_BLOCK_SIZE >> 2)
+
struct omap_aes_ctx {
struct omap_aes_dev *dd;
@@ -119,6 +129,8 @@ struct omap_aes_pdata {
u32 data_ofs;
u32 rev_ofs;
u32 mask_ofs;
+ u32 irq_enable_ofs;
+ u32 irq_status_ofs;
u32 dma_enable_in;
u32 dma_enable_out;
@@ -146,25 +158,32 @@ struct omap_aes_dev {
struct tasklet_struct queue_task;
struct ablkcipher_request *req;
+
+ /*
+ * total is used by PIO mode for book keeping so introduce
+ * variable total_save as need it to calc page_order
+ */
size_t total;
+ size_t total_save;
+
struct scatterlist *in_sg;
- struct scatterlist in_sgl;
- size_t in_offset;
struct scatterlist *out_sg;
+
+ /* Buffers for copying for unaligned cases */
+ struct scatterlist in_sgl;
struct scatterlist out_sgl;
- size_t out_offset;
+ struct scatterlist *orig_out;
+ int sgs_copied;
- size_t buflen;
- void *buf_in;
- size_t dma_size;
+ struct scatter_walk in_walk;
+ struct scatter_walk out_walk;
int dma_in;
struct dma_chan *dma_lch_in;
- dma_addr_t dma_addr_in;
- void *buf_out;
int dma_out;
struct dma_chan *dma_lch_out;
- dma_addr_t dma_addr_out;
-
+ int in_sg_len;
+ int out_sg_len;
+ int pio_only;
const struct omap_aes_pdata *pdata;
};
@@ -172,16 +191,36 @@ struct omap_aes_dev {
static LIST_HEAD(dev_list);
static DEFINE_SPINLOCK(list_lock);
+#ifdef DEBUG
+#define omap_aes_read(dd, offset) \
+({ \
+ int _read_ret; \
+ _read_ret = __raw_readl(dd->io_base + offset); \
+ pr_debug("omap_aes_read(" #offset "=%#x)= %#x\n", \
+ offset, _read_ret); \
+ _read_ret; \
+})
+#else
static inline u32 omap_aes_read(struct omap_aes_dev *dd, u32 offset)
{
return __raw_readl(dd->io_base + offset);
}
+#endif
+#ifdef DEBUG
+#define omap_aes_write(dd, offset, value) \
+ do { \
+ pr_debug("omap_aes_write(" #offset "=%#x) value=%#x\n", \
+ offset, value); \
+ __raw_writel(value, dd->io_base + offset); \
+ } while (0)
+#else
static inline void omap_aes_write(struct omap_aes_dev *dd, u32 offset,
u32 value)
{
__raw_writel(value, dd->io_base + offset);
}
+#endif
static inline void omap_aes_write_mask(struct omap_aes_dev *dd, u32 offset,
u32 value, u32 mask)
@@ -323,33 +362,6 @@ static int omap_aes_dma_init(struct omap_aes_dev *dd)
dd->dma_lch_out = NULL;
dd->dma_lch_in = NULL;
- dd->buf_in = (void *)__get_free_pages(GFP_KERNEL, OMAP_AES_CACHE_SIZE);
- dd->buf_out = (void *)__get_free_pages(GFP_KERNEL, OMAP_AES_CACHE_SIZE);
- dd->buflen = PAGE_SIZE << OMAP_AES_CACHE_SIZE;
- dd->buflen &= ~(AES_BLOCK_SIZE - 1);
-
- if (!dd->buf_in || !dd->buf_out) {
- dev_err(dd->dev, "unable to alloc pages.\n");
- goto err_alloc;
- }
-
- /* MAP here */
- dd->dma_addr_in = dma_map_single(dd->dev, dd->buf_in, dd->buflen,
- DMA_TO_DEVICE);
- if (dma_mapping_error(dd->dev, dd->dma_addr_in)) {
- dev_err(dd->dev, "dma %d bytes error\n", dd->buflen);
- err = -EINVAL;
- goto err_map_in;
- }
-
- dd->dma_addr_out = dma_map_single(dd->dev, dd->buf_out, dd->buflen,
- DMA_FROM_DEVICE);
- if (dma_mapping_error(dd->dev, dd->dma_addr_out)) {
- dev_err(dd->dev, "dma %d bytes error\n", dd->buflen);
- err = -EINVAL;
- goto err_map_out;
- }
-
dma_cap_zero(mask);
dma_cap_set(DMA_SLAVE, mask);
@@ -376,14 +388,6 @@ static int omap_aes_dma_init(struct omap_aes_dev *dd)
err_dma_out:
dma_release_channel(dd->dma_lch_in);
err_dma_in:
- dma_unmap_single(dd->dev, dd->dma_addr_out, dd->buflen,
- DMA_FROM_DEVICE);
-err_map_out:
- dma_unmap_single(dd->dev, dd->dma_addr_in, dd->buflen, DMA_TO_DEVICE);
-err_map_in:
- free_pages((unsigned long)dd->buf_out, OMAP_AES_CACHE_SIZE);
- free_pages((unsigned long)dd->buf_in, OMAP_AES_CACHE_SIZE);
-err_alloc:
if (err)
pr_err("error: %d\n", err);
return err;
@@ -393,11 +397,6 @@ static void omap_aes_dma_cleanup(struct omap_aes_dev *dd)
{
dma_release_channel(dd->dma_lch_out);
dma_release_channel(dd->dma_lch_in);
- dma_unmap_single(dd->dev, dd->dma_addr_out, dd->buflen,
- DMA_FROM_DEVICE);
- dma_unmap_single(dd->dev, dd->dma_addr_in, dd->buflen, DMA_TO_DEVICE);
- free_pages((unsigned long)dd->buf_out, OMAP_AES_CACHE_SIZE);
- free_pages((unsigned long)dd->buf_in, OMAP_AES_CACHE_SIZE);
}
static void sg_copy_buf(void *buf, struct scatterlist *sg,
@@ -414,59 +413,27 @@ static void sg_copy_buf(void *buf, struct scatterlist *sg,
scatterwalk_done(&walk, out, 0);
}
-static int sg_copy(struct scatterlist **sg, size_t *offset, void *buf,
- size_t buflen, size_t total, int out)
-{
- unsigned int count, off = 0;
-
- while (buflen && total) {
- count = min((*sg)->length - *offset, total);
- count = min(count, buflen);
-
- if (!count)
- return off;
-
- /*
- * buflen and total are AES_BLOCK_SIZE size aligned,
- * so count should be also aligned
- */
-
- sg_copy_buf(buf + off, *sg, *offset, count, out);
-
- off += count;
- buflen -= count;
- *offset += count;
- total -= count;
-
- if (*offset == (*sg)->length) {
- *sg = sg_next(*sg);
- if (*sg)
- *offset = 0;
- else
- total = 0;
- }
- }
-
- return off;
-}
-
static int omap_aes_crypt_dma(struct crypto_tfm *tfm,
- struct scatterlist *in_sg, struct scatterlist *out_sg)
+ struct scatterlist *in_sg, struct scatterlist *out_sg,
+ int in_sg_len, int out_sg_len)
{
struct omap_aes_ctx *ctx = crypto_tfm_ctx(tfm);
struct omap_aes_dev *dd = ctx->dd;
struct dma_async_tx_descriptor *tx_in, *tx_out;
struct dma_slave_config cfg;
- dma_addr_t dma_addr_in = sg_dma_address(in_sg);
- int ret, length = sg_dma_len(in_sg);
+ int ret;
- pr_debug("len: %d\n", length);
+ if (dd->pio_only) {
+ scatterwalk_start(&dd->in_walk, dd->in_sg);
+ scatterwalk_start(&dd->out_walk, dd->out_sg);
- dd->dma_size = length;
+ /* Enable DATAIN interrupt and let it take
+ care of the rest */
+ omap_aes_write(dd, AES_REG_IRQ_ENABLE(dd), 0x2);
+ return 0;
+ }
- if (!(dd->flags & FLAGS_FAST))
- dma_sync_single_for_device(dd->dev, dma_addr_in, length,
- DMA_TO_DEVICE);
+ dma_sync_sg_for_device(dd->dev, dd->in_sg, in_sg_len, DMA_TO_DEVICE);
memset(&cfg, 0, sizeof(cfg));
@@ -485,7 +452,7 @@ static int omap_aes_crypt_dma(struct crypto_tfm *tfm,
return ret;
}
- tx_in = dmaengine_prep_slave_sg(dd->dma_lch_in, in_sg, 1,
+ tx_in = dmaengine_prep_slave_sg(dd->dma_lch_in, in_sg, in_sg_len,
DMA_MEM_TO_DEV,
DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
if (!tx_in) {
@@ -504,7 +471,7 @@ static int omap_aes_crypt_dma(struct crypto_tfm *tfm,
return ret;
}
- tx_out = dmaengine_prep_slave_sg(dd->dma_lch_out, out_sg, 1,
+ tx_out = dmaengine_prep_slave_sg(dd->dma_lch_out, out_sg, out_sg_len,
DMA_DEV_TO_MEM,
DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
if (!tx_out) {
@@ -522,7 +489,7 @@ static int omap_aes_crypt_dma(struct crypto_tfm *tfm,
dma_async_issue_pending(dd->dma_lch_out);
/* start DMA */
- dd->pdata->trigger(dd, length);
+ dd->pdata->trigger(dd, dd->total);
return 0;
}
@@ -531,93 +498,32 @@ static int omap_aes_crypt_dma_start(struct omap_aes_dev *dd)
{
struct crypto_tfm *tfm = crypto_ablkcipher_tfm(
crypto_ablkcipher_reqtfm(dd->req));
- int err, fast = 0, in, out;
- size_t count;
- dma_addr_t addr_in, addr_out;
- struct scatterlist *in_sg, *out_sg;
- int len32;
+ int err;
pr_debug("total: %d\n", dd->total);
- if (sg_is_last(dd->in_sg) && sg_is_last(dd->out_sg)) {
- /* check for alignment */
- in = IS_ALIGNED((u32)dd->in_sg->offset, sizeof(u32));
- out = IS_ALIGNED((u32)dd->out_sg->offset, sizeof(u32));
-
- fast = in && out;
- }
-
- if (fast) {
- count = min(dd->total, sg_dma_len(dd->in_sg));
- count = min(count, sg_dma_len(dd->out_sg));
-
- if (count != dd->total) {
- pr_err("request length != buffer length\n");
- return -EINVAL;
- }
-
- pr_debug("fast\n");
-
- err = dma_map_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE);
+ if (!dd->pio_only) {
+ err = dma_map_sg(dd->dev, dd->in_sg, dd->in_sg_len,
+ DMA_TO_DEVICE);
if (!err) {
dev_err(dd->dev, "dma_map_sg() error\n");
return -EINVAL;
}
- err = dma_map_sg(dd->dev, dd->out_sg, 1, DMA_FROM_DEVICE);
+ err = dma_map_sg(dd->dev, dd->out_sg, dd->out_sg_len,
+ DMA_FROM_DEVICE);
if (!err) {
dev_err(dd->dev, "dma_map_sg() error\n");
- dma_unmap_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE);
return -EINVAL;
}
-
- addr_in = sg_dma_address(dd->in_sg);
- addr_out = sg_dma_address(dd->out_sg);
-
- in_sg = dd->in_sg;
- out_sg = dd->out_sg;
-
- dd->flags |= FLAGS_FAST;
-
- } else {
- /* use cache buffers */
- count = sg_copy(&dd->in_sg, &dd->in_offset, dd->buf_in,
- dd->buflen, dd->total, 0);
-
- len32 = DIV_ROUND_UP(count, DMA_MIN) * DMA_MIN;
-
- /*
- * The data going into the AES module has been copied
- * to a local buffer and the data coming out will go
- * into a local buffer so set up local SG entries for
- * both.
- */
- sg_init_table(&dd->in_sgl, 1);
- dd->in_sgl.offset = dd->in_offset;
- sg_dma_len(&dd->in_sgl) = len32;
- sg_dma_address(&dd->in_sgl) = dd->dma_addr_in;
-
- sg_init_table(&dd->out_sgl, 1);
- dd->out_sgl.offset = dd->out_offset;
- sg_dma_len(&dd->out_sgl) = len32;
- sg_dma_address(&dd->out_sgl) = dd->dma_addr_out;
-
- in_sg = &dd->in_sgl;
- out_sg = &dd->out_sgl;
-
- addr_in = dd->dma_addr_in;
- addr_out = dd->dma_addr_out;
-
- dd->flags &= ~FLAGS_FAST;
-
}
- dd->total -= count;
-
- err = omap_aes_crypt_dma(tfm, in_sg, out_sg);
- if (err) {
- dma_unmap_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE);
- dma_unmap_sg(dd->dev, dd->out_sg, 1, DMA_TO_DEVICE);
+ err = omap_aes_crypt_dma(tfm, dd->in_sg, dd->out_sg, dd->in_sg_len,
+ dd->out_sg_len);
+ if (err && !dd->pio_only) {
+ dma_unmap_sg(dd->dev, dd->in_sg, dd->in_sg_len, DMA_TO_DEVICE);
+ dma_unmap_sg(dd->dev, dd->out_sg, dd->out_sg_len,
+ DMA_FROM_DEVICE);
}
return err;
@@ -637,7 +543,6 @@ static void omap_aes_finish_req(struct omap_aes_dev *dd, int err)
static int omap_aes_crypt_dma_stop(struct omap_aes_dev *dd)
{
int err = 0;
- size_t count;
pr_debug("total: %d\n", dd->total);
@@ -646,23 +551,49 @@ static int omap_aes_crypt_dma_stop(struct omap_aes_dev *dd)
dmaengine_terminate_all(dd->dma_lch_in);
dmaengine_terminate_all(dd->dma_lch_out);
- if (dd->flags & FLAGS_FAST) {
- dma_unmap_sg(dd->dev, dd->out_sg, 1, DMA_FROM_DEVICE);
- dma_unmap_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE);
- } else {
- dma_sync_single_for_device(dd->dev, dd->dma_addr_out,
- dd->dma_size, DMA_FROM_DEVICE);
-
- /* copy data */
- count = sg_copy(&dd->out_sg, &dd->out_offset, dd->buf_out,
- dd->buflen, dd->dma_size, 1);
- if (count != dd->dma_size) {
- err = -EINVAL;
- pr_err("not all data converted: %u\n", count);
- }
+ return err;
+}
+
+int omap_aes_check_aligned(struct scatterlist *sg)
+{
+ while (sg) {
+ if (!IS_ALIGNED(sg->offset, 4))
+ return -1;
+ if (!IS_ALIGNED(sg->length, AES_BLOCK_SIZE))
+ return -1;
+ sg = sg_next(sg);
}
+ return 0;
+}
- return err;
+int omap_aes_copy_sgs(struct omap_aes_dev *dd)
+{
+ void *buf_in, *buf_out;
+ int pages;
+
+ pages = get_order(dd->total);
+
+ buf_in = (void *)__get_free_pages(GFP_ATOMIC, pages);
+ buf_out = (void *)__get_free_pages(GFP_ATOMIC, pages);
+
+ if (!buf_in || !buf_out) {
+ pr_err("Couldn't allocated pages for unaligned cases.\n");
+ return -1;
+ }
+
+ dd->orig_out = dd->out_sg;
+
+ sg_copy_buf(buf_in, dd->in_sg, 0, dd->total, 0);
+
+ sg_init_table(&dd->in_sgl, 1);
+ sg_set_buf(&dd->in_sgl, buf_in, dd->total);
+ dd->in_sg = &dd->in_sgl;
+
+ sg_init_table(&dd->out_sgl, 1);
+ sg_set_buf(&dd->out_sgl, buf_out, dd->total);
+ dd->out_sg = &dd->out_sgl;
+
+ return 0;
}
static int omap_aes_handle_queue(struct omap_aes_dev *dd,
@@ -698,11 +629,23 @@ static int omap_aes_handle_queue(struct omap_aes_dev *dd,
/* assign new request to device */
dd->req = req;
dd->total = req->nbytes;
- dd->in_offset = 0;
+ dd->total_save = req->nbytes;
dd->in_sg = req->src;
- dd->out_offset = 0;
dd->out_sg = req->dst;
+ if (omap_aes_check_aligned(dd->in_sg) ||
+ omap_aes_check_aligned(dd->out_sg)) {
+ if (omap_aes_copy_sgs(dd))
+ pr_err("Failed to copy SGs for unaligned cases\n");
+ dd->sgs_copied = 1;
+ } else {
+ dd->sgs_copied = 0;
+ }
+
+ dd->in_sg_len = scatterwalk_bytes_sglen(dd->in_sg, dd->total);
+ dd->out_sg_len = scatterwalk_bytes_sglen(dd->out_sg, dd->total);
+ BUG_ON(dd->in_sg_len < 0 || dd->out_sg_len < 0);
+
rctx = ablkcipher_request_ctx(req);
ctx = crypto_ablkcipher_ctx(crypto_ablkcipher_reqtfm(req));
rctx->mode &= FLAGS_MODE_MASK;
@@ -726,21 +669,32 @@ static int omap_aes_handle_queue(struct omap_aes_dev *dd,
static void omap_aes_done_task(unsigned long data)
{
struct omap_aes_dev *dd = (struct omap_aes_dev *)data;
- int err;
-
- pr_debug("enter\n");
+ void *buf_in, *buf_out;
+ int pages;
+
+ pr_debug("enter done_task\n");
+
+ if (!dd->pio_only) {
+ dma_sync_sg_for_device(dd->dev, dd->out_sg, dd->out_sg_len,
+ DMA_FROM_DEVICE);
+ dma_unmap_sg(dd->dev, dd->in_sg, dd->in_sg_len, DMA_TO_DEVICE);
+ dma_unmap_sg(dd->dev, dd->out_sg, dd->out_sg_len,
+ DMA_FROM_DEVICE);
+ omap_aes_crypt_dma_stop(dd);
+ }
- err = omap_aes_crypt_dma_stop(dd);
+ if (dd->sgs_copied) {
+ buf_in = sg_virt(&dd->in_sgl);
+ buf_out = sg_virt(&dd->out_sgl);
- err = dd->err ? : err;
+ sg_copy_buf(buf_out, dd->orig_out, 0, dd->total_save, 1);
- if (dd->total && !err) {
- err = omap_aes_crypt_dma_start(dd);
- if (!err)
- return; /* DMA started. Not fininishing. */
+ pages = get_order(dd->total_save);
+ free_pages((unsigned long)buf_in, pages);
+ free_pages((unsigned long)buf_out, pages);
}
- omap_aes_finish_req(dd, err);
+ omap_aes_finish_req(dd, 0);
omap_aes_handle_queue(dd, NULL);
pr_debug("exit\n");
@@ -1002,6 +956,8 @@ static const struct omap_aes_pdata omap_aes_pdata_omap4 = {
.data_ofs = 0x60,
.rev_ofs = 0x80,
.mask_ofs = 0x84,
+ .irq_status_ofs = 0x8c,
+ .irq_enable_ofs = 0x90,
.dma_enable_in = BIT(5),
.dma_enable_out = BIT(6),
.major_mask = 0x0700,
@@ -1010,6 +966,90 @@ static const struct omap_aes_pdata omap_aes_pdata_omap4 = {
.minor_shift = 0,
};
+static irqreturn_t omap_aes_irq(int irq, void *dev_id)
+{
+ struct omap_aes_dev *dd = dev_id;
+ u32 status, i;
+ u32 *src, *dst;
+
+ status = omap_aes_read(dd, AES_REG_IRQ_STATUS(dd));
+ if (status & AES_REG_IRQ_DATA_IN) {
+ omap_aes_write(dd, AES_REG_IRQ_ENABLE(dd), 0x0);
+
+ BUG_ON(!dd->in_sg);
+
+ BUG_ON(_calc_walked(in) > dd->in_sg->length);
+
+ src = sg_virt(dd->in_sg) + _calc_walked(in);
+
+ for (i = 0; i < AES_BLOCK_WORDS; i++) {
+ omap_aes_write(dd, AES_REG_DATA_N(dd, i), *src);
+
+ scatterwalk_advance(&dd->in_walk, 4);
+ if (dd->in_sg->length == _calc_walked(in)) {
+ dd->in_sg = scatterwalk_sg_next(dd->in_sg);
+ if (dd->in_sg) {
+ scatterwalk_start(&dd->in_walk,
+ dd->in_sg);
+ src = sg_virt(dd->in_sg) +
+ _calc_walked(in);
+ }
+ } else {
+ src++;
+ }
+ }
+
+ /* Clear IRQ status */
+ status &= ~AES_REG_IRQ_DATA_IN;
+ omap_aes_write(dd, AES_REG_IRQ_STATUS(dd), status);
+
+ /* Enable DATA_OUT interrupt */
+ omap_aes_write(dd, AES_REG_IRQ_ENABLE(dd), 0x4);
+
+ } else if (status & AES_REG_IRQ_DATA_OUT) {
+ omap_aes_write(dd, AES_REG_IRQ_ENABLE(dd), 0x0);
+
+ BUG_ON(!dd->out_sg);
+
+ BUG_ON(_calc_walked(out) > dd->out_sg->length);
+
+ dst = sg_virt(dd->out_sg) + _calc_walked(out);
+
+ for (i = 0; i < AES_BLOCK_WORDS; i++) {
+ *dst = omap_aes_read(dd, AES_REG_DATA_N(dd, i));
+ scatterwalk_advance(&dd->out_walk, 4);
+ if (dd->out_sg->length == _calc_walked(out)) {
+ dd->out_sg = scatterwalk_sg_next(dd->out_sg);
+ if (dd->out_sg) {
+ scatterwalk_start(&dd->out_walk,
+ dd->out_sg);
+ dst = sg_virt(dd->out_sg) +
+ _calc_walked(out);
+ }
+ } else {
+ dst++;
+ }
+ }
+
+ dd->total -= AES_BLOCK_SIZE;
+
+ BUG_ON(dd->total < 0);
+
+ /* Clear IRQ status */
+ status &= ~AES_REG_IRQ_DATA_OUT;
+ omap_aes_write(dd, AES_REG_IRQ_STATUS(dd), status);
+
+ if (!dd->total)
+ /* All bytes read! */
+ tasklet_schedule(&dd->done_task);
+ else
+ /* Enable DATA_IN interrupt for next block */
+ omap_aes_write(dd, AES_REG_IRQ_ENABLE(dd), 0x2);
+ }
+
+ return IRQ_HANDLED;
+}
+
static const struct of_device_id omap_aes_of_match[] = {
{
.compatible = "ti,omap2-aes",
@@ -1115,10 +1155,10 @@ static int omap_aes_probe(struct platform_device *pdev)
struct omap_aes_dev *dd;
struct crypto_alg *algp;
struct resource res;
- int err = -ENOMEM, i, j;
+ int err = -ENOMEM, i, j, irq = -1;
u32 reg;
- dd = kzalloc(sizeof(struct omap_aes_dev), GFP_KERNEL);
+ dd = devm_kzalloc(dev, sizeof(struct omap_aes_dev), GFP_KERNEL);
if (dd == NULL) {
dev_err(dev, "unable to alloc data struct.\n");
goto err_data;
@@ -1158,8 +1198,23 @@ static int omap_aes_probe(struct platform_device *pdev)
tasklet_init(&dd->queue_task, omap_aes_queue_task, (unsigned long)dd);
err = omap_aes_dma_init(dd);
- if (err)
- goto err_dma;
+ if (err && AES_REG_IRQ_STATUS(dd) && AES_REG_IRQ_ENABLE(dd)) {
+ dd->pio_only = 1;
+
+ irq = platform_get_irq(pdev, 0);
+ if (irq < 0) {
+ dev_err(dev, "can't get IRQ resource\n");
+ goto err_irq;
+ }
+
+ err = devm_request_irq(dev, irq, omap_aes_irq, 0,
+ dev_name(dev), dd);
+ if (err) {
+ dev_err(dev, "Unable to grab omap-aes IRQ\n");
+ goto err_irq;
+ }
+ }
+
INIT_LIST_HEAD(&dd->list);
spin_lock(&list_lock);
@@ -1187,13 +1242,13 @@ err_algs:
for (j = dd->pdata->algs_info[i].registered - 1; j >= 0; j--)
crypto_unregister_alg(
&dd->pdata->algs_info[i].algs_list[j]);
- omap_aes_dma_cleanup(dd);
-err_dma:
+ if (!dd->pio_only)
+ omap_aes_dma_cleanup(dd);
+err_irq:
tasklet_kill(&dd->done_task);
tasklet_kill(&dd->queue_task);
pm_runtime_disable(dev);
err_res:
- kfree(dd);
dd = NULL;
err_data:
dev_err(dev, "initialization failed.\n");
@@ -1221,7 +1276,6 @@ static int omap_aes_remove(struct platform_device *pdev)
tasklet_kill(&dd->queue_task);
omap_aes_dma_cleanup(dd);
pm_runtime_disable(dd->dev);
- kfree(dd);
dd = NULL;
return 0;
diff --git a/drivers/crypto/omap-sham.c b/drivers/crypto/omap-sham.c
index 4bb67652c200..8bdde57f6bb1 100644
--- a/drivers/crypto/omap-sham.c
+++ b/drivers/crypto/omap-sham.c
@@ -44,17 +44,13 @@
#include <crypto/hash.h>
#include <crypto/internal/hash.h>
-#define SHA1_MD5_BLOCK_SIZE SHA1_BLOCK_SIZE
#define MD5_DIGEST_SIZE 16
-#define DST_MAXBURST 16
-#define DMA_MIN (DST_MAXBURST * sizeof(u32))
-
#define SHA_REG_IDIGEST(dd, x) ((dd)->pdata->idigest_ofs + ((x)*0x04))
#define SHA_REG_DIN(dd, x) ((dd)->pdata->din_ofs + ((x) * 0x04))
#define SHA_REG_DIGCNT(dd) ((dd)->pdata->digcnt_ofs)
-#define SHA_REG_ODIGEST(x) (0x00 + ((x) * 0x04))
+#define SHA_REG_ODIGEST(dd, x) ((dd)->pdata->odigest_ofs + (x * 0x04))
#define SHA_REG_CTRL 0x18
#define SHA_REG_CTRL_LENGTH (0xFFFFFFFF << 5)
@@ -75,18 +71,21 @@
#define SHA_REG_SYSSTATUS(dd) ((dd)->pdata->sysstatus_ofs)
#define SHA_REG_SYSSTATUS_RESETDONE (1 << 0)
-#define SHA_REG_MODE 0x44
+#define SHA_REG_MODE(dd) ((dd)->pdata->mode_ofs)
#define SHA_REG_MODE_HMAC_OUTER_HASH (1 << 7)
#define SHA_REG_MODE_HMAC_KEY_PROC (1 << 5)
#define SHA_REG_MODE_CLOSE_HASH (1 << 4)
#define SHA_REG_MODE_ALGO_CONSTANT (1 << 3)
-#define SHA_REG_MODE_ALGO_MASK (3 << 1)
-#define SHA_REG_MODE_ALGO_MD5_128 (0 << 1)
-#define SHA_REG_MODE_ALGO_SHA1_160 (1 << 1)
-#define SHA_REG_MODE_ALGO_SHA2_224 (2 << 1)
-#define SHA_REG_MODE_ALGO_SHA2_256 (3 << 1)
-#define SHA_REG_LENGTH 0x48
+#define SHA_REG_MODE_ALGO_MASK (7 << 0)
+#define SHA_REG_MODE_ALGO_MD5_128 (0 << 1)
+#define SHA_REG_MODE_ALGO_SHA1_160 (1 << 1)
+#define SHA_REG_MODE_ALGO_SHA2_224 (2 << 1)
+#define SHA_REG_MODE_ALGO_SHA2_256 (3 << 1)
+#define SHA_REG_MODE_ALGO_SHA2_384 (1 << 0)
+#define SHA_REG_MODE_ALGO_SHA2_512 (3 << 0)
+
+#define SHA_REG_LENGTH(dd) ((dd)->pdata->length_ofs)
#define SHA_REG_IRQSTATUS 0x118
#define SHA_REG_IRQSTATUS_CTX_RDY (1 << 3)
@@ -117,18 +116,16 @@
#define FLAGS_SG 17
#define FLAGS_MODE_SHIFT 18
-#define FLAGS_MODE_MASK (SHA_REG_MODE_ALGO_MASK \
- << (FLAGS_MODE_SHIFT - 1))
-#define FLAGS_MODE_MD5 (SHA_REG_MODE_ALGO_MD5_128 \
- << (FLAGS_MODE_SHIFT - 1))
-#define FLAGS_MODE_SHA1 (SHA_REG_MODE_ALGO_SHA1_160 \
- << (FLAGS_MODE_SHIFT - 1))
-#define FLAGS_MODE_SHA224 (SHA_REG_MODE_ALGO_SHA2_224 \
- << (FLAGS_MODE_SHIFT - 1))
-#define FLAGS_MODE_SHA256 (SHA_REG_MODE_ALGO_SHA2_256 \
- << (FLAGS_MODE_SHIFT - 1))
-#define FLAGS_HMAC 20
-#define FLAGS_ERROR 21
+#define FLAGS_MODE_MASK (SHA_REG_MODE_ALGO_MASK << FLAGS_MODE_SHIFT)
+#define FLAGS_MODE_MD5 (SHA_REG_MODE_ALGO_MD5_128 << FLAGS_MODE_SHIFT)
+#define FLAGS_MODE_SHA1 (SHA_REG_MODE_ALGO_SHA1_160 << FLAGS_MODE_SHIFT)
+#define FLAGS_MODE_SHA224 (SHA_REG_MODE_ALGO_SHA2_224 << FLAGS_MODE_SHIFT)
+#define FLAGS_MODE_SHA256 (SHA_REG_MODE_ALGO_SHA2_256 << FLAGS_MODE_SHIFT)
+#define FLAGS_MODE_SHA384 (SHA_REG_MODE_ALGO_SHA2_384 << FLAGS_MODE_SHIFT)
+#define FLAGS_MODE_SHA512 (SHA_REG_MODE_ALGO_SHA2_512 << FLAGS_MODE_SHIFT)
+
+#define FLAGS_HMAC 21
+#define FLAGS_ERROR 22
#define OP_UPDATE 1
#define OP_FINAL 2
@@ -145,7 +142,7 @@ struct omap_sham_reqctx {
unsigned long flags;
unsigned long op;
- u8 digest[SHA256_DIGEST_SIZE] OMAP_ALIGNED;
+ u8 digest[SHA512_DIGEST_SIZE] OMAP_ALIGNED;
size_t digcnt;
size_t bufcnt;
size_t buflen;
@@ -162,8 +159,8 @@ struct omap_sham_reqctx {
struct omap_sham_hmac_ctx {
struct crypto_shash *shash;
- u8 ipad[SHA1_MD5_BLOCK_SIZE] OMAP_ALIGNED;
- u8 opad[SHA1_MD5_BLOCK_SIZE] OMAP_ALIGNED;
+ u8 ipad[SHA512_BLOCK_SIZE] OMAP_ALIGNED;
+ u8 opad[SHA512_BLOCK_SIZE] OMAP_ALIGNED;
};
struct omap_sham_ctx {
@@ -205,6 +202,8 @@ struct omap_sham_pdata {
u32 rev_ofs;
u32 mask_ofs;
u32 sysstatus_ofs;
+ u32 mode_ofs;
+ u32 length_ofs;
u32 major_mask;
u32 major_shift;
@@ -223,6 +222,7 @@ struct omap_sham_dev {
unsigned int dma;
struct dma_chan *dma_lch;
struct tasklet_struct done_task;
+ u8 polling_mode;
unsigned long flags;
struct crypto_queue queue;
@@ -306,9 +306,9 @@ static void omap_sham_copy_hash_omap4(struct ahash_request *req, int out)
for (i = 0; i < dd->pdata->digest_size / sizeof(u32); i++) {
if (out)
opad[i] = omap_sham_read(dd,
- SHA_REG_ODIGEST(i));
+ SHA_REG_ODIGEST(dd, i));
else
- omap_sham_write(dd, SHA_REG_ODIGEST(i),
+ omap_sham_write(dd, SHA_REG_ODIGEST(dd, i),
opad[i]);
}
}
@@ -342,6 +342,12 @@ static void omap_sham_copy_ready_hash(struct ahash_request *req)
case FLAGS_MODE_SHA256:
d = SHA256_DIGEST_SIZE / sizeof(u32);
break;
+ case FLAGS_MODE_SHA384:
+ d = SHA384_DIGEST_SIZE / sizeof(u32);
+ break;
+ case FLAGS_MODE_SHA512:
+ d = SHA512_DIGEST_SIZE / sizeof(u32);
+ break;
default:
d = 0;
}
@@ -404,6 +410,30 @@ static int omap_sham_poll_irq_omap2(struct omap_sham_dev *dd)
return omap_sham_wait(dd, SHA_REG_CTRL, SHA_REG_CTRL_INPUT_READY);
}
+static int get_block_size(struct omap_sham_reqctx *ctx)
+{
+ int d;
+
+ switch (ctx->flags & FLAGS_MODE_MASK) {
+ case FLAGS_MODE_MD5:
+ case FLAGS_MODE_SHA1:
+ d = SHA1_BLOCK_SIZE;
+ break;
+ case FLAGS_MODE_SHA224:
+ case FLAGS_MODE_SHA256:
+ d = SHA256_BLOCK_SIZE;
+ break;
+ case FLAGS_MODE_SHA384:
+ case FLAGS_MODE_SHA512:
+ d = SHA512_BLOCK_SIZE;
+ break;
+ default:
+ d = 0;
+ }
+
+ return d;
+}
+
static void omap_sham_write_n(struct omap_sham_dev *dd, u32 offset,
u32 *value, int count)
{
@@ -422,20 +452,24 @@ static void omap_sham_write_ctrl_omap4(struct omap_sham_dev *dd, size_t length,
* CLOSE_HASH only for the last one. Note that flags mode bits
* correspond to algorithm encoding in mode register.
*/
- val = (ctx->flags & FLAGS_MODE_MASK) >> (FLAGS_MODE_SHIFT - 1);
+ val = (ctx->flags & FLAGS_MODE_MASK) >> (FLAGS_MODE_SHIFT);
if (!ctx->digcnt) {
struct crypto_ahash *tfm = crypto_ahash_reqtfm(dd->req);
struct omap_sham_ctx *tctx = crypto_ahash_ctx(tfm);
struct omap_sham_hmac_ctx *bctx = tctx->base;
+ int bs, nr_dr;
val |= SHA_REG_MODE_ALGO_CONSTANT;
if (ctx->flags & BIT(FLAGS_HMAC)) {
+ bs = get_block_size(ctx);
+ nr_dr = bs / (2 * sizeof(u32));
val |= SHA_REG_MODE_HMAC_KEY_PROC;
- omap_sham_write_n(dd, SHA_REG_ODIGEST(0),
- (u32 *)bctx->ipad,
- SHA1_BLOCK_SIZE / sizeof(u32));
- ctx->digcnt += SHA1_BLOCK_SIZE;
+ omap_sham_write_n(dd, SHA_REG_ODIGEST(dd, 0),
+ (u32 *)bctx->ipad, nr_dr);
+ omap_sham_write_n(dd, SHA_REG_IDIGEST(dd, 0),
+ (u32 *)bctx->ipad + nr_dr, nr_dr);
+ ctx->digcnt += bs;
}
}
@@ -451,7 +485,7 @@ static void omap_sham_write_ctrl_omap4(struct omap_sham_dev *dd, size_t length,
SHA_REG_MODE_HMAC_KEY_PROC;
dev_dbg(dd->dev, "ctrl: %08x, flags: %08lx\n", val, ctx->flags);
- omap_sham_write_mask(dd, SHA_REG_MODE, val, mask);
+ omap_sham_write_mask(dd, SHA_REG_MODE(dd), val, mask);
omap_sham_write(dd, SHA_REG_IRQENA, SHA_REG_IRQENA_OUTPUT_RDY);
omap_sham_write_mask(dd, SHA_REG_MASK(dd),
SHA_REG_MASK_IT_EN |
@@ -461,7 +495,7 @@ static void omap_sham_write_ctrl_omap4(struct omap_sham_dev *dd, size_t length,
static void omap_sham_trigger_omap4(struct omap_sham_dev *dd, size_t length)
{
- omap_sham_write(dd, SHA_REG_LENGTH, length);
+ omap_sham_write(dd, SHA_REG_LENGTH(dd), length);
}
static int omap_sham_poll_irq_omap4(struct omap_sham_dev *dd)
@@ -474,7 +508,7 @@ static int omap_sham_xmit_cpu(struct omap_sham_dev *dd, const u8 *buf,
size_t length, int final)
{
struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);
- int count, len32;
+ int count, len32, bs32, offset = 0;
const u32 *buffer = (const u32 *)buf;
dev_dbg(dd->dev, "xmit_cpu: digcnt: %d, length: %d, final: %d\n",
@@ -486,18 +520,23 @@ static int omap_sham_xmit_cpu(struct omap_sham_dev *dd, const u8 *buf,
/* should be non-zero before next lines to disable clocks later */
ctx->digcnt += length;
- if (dd->pdata->poll_irq(dd))
- return -ETIMEDOUT;
-
if (final)
set_bit(FLAGS_FINAL, &dd->flags); /* catch last interrupt */
set_bit(FLAGS_CPU, &dd->flags);
len32 = DIV_ROUND_UP(length, sizeof(u32));
+ bs32 = get_block_size(ctx) / sizeof(u32);
- for (count = 0; count < len32; count++)
- omap_sham_write(dd, SHA_REG_DIN(dd, count), buffer[count]);
+ while (len32) {
+ if (dd->pdata->poll_irq(dd))
+ return -ETIMEDOUT;
+
+ for (count = 0; count < min(len32, bs32); count++, offset++)
+ omap_sham_write(dd, SHA_REG_DIN(dd, count),
+ buffer[offset]);
+ len32 -= min(len32, bs32);
+ }
return -EINPROGRESS;
}
@@ -516,7 +555,7 @@ static int omap_sham_xmit_dma(struct omap_sham_dev *dd, dma_addr_t dma_addr,
struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);
struct dma_async_tx_descriptor *tx;
struct dma_slave_config cfg;
- int len32, ret;
+ int len32, ret, dma_min = get_block_size(ctx);
dev_dbg(dd->dev, "xmit_dma: digcnt: %d, length: %d, final: %d\n",
ctx->digcnt, length, final);
@@ -525,7 +564,7 @@ static int omap_sham_xmit_dma(struct omap_sham_dev *dd, dma_addr_t dma_addr,
cfg.dst_addr = dd->phys_base + SHA_REG_DIN(dd, 0);
cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
- cfg.dst_maxburst = DST_MAXBURST;
+ cfg.dst_maxburst = dma_min / DMA_SLAVE_BUSWIDTH_4_BYTES;
ret = dmaengine_slave_config(dd->dma_lch, &cfg);
if (ret) {
@@ -533,7 +572,7 @@ static int omap_sham_xmit_dma(struct omap_sham_dev *dd, dma_addr_t dma_addr,
return ret;
}
- len32 = DIV_ROUND_UP(length, DMA_MIN) * DMA_MIN;
+ len32 = DIV_ROUND_UP(length, dma_min) * dma_min;
if (is_sg) {
/*
@@ -666,14 +705,14 @@ static int omap_sham_update_dma_slow(struct omap_sham_dev *dd)
/* Start address alignment */
#define SG_AA(sg) (IS_ALIGNED(sg->offset, sizeof(u32)))
/* SHA1 block size alignment */
-#define SG_SA(sg) (IS_ALIGNED(sg->length, SHA1_MD5_BLOCK_SIZE))
+#define SG_SA(sg, bs) (IS_ALIGNED(sg->length, bs))
static int omap_sham_update_dma_start(struct omap_sham_dev *dd)
{
struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);
unsigned int length, final, tail;
struct scatterlist *sg;
- int ret;
+ int ret, bs;
if (!ctx->total)
return 0;
@@ -687,30 +726,31 @@ static int omap_sham_update_dma_start(struct omap_sham_dev *dd)
* the dmaengine infrastructure will calculate that it needs
* to transfer 0 frames which ultimately fails.
*/
- if (ctx->total < (DST_MAXBURST * sizeof(u32)))
+ if (ctx->total < get_block_size(ctx))
return omap_sham_update_dma_slow(dd);
dev_dbg(dd->dev, "fast: digcnt: %d, bufcnt: %u, total: %u\n",
ctx->digcnt, ctx->bufcnt, ctx->total);
sg = ctx->sg;
+ bs = get_block_size(ctx);
if (!SG_AA(sg))
return omap_sham_update_dma_slow(dd);
- if (!sg_is_last(sg) && !SG_SA(sg))
- /* size is not SHA1_BLOCK_SIZE aligned */
+ if (!sg_is_last(sg) && !SG_SA(sg, bs))
+ /* size is not BLOCK_SIZE aligned */
return omap_sham_update_dma_slow(dd);
length = min(ctx->total, sg->length);
if (sg_is_last(sg)) {
if (!(ctx->flags & BIT(FLAGS_FINUP))) {
- /* not last sg must be SHA1_MD5_BLOCK_SIZE aligned */
- tail = length & (SHA1_MD5_BLOCK_SIZE - 1);
+ /* not last sg must be BLOCK_SIZE aligned */
+ tail = length & (bs - 1);
/* without finup() we need one block to close hash */
if (!tail)
- tail = SHA1_MD5_BLOCK_SIZE;
+ tail = bs;
length -= tail;
}
}
@@ -737,13 +777,22 @@ static int omap_sham_update_dma_start(struct omap_sham_dev *dd)
static int omap_sham_update_cpu(struct omap_sham_dev *dd)
{
struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);
- int bufcnt;
+ int bufcnt, final;
+
+ if (!ctx->total)
+ return 0;
omap_sham_append_sg(ctx);
+
+ final = (ctx->flags & BIT(FLAGS_FINUP)) && !ctx->total;
+
+ dev_dbg(dd->dev, "cpu: bufcnt: %u, digcnt: %d, final: %d\n",
+ ctx->bufcnt, ctx->digcnt, final);
+
bufcnt = ctx->bufcnt;
ctx->bufcnt = 0;
- return omap_sham_xmit_cpu(dd, ctx->buffer, bufcnt, 1);
+ return omap_sham_xmit_cpu(dd, ctx->buffer, bufcnt, final);
}
static int omap_sham_update_dma_stop(struct omap_sham_dev *dd)
@@ -773,6 +822,7 @@ static int omap_sham_init(struct ahash_request *req)
struct omap_sham_ctx *tctx = crypto_ahash_ctx(tfm);
struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
struct omap_sham_dev *dd = NULL, *tmp;
+ int bs = 0;
spin_lock_bh(&sham.lock);
if (!tctx->dd) {
@@ -796,15 +846,27 @@ static int omap_sham_init(struct ahash_request *req)
switch (crypto_ahash_digestsize(tfm)) {
case MD5_DIGEST_SIZE:
ctx->flags |= FLAGS_MODE_MD5;
+ bs = SHA1_BLOCK_SIZE;
break;
case SHA1_DIGEST_SIZE:
ctx->flags |= FLAGS_MODE_SHA1;
+ bs = SHA1_BLOCK_SIZE;
break;
case SHA224_DIGEST_SIZE:
ctx->flags |= FLAGS_MODE_SHA224;
+ bs = SHA224_BLOCK_SIZE;
break;
case SHA256_DIGEST_SIZE:
ctx->flags |= FLAGS_MODE_SHA256;
+ bs = SHA256_BLOCK_SIZE;
+ break;
+ case SHA384_DIGEST_SIZE:
+ ctx->flags |= FLAGS_MODE_SHA384;
+ bs = SHA384_BLOCK_SIZE;
+ break;
+ case SHA512_DIGEST_SIZE:
+ ctx->flags |= FLAGS_MODE_SHA512;
+ bs = SHA512_BLOCK_SIZE;
break;
}
@@ -816,8 +878,8 @@ static int omap_sham_init(struct ahash_request *req)
if (!test_bit(FLAGS_AUTO_XOR, &dd->flags)) {
struct omap_sham_hmac_ctx *bctx = tctx->base;
- memcpy(ctx->buffer, bctx->ipad, SHA1_MD5_BLOCK_SIZE);
- ctx->bufcnt = SHA1_MD5_BLOCK_SIZE;
+ memcpy(ctx->buffer, bctx->ipad, bs);
+ ctx->bufcnt = bs;
}
ctx->flags |= BIT(FLAGS_HMAC);
@@ -853,8 +915,11 @@ static int omap_sham_final_req(struct omap_sham_dev *dd)
struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
int err = 0, use_dma = 1;
- if (ctx->bufcnt <= DMA_MIN)
- /* faster to handle last block with cpu */
+ if ((ctx->bufcnt <= get_block_size(ctx)) || dd->polling_mode)
+ /*
+ * faster to handle last block with cpu or
+ * use cpu when dma is not present.
+ */
use_dma = 0;
if (use_dma)
@@ -1006,6 +1071,8 @@ static int omap_sham_enqueue(struct ahash_request *req, unsigned int op)
static int omap_sham_update(struct ahash_request *req)
{
struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
+ struct omap_sham_dev *dd = ctx->dd;
+ int bs = get_block_size(ctx);
if (!req->nbytes)
return 0;
@@ -1023,10 +1090,12 @@ static int omap_sham_update(struct ahash_request *req)
*/
omap_sham_append_sg(ctx);
return 0;
- } else if (ctx->bufcnt + ctx->total <= SHA1_MD5_BLOCK_SIZE) {
+ } else if ((ctx->bufcnt + ctx->total <= bs) ||
+ dd->polling_mode) {
/*
- * faster to use CPU for short transfers
- */
+ * faster to use CPU for short transfers or
+ * use cpu when dma is not present.
+ */
ctx->flags |= BIT(FLAGS_CPU);
}
} else if (ctx->bufcnt + ctx->total < ctx->buflen) {
@@ -1214,6 +1283,16 @@ static int omap_sham_cra_md5_init(struct crypto_tfm *tfm)
return omap_sham_cra_init_alg(tfm, "md5");
}
+static int omap_sham_cra_sha384_init(struct crypto_tfm *tfm)
+{
+ return omap_sham_cra_init_alg(tfm, "sha384");
+}
+
+static int omap_sham_cra_sha512_init(struct crypto_tfm *tfm)
+{
+ return omap_sham_cra_init_alg(tfm, "sha512");
+}
+
static void omap_sham_cra_exit(struct crypto_tfm *tfm)
{
struct omap_sham_ctx *tctx = crypto_tfm_ctx(tfm);
@@ -1422,6 +1501,101 @@ static struct ahash_alg algs_sha224_sha256[] = {
},
};
+static struct ahash_alg algs_sha384_sha512[] = {
+{
+ .init = omap_sham_init,
+ .update = omap_sham_update,
+ .final = omap_sham_final,
+ .finup = omap_sham_finup,
+ .digest = omap_sham_digest,
+ .halg.digestsize = SHA384_DIGEST_SIZE,
+ .halg.base = {
+ .cra_name = "sha384",
+ .cra_driver_name = "omap-sha384",
+ .cra_priority = 100,
+ .cra_flags = CRYPTO_ALG_TYPE_AHASH |
+ CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_NEED_FALLBACK,
+ .cra_blocksize = SHA384_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct omap_sham_ctx),
+ .cra_alignmask = 0,
+ .cra_module = THIS_MODULE,
+ .cra_init = omap_sham_cra_init,
+ .cra_exit = omap_sham_cra_exit,
+ }
+},
+{
+ .init = omap_sham_init,
+ .update = omap_sham_update,
+ .final = omap_sham_final,
+ .finup = omap_sham_finup,
+ .digest = omap_sham_digest,
+ .halg.digestsize = SHA512_DIGEST_SIZE,
+ .halg.base = {
+ .cra_name = "sha512",
+ .cra_driver_name = "omap-sha512",
+ .cra_priority = 100,
+ .cra_flags = CRYPTO_ALG_TYPE_AHASH |
+ CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_NEED_FALLBACK,
+ .cra_blocksize = SHA512_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct omap_sham_ctx),
+ .cra_alignmask = 0,
+ .cra_module = THIS_MODULE,
+ .cra_init = omap_sham_cra_init,
+ .cra_exit = omap_sham_cra_exit,
+ }
+},
+{
+ .init = omap_sham_init,
+ .update = omap_sham_update,
+ .final = omap_sham_final,
+ .finup = omap_sham_finup,
+ .digest = omap_sham_digest,
+ .setkey = omap_sham_setkey,
+ .halg.digestsize = SHA384_DIGEST_SIZE,
+ .halg.base = {
+ .cra_name = "hmac(sha384)",
+ .cra_driver_name = "omap-hmac-sha384",
+ .cra_priority = 100,
+ .cra_flags = CRYPTO_ALG_TYPE_AHASH |
+ CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_NEED_FALLBACK,
+ .cra_blocksize = SHA384_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct omap_sham_ctx) +
+ sizeof(struct omap_sham_hmac_ctx),
+ .cra_alignmask = OMAP_ALIGN_MASK,
+ .cra_module = THIS_MODULE,
+ .cra_init = omap_sham_cra_sha384_init,
+ .cra_exit = omap_sham_cra_exit,
+ }
+},
+{
+ .init = omap_sham_init,
+ .update = omap_sham_update,
+ .final = omap_sham_final,
+ .finup = omap_sham_finup,
+ .digest = omap_sham_digest,
+ .setkey = omap_sham_setkey,
+ .halg.digestsize = SHA512_DIGEST_SIZE,
+ .halg.base = {
+ .cra_name = "hmac(sha512)",
+ .cra_driver_name = "omap-hmac-sha512",
+ .cra_priority = 100,
+ .cra_flags = CRYPTO_ALG_TYPE_AHASH |
+ CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_NEED_FALLBACK,
+ .cra_blocksize = SHA512_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct omap_sham_ctx) +
+ sizeof(struct omap_sham_hmac_ctx),
+ .cra_alignmask = OMAP_ALIGN_MASK,
+ .cra_module = THIS_MODULE,
+ .cra_init = omap_sham_cra_sha512_init,
+ .cra_exit = omap_sham_cra_exit,
+ }
+},
+};
+
static void omap_sham_done_task(unsigned long data)
{
struct omap_sham_dev *dd = (struct omap_sham_dev *)data;
@@ -1433,8 +1607,12 @@ static void omap_sham_done_task(unsigned long data)
}
if (test_bit(FLAGS_CPU, &dd->flags)) {
- if (test_and_clear_bit(FLAGS_OUTPUT_READY, &dd->flags))
- goto finish;
+ if (test_and_clear_bit(FLAGS_OUTPUT_READY, &dd->flags)) {
+ /* hash or semi-hash ready */
+ err = omap_sham_update_cpu(dd);
+ if (err != -EINPROGRESS)
+ goto finish;
+ }
} else if (test_bit(FLAGS_DMA_READY, &dd->flags)) {
if (test_and_clear_bit(FLAGS_DMA_ACTIVE, &dd->flags)) {
omap_sham_update_dma_stop(dd);
@@ -1548,11 +1726,54 @@ static const struct omap_sham_pdata omap_sham_pdata_omap4 = {
.poll_irq = omap_sham_poll_irq_omap4,
.intr_hdlr = omap_sham_irq_omap4,
.idigest_ofs = 0x020,
+ .odigest_ofs = 0x0,
.din_ofs = 0x080,
.digcnt_ofs = 0x040,
.rev_ofs = 0x100,
.mask_ofs = 0x110,
.sysstatus_ofs = 0x114,
+ .mode_ofs = 0x44,
+ .length_ofs = 0x48,
+ .major_mask = 0x0700,
+ .major_shift = 8,
+ .minor_mask = 0x003f,
+ .minor_shift = 0,
+};
+
+static struct omap_sham_algs_info omap_sham_algs_info_omap5[] = {
+ {
+ .algs_list = algs_sha1_md5,
+ .size = ARRAY_SIZE(algs_sha1_md5),
+ },
+ {
+ .algs_list = algs_sha224_sha256,
+ .size = ARRAY_SIZE(algs_sha224_sha256),
+ },
+ {
+ .algs_list = algs_sha384_sha512,
+ .size = ARRAY_SIZE(algs_sha384_sha512),
+ },
+};
+
+static const struct omap_sham_pdata omap_sham_pdata_omap5 = {
+ .algs_info = omap_sham_algs_info_omap5,
+ .algs_info_size = ARRAY_SIZE(omap_sham_algs_info_omap5),
+ .flags = BIT(FLAGS_AUTO_XOR),
+ .digest_size = SHA512_DIGEST_SIZE,
+ .copy_hash = omap_sham_copy_hash_omap4,
+ .write_ctrl = omap_sham_write_ctrl_omap4,
+ .trigger = omap_sham_trigger_omap4,
+ .poll_irq = omap_sham_poll_irq_omap4,
+ .intr_hdlr = omap_sham_irq_omap4,
+ .idigest_ofs = 0x240,
+ .odigest_ofs = 0x200,
+ .din_ofs = 0x080,
+ .digcnt_ofs = 0x280,
+ .rev_ofs = 0x100,
+ .mask_ofs = 0x110,
+ .sysstatus_ofs = 0x114,
+ .mode_ofs = 0x284,
+ .length_ofs = 0x288,
.major_mask = 0x0700,
.major_shift = 8,
.minor_mask = 0x003f,
@@ -1568,6 +1789,10 @@ static const struct of_device_id omap_sham_of_match[] = {
.compatible = "ti,omap4-sham",
.data = &omap_sham_pdata_omap4,
},
+ {
+ .compatible = "ti,omap5-sham",
+ .data = &omap_sham_pdata_omap5,
+ },
{},
};
MODULE_DEVICE_TABLE(of, omap_sham_of_match);
@@ -1667,7 +1892,7 @@ static int omap_sham_probe(struct platform_device *pdev)
int err, i, j;
u32 rev;
- dd = kzalloc(sizeof(struct omap_sham_dev), GFP_KERNEL);
+ dd = devm_kzalloc(dev, sizeof(struct omap_sham_dev), GFP_KERNEL);
if (dd == NULL) {
dev_err(dev, "unable to alloc data struct.\n");
err = -ENOMEM;
@@ -1684,20 +1909,21 @@ static int omap_sham_probe(struct platform_device *pdev)
err = (dev->of_node) ? omap_sham_get_res_of(dd, dev, &res) :
omap_sham_get_res_pdev(dd, pdev, &res);
if (err)
- goto res_err;
+ goto data_err;
dd->io_base = devm_ioremap_resource(dev, &res);
if (IS_ERR(dd->io_base)) {
err = PTR_ERR(dd->io_base);
- goto res_err;
+ goto data_err;
}
dd->phys_base = res.start;
- err = request_irq(dd->irq, dd->pdata->intr_hdlr, IRQF_TRIGGER_LOW,
- dev_name(dev), dd);
+ err = devm_request_irq(dev, dd->irq, dd->pdata->intr_hdlr,
+ IRQF_TRIGGER_NONE, dev_name(dev), dd);
if (err) {
- dev_err(dev, "unable to request irq.\n");
- goto res_err;
+ dev_err(dev, "unable to request irq %d, err = %d\n",
+ dd->irq, err);
+ goto data_err;
}
dma_cap_zero(mask);
@@ -1706,10 +1932,8 @@ static int omap_sham_probe(struct platform_device *pdev)
dd->dma_lch = dma_request_slave_channel_compat(mask, omap_dma_filter_fn,
&dd->dma, dev, "rx");
if (!dd->dma_lch) {
- dev_err(dev, "unable to obtain RX DMA engine channel %u\n",
- dd->dma);
- err = -ENXIO;
- goto dma_err;
+ dd->polling_mode = 1;
+ dev_dbg(dev, "using polling mode instead of dma\n");
}
dd->flags |= dd->pdata->flags;
@@ -1747,11 +1971,6 @@ err_algs:
&dd->pdata->algs_info[i].algs_list[j]);
pm_runtime_disable(dev);
dma_release_channel(dd->dma_lch);
-dma_err:
- free_irq(dd->irq, dd);
-res_err:
- kfree(dd);
- dd = NULL;
data_err:
dev_err(dev, "initialization failed.\n");
@@ -1776,9 +1995,6 @@ static int omap_sham_remove(struct platform_device *pdev)
tasklet_kill(&dd->done_task);
pm_runtime_disable(&pdev->dev);
dma_release_channel(dd->dma_lch);
- free_irq(dd->irq, dd);
- kfree(dd);
- dd = NULL;
return 0;
}
diff --git a/drivers/crypto/sahara.c b/drivers/crypto/sahara.c
index c3dc1c04a5df..d7bb8bac36e9 100644
--- a/drivers/crypto/sahara.c
+++ b/drivers/crypto/sahara.c
@@ -417,7 +417,7 @@ static void sahara_aes_done_task(unsigned long data)
dev->req->base.complete(&dev->req->base, dev->error);
}
-void sahara_watchdog(unsigned long data)
+static void sahara_watchdog(unsigned long data)
{
struct sahara_dev *dev = (struct sahara_dev *)data;
unsigned int err = sahara_read(dev, SAHARA_REG_ERRSTATUS);
@@ -955,7 +955,7 @@ static int sahara_probe(struct platform_device *pdev)
dev->hw_link[0] = dma_alloc_coherent(&pdev->dev,
SAHARA_MAX_HW_LINK * sizeof(struct sahara_hw_link),
&dev->hw_phys_link[0], GFP_KERNEL);
- if (!dev->hw_link) {
+ if (!dev->hw_link[0]) {
dev_err(&pdev->dev, "Could not allocate hw links\n");
err = -ENOMEM;
goto err_link;
diff --git a/drivers/crypto/tegra-aes.c b/drivers/crypto/tegra-aes.c
index 85ea7525fa36..2d58da972ae2 100644
--- a/drivers/crypto/tegra-aes.c
+++ b/drivers/crypto/tegra-aes.c
@@ -275,7 +275,7 @@ static int aes_start_crypt(struct tegra_aes_dev *dd, u32 in_addr, u32 out_addr,
value = aes_readl(dd, TEGRA_AES_INTR_STATUS);
eng_busy = value & TEGRA_AES_ENGINE_BUSY_FIELD;
icq_empty = value & TEGRA_AES_ICQ_EMPTY_FIELD;
- } while (eng_busy & (!icq_empty));
+ } while (eng_busy && !icq_empty);
aes_writel(dd, cmdq[i], TEGRA_AES_ICMDQUE_WR);
}
@@ -365,7 +365,7 @@ static int aes_set_key(struct tegra_aes_dev *dd)
eng_busy = value & TEGRA_AES_ENGINE_BUSY_FIELD;
icq_empty = value & TEGRA_AES_ICQ_EMPTY_FIELD;
dma_busy = value & TEGRA_AES_DMA_BUSY_FIELD;
- } while (eng_busy & (!icq_empty) & dma_busy);
+ } while (eng_busy && !icq_empty && dma_busy);
/* settable command to get key into internal registers */
value = CMD_SETTABLE << CMDQ_OPCODE_SHIFT |
@@ -379,7 +379,7 @@ static int aes_set_key(struct tegra_aes_dev *dd)
value = aes_readl(dd, TEGRA_AES_INTR_STATUS);
eng_busy = value & TEGRA_AES_ENGINE_BUSY_FIELD;
icq_empty = value & TEGRA_AES_ICQ_EMPTY_FIELD;
- } while (eng_busy & (!icq_empty));
+ } while (eng_busy && !icq_empty);
return 0;
}
diff --git a/drivers/crypto/ux500/hash/hash_core.c b/drivers/crypto/ux500/hash/hash_core.c
index 33693d966b6a..1c73f4fbc252 100644
--- a/drivers/crypto/ux500/hash/hash_core.c
+++ b/drivers/crypto/ux500/hash/hash_core.c
@@ -11,6 +11,8 @@
* License terms: GNU General Public License (GPL) version 2
*/
+#define pr_fmt(fmt) "hashX hashX: " fmt
+
#include <linux/clk.h>
#include <linux/device.h>
#include <linux/err.h>
@@ -35,8 +37,6 @@
#include "hash_alg.h"
-#define DEV_DBG_NAME "hashX hashX:"
-
static int hash_mode;
module_param(hash_mode, int, 0);
MODULE_PARM_DESC(hash_mode, "CPU or DMA mode. CPU = 0 (default), DMA = 1");
@@ -44,13 +44,13 @@ MODULE_PARM_DESC(hash_mode, "CPU or DMA mode. CPU = 0 (default), DMA = 1");
/**
* Pre-calculated empty message digests.
*/
-static u8 zero_message_hash_sha1[SHA1_DIGEST_SIZE] = {
+static const u8 zero_message_hash_sha1[SHA1_DIGEST_SIZE] = {
0xda, 0x39, 0xa3, 0xee, 0x5e, 0x6b, 0x4b, 0x0d,
0x32, 0x55, 0xbf, 0xef, 0x95, 0x60, 0x18, 0x90,
0xaf, 0xd8, 0x07, 0x09
};
-static u8 zero_message_hash_sha256[SHA256_DIGEST_SIZE] = {
+static const u8 zero_message_hash_sha256[SHA256_DIGEST_SIZE] = {
0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14,
0x9a, 0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24,
0x27, 0xae, 0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c,
@@ -58,14 +58,14 @@ static u8 zero_message_hash_sha256[SHA256_DIGEST_SIZE] = {
};
/* HMAC-SHA1, no key */
-static u8 zero_message_hmac_sha1[SHA1_DIGEST_SIZE] = {
+static const u8 zero_message_hmac_sha1[SHA1_DIGEST_SIZE] = {
0xfb, 0xdb, 0x1d, 0x1b, 0x18, 0xaa, 0x6c, 0x08,
0x32, 0x4b, 0x7d, 0x64, 0xb7, 0x1f, 0xb7, 0x63,
0x70, 0x69, 0x0e, 0x1d
};
/* HMAC-SHA256, no key */
-static u8 zero_message_hmac_sha256[SHA256_DIGEST_SIZE] = {
+static const u8 zero_message_hmac_sha256[SHA256_DIGEST_SIZE] = {
0xb6, 0x13, 0x67, 0x9a, 0x08, 0x14, 0xd9, 0xec,
0x77, 0x2f, 0x95, 0xd7, 0x78, 0xc3, 0x5f, 0xc5,
0xff, 0x16, 0x97, 0xc4, 0x93, 0x71, 0x56, 0x53,
@@ -97,7 +97,7 @@ static struct hash_driver_data driver_data;
*
*/
static void hash_messagepad(struct hash_device_data *device_data,
- const u32 *message, u8 index_bytes);
+ const u32 *message, u8 index_bytes);
/**
* release_hash_device - Releases a previously allocated hash device.
@@ -119,7 +119,7 @@ static void release_hash_device(struct hash_device_data *device_data)
}
static void hash_dma_setup_channel(struct hash_device_data *device_data,
- struct device *dev)
+ struct device *dev)
{
struct hash_platform_data *platform_data = dev->platform_data;
struct dma_slave_config conf = {
@@ -127,7 +127,7 @@ static void hash_dma_setup_channel(struct hash_device_data *device_data,
.dst_addr = device_data->phybase + HASH_DMA_FIFO,
.dst_addr_width = DMA_SLAVE_BUSWIDTH_2_BYTES,
.dst_maxburst = 16,
- };
+ };
dma_cap_zero(device_data->dma.mask);
dma_cap_set(DMA_SLAVE, device_data->dma.mask);
@@ -135,8 +135,8 @@ static void hash_dma_setup_channel(struct hash_device_data *device_data,
device_data->dma.cfg_mem2hash = platform_data->mem_to_engine;
device_data->dma.chan_mem2hash =
dma_request_channel(device_data->dma.mask,
- platform_data->dma_filter,
- device_data->dma.cfg_mem2hash);
+ platform_data->dma_filter,
+ device_data->dma.cfg_mem2hash);
dmaengine_slave_config(device_data->dma.chan_mem2hash, &conf);
@@ -145,21 +145,21 @@ static void hash_dma_setup_channel(struct hash_device_data *device_data,
static void hash_dma_callback(void *data)
{
- struct hash_ctx *ctx = (struct hash_ctx *) data;
+ struct hash_ctx *ctx = data;
complete(&ctx->device->dma.complete);
}
static int hash_set_dma_transfer(struct hash_ctx *ctx, struct scatterlist *sg,
- int len, enum dma_data_direction direction)
+ int len, enum dma_data_direction direction)
{
struct dma_async_tx_descriptor *desc = NULL;
struct dma_chan *channel = NULL;
dma_cookie_t cookie;
if (direction != DMA_TO_DEVICE) {
- dev_err(ctx->device->dev, "[%s] Invalid DMA direction",
- __func__);
+ dev_err(ctx->device->dev, "%s: Invalid DMA direction\n",
+ __func__);
return -EFAULT;
}
@@ -172,20 +172,19 @@ static int hash_set_dma_transfer(struct hash_ctx *ctx, struct scatterlist *sg,
direction);
if (!ctx->device->dma.sg_len) {
- dev_err(ctx->device->dev,
- "[%s]: Could not map the sg list (TO_DEVICE)",
- __func__);
+ dev_err(ctx->device->dev, "%s: Could not map the sg list (TO_DEVICE)\n",
+ __func__);
return -EFAULT;
}
- dev_dbg(ctx->device->dev, "[%s]: Setting up DMA for buffer "
- "(TO_DEVICE)", __func__);
+ dev_dbg(ctx->device->dev, "%s: Setting up DMA for buffer (TO_DEVICE)\n",
+ __func__);
desc = dmaengine_prep_slave_sg(channel,
ctx->device->dma.sg, ctx->device->dma.sg_len,
direction, DMA_CTRL_ACK | DMA_PREP_INTERRUPT);
if (!desc) {
dev_err(ctx->device->dev,
- "[%s]: device_prep_slave_sg() failed!", __func__);
+ "%s: device_prep_slave_sg() failed!\n", __func__);
return -EFAULT;
}
@@ -205,17 +204,16 @@ static void hash_dma_done(struct hash_ctx *ctx)
chan = ctx->device->dma.chan_mem2hash;
dmaengine_device_control(chan, DMA_TERMINATE_ALL, 0);
dma_unmap_sg(chan->device->dev, ctx->device->dma.sg,
- ctx->device->dma.sg_len, DMA_TO_DEVICE);
-
+ ctx->device->dma.sg_len, DMA_TO_DEVICE);
}
static int hash_dma_write(struct hash_ctx *ctx,
- struct scatterlist *sg, int len)
+ struct scatterlist *sg, int len)
{
int error = hash_set_dma_transfer(ctx, sg, len, DMA_TO_DEVICE);
if (error) {
- dev_dbg(ctx->device->dev, "[%s]: hash_set_dma_transfer() "
- "failed", __func__);
+ dev_dbg(ctx->device->dev,
+ "%s: hash_set_dma_transfer() failed\n", __func__);
return error;
}
@@ -245,19 +243,18 @@ static int get_empty_message_digest(
if (HASH_OPER_MODE_HASH == ctx->config.oper_mode) {
if (HASH_ALGO_SHA1 == ctx->config.algorithm) {
memcpy(zero_hash, &zero_message_hash_sha1[0],
- SHA1_DIGEST_SIZE);
+ SHA1_DIGEST_SIZE);
*zero_hash_size = SHA1_DIGEST_SIZE;
*zero_digest = true;
} else if (HASH_ALGO_SHA256 ==
ctx->config.algorithm) {
memcpy(zero_hash, &zero_message_hash_sha256[0],
- SHA256_DIGEST_SIZE);
+ SHA256_DIGEST_SIZE);
*zero_hash_size = SHA256_DIGEST_SIZE;
*zero_digest = true;
} else {
- dev_err(device_data->dev, "[%s] "
- "Incorrect algorithm!"
- , __func__);
+ dev_err(device_data->dev, "%s: Incorrect algorithm!\n",
+ __func__);
ret = -EINVAL;
goto out;
}
@@ -265,25 +262,24 @@ static int get_empty_message_digest(
if (!ctx->keylen) {
if (HASH_ALGO_SHA1 == ctx->config.algorithm) {
memcpy(zero_hash, &zero_message_hmac_sha1[0],
- SHA1_DIGEST_SIZE);
+ SHA1_DIGEST_SIZE);
*zero_hash_size = SHA1_DIGEST_SIZE;
*zero_digest = true;
} else if (HASH_ALGO_SHA256 == ctx->config.algorithm) {
memcpy(zero_hash, &zero_message_hmac_sha256[0],
- SHA256_DIGEST_SIZE);
+ SHA256_DIGEST_SIZE);
*zero_hash_size = SHA256_DIGEST_SIZE;
*zero_digest = true;
} else {
- dev_err(device_data->dev, "[%s] "
- "Incorrect algorithm!"
- , __func__);
+ dev_err(device_data->dev, "%s: Incorrect algorithm!\n",
+ __func__);
ret = -EINVAL;
goto out;
}
} else {
- dev_dbg(device_data->dev, "[%s] Continue hash "
- "calculation, since hmac key available",
- __func__);
+ dev_dbg(device_data->dev,
+ "%s: Continue hash calculation, since hmac key available\n",
+ __func__);
}
}
out:
@@ -299,9 +295,8 @@ out:
* This function request for disabling power (regulator) and clock,
* and could also save current hw state.
*/
-static int hash_disable_power(
- struct hash_device_data *device_data,
- bool save_device_state)
+static int hash_disable_power(struct hash_device_data *device_data,
+ bool save_device_state)
{
int ret = 0;
struct device *dev = device_data->dev;
@@ -319,7 +314,7 @@ static int hash_disable_power(
clk_disable(device_data->clk);
ret = regulator_disable(device_data->regulator);
if (ret)
- dev_err(dev, "[%s] regulator_disable() failed!", __func__);
+ dev_err(dev, "%s: regulator_disable() failed!\n", __func__);
device_data->power_state = false;
@@ -337,9 +332,8 @@ out:
* This function request for enabling power (regulator) and clock,
* and could also restore a previously saved hw state.
*/
-static int hash_enable_power(
- struct hash_device_data *device_data,
- bool restore_device_state)
+static int hash_enable_power(struct hash_device_data *device_data,
+ bool restore_device_state)
{
int ret = 0;
struct device *dev = device_data->dev;
@@ -348,14 +342,13 @@ static int hash_enable_power(
if (!device_data->power_state) {
ret = regulator_enable(device_data->regulator);
if (ret) {
- dev_err(dev, "[%s]: regulator_enable() failed!",
- __func__);
+ dev_err(dev, "%s: regulator_enable() failed!\n",
+ __func__);
goto out;
}
ret = clk_enable(device_data->clk);
if (ret) {
- dev_err(dev, "[%s]: clk_enable() failed!",
- __func__);
+ dev_err(dev, "%s: clk_enable() failed!\n", __func__);
ret = regulator_disable(
device_data->regulator);
goto out;
@@ -366,8 +359,7 @@ static int hash_enable_power(
if (device_data->restore_dev_state) {
if (restore_device_state) {
device_data->restore_dev_state = false;
- hash_resume_state(device_data,
- &device_data->state);
+ hash_resume_state(device_data, &device_data->state);
}
}
out:
@@ -447,7 +439,7 @@ static int hash_get_device_data(struct hash_ctx *ctx,
* spec or due to a bug in the hw.
*/
static void hash_hw_write_key(struct hash_device_data *device_data,
- const u8 *key, unsigned int keylen)
+ const u8 *key, unsigned int keylen)
{
u32 word = 0;
int nwords = 1;
@@ -491,14 +483,14 @@ static void hash_hw_write_key(struct hash_device_data *device_data,
* calculation.
*/
static int init_hash_hw(struct hash_device_data *device_data,
- struct hash_ctx *ctx)
+ struct hash_ctx *ctx)
{
int ret = 0;
ret = hash_setconfiguration(device_data, &ctx->config);
if (ret) {
- dev_err(device_data->dev, "[%s] hash_setconfiguration() "
- "failed!", __func__);
+ dev_err(device_data->dev, "%s: hash_setconfiguration() failed!\n",
+ __func__);
return ret;
}
@@ -528,9 +520,8 @@ static int hash_get_nents(struct scatterlist *sg, int size, bool *aligned)
size -= sg->length;
/* hash_set_dma_transfer will align last nent */
- if ((aligned && !IS_ALIGNED(sg->offset, HASH_DMA_ALIGN_SIZE))
- || (!IS_ALIGNED(sg->length, HASH_DMA_ALIGN_SIZE) &&
- size > 0))
+ if ((aligned && !IS_ALIGNED(sg->offset, HASH_DMA_ALIGN_SIZE)) ||
+ (!IS_ALIGNED(sg->length, HASH_DMA_ALIGN_SIZE) && size > 0))
aligned_data = false;
sg = sg_next(sg);
@@ -585,21 +576,17 @@ static int hash_init(struct ahash_request *req)
if (req->nbytes < HASH_DMA_ALIGN_SIZE) {
req_ctx->dma_mode = false; /* Don't use DMA */
- pr_debug(DEV_DBG_NAME " [%s] DMA mode, but direct "
- "to CPU mode for data size < %d",
- __func__, HASH_DMA_ALIGN_SIZE);
+ pr_debug("%s: DMA mode, but direct to CPU mode for data size < %d\n",
+ __func__, HASH_DMA_ALIGN_SIZE);
} else {
if (req->nbytes >= HASH_DMA_PERFORMANCE_MIN_SIZE &&
- hash_dma_valid_data(req->src,
- req->nbytes)) {
+ hash_dma_valid_data(req->src, req->nbytes)) {
req_ctx->dma_mode = true;
} else {
req_ctx->dma_mode = false;
- pr_debug(DEV_DBG_NAME " [%s] DMA mode, but use"
- " CPU mode for datalength < %d"
- " or non-aligned data, except "
- "in last nent", __func__,
- HASH_DMA_PERFORMANCE_MIN_SIZE);
+ pr_debug("%s: DMA mode, but use CPU mode for datalength < %d or non-aligned data, except in last nent\n",
+ __func__,
+ HASH_DMA_PERFORMANCE_MIN_SIZE);
}
}
}
@@ -614,9 +601,8 @@ static int hash_init(struct ahash_request *req)
* the HASH hardware.
*
*/
-static void hash_processblock(
- struct hash_device_data *device_data,
- const u32 *message, int length)
+static void hash_processblock(struct hash_device_data *device_data,
+ const u32 *message, int length)
{
int len = length / HASH_BYTES_PER_WORD;
/*
@@ -641,7 +627,7 @@ static void hash_processblock(
*
*/
static void hash_messagepad(struct hash_device_data *device_data,
- const u32 *message, u8 index_bytes)
+ const u32 *message, u8 index_bytes)
{
int nwords = 1;
@@ -666,15 +652,13 @@ static void hash_messagepad(struct hash_device_data *device_data,
/* num_of_bytes == 0 => NBLW <- 0 (32 bits valid in DATAIN) */
HASH_SET_NBLW(index_bytes * 8);
- dev_dbg(device_data->dev, "[%s] DIN=0x%08x NBLW=%d", __func__,
- readl_relaxed(&device_data->base->din),
- (int)(readl_relaxed(&device_data->base->str) &
- HASH_STR_NBLW_MASK));
+ dev_dbg(device_data->dev, "%s: DIN=0x%08x NBLW=%lu\n",
+ __func__, readl_relaxed(&device_data->base->din),
+ readl_relaxed(&device_data->base->str) & HASH_STR_NBLW_MASK);
HASH_SET_DCAL;
- dev_dbg(device_data->dev, "[%s] after dcal -> DIN=0x%08x NBLW=%d",
- __func__, readl_relaxed(&device_data->base->din),
- (int)(readl_relaxed(&device_data->base->str) &
- HASH_STR_NBLW_MASK));
+ dev_dbg(device_data->dev, "%s: after dcal -> DIN=0x%08x NBLW=%lu\n",
+ __func__, readl_relaxed(&device_data->base->din),
+ readl_relaxed(&device_data->base->str) & HASH_STR_NBLW_MASK);
while (readl(&device_data->base->str) & HASH_STR_DCAL_MASK)
cpu_relax();
@@ -704,7 +688,7 @@ static void hash_incrementlength(struct hash_req_ctx *ctx, u32 incr)
* @config: Pointer to a configuration structure.
*/
int hash_setconfiguration(struct hash_device_data *device_data,
- struct hash_config *config)
+ struct hash_config *config)
{
int ret = 0;
@@ -731,8 +715,8 @@ int hash_setconfiguration(struct hash_device_data *device_data,
break;
default:
- dev_err(device_data->dev, "[%s] Incorrect algorithm.",
- __func__);
+ dev_err(device_data->dev, "%s: Incorrect algorithm\n",
+ __func__);
return -EPERM;
}
@@ -744,23 +728,22 @@ int hash_setconfiguration(struct hash_device_data *device_data,
HASH_CLEAR_BITS(&device_data->base->cr,
HASH_CR_MODE_MASK);
else if (HASH_OPER_MODE_HMAC == config->oper_mode) {
- HASH_SET_BITS(&device_data->base->cr,
- HASH_CR_MODE_MASK);
+ HASH_SET_BITS(&device_data->base->cr, HASH_CR_MODE_MASK);
if (device_data->current_ctx->keylen > HASH_BLOCK_SIZE) {
/* Truncate key to blocksize */
- dev_dbg(device_data->dev, "[%s] LKEY set", __func__);
+ dev_dbg(device_data->dev, "%s: LKEY set\n", __func__);
HASH_SET_BITS(&device_data->base->cr,
- HASH_CR_LKEY_MASK);
+ HASH_CR_LKEY_MASK);
} else {
- dev_dbg(device_data->dev, "[%s] LKEY cleared",
- __func__);
+ dev_dbg(device_data->dev, "%s: LKEY cleared\n",
+ __func__);
HASH_CLEAR_BITS(&device_data->base->cr,
HASH_CR_LKEY_MASK);
}
} else { /* Wrong hash mode */
ret = -EPERM;
- dev_err(device_data->dev, "[%s] HASH_INVALID_PARAMETER!",
- __func__);
+ dev_err(device_data->dev, "%s: HASH_INVALID_PARAMETER!\n",
+ __func__);
}
return ret;
}
@@ -793,8 +776,9 @@ void hash_begin(struct hash_device_data *device_data, struct hash_ctx *ctx)
}
static int hash_process_data(struct hash_device_data *device_data,
- struct hash_ctx *ctx, struct hash_req_ctx *req_ctx,
- int msg_length, u8 *data_buffer, u8 *buffer, u8 *index)
+ struct hash_ctx *ctx, struct hash_req_ctx *req_ctx,
+ int msg_length, u8 *data_buffer, u8 *buffer,
+ u8 *index)
{
int ret = 0;
u32 count;
@@ -809,24 +793,23 @@ static int hash_process_data(struct hash_device_data *device_data,
msg_length = 0;
} else {
if (req_ctx->updated) {
-
ret = hash_resume_state(device_data,
&device_data->state);
memmove(req_ctx->state.buffer,
- device_data->state.buffer,
- HASH_BLOCK_SIZE / sizeof(u32));
+ device_data->state.buffer,
+ HASH_BLOCK_SIZE / sizeof(u32));
if (ret) {
- dev_err(device_data->dev, "[%s] "
- "hash_resume_state()"
- " failed!", __func__);
+ dev_err(device_data->dev,
+ "%s: hash_resume_state() failed!\n",
+ __func__);
goto out;
}
} else {
ret = init_hash_hw(device_data, ctx);
if (ret) {
- dev_err(device_data->dev, "[%s] "
- "init_hash_hw()"
- " failed!", __func__);
+ dev_err(device_data->dev,
+ "%s: init_hash_hw() failed!\n",
+ __func__);
goto out;
}
req_ctx->updated = 1;
@@ -838,22 +821,21 @@ static int hash_process_data(struct hash_device_data *device_data,
* HW peripheral, otherwise we first copy data
* to a local buffer
*/
- if ((0 == (((u32)data_buffer) % 4))
- && (0 == *index))
+ if ((0 == (((u32)data_buffer) % 4)) &&
+ (0 == *index))
hash_processblock(device_data,
- (const u32 *)
- data_buffer, HASH_BLOCK_SIZE);
+ (const u32 *)data_buffer,
+ HASH_BLOCK_SIZE);
else {
- for (count = 0; count <
- (u32)(HASH_BLOCK_SIZE -
- *index);
- count++) {
+ for (count = 0;
+ count < (u32)(HASH_BLOCK_SIZE - *index);
+ count++) {
buffer[*index + count] =
*(data_buffer + count);
}
hash_processblock(device_data,
- (const u32 *)buffer,
- HASH_BLOCK_SIZE);
+ (const u32 *)buffer,
+ HASH_BLOCK_SIZE);
}
hash_incrementlength(req_ctx, HASH_BLOCK_SIZE);
data_buffer += (HASH_BLOCK_SIZE - *index);
@@ -865,12 +847,11 @@ static int hash_process_data(struct hash_device_data *device_data,
&device_data->state);
memmove(device_data->state.buffer,
- req_ctx->state.buffer,
- HASH_BLOCK_SIZE / sizeof(u32));
+ req_ctx->state.buffer,
+ HASH_BLOCK_SIZE / sizeof(u32));
if (ret) {
- dev_err(device_data->dev, "[%s] "
- "hash_save_state()"
- " failed!", __func__);
+ dev_err(device_data->dev, "%s: hash_save_state() failed!\n",
+ __func__);
goto out;
}
}
@@ -898,25 +879,24 @@ static int hash_dma_final(struct ahash_request *req)
if (ret)
return ret;
- dev_dbg(device_data->dev, "[%s] (ctx=0x%x)!", __func__, (u32) ctx);
+ dev_dbg(device_data->dev, "%s: (ctx=0x%x)!\n", __func__, (u32) ctx);
if (req_ctx->updated) {
ret = hash_resume_state(device_data, &device_data->state);
if (ret) {
- dev_err(device_data->dev, "[%s] hash_resume_state() "
- "failed!", __func__);
+ dev_err(device_data->dev, "%s: hash_resume_state() failed!\n",
+ __func__);
goto out;
}
-
}
if (!req_ctx->updated) {
ret = hash_setconfiguration(device_data, &ctx->config);
if (ret) {
- dev_err(device_data->dev, "[%s] "
- "hash_setconfiguration() failed!",
- __func__);
+ dev_err(device_data->dev,
+ "%s: hash_setconfiguration() failed!\n",
+ __func__);
goto out;
}
@@ -926,9 +906,9 @@ static int hash_dma_final(struct ahash_request *req)
HASH_CR_DMAE_MASK);
} else {
HASH_SET_BITS(&device_data->base->cr,
- HASH_CR_DMAE_MASK);
+ HASH_CR_DMAE_MASK);
HASH_SET_BITS(&device_data->base->cr,
- HASH_CR_PRIVN_MASK);
+ HASH_CR_PRIVN_MASK);
}
HASH_INITIALIZE;
@@ -944,16 +924,16 @@ static int hash_dma_final(struct ahash_request *req)
/* Store the nents in the dma struct. */
ctx->device->dma.nents = hash_get_nents(req->src, req->nbytes, NULL);
if (!ctx->device->dma.nents) {
- dev_err(device_data->dev, "[%s] "
- "ctx->device->dma.nents = 0", __func__);
+ dev_err(device_data->dev, "%s: ctx->device->dma.nents = 0\n",
+ __func__);
ret = ctx->device->dma.nents;
goto out;
}
bytes_written = hash_dma_write(ctx, req->src, req->nbytes);
if (bytes_written != req->nbytes) {
- dev_err(device_data->dev, "[%s] "
- "hash_dma_write() failed!", __func__);
+ dev_err(device_data->dev, "%s: hash_dma_write() failed!\n",
+ __func__);
ret = bytes_written;
goto out;
}
@@ -968,8 +948,8 @@ static int hash_dma_final(struct ahash_request *req)
unsigned int keylen = ctx->keylen;
u8 *key = ctx->key;
- dev_dbg(device_data->dev, "[%s] keylen: %d", __func__,
- ctx->keylen);
+ dev_dbg(device_data->dev, "%s: keylen: %d\n",
+ __func__, ctx->keylen);
hash_hw_write_key(device_data, key, keylen);
}
@@ -1004,14 +984,14 @@ static int hash_hw_final(struct ahash_request *req)
if (ret)
return ret;
- dev_dbg(device_data->dev, "[%s] (ctx=0x%x)!", __func__, (u32) ctx);
+ dev_dbg(device_data->dev, "%s: (ctx=0x%x)!\n", __func__, (u32) ctx);
if (req_ctx->updated) {
ret = hash_resume_state(device_data, &device_data->state);
if (ret) {
- dev_err(device_data->dev, "[%s] hash_resume_state() "
- "failed!", __func__);
+ dev_err(device_data->dev,
+ "%s: hash_resume_state() failed!\n", __func__);
goto out;
}
} else if (req->nbytes == 0 && ctx->keylen == 0) {
@@ -1025,31 +1005,33 @@ static int hash_hw_final(struct ahash_request *req)
ret = get_empty_message_digest(device_data, &zero_hash[0],
&zero_hash_size, &zero_digest);
if (!ret && likely(zero_hash_size == ctx->digestsize) &&
- zero_digest) {
+ zero_digest) {
memcpy(req->result, &zero_hash[0], ctx->digestsize);
goto out;
} else if (!ret && !zero_digest) {
- dev_dbg(device_data->dev, "[%s] HMAC zero msg with "
- "key, continue...", __func__);
+ dev_dbg(device_data->dev,
+ "%s: HMAC zero msg with key, continue...\n",
+ __func__);
} else {
- dev_err(device_data->dev, "[%s] ret=%d, or wrong "
- "digest size? %s", __func__, ret,
- (zero_hash_size == ctx->digestsize) ?
- "true" : "false");
+ dev_err(device_data->dev,
+ "%s: ret=%d, or wrong digest size? %s\n",
+ __func__, ret,
+ zero_hash_size == ctx->digestsize ?
+ "true" : "false");
/* Return error */
goto out;
}
} else if (req->nbytes == 0 && ctx->keylen > 0) {
- dev_err(device_data->dev, "[%s] Empty message with "
- "keylength > 0, NOT supported.", __func__);
+ dev_err(device_data->dev, "%s: Empty message with keylength > 0, NOT supported\n",
+ __func__);
goto out;
}
if (!req_ctx->updated) {
ret = init_hash_hw(device_data, ctx);
if (ret) {
- dev_err(device_data->dev, "[%s] init_hash_hw() "
- "failed!", __func__);
+ dev_err(device_data->dev,
+ "%s: init_hash_hw() failed!\n", __func__);
goto out;
}
}
@@ -1067,8 +1049,8 @@ static int hash_hw_final(struct ahash_request *req)
unsigned int keylen = ctx->keylen;
u8 *key = ctx->key;
- dev_dbg(device_data->dev, "[%s] keylen: %d", __func__,
- ctx->keylen);
+ dev_dbg(device_data->dev, "%s: keylen: %d\n",
+ __func__, ctx->keylen);
hash_hw_write_key(device_data, key, keylen);
}
@@ -1115,10 +1097,8 @@ int hash_hw_update(struct ahash_request *req)
/* Check if ctx->state.length + msg_length
overflows */
if (msg_length > (req_ctx->state.length.low_word + msg_length) &&
- HASH_HIGH_WORD_MAX_VAL ==
- req_ctx->state.length.high_word) {
- pr_err(DEV_DBG_NAME " [%s] HASH_MSG_LENGTH_OVERFLOW!",
- __func__);
+ HASH_HIGH_WORD_MAX_VAL == req_ctx->state.length.high_word) {
+ pr_err("%s: HASH_MSG_LENGTH_OVERFLOW!\n", __func__);
return -EPERM;
}
@@ -1133,8 +1113,8 @@ int hash_hw_update(struct ahash_request *req)
data_buffer, buffer, &index);
if (ret) {
- dev_err(device_data->dev, "[%s] hash_internal_hw_"
- "update() failed!", __func__);
+ dev_err(device_data->dev, "%s: hash_internal_hw_update() failed!\n",
+ __func__);
goto out;
}
@@ -1142,9 +1122,8 @@ int hash_hw_update(struct ahash_request *req)
}
req_ctx->state.index = index;
- dev_dbg(device_data->dev, "[%s] indata length=%d, bin=%d))",
- __func__, req_ctx->state.index,
- req_ctx->state.bit_index);
+ dev_dbg(device_data->dev, "%s: indata length=%d, bin=%d\n",
+ __func__, req_ctx->state.index, req_ctx->state.bit_index);
out:
release_hash_device(device_data);
@@ -1158,23 +1137,23 @@ out:
* @device_state: The state to be restored in the hash hardware
*/
int hash_resume_state(struct hash_device_data *device_data,
- const struct hash_state *device_state)
+ const struct hash_state *device_state)
{
u32 temp_cr;
s32 count;
int hash_mode = HASH_OPER_MODE_HASH;
if (NULL == device_state) {
- dev_err(device_data->dev, "[%s] HASH_INVALID_PARAMETER!",
- __func__);
+ dev_err(device_data->dev, "%s: HASH_INVALID_PARAMETER!\n",
+ __func__);
return -EPERM;
}
/* Check correctness of index and length members */
- if (device_state->index > HASH_BLOCK_SIZE
- || (device_state->length.low_word % HASH_BLOCK_SIZE) != 0) {
- dev_err(device_data->dev, "[%s] HASH_INVALID_PARAMETER!",
- __func__);
+ if (device_state->index > HASH_BLOCK_SIZE ||
+ (device_state->length.low_word % HASH_BLOCK_SIZE) != 0) {
+ dev_err(device_data->dev, "%s: HASH_INVALID_PARAMETER!\n",
+ __func__);
return -EPERM;
}
@@ -1198,7 +1177,7 @@ int hash_resume_state(struct hash_device_data *device_data,
break;
writel_relaxed(device_state->csr[count],
- &device_data->base->csrx[count]);
+ &device_data->base->csrx[count]);
}
writel_relaxed(device_state->csfull, &device_data->base->csfull);
@@ -1216,15 +1195,15 @@ int hash_resume_state(struct hash_device_data *device_data,
* @device_state: The strucure where the hardware state should be saved.
*/
int hash_save_state(struct hash_device_data *device_data,
- struct hash_state *device_state)
+ struct hash_state *device_state)
{
u32 temp_cr;
u32 count;
int hash_mode = HASH_OPER_MODE_HASH;
if (NULL == device_state) {
- dev_err(device_data->dev, "[%s] HASH_INVALID_PARAMETER!",
- __func__);
+ dev_err(device_data->dev, "%s: HASH_INVALID_PARAMETER!\n",
+ __func__);
return -ENOTSUPP;
}
@@ -1270,20 +1249,18 @@ int hash_save_state(struct hash_device_data *device_data,
int hash_check_hw(struct hash_device_data *device_data)
{
/* Checking Peripheral Ids */
- if (HASH_P_ID0 == readl_relaxed(&device_data->base->periphid0)
- && HASH_P_ID1 == readl_relaxed(&device_data->base->periphid1)
- && HASH_P_ID2 == readl_relaxed(&device_data->base->periphid2)
- && HASH_P_ID3 == readl_relaxed(&device_data->base->periphid3)
- && HASH_CELL_ID0 == readl_relaxed(&device_data->base->cellid0)
- && HASH_CELL_ID1 == readl_relaxed(&device_data->base->cellid1)
- && HASH_CELL_ID2 == readl_relaxed(&device_data->base->cellid2)
- && HASH_CELL_ID3 == readl_relaxed(&device_data->base->cellid3)
- ) {
+ if (HASH_P_ID0 == readl_relaxed(&device_data->base->periphid0) &&
+ HASH_P_ID1 == readl_relaxed(&device_data->base->periphid1) &&
+ HASH_P_ID2 == readl_relaxed(&device_data->base->periphid2) &&
+ HASH_P_ID3 == readl_relaxed(&device_data->base->periphid3) &&
+ HASH_CELL_ID0 == readl_relaxed(&device_data->base->cellid0) &&
+ HASH_CELL_ID1 == readl_relaxed(&device_data->base->cellid1) &&
+ HASH_CELL_ID2 == readl_relaxed(&device_data->base->cellid2) &&
+ HASH_CELL_ID3 == readl_relaxed(&device_data->base->cellid3)) {
return 0;
}
- dev_err(device_data->dev, "[%s] HASH_UNSUPPORTED_HW!",
- __func__);
+ dev_err(device_data->dev, "%s: HASH_UNSUPPORTED_HW!\n", __func__);
return -ENOTSUPP;
}
@@ -1294,14 +1271,14 @@ int hash_check_hw(struct hash_device_data *device_data)
* @algorithm: The algorithm in use.
*/
void hash_get_digest(struct hash_device_data *device_data,
- u8 *digest, int algorithm)
+ u8 *digest, int algorithm)
{
u32 temp_hx_val, count;
int loop_ctr;
if (algorithm != HASH_ALGO_SHA1 && algorithm != HASH_ALGO_SHA256) {
- dev_err(device_data->dev, "[%s] Incorrect algorithm %d",
- __func__, algorithm);
+ dev_err(device_data->dev, "%s: Incorrect algorithm %d\n",
+ __func__, algorithm);
return;
}
@@ -1310,8 +1287,8 @@ void hash_get_digest(struct hash_device_data *device_data,
else
loop_ctr = SHA256_DIGEST_SIZE / sizeof(u32);
- dev_dbg(device_data->dev, "[%s] digest array:(0x%x)",
- __func__, (u32) digest);
+ dev_dbg(device_data->dev, "%s: digest array:(0x%x)\n",
+ __func__, (u32) digest);
/* Copy result into digest array */
for (count = 0; count < loop_ctr; count++) {
@@ -1337,8 +1314,7 @@ static int ahash_update(struct ahash_request *req)
/* Skip update for DMA, all data will be passed to DMA in final */
if (ret) {
- pr_err(DEV_DBG_NAME " [%s] hash_hw_update() failed!",
- __func__);
+ pr_err("%s: hash_hw_update() failed!\n", __func__);
}
return ret;
@@ -1353,7 +1329,7 @@ static int ahash_final(struct ahash_request *req)
int ret = 0;
struct hash_req_ctx *req_ctx = ahash_request_ctx(req);
- pr_debug(DEV_DBG_NAME " [%s] data size: %d", __func__, req->nbytes);
+ pr_debug("%s: data size: %d\n", __func__, req->nbytes);
if ((hash_mode == HASH_MODE_DMA) && req_ctx->dma_mode)
ret = hash_dma_final(req);
@@ -1361,15 +1337,14 @@ static int ahash_final(struct ahash_request *req)
ret = hash_hw_final(req);
if (ret) {
- pr_err(DEV_DBG_NAME " [%s] hash_hw/dma_final() failed",
- __func__);
+ pr_err("%s: hash_hw/dma_final() failed\n", __func__);
}
return ret;
}
static int hash_setkey(struct crypto_ahash *tfm,
- const u8 *key, unsigned int keylen, int alg)
+ const u8 *key, unsigned int keylen, int alg)
{
int ret = 0;
struct hash_ctx *ctx = crypto_ahash_ctx(tfm);
@@ -1379,8 +1354,8 @@ static int hash_setkey(struct crypto_ahash *tfm,
*/
ctx->key = kmemdup(key, keylen, GFP_KERNEL);
if (!ctx->key) {
- pr_err(DEV_DBG_NAME " [%s] Failed to allocate ctx->key "
- "for %d\n", __func__, alg);
+ pr_err("%s: Failed to allocate ctx->key for %d\n",
+ __func__, alg);
return -ENOMEM;
}
ctx->keylen = keylen;
@@ -1501,13 +1476,13 @@ out:
}
static int hmac_sha1_setkey(struct crypto_ahash *tfm,
- const u8 *key, unsigned int keylen)
+ const u8 *key, unsigned int keylen)
{
return hash_setkey(tfm, key, keylen, HASH_ALGO_SHA1);
}
static int hmac_sha256_setkey(struct crypto_ahash *tfm,
- const u8 *key, unsigned int keylen)
+ const u8 *key, unsigned int keylen)
{
return hash_setkey(tfm, key, keylen, HASH_ALGO_SHA256);
}
@@ -1528,7 +1503,7 @@ static int hash_cra_init(struct crypto_tfm *tfm)
hash);
crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
- sizeof(struct hash_req_ctx));
+ sizeof(struct hash_req_ctx));
ctx->config.data_format = HASH_DATA_8_BITS;
ctx->config.algorithm = hash_alg->conf.algorithm;
@@ -1541,98 +1516,97 @@ static int hash_cra_init(struct crypto_tfm *tfm)
static struct hash_algo_template hash_algs[] = {
{
- .conf.algorithm = HASH_ALGO_SHA1,
- .conf.oper_mode = HASH_OPER_MODE_HASH,
- .hash = {
- .init = hash_init,
- .update = ahash_update,
- .final = ahash_final,
- .digest = ahash_sha1_digest,
- .halg.digestsize = SHA1_DIGEST_SIZE,
- .halg.statesize = sizeof(struct hash_ctx),
- .halg.base = {
- .cra_name = "sha1",
- .cra_driver_name = "sha1-ux500",
- .cra_flags = CRYPTO_ALG_TYPE_AHASH |
- CRYPTO_ALG_ASYNC,
- .cra_blocksize = SHA1_BLOCK_SIZE,
- .cra_ctxsize = sizeof(struct hash_ctx),
- .cra_init = hash_cra_init,
- .cra_module = THIS_MODULE,
+ .conf.algorithm = HASH_ALGO_SHA1,
+ .conf.oper_mode = HASH_OPER_MODE_HASH,
+ .hash = {
+ .init = hash_init,
+ .update = ahash_update,
+ .final = ahash_final,
+ .digest = ahash_sha1_digest,
+ .halg.digestsize = SHA1_DIGEST_SIZE,
+ .halg.statesize = sizeof(struct hash_ctx),
+ .halg.base = {
+ .cra_name = "sha1",
+ .cra_driver_name = "sha1-ux500",
+ .cra_flags = (CRYPTO_ALG_TYPE_AHASH |
+ CRYPTO_ALG_ASYNC),
+ .cra_blocksize = SHA1_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct hash_ctx),
+ .cra_init = hash_cra_init,
+ .cra_module = THIS_MODULE,
}
}
},
{
- .conf.algorithm = HASH_ALGO_SHA256,
- .conf.oper_mode = HASH_OPER_MODE_HASH,
- .hash = {
- .init = hash_init,
- .update = ahash_update,
- .final = ahash_final,
- .digest = ahash_sha256_digest,
- .halg.digestsize = SHA256_DIGEST_SIZE,
- .halg.statesize = sizeof(struct hash_ctx),
- .halg.base = {
- .cra_name = "sha256",
- .cra_driver_name = "sha256-ux500",
- .cra_flags = CRYPTO_ALG_TYPE_AHASH |
- CRYPTO_ALG_ASYNC,
- .cra_blocksize = SHA256_BLOCK_SIZE,
- .cra_ctxsize = sizeof(struct hash_ctx),
- .cra_type = &crypto_ahash_type,
- .cra_init = hash_cra_init,
- .cra_module = THIS_MODULE,
- }
+ .conf.algorithm = HASH_ALGO_SHA256,
+ .conf.oper_mode = HASH_OPER_MODE_HASH,
+ .hash = {
+ .init = hash_init,
+ .update = ahash_update,
+ .final = ahash_final,
+ .digest = ahash_sha256_digest,
+ .halg.digestsize = SHA256_DIGEST_SIZE,
+ .halg.statesize = sizeof(struct hash_ctx),
+ .halg.base = {
+ .cra_name = "sha256",
+ .cra_driver_name = "sha256-ux500",
+ .cra_flags = (CRYPTO_ALG_TYPE_AHASH |
+ CRYPTO_ALG_ASYNC),
+ .cra_blocksize = SHA256_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct hash_ctx),
+ .cra_type = &crypto_ahash_type,
+ .cra_init = hash_cra_init,
+ .cra_module = THIS_MODULE,
}
-
+ }
},
{
- .conf.algorithm = HASH_ALGO_SHA1,
- .conf.oper_mode = HASH_OPER_MODE_HMAC,
+ .conf.algorithm = HASH_ALGO_SHA1,
+ .conf.oper_mode = HASH_OPER_MODE_HMAC,
.hash = {
- .init = hash_init,
- .update = ahash_update,
- .final = ahash_final,
- .digest = hmac_sha1_digest,
- .setkey = hmac_sha1_setkey,
- .halg.digestsize = SHA1_DIGEST_SIZE,
- .halg.statesize = sizeof(struct hash_ctx),
- .halg.base = {
- .cra_name = "hmac(sha1)",
- .cra_driver_name = "hmac-sha1-ux500",
- .cra_flags = CRYPTO_ALG_TYPE_AHASH |
- CRYPTO_ALG_ASYNC,
- .cra_blocksize = SHA1_BLOCK_SIZE,
- .cra_ctxsize = sizeof(struct hash_ctx),
- .cra_type = &crypto_ahash_type,
- .cra_init = hash_cra_init,
- .cra_module = THIS_MODULE,
- }
+ .init = hash_init,
+ .update = ahash_update,
+ .final = ahash_final,
+ .digest = hmac_sha1_digest,
+ .setkey = hmac_sha1_setkey,
+ .halg.digestsize = SHA1_DIGEST_SIZE,
+ .halg.statesize = sizeof(struct hash_ctx),
+ .halg.base = {
+ .cra_name = "hmac(sha1)",
+ .cra_driver_name = "hmac-sha1-ux500",
+ .cra_flags = (CRYPTO_ALG_TYPE_AHASH |
+ CRYPTO_ALG_ASYNC),
+ .cra_blocksize = SHA1_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct hash_ctx),
+ .cra_type = &crypto_ahash_type,
+ .cra_init = hash_cra_init,
+ .cra_module = THIS_MODULE,
}
+ }
},
{
- .conf.algorithm = HASH_ALGO_SHA256,
- .conf.oper_mode = HASH_OPER_MODE_HMAC,
- .hash = {
- .init = hash_init,
- .update = ahash_update,
- .final = ahash_final,
- .digest = hmac_sha256_digest,
- .setkey = hmac_sha256_setkey,
- .halg.digestsize = SHA256_DIGEST_SIZE,
- .halg.statesize = sizeof(struct hash_ctx),
- .halg.base = {
- .cra_name = "hmac(sha256)",
- .cra_driver_name = "hmac-sha256-ux500",
- .cra_flags = CRYPTO_ALG_TYPE_AHASH |
- CRYPTO_ALG_ASYNC,
- .cra_blocksize = SHA256_BLOCK_SIZE,
- .cra_ctxsize = sizeof(struct hash_ctx),
- .cra_type = &crypto_ahash_type,
- .cra_init = hash_cra_init,
- .cra_module = THIS_MODULE,
- }
+ .conf.algorithm = HASH_ALGO_SHA256,
+ .conf.oper_mode = HASH_OPER_MODE_HMAC,
+ .hash = {
+ .init = hash_init,
+ .update = ahash_update,
+ .final = ahash_final,
+ .digest = hmac_sha256_digest,
+ .setkey = hmac_sha256_setkey,
+ .halg.digestsize = SHA256_DIGEST_SIZE,
+ .halg.statesize = sizeof(struct hash_ctx),
+ .halg.base = {
+ .cra_name = "hmac(sha256)",
+ .cra_driver_name = "hmac-sha256-ux500",
+ .cra_flags = (CRYPTO_ALG_TYPE_AHASH |
+ CRYPTO_ALG_ASYNC),
+ .cra_blocksize = SHA256_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct hash_ctx),
+ .cra_type = &crypto_ahash_type,
+ .cra_init = hash_cra_init,
+ .cra_module = THIS_MODULE,
}
+ }
}
};
@@ -1649,7 +1623,7 @@ static int ahash_algs_register_all(struct hash_device_data *device_data)
ret = crypto_register_ahash(&hash_algs[i].hash);
if (ret) {
count = i;
- dev_err(device_data->dev, "[%s] alg registration failed",
+ dev_err(device_data->dev, "%s: alg registration failed\n",
hash_algs[i].hash.halg.base.cra_driver_name);
goto unreg;
}
@@ -1683,9 +1657,8 @@ static int ux500_hash_probe(struct platform_device *pdev)
struct hash_device_data *device_data;
struct device *dev = &pdev->dev;
- device_data = kzalloc(sizeof(struct hash_device_data), GFP_ATOMIC);
+ device_data = kzalloc(sizeof(*device_data), GFP_ATOMIC);
if (!device_data) {
- dev_dbg(dev, "[%s] kzalloc() failed!", __func__);
ret = -ENOMEM;
goto out;
}
@@ -1695,14 +1668,14 @@ static int ux500_hash_probe(struct platform_device *pdev)
res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
if (!res) {
- dev_dbg(dev, "[%s] platform_get_resource() failed!", __func__);
+ dev_dbg(dev, "%s: platform_get_resource() failed!\n", __func__);
ret = -ENODEV;
goto out_kfree;
}
res = request_mem_region(res->start, resource_size(res), pdev->name);
if (res == NULL) {
- dev_dbg(dev, "[%s] request_mem_region() failed!", __func__);
+ dev_dbg(dev, "%s: request_mem_region() failed!\n", __func__);
ret = -EBUSY;
goto out_kfree;
}
@@ -1710,8 +1683,7 @@ static int ux500_hash_probe(struct platform_device *pdev)
device_data->phybase = res->start;
device_data->base = ioremap(res->start, resource_size(res));
if (!device_data->base) {
- dev_err(dev, "[%s] ioremap() failed!",
- __func__);
+ dev_err(dev, "%s: ioremap() failed!\n", __func__);
ret = -ENOMEM;
goto out_free_mem;
}
@@ -1721,7 +1693,7 @@ static int ux500_hash_probe(struct platform_device *pdev)
/* Enable power for HASH1 hardware block */
device_data->regulator = regulator_get(dev, "v-ape");
if (IS_ERR(device_data->regulator)) {
- dev_err(dev, "[%s] regulator_get() failed!", __func__);
+ dev_err(dev, "%s: regulator_get() failed!\n", __func__);
ret = PTR_ERR(device_data->regulator);
device_data->regulator = NULL;
goto out_unmap;
@@ -1730,27 +1702,27 @@ static int ux500_hash_probe(struct platform_device *pdev)
/* Enable the clock for HASH1 hardware block */
device_data->clk = clk_get(dev, NULL);
if (IS_ERR(device_data->clk)) {
- dev_err(dev, "[%s] clk_get() failed!", __func__);
+ dev_err(dev, "%s: clk_get() failed!\n", __func__);
ret = PTR_ERR(device_data->clk);
goto out_regulator;
}
ret = clk_prepare(device_data->clk);
if (ret) {
- dev_err(dev, "[%s] clk_prepare() failed!", __func__);
+ dev_err(dev, "%s: clk_prepare() failed!\n", __func__);
goto out_clk;
}
/* Enable device power (and clock) */
ret = hash_enable_power(device_data, false);
if (ret) {
- dev_err(dev, "[%s]: hash_enable_power() failed!", __func__);
+ dev_err(dev, "%s: hash_enable_power() failed!\n", __func__);
goto out_clk_unprepare;
}
ret = hash_check_hw(device_data);
if (ret) {
- dev_err(dev, "[%s] hash_check_hw() failed!", __func__);
+ dev_err(dev, "%s: hash_check_hw() failed!\n", __func__);
goto out_power;
}
@@ -1766,8 +1738,8 @@ static int ux500_hash_probe(struct platform_device *pdev)
ret = ahash_algs_register_all(device_data);
if (ret) {
- dev_err(dev, "[%s] ahash_algs_register_all() "
- "failed!", __func__);
+ dev_err(dev, "%s: ahash_algs_register_all() failed!\n",
+ __func__);
goto out_power;
}
@@ -1810,8 +1782,7 @@ static int ux500_hash_remove(struct platform_device *pdev)
device_data = platform_get_drvdata(pdev);
if (!device_data) {
- dev_err(dev, "[%s]: platform_get_drvdata() failed!",
- __func__);
+ dev_err(dev, "%s: platform_get_drvdata() failed!\n", __func__);
return -ENOMEM;
}
@@ -1841,7 +1812,7 @@ static int ux500_hash_remove(struct platform_device *pdev)
ahash_algs_unregister_all(device_data);
if (hash_disable_power(device_data, false))
- dev_err(dev, "[%s]: hash_disable_power() failed",
+ dev_err(dev, "%s: hash_disable_power() failed\n",
__func__);
clk_unprepare(device_data->clk);
@@ -1870,8 +1841,8 @@ static void ux500_hash_shutdown(struct platform_device *pdev)
device_data = platform_get_drvdata(pdev);
if (!device_data) {
- dev_err(&pdev->dev, "[%s] platform_get_drvdata() failed!",
- __func__);
+ dev_err(&pdev->dev, "%s: platform_get_drvdata() failed!\n",
+ __func__);
return;
}
@@ -1880,8 +1851,8 @@ static void ux500_hash_shutdown(struct platform_device *pdev)
/* current_ctx allocates a device, NULL = unallocated */
if (!device_data->current_ctx) {
if (down_trylock(&driver_data.device_allocation))
- dev_dbg(&pdev->dev, "[%s]: Cryp still in use!"
- "Shutting down anyway...", __func__);
+ dev_dbg(&pdev->dev, "%s: Cryp still in use! Shutting down anyway...\n",
+ __func__);
/**
* (Allocate the device)
* Need to set this to non-null (dummy) value,
@@ -1906,8 +1877,8 @@ static void ux500_hash_shutdown(struct platform_device *pdev)
release_mem_region(res->start, resource_size(res));
if (hash_disable_power(device_data, false))
- dev_err(&pdev->dev, "[%s] hash_disable_power() failed",
- __func__);
+ dev_err(&pdev->dev, "%s: hash_disable_power() failed\n",
+ __func__);
}
/**
@@ -1922,7 +1893,7 @@ static int ux500_hash_suspend(struct device *dev)
device_data = dev_get_drvdata(dev);
if (!device_data) {
- dev_err(dev, "[%s] platform_get_drvdata() failed!", __func__);
+ dev_err(dev, "%s: platform_get_drvdata() failed!\n", __func__);
return -ENOMEM;
}
@@ -1933,15 +1904,16 @@ static int ux500_hash_suspend(struct device *dev)
if (device_data->current_ctx == ++temp_ctx) {
if (down_interruptible(&driver_data.device_allocation))
- dev_dbg(dev, "[%s]: down_interruptible() failed",
+ dev_dbg(dev, "%s: down_interruptible() failed\n",
__func__);
ret = hash_disable_power(device_data, false);
- } else
+ } else {
ret = hash_disable_power(device_data, true);
+ }
if (ret)
- dev_err(dev, "[%s]: hash_disable_power()", __func__);
+ dev_err(dev, "%s: hash_disable_power()\n", __func__);
return ret;
}
@@ -1958,7 +1930,7 @@ static int ux500_hash_resume(struct device *dev)
device_data = dev_get_drvdata(dev);
if (!device_data) {
- dev_err(dev, "[%s] platform_get_drvdata() failed!", __func__);
+ dev_err(dev, "%s: platform_get_drvdata() failed!\n", __func__);
return -ENOMEM;
}
@@ -1973,7 +1945,7 @@ static int ux500_hash_resume(struct device *dev)
ret = hash_enable_power(device_data, true);
if (ret)
- dev_err(dev, "[%s]: hash_enable_power() failed!", __func__);
+ dev_err(dev, "%s: hash_enable_power() failed!\n", __func__);
return ret;
}
@@ -1981,8 +1953,8 @@ static int ux500_hash_resume(struct device *dev)
static SIMPLE_DEV_PM_OPS(ux500_hash_pm, ux500_hash_suspend, ux500_hash_resume);
static const struct of_device_id ux500_hash_match[] = {
- { .compatible = "stericsson,ux500-hash" },
- { },
+ { .compatible = "stericsson,ux500-hash" },
+ { },
};
static struct platform_driver hash_driver = {
diff --git a/include/crypto/scatterwalk.h b/include/crypto/scatterwalk.h
index 3744d2a642df..13621cc8cf4c 100644
--- a/include/crypto/scatterwalk.h
+++ b/include/crypto/scatterwalk.h
@@ -113,4 +113,6 @@ void scatterwalk_done(struct scatter_walk *walk, int out, int more);
void scatterwalk_map_and_copy(void *buf, struct scatterlist *sg,
unsigned int start, unsigned int nbytes, int out);
+int scatterwalk_bytes_sglen(struct scatterlist *sg, int num_bytes);
+
#endif /* _CRYPTO_SCATTERWALK_H */
diff --git a/include/linux/crc-t10dif.h b/include/linux/crc-t10dif.h
index a9c96d865ee7..b3cb71f0d3b0 100644
--- a/include/linux/crc-t10dif.h
+++ b/include/linux/crc-t10dif.h
@@ -3,6 +3,10 @@
#include <linux/types.h>
+#define CRC_T10DIF_DIGEST_SIZE 2
+#define CRC_T10DIF_BLOCK_SIZE 1
+
+__u16 crc_t10dif_generic(__u16 crc, const unsigned char *buffer, size_t len);
__u16 crc_t10dif(unsigned char const *, size_t);
#endif
diff --git a/kernel/padata.c b/kernel/padata.c
index 072f4ee4eb89..07af2c95dcfe 100644
--- a/kernel/padata.c
+++ b/kernel/padata.c
@@ -846,6 +846,8 @@ static int padata_cpu_callback(struct notifier_block *nfb,
switch (action) {
case CPU_ONLINE:
case CPU_ONLINE_FROZEN:
+ case CPU_DOWN_FAILED:
+ case CPU_DOWN_FAILED_FROZEN:
if (!pinst_has_cpu(pinst, cpu))
break;
mutex_lock(&pinst->lock);
@@ -857,6 +859,8 @@ static int padata_cpu_callback(struct notifier_block *nfb,
case CPU_DOWN_PREPARE:
case CPU_DOWN_PREPARE_FROZEN:
+ case CPU_UP_CANCELED:
+ case CPU_UP_CANCELED_FROZEN:
if (!pinst_has_cpu(pinst, cpu))
break;
mutex_lock(&pinst->lock);
@@ -865,22 +869,6 @@ static int padata_cpu_callback(struct notifier_block *nfb,
if (err)
return notifier_from_errno(err);
break;
-
- case CPU_UP_CANCELED:
- case CPU_UP_CANCELED_FROZEN:
- if (!pinst_has_cpu(pinst, cpu))
- break;
- mutex_lock(&pinst->lock);
- __padata_remove_cpu(pinst, cpu);
- mutex_unlock(&pinst->lock);
-
- case CPU_DOWN_FAILED:
- case CPU_DOWN_FAILED_FROZEN:
- if (!pinst_has_cpu(pinst, cpu))
- break;
- mutex_lock(&pinst->lock);
- __padata_add_cpu(pinst, cpu);
- mutex_unlock(&pinst->lock);
}
return NOTIFY_OK;
@@ -1086,18 +1074,18 @@ struct padata_instance *padata_alloc(struct workqueue_struct *wq,
pinst->flags = 0;
-#ifdef CONFIG_HOTPLUG_CPU
- pinst->cpu_notifier.notifier_call = padata_cpu_callback;
- pinst->cpu_notifier.priority = 0;
- register_hotcpu_notifier(&pinst->cpu_notifier);
-#endif
-
put_online_cpus();
BLOCKING_INIT_NOTIFIER_HEAD(&pinst->cpumask_change_notifier);
kobject_init(&pinst->kobj, &padata_attr_type);
mutex_init(&pinst->lock);
+#ifdef CONFIG_HOTPLUG_CPU
+ pinst->cpu_notifier.notifier_call = padata_cpu_callback;
+ pinst->cpu_notifier.priority = 0;
+ register_hotcpu_notifier(&pinst->cpu_notifier);
+#endif
+
return pinst;
err_free_masks:
diff --git a/lib/Kconfig b/lib/Kconfig
index 65561716c16c..b3c8be0da17f 100644
--- a/lib/Kconfig
+++ b/lib/Kconfig
@@ -76,6 +76,8 @@ config CRC16
config CRC_T10DIF
tristate "CRC calculation for the T10 Data Integrity Field"
+ select CRYPTO
+ select CRYPTO_CRCT10DIF
help
This option is only needed if a module that's not in the
kernel tree needs to calculate CRC checks for use with the
diff --git a/lib/crc-t10dif.c b/lib/crc-t10dif.c
index fbbd66ed86cd..43bc5b071f96 100644
--- a/lib/crc-t10dif.c
+++ b/lib/crc-t10dif.c
@@ -11,57 +11,45 @@
#include <linux/types.h>
#include <linux/module.h>
#include <linux/crc-t10dif.h>
+#include <linux/err.h>
+#include <linux/init.h>
+#include <crypto/hash.h>
-/* Table generated using the following polynomium:
- * x^16 + x^15 + x^11 + x^9 + x^8 + x^7 + x^5 + x^4 + x^2 + x + 1
- * gt: 0x8bb7
- */
-static const __u16 t10_dif_crc_table[256] = {
- 0x0000, 0x8BB7, 0x9CD9, 0x176E, 0xB205, 0x39B2, 0x2EDC, 0xA56B,
- 0xEFBD, 0x640A, 0x7364, 0xF8D3, 0x5DB8, 0xD60F, 0xC161, 0x4AD6,
- 0x54CD, 0xDF7A, 0xC814, 0x43A3, 0xE6C8, 0x6D7F, 0x7A11, 0xF1A6,
- 0xBB70, 0x30C7, 0x27A9, 0xAC1E, 0x0975, 0x82C2, 0x95AC, 0x1E1B,
- 0xA99A, 0x222D, 0x3543, 0xBEF4, 0x1B9F, 0x9028, 0x8746, 0x0CF1,
- 0x4627, 0xCD90, 0xDAFE, 0x5149, 0xF422, 0x7F95, 0x68FB, 0xE34C,
- 0xFD57, 0x76E0, 0x618E, 0xEA39, 0x4F52, 0xC4E5, 0xD38B, 0x583C,
- 0x12EA, 0x995D, 0x8E33, 0x0584, 0xA0EF, 0x2B58, 0x3C36, 0xB781,
- 0xD883, 0x5334, 0x445A, 0xCFED, 0x6A86, 0xE131, 0xF65F, 0x7DE8,
- 0x373E, 0xBC89, 0xABE7, 0x2050, 0x853B, 0x0E8C, 0x19E2, 0x9255,
- 0x8C4E, 0x07F9, 0x1097, 0x9B20, 0x3E4B, 0xB5FC, 0xA292, 0x2925,
- 0x63F3, 0xE844, 0xFF2A, 0x749D, 0xD1F6, 0x5A41, 0x4D2F, 0xC698,
- 0x7119, 0xFAAE, 0xEDC0, 0x6677, 0xC31C, 0x48AB, 0x5FC5, 0xD472,
- 0x9EA4, 0x1513, 0x027D, 0x89CA, 0x2CA1, 0xA716, 0xB078, 0x3BCF,
- 0x25D4, 0xAE63, 0xB90D, 0x32BA, 0x97D1, 0x1C66, 0x0B08, 0x80BF,
- 0xCA69, 0x41DE, 0x56B0, 0xDD07, 0x786C, 0xF3DB, 0xE4B5, 0x6F02,
- 0x3AB1, 0xB106, 0xA668, 0x2DDF, 0x88B4, 0x0303, 0x146D, 0x9FDA,
- 0xD50C, 0x5EBB, 0x49D5, 0xC262, 0x6709, 0xECBE, 0xFBD0, 0x7067,
- 0x6E7C, 0xE5CB, 0xF2A5, 0x7912, 0xDC79, 0x57CE, 0x40A0, 0xCB17,
- 0x81C1, 0x0A76, 0x1D18, 0x96AF, 0x33C4, 0xB873, 0xAF1D, 0x24AA,
- 0x932B, 0x189C, 0x0FF2, 0x8445, 0x212E, 0xAA99, 0xBDF7, 0x3640,
- 0x7C96, 0xF721, 0xE04F, 0x6BF8, 0xCE93, 0x4524, 0x524A, 0xD9FD,
- 0xC7E6, 0x4C51, 0x5B3F, 0xD088, 0x75E3, 0xFE54, 0xE93A, 0x628D,
- 0x285B, 0xA3EC, 0xB482, 0x3F35, 0x9A5E, 0x11E9, 0x0687, 0x8D30,
- 0xE232, 0x6985, 0x7EEB, 0xF55C, 0x5037, 0xDB80, 0xCCEE, 0x4759,
- 0x0D8F, 0x8638, 0x9156, 0x1AE1, 0xBF8A, 0x343D, 0x2353, 0xA8E4,
- 0xB6FF, 0x3D48, 0x2A26, 0xA191, 0x04FA, 0x8F4D, 0x9823, 0x1394,
- 0x5942, 0xD2F5, 0xC59B, 0x4E2C, 0xEB47, 0x60F0, 0x779E, 0xFC29,
- 0x4BA8, 0xC01F, 0xD771, 0x5CC6, 0xF9AD, 0x721A, 0x6574, 0xEEC3,
- 0xA415, 0x2FA2, 0x38CC, 0xB37B, 0x1610, 0x9DA7, 0x8AC9, 0x017E,
- 0x1F65, 0x94D2, 0x83BC, 0x080B, 0xAD60, 0x26D7, 0x31B9, 0xBA0E,
- 0xF0D8, 0x7B6F, 0x6C01, 0xE7B6, 0x42DD, 0xC96A, 0xDE04, 0x55B3
-};
+static struct crypto_shash *crct10dif_tfm;
__u16 crc_t10dif(const unsigned char *buffer, size_t len)
{
- __u16 crc = 0;
- unsigned int i;
+ struct {
+ struct shash_desc shash;
+ char ctx[2];
+ } desc;
+ int err;
+
+ desc.shash.tfm = crct10dif_tfm;
+ desc.shash.flags = 0;
+ *(__u16 *)desc.ctx = 0;
- for (i = 0 ; i < len ; i++)
- crc = (crc << 8) ^ t10_dif_crc_table[((crc >> 8) ^ buffer[i]) & 0xff];
+ err = crypto_shash_update(&desc.shash, buffer, len);
+ BUG_ON(err);
- return crc;
+ return *(__u16 *)desc.ctx;
}
EXPORT_SYMBOL(crc_t10dif);
+static int __init crc_t10dif_mod_init(void)
+{
+ crct10dif_tfm = crypto_alloc_shash("crct10dif", 0, 0);
+ return PTR_RET(crct10dif_tfm);
+}
+
+static void __exit crc_t10dif_mod_fini(void)
+{
+ crypto_free_shash(crct10dif_tfm);
+}
+
+module_init(crc_t10dif_mod_init);
+module_exit(crc_t10dif_mod_fini);
+
MODULE_DESCRIPTION("T10 DIF CRC calculation");
MODULE_LICENSE("GPL");
+MODULE_SOFTDEP("pre: crct10dif");