diff options
154 files changed, 15624 insertions, 7328 deletions
diff --git a/Documentation/DocBook/crypto-API.tmpl b/Documentation/DocBook/crypto-API.tmpl index 0992531ffefb..07df23ea06e4 100644 --- a/Documentation/DocBook/crypto-API.tmpl +++ b/Documentation/DocBook/crypto-API.tmpl @@ -585,7 +585,7 @@ kernel crypto API | IPSEC Layer +-----------+ | | | (1) | aead | <----------------------------------- esp_output -| (seqniv) | ---+ +| (seqiv) | ---+ +-----------+ | | (2) +-----------+ | @@ -1101,7 +1101,7 @@ kernel crypto API | Caller </para> <para> - [1] http://www.chronox.de/libkcapi.html + [1] <ulink url="http://www.chronox.de/libkcapi.html">http://www.chronox.de/libkcapi.html</ulink> </para> </sect1> @@ -1661,7 +1661,7 @@ read(opfd, out, outlen); </para> <para> - [1] http://www.chronox.de/libkcapi.html + [1] <ulink url="http://www.chronox.de/libkcapi.html">http://www.chronox.de/libkcapi.html</ulink> </para> </sect1> @@ -1687,7 +1687,7 @@ read(opfd, out, outlen); !Pinclude/linux/crypto.h Block Cipher Algorithm Definitions !Finclude/linux/crypto.h crypto_alg !Finclude/linux/crypto.h ablkcipher_alg -!Finclude/linux/crypto.h aead_alg +!Finclude/crypto/aead.h aead_alg !Finclude/linux/crypto.h blkcipher_alg !Finclude/linux/crypto.h cipher_alg !Finclude/crypto/rng.h rng_alg diff --git a/Documentation/devicetree/bindings/crypto/fsl-sec4.txt b/Documentation/devicetree/bindings/crypto/fsl-sec4.txt index e4022776ac6e..100307304766 100644 --- a/Documentation/devicetree/bindings/crypto/fsl-sec4.txt +++ b/Documentation/devicetree/bindings/crypto/fsl-sec4.txt @@ -106,6 +106,18 @@ PROPERTIES to the interrupt parent to which the child domain is being mapped. + - clocks + Usage: required if SEC 4.0 requires explicit enablement of clocks + Value type: <prop_encoded-array> + Definition: A list of phandle and clock specifier pairs describing + the clocks required for enabling and disabling SEC 4.0. + + - clock-names + Usage: required if SEC 4.0 requires explicit enablement of clocks + Value type: <string> + Definition: A list of clock name strings in the same order as the + clocks property. + Note: All other standard properties (see the ePAPR) are allowed but are optional. @@ -120,6 +132,11 @@ EXAMPLE ranges = <0 0x300000 0x10000>; interrupt-parent = <&mpic>; interrupts = <92 2>; + clocks = <&clks IMX6QDL_CLK_CAAM_MEM>, + <&clks IMX6QDL_CLK_CAAM_ACLK>, + <&clks IMX6QDL_CLK_CAAM_IPG>, + <&clks IMX6QDL_CLK_EIM_SLOW>; + clock-names = "mem", "aclk", "ipg", "emi_slow"; }; ===================================================================== diff --git a/Documentation/devicetree/bindings/crypto/sun4i-ss.txt b/Documentation/devicetree/bindings/crypto/sun4i-ss.txt new file mode 100644 index 000000000000..5d38e9b7033f --- /dev/null +++ b/Documentation/devicetree/bindings/crypto/sun4i-ss.txt @@ -0,0 +1,23 @@ +* Allwinner Security System found on A20 SoC + +Required properties: +- compatible : Should be "allwinner,sun4i-a10-crypto". +- reg: Should contain the Security System register location and length. +- interrupts: Should contain the IRQ line for the Security System. +- clocks : List of clock specifiers, corresponding to ahb and ss. +- clock-names : Name of the functional clock, should be + * "ahb" : AHB gating clock + * "mod" : SS controller clock + +Optional properties: + - resets : phandle + reset specifier pair + - reset-names : must contain "ahb" + +Example: + crypto: crypto-engine@01c15000 { + compatible = "allwinner,sun4i-a10-crypto"; + reg = <0x01c15000 0x1000>; + interrupts = <GIC_SPI 86 IRQ_TYPE_LEVEL_HIGH>; + clocks = <&ahb_gates 5>, <&ss_clk>; + clock-names = "ahb", "mod"; + }; diff --git a/MAINTAINERS b/MAINTAINERS index cdcd9c5d3c0e..e16584a5091a 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -556,6 +556,12 @@ S: Maintained F: Documentation/i2c/busses/i2c-ali1563 F: drivers/i2c/busses/i2c-ali1563.c +ALLWINNER SECURITY SYSTEM +M: Corentin Labbe <clabbe.montjoie@gmail.com> +L: linux-crypto@vger.kernel.org +S: Maintained +F: drivers/crypto/sunxi-ss/ + ALPHA PORT M: Richard Henderson <rth@twiddle.net> M: Ivan Kokshaysky <ink@jurassic.park.msu.ru> @@ -5078,9 +5084,21 @@ T: git git://git.kernel.org/pub/scm/linux/kernel/git/aegl/linux.git S: Maintained F: arch/ia64/ +IBM Power VMX Cryptographic instructions +M: Leonidas S. Barbosa <leosilva@linux.vnet.ibm.com> +M: Paulo Flabiano Smorigo <pfsmorigo@linux.vnet.ibm.com> +L: linux-crypto@vger.kernel.org +S: Supported +F: drivers/crypto/vmx/Makefile +F: drivers/crypto/vmx/Kconfig +F: drivers/crypto/vmx/vmx.c +F: drivers/crypto/vmx/aes* +F: drivers/crypto/vmx/ghash* +F: drivers/crypto/vmx/ppc-xlate.pl + IBM Power in-Nest Crypto Acceleration -M: Marcelo Henrique Cerri <mhcerri@linux.vnet.ibm.com> -M: Fionnuala Gunter <fin@linux.vnet.ibm.com> +M: Leonidas S. Barbosa <leosilva@linux.vnet.ibm.com> +M: Paulo Flabiano Smorigo <pfsmorigo@linux.vnet.ibm.com> L: linux-crypto@vger.kernel.org S: Supported F: drivers/crypto/nx/Makefile @@ -5092,7 +5110,7 @@ F: drivers/crypto/nx/nx_csbcpb.h F: drivers/crypto/nx/nx_debugfs.h IBM Power 842 compression accelerator -M: Dan Streetman <ddstreet@us.ibm.com> +M: Dan Streetman <ddstreet@ieee.org> S: Supported F: drivers/crypto/nx/Makefile F: drivers/crypto/nx/Kconfig diff --git a/arch/arm/boot/dts/imx6qdl.dtsi b/arch/arm/boot/dts/imx6qdl.dtsi index b57033e8c633..10d0b26c93f1 100644 --- a/arch/arm/boot/dts/imx6qdl.dtsi +++ b/arch/arm/boot/dts/imx6qdl.dtsi @@ -836,10 +836,31 @@ reg = <0x02100000 0x100000>; ranges; - caam@02100000 { - reg = <0x02100000 0x40000>; - interrupts = <0 105 IRQ_TYPE_LEVEL_HIGH>, - <0 106 IRQ_TYPE_LEVEL_HIGH>; + crypto: caam@2100000 { + compatible = "fsl,sec-v4.0"; + fsl,sec-era = <4>; + #address-cells = <1>; + #size-cells = <1>; + reg = <0x2100000 0x10000>; + ranges = <0 0x2100000 0x10000>; + interrupt-parent = <&intc>; + clocks = <&clks IMX6QDL_CLK_CAAM_MEM>, + <&clks IMX6QDL_CLK_CAAM_ACLK>, + <&clks IMX6QDL_CLK_CAAM_IPG>, + <&clks IMX6QDL_CLK_EIM_SLOW>; + clock-names = "mem", "aclk", "ipg", "emi_slow"; + + sec_jr0: jr0@1000 { + compatible = "fsl,sec-v4.0-job-ring"; + reg = <0x1000 0x1000>; + interrupts = <GIC_SPI 105 IRQ_TYPE_LEVEL_HIGH>; + }; + + sec_jr1: jr1@2000 { + compatible = "fsl,sec-v4.0-job-ring"; + reg = <0x2000 0x1000>; + interrupts = <GIC_SPI 106 IRQ_TYPE_LEVEL_HIGH>; + }; }; aipstz@0217c000 { /* AIPSTZ2 */ diff --git a/arch/arm/boot/dts/imx6sx.dtsi b/arch/arm/boot/dts/imx6sx.dtsi index 708175d59b9c..e6223d8e79af 100644 --- a/arch/arm/boot/dts/imx6sx.dtsi +++ b/arch/arm/boot/dts/imx6sx.dtsi @@ -738,6 +738,33 @@ reg = <0x02100000 0x100000>; ranges; + crypto: caam@2100000 { + compatible = "fsl,sec-v4.0"; + fsl,sec-era = <4>; + #address-cells = <1>; + #size-cells = <1>; + reg = <0x2100000 0x10000>; + ranges = <0 0x2100000 0x10000>; + interrupt-parent = <&intc>; + clocks = <&clks IMX6SX_CLK_CAAM_MEM>, + <&clks IMX6SX_CLK_CAAM_ACLK>, + <&clks IMX6SX_CLK_CAAM_IPG>, + <&clks IMX6SX_CLK_EIM_SLOW>; + clock-names = "mem", "aclk", "ipg", "emi_slow"; + + sec_jr0: jr0@1000 { + compatible = "fsl,sec-v4.0-job-ring"; + reg = <0x1000 0x1000>; + interrupts = <GIC_SPI 105 IRQ_TYPE_LEVEL_HIGH>; + }; + + sec_jr1: jr1@2000 { + compatible = "fsl,sec-v4.0-job-ring"; + reg = <0x2000 0x1000>; + interrupts = <GIC_SPI 106 IRQ_TYPE_LEVEL_HIGH>; + }; + }; + usbotg1: usb@02184000 { compatible = "fsl,imx6sx-usb", "fsl,imx27-usb"; reg = <0x02184000 0x200>; diff --git a/arch/arm/boot/dts/sun4i-a10.dtsi b/arch/arm/boot/dts/sun4i-a10.dtsi index ab0e131587bb..adaa57b7a943 100644 --- a/arch/arm/boot/dts/sun4i-a10.dtsi +++ b/arch/arm/boot/dts/sun4i-a10.dtsi @@ -678,6 +678,14 @@ status = "disabled"; }; + crypto: crypto-engine@01c15000 { + compatible = "allwinner,sun4i-a10-crypto"; + reg = <0x01c15000 0x1000>; + interrupts = <86>; + clocks = <&ahb_gates 5>, <&ss_clk>; + clock-names = "ahb", "mod"; + }; + spi2: spi@01c17000 { compatible = "allwinner,sun4i-a10-spi"; reg = <0x01c17000 0x1000>; diff --git a/arch/arm/boot/dts/sun6i-a31.dtsi b/arch/arm/boot/dts/sun6i-a31.dtsi index 3ec456fa03a4..e4d3484d97bd 100644 --- a/arch/arm/boot/dts/sun6i-a31.dtsi +++ b/arch/arm/boot/dts/sun6i-a31.dtsi @@ -367,6 +367,14 @@ "mmc3_sample"; }; + ss_clk: clk@01c2009c { + #clock-cells = <0>; + compatible = "allwinner,sun4i-a10-mod0-clk"; + reg = <0x01c2009c 0x4>; + clocks = <&osc24M>, <&pll6 0>; + clock-output-names = "ss"; + }; + spi0_clk: clk@01c200a0 { #clock-cells = <0>; compatible = "allwinner,sun4i-a10-mod0-clk"; @@ -894,6 +902,16 @@ #size-cells = <0>; }; + crypto: crypto-engine@01c15000 { + compatible = "allwinner,sun4i-a10-crypto"; + reg = <0x01c15000 0x1000>; + interrupts = <GIC_SPI 80 IRQ_TYPE_LEVEL_HIGH>; + clocks = <&ahb1_gates 5>, <&ss_clk>; + clock-names = "ahb", "mod"; + resets = <&ahb1_rst 5>; + reset-names = "ahb"; + }; + timer@01c60000 { compatible = "allwinner,sun6i-a31-hstimer", "allwinner,sun7i-a20-hstimer"; diff --git a/arch/arm/boot/dts/sun7i-a20.dtsi b/arch/arm/boot/dts/sun7i-a20.dtsi index ca0b01a96c52..d3b2f26417aa 100644 --- a/arch/arm/boot/dts/sun7i-a20.dtsi +++ b/arch/arm/boot/dts/sun7i-a20.dtsi @@ -754,6 +754,14 @@ status = "disabled"; }; + crypto: crypto-engine@01c15000 { + compatible = "allwinner,sun4i-a10-crypto"; + reg = <0x01c15000 0x1000>; + interrupts = <GIC_SPI 86 IRQ_TYPE_LEVEL_HIGH>; + clocks = <&ahb_gates 5>, <&ss_clk>; + clock-names = "ahb", "mod"; + }; + spi2: spi@01c17000 { compatible = "allwinner,sun4i-a10-spi"; reg = <0x01c17000 0x1000>; diff --git a/arch/arm/configs/imx_v6_v7_defconfig b/arch/arm/configs/imx_v6_v7_defconfig index b47863d49ac6..7569b391704e 100644 --- a/arch/arm/configs/imx_v6_v7_defconfig +++ b/arch/arm/configs/imx_v6_v7_defconfig @@ -354,8 +354,7 @@ CONFIG_PROVE_LOCKING=y # CONFIG_FTRACE is not set # CONFIG_ARM_UNWIND is not set CONFIG_SECURITYFS=y -# CONFIG_CRYPTO_ANSI_CPRNG is not set -# CONFIG_CRYPTO_HW is not set +CONFIG_CRYPTO_DEV_FSL_CAAM=y CONFIG_CRC_CCITT=m CONFIG_CRC_T10DIF=y CONFIG_CRC7=m diff --git a/arch/arm/crypto/.gitignore b/arch/arm/crypto/.gitignore index 6231d36b3635..31e1f538df7d 100644 --- a/arch/arm/crypto/.gitignore +++ b/arch/arm/crypto/.gitignore @@ -1 +1,3 @@ aesbs-core.S +sha256-core.S +sha512-core.S diff --git a/arch/arm64/crypto/aes-ce-ccm-glue.c b/arch/arm64/crypto/aes-ce-ccm-glue.c index 3303e8a7b837..f4bf2f2a014c 100644 --- a/arch/arm64/crypto/aes-ce-ccm-glue.c +++ b/arch/arm64/crypto/aes-ce-ccm-glue.c @@ -124,7 +124,7 @@ static void ccm_calculate_auth_mac(struct aead_request *req, u8 mac[]) ce_aes_ccm_auth_data(mac, (u8 *)<ag, ltag.len, &macp, ctx->key_enc, num_rounds(ctx)); - scatterwalk_start(&walk, req->assoc); + scatterwalk_start(&walk, req->src); do { u32 n = scatterwalk_clamp(&walk, len); @@ -151,6 +151,10 @@ static int ccm_encrypt(struct aead_request *req) struct crypto_aes_ctx *ctx = crypto_aead_ctx(aead); struct blkcipher_desc desc = { .info = req->iv }; struct blkcipher_walk walk; + struct scatterlist srcbuf[2]; + struct scatterlist dstbuf[2]; + struct scatterlist *src; + struct scatterlist *dst; u8 __aligned(8) mac[AES_BLOCK_SIZE]; u8 buf[AES_BLOCK_SIZE]; u32 len = req->cryptlen; @@ -168,7 +172,12 @@ static int ccm_encrypt(struct aead_request *req) /* preserve the original iv for the final round */ memcpy(buf, req->iv, AES_BLOCK_SIZE); - blkcipher_walk_init(&walk, req->dst, req->src, len); + src = scatterwalk_ffwd(srcbuf, req->src, req->assoclen); + dst = src; + if (req->src != req->dst) + dst = scatterwalk_ffwd(dstbuf, req->dst, req->assoclen); + + blkcipher_walk_init(&walk, dst, src, len); err = blkcipher_aead_walk_virt_block(&desc, &walk, aead, AES_BLOCK_SIZE); @@ -194,7 +203,7 @@ static int ccm_encrypt(struct aead_request *req) return err; /* copy authtag to end of dst */ - scatterwalk_map_and_copy(mac, req->dst, req->cryptlen, + scatterwalk_map_and_copy(mac, dst, req->cryptlen, crypto_aead_authsize(aead), 1); return 0; @@ -207,6 +216,10 @@ static int ccm_decrypt(struct aead_request *req) unsigned int authsize = crypto_aead_authsize(aead); struct blkcipher_desc desc = { .info = req->iv }; struct blkcipher_walk walk; + struct scatterlist srcbuf[2]; + struct scatterlist dstbuf[2]; + struct scatterlist *src; + struct scatterlist *dst; u8 __aligned(8) mac[AES_BLOCK_SIZE]; u8 buf[AES_BLOCK_SIZE]; u32 len = req->cryptlen - authsize; @@ -224,7 +237,12 @@ static int ccm_decrypt(struct aead_request *req) /* preserve the original iv for the final round */ memcpy(buf, req->iv, AES_BLOCK_SIZE); - blkcipher_walk_init(&walk, req->dst, req->src, len); + src = scatterwalk_ffwd(srcbuf, req->src, req->assoclen); + dst = src; + if (req->src != req->dst) + dst = scatterwalk_ffwd(dstbuf, req->dst, req->assoclen); + + blkcipher_walk_init(&walk, dst, src, len); err = blkcipher_aead_walk_virt_block(&desc, &walk, aead, AES_BLOCK_SIZE); @@ -250,44 +268,42 @@ static int ccm_decrypt(struct aead_request *req) return err; /* compare calculated auth tag with the stored one */ - scatterwalk_map_and_copy(buf, req->src, req->cryptlen - authsize, + scatterwalk_map_and_copy(buf, src, req->cryptlen - authsize, authsize, 0); - if (memcmp(mac, buf, authsize)) + if (crypto_memneq(mac, buf, authsize)) return -EBADMSG; return 0; } -static struct crypto_alg ccm_aes_alg = { - .cra_name = "ccm(aes)", - .cra_driver_name = "ccm-aes-ce", - .cra_priority = 300, - .cra_flags = CRYPTO_ALG_TYPE_AEAD, - .cra_blocksize = 1, - .cra_ctxsize = sizeof(struct crypto_aes_ctx), - .cra_alignmask = 7, - .cra_type = &crypto_aead_type, - .cra_module = THIS_MODULE, - .cra_aead = { - .ivsize = AES_BLOCK_SIZE, - .maxauthsize = AES_BLOCK_SIZE, - .setkey = ccm_setkey, - .setauthsize = ccm_setauthsize, - .encrypt = ccm_encrypt, - .decrypt = ccm_decrypt, - } +static struct aead_alg ccm_aes_alg = { + .base = { + .cra_name = "ccm(aes)", + .cra_driver_name = "ccm-aes-ce", + .cra_priority = 300, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct crypto_aes_ctx), + .cra_alignmask = 7, + .cra_module = THIS_MODULE, + }, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = AES_BLOCK_SIZE, + .setkey = ccm_setkey, + .setauthsize = ccm_setauthsize, + .encrypt = ccm_encrypt, + .decrypt = ccm_decrypt, }; static int __init aes_mod_init(void) { if (!(elf_hwcap & HWCAP_AES)) return -ENODEV; - return crypto_register_alg(&ccm_aes_alg); + return crypto_register_aead(&ccm_aes_alg); } static void __exit aes_mod_exit(void) { - crypto_unregister_alg(&ccm_aes_alg); + crypto_unregister_aead(&ccm_aes_alg); } module_init(aes_mod_init); diff --git a/arch/powerpc/include/asm/switch_to.h b/arch/powerpc/include/asm/switch_to.h index 58abeda64cb7..15cca17cba4b 100644 --- a/arch/powerpc/include/asm/switch_to.h +++ b/arch/powerpc/include/asm/switch_to.h @@ -29,6 +29,7 @@ static inline void save_early_sprs(struct thread_struct *prev) {} extern void enable_kernel_fp(void); extern void enable_kernel_altivec(void); +extern void enable_kernel_vsx(void); extern int emulate_altivec(struct pt_regs *); extern void __giveup_vsx(struct task_struct *); extern void giveup_vsx(struct task_struct *); diff --git a/arch/powerpc/kernel/process.c b/arch/powerpc/kernel/process.c index 8005e18d1b40..64e6e9d9e656 100644 --- a/arch/powerpc/kernel/process.c +++ b/arch/powerpc/kernel/process.c @@ -204,8 +204,6 @@ EXPORT_SYMBOL_GPL(flush_altivec_to_thread); #endif /* CONFIG_ALTIVEC */ #ifdef CONFIG_VSX -#if 0 -/* not currently used, but some crazy RAID module might want to later */ void enable_kernel_vsx(void) { WARN_ON(preemptible()); @@ -220,7 +218,6 @@ void enable_kernel_vsx(void) #endif /* CONFIG_SMP */ } EXPORT_SYMBOL(enable_kernel_vsx); -#endif void giveup_vsx(struct task_struct *tsk) { diff --git a/arch/x86/crypto/Makefile b/arch/x86/crypto/Makefile index 5a4a089e8b1f..9a2838cf0591 100644 --- a/arch/x86/crypto/Makefile +++ b/arch/x86/crypto/Makefile @@ -20,6 +20,7 @@ obj-$(CONFIG_CRYPTO_BLOWFISH_X86_64) += blowfish-x86_64.o obj-$(CONFIG_CRYPTO_TWOFISH_X86_64) += twofish-x86_64.o obj-$(CONFIG_CRYPTO_TWOFISH_X86_64_3WAY) += twofish-x86_64-3way.o obj-$(CONFIG_CRYPTO_SALSA20_X86_64) += salsa20-x86_64.o +obj-$(CONFIG_CRYPTO_CHACHA20_X86_64) += chacha20-x86_64.o obj-$(CONFIG_CRYPTO_SERPENT_SSE2_X86_64) += serpent-sse2-x86_64.o obj-$(CONFIG_CRYPTO_AES_NI_INTEL) += aesni-intel.o obj-$(CONFIG_CRYPTO_GHASH_CLMUL_NI_INTEL) += ghash-clmulni-intel.o @@ -30,6 +31,7 @@ obj-$(CONFIG_CRYPTO_CRC32_PCLMUL) += crc32-pclmul.o obj-$(CONFIG_CRYPTO_SHA256_SSSE3) += sha256-ssse3.o obj-$(CONFIG_CRYPTO_SHA512_SSSE3) += sha512-ssse3.o obj-$(CONFIG_CRYPTO_CRCT10DIF_PCLMUL) += crct10dif-pclmul.o +obj-$(CONFIG_CRYPTO_POLY1305_X86_64) += poly1305-x86_64.o # These modules require assembler to support AVX. ifeq ($(avx_supported),yes) @@ -60,6 +62,7 @@ blowfish-x86_64-y := blowfish-x86_64-asm_64.o blowfish_glue.o twofish-x86_64-y := twofish-x86_64-asm_64.o twofish_glue.o twofish-x86_64-3way-y := twofish-x86_64-asm_64-3way.o twofish_glue_3way.o salsa20-x86_64-y := salsa20-x86_64-asm_64.o salsa20_glue.o +chacha20-x86_64-y := chacha20-ssse3-x86_64.o chacha20_glue.o serpent-sse2-x86_64-y := serpent-sse2-x86_64-asm_64.o serpent_sse2_glue.o ifeq ($(avx_supported),yes) @@ -75,6 +78,7 @@ endif ifeq ($(avx2_supported),yes) camellia-aesni-avx2-y := camellia-aesni-avx2-asm_64.o camellia_aesni_avx2_glue.o + chacha20-x86_64-y += chacha20-avx2-x86_64.o serpent-avx2-y := serpent-avx2-asm_64.o serpent_avx2_glue.o endif @@ -82,8 +86,10 @@ aesni-intel-y := aesni-intel_asm.o aesni-intel_glue.o fpu.o aesni-intel-$(CONFIG_64BIT) += aesni-intel_avx-x86_64.o aes_ctrby8_avx-x86_64.o ghash-clmulni-intel-y := ghash-clmulni-intel_asm.o ghash-clmulni-intel_glue.o sha1-ssse3-y := sha1_ssse3_asm.o sha1_ssse3_glue.o +poly1305-x86_64-y := poly1305-sse2-x86_64.o poly1305_glue.o ifeq ($(avx2_supported),yes) sha1-ssse3-y += sha1_avx2_x86_64_asm.o +poly1305-x86_64-y += poly1305-avx2-x86_64.o endif crc32c-intel-y := crc32c-intel_glue.o crc32c-intel-$(CONFIG_64BIT) += crc32c-pcl-intel-asm_64.o diff --git a/arch/x86/crypto/aesni-intel_glue.c b/arch/x86/crypto/aesni-intel_glue.c index dccad38b59a8..3633ad6145c5 100644 --- a/arch/x86/crypto/aesni-intel_glue.c +++ b/arch/x86/crypto/aesni-intel_glue.c @@ -803,10 +803,7 @@ static int rfc4106_init(struct crypto_aead *aead) return PTR_ERR(cryptd_tfm); *ctx = cryptd_tfm; - crypto_aead_set_reqsize( - aead, - sizeof(struct aead_request) + - crypto_aead_reqsize(&cryptd_tfm->base)); + crypto_aead_set_reqsize(aead, crypto_aead_reqsize(&cryptd_tfm->base)); return 0; } @@ -955,8 +952,8 @@ static int helper_rfc4106_encrypt(struct aead_request *req) /* Assuming we are supporting rfc4106 64-bit extended */ /* sequence numbers We need to have the AAD length equal */ - /* to 8 or 12 bytes */ - if (unlikely(req->assoclen != 8 && req->assoclen != 12)) + /* to 16 or 20 bytes */ + if (unlikely(req->assoclen != 16 && req->assoclen != 20)) return -EINVAL; /* IV below built */ @@ -992,9 +989,9 @@ static int helper_rfc4106_encrypt(struct aead_request *req) } kernel_fpu_begin(); - aesni_gcm_enc_tfm(aes_ctx, dst, src, (unsigned long)req->cryptlen, iv, - ctx->hash_subkey, assoc, (unsigned long)req->assoclen, dst - + ((unsigned long)req->cryptlen), auth_tag_len); + aesni_gcm_enc_tfm(aes_ctx, dst, src, req->cryptlen, iv, + ctx->hash_subkey, assoc, req->assoclen - 8, + dst + req->cryptlen, auth_tag_len); kernel_fpu_end(); /* The authTag (aka the Integrity Check Value) needs to be written @@ -1033,12 +1030,12 @@ static int helper_rfc4106_decrypt(struct aead_request *req) struct scatter_walk dst_sg_walk; unsigned int i; - if (unlikely(req->assoclen != 8 && req->assoclen != 12)) + if (unlikely(req->assoclen != 16 && req->assoclen != 20)) return -EINVAL; /* Assuming we are supporting rfc4106 64-bit extended */ /* sequence numbers We need to have the AAD length */ - /* equal to 8 or 12 bytes */ + /* equal to 16 or 20 bytes */ tempCipherLen = (unsigned long)(req->cryptlen - auth_tag_len); /* IV below built */ @@ -1075,8 +1072,8 @@ static int helper_rfc4106_decrypt(struct aead_request *req) kernel_fpu_begin(); aesni_gcm_dec_tfm(aes_ctx, dst, src, tempCipherLen, iv, - ctx->hash_subkey, assoc, (unsigned long)req->assoclen, - authTag, auth_tag_len); + ctx->hash_subkey, assoc, req->assoclen - 8, + authTag, auth_tag_len); kernel_fpu_end(); /* Compare generated tag with passed in tag. */ @@ -1105,19 +1102,12 @@ static int rfc4106_encrypt(struct aead_request *req) struct crypto_aead *tfm = crypto_aead_reqtfm(req); struct cryptd_aead **ctx = crypto_aead_ctx(tfm); struct cryptd_aead *cryptd_tfm = *ctx; - struct aead_request *subreq = aead_request_ctx(req); - aead_request_set_tfm(subreq, irq_fpu_usable() ? - cryptd_aead_child(cryptd_tfm) : - &cryptd_tfm->base); + aead_request_set_tfm(req, irq_fpu_usable() ? + cryptd_aead_child(cryptd_tfm) : + &cryptd_tfm->base); - aead_request_set_callback(subreq, req->base.flags, - req->base.complete, req->base.data); - aead_request_set_crypt(subreq, req->src, req->dst, - req->cryptlen, req->iv); - aead_request_set_ad(subreq, req->assoclen); - - return crypto_aead_encrypt(subreq); + return crypto_aead_encrypt(req); } static int rfc4106_decrypt(struct aead_request *req) @@ -1125,19 +1115,12 @@ static int rfc4106_decrypt(struct aead_request *req) struct crypto_aead *tfm = crypto_aead_reqtfm(req); struct cryptd_aead **ctx = crypto_aead_ctx(tfm); struct cryptd_aead *cryptd_tfm = *ctx; - struct aead_request *subreq = aead_request_ctx(req); - - aead_request_set_tfm(subreq, irq_fpu_usable() ? - cryptd_aead_child(cryptd_tfm) : - &cryptd_tfm->base); - aead_request_set_callback(subreq, req->base.flags, - req->base.complete, req->base.data); - aead_request_set_crypt(subreq, req->src, req->dst, - req->cryptlen, req->iv); - aead_request_set_ad(subreq, req->assoclen); + aead_request_set_tfm(req, irq_fpu_usable() ? + cryptd_aead_child(cryptd_tfm) : + &cryptd_tfm->base); - return crypto_aead_decrypt(subreq); + return crypto_aead_decrypt(req); } #endif diff --git a/arch/x86/crypto/chacha20-avx2-x86_64.S b/arch/x86/crypto/chacha20-avx2-x86_64.S new file mode 100644 index 000000000000..16694e625f77 --- /dev/null +++ b/arch/x86/crypto/chacha20-avx2-x86_64.S @@ -0,0 +1,443 @@ +/* + * ChaCha20 256-bit cipher algorithm, RFC7539, x64 AVX2 functions + * + * Copyright (C) 2015 Martin Willi + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + */ + +#include <linux/linkage.h> + +.data +.align 32 + +ROT8: .octa 0x0e0d0c0f0a09080b0605040702010003 + .octa 0x0e0d0c0f0a09080b0605040702010003 +ROT16: .octa 0x0d0c0f0e09080b0a0504070601000302 + .octa 0x0d0c0f0e09080b0a0504070601000302 +CTRINC: .octa 0x00000003000000020000000100000000 + .octa 0x00000007000000060000000500000004 + +.text + +ENTRY(chacha20_8block_xor_avx2) + # %rdi: Input state matrix, s + # %rsi: 8 data blocks output, o + # %rdx: 8 data blocks input, i + + # This function encrypts eight consecutive ChaCha20 blocks by loading + # the state matrix in AVX registers eight times. As we need some + # scratch registers, we save the first four registers on the stack. The + # algorithm performs each operation on the corresponding word of each + # state matrix, hence requires no word shuffling. For final XORing step + # we transpose the matrix by interleaving 32-, 64- and then 128-bit + # words, which allows us to do XOR in AVX registers. 8/16-bit word + # rotation is done with the slightly better performing byte shuffling, + # 7/12-bit word rotation uses traditional shift+OR. + + vzeroupper + # 4 * 32 byte stack, 32-byte aligned + mov %rsp, %r8 + and $~31, %rsp + sub $0x80, %rsp + + # x0..15[0-7] = s[0..15] + vpbroadcastd 0x00(%rdi),%ymm0 + vpbroadcastd 0x04(%rdi),%ymm1 + vpbroadcastd 0x08(%rdi),%ymm2 + vpbroadcastd 0x0c(%rdi),%ymm3 + vpbroadcastd 0x10(%rdi),%ymm4 + vpbroadcastd 0x14(%rdi),%ymm5 + vpbroadcastd 0x18(%rdi),%ymm6 + vpbroadcastd 0x1c(%rdi),%ymm7 + vpbroadcastd 0x20(%rdi),%ymm8 + vpbroadcastd 0x24(%rdi),%ymm9 + vpbroadcastd 0x28(%rdi),%ymm10 + vpbroadcastd 0x2c(%rdi),%ymm11 + vpbroadcastd 0x30(%rdi),%ymm12 + vpbroadcastd 0x34(%rdi),%ymm13 + vpbroadcastd 0x38(%rdi),%ymm14 + vpbroadcastd 0x3c(%rdi),%ymm15 + # x0..3 on stack + vmovdqa %ymm0,0x00(%rsp) + vmovdqa %ymm1,0x20(%rsp) + vmovdqa %ymm2,0x40(%rsp) + vmovdqa %ymm3,0x60(%rsp) + + vmovdqa CTRINC(%rip),%ymm1 + vmovdqa ROT8(%rip),%ymm2 + vmovdqa ROT16(%rip),%ymm3 + + # x12 += counter values 0-3 + vpaddd %ymm1,%ymm12,%ymm12 + + mov $10,%ecx + +.Ldoubleround8: + # x0 += x4, x12 = rotl32(x12 ^ x0, 16) + vpaddd 0x00(%rsp),%ymm4,%ymm0 + vmovdqa %ymm0,0x00(%rsp) + vpxor %ymm0,%ymm12,%ymm12 + vpshufb %ymm3,%ymm12,%ymm12 + # x1 += x5, x13 = rotl32(x13 ^ x1, 16) + vpaddd 0x20(%rsp),%ymm5,%ymm0 + vmovdqa %ymm0,0x20(%rsp) + vpxor %ymm0,%ymm13,%ymm13 + vpshufb %ymm3,%ymm13,%ymm13 + # x2 += x6, x14 = rotl32(x14 ^ x2, 16) + vpaddd 0x40(%rsp),%ymm6,%ymm0 + vmovdqa %ymm0,0x40(%rsp) + vpxor %ymm0,%ymm14,%ymm14 + vpshufb %ymm3,%ymm14,%ymm14 + # x3 += x7, x15 = rotl32(x15 ^ x3, 16) + vpaddd 0x60(%rsp),%ymm7,%ymm0 + vmovdqa %ymm0,0x60(%rsp) + vpxor %ymm0,%ymm15,%ymm15 + vpshufb %ymm3,%ymm15,%ymm15 + + # x8 += x12, x4 = rotl32(x4 ^ x8, 12) + vpaddd %ymm12,%ymm8,%ymm8 + vpxor %ymm8,%ymm4,%ymm4 + vpslld $12,%ymm4,%ymm0 + vpsrld $20,%ymm4,%ymm4 + vpor %ymm0,%ymm4,%ymm4 + # x9 += x13, x5 = rotl32(x5 ^ x9, 12) + vpaddd %ymm13,%ymm9,%ymm9 + vpxor %ymm9,%ymm5,%ymm5 + vpslld $12,%ymm5,%ymm0 + vpsrld $20,%ymm5,%ymm5 + vpor %ymm0,%ymm5,%ymm5 + # x10 += x14, x6 = rotl32(x6 ^ x10, 12) + vpaddd %ymm14,%ymm10,%ymm10 + vpxor %ymm10,%ymm6,%ymm6 + vpslld $12,%ymm6,%ymm0 + vpsrld $20,%ymm6,%ymm6 + vpor %ymm0,%ymm6,%ymm6 + # x11 += x15, x7 = rotl32(x7 ^ x11, 12) + vpaddd %ymm15,%ymm11,%ymm11 + vpxor %ymm11,%ymm7,%ymm7 + vpslld $12,%ymm7,%ymm0 + vpsrld $20,%ymm7,%ymm7 + vpor %ymm0,%ymm7,%ymm7 + + # x0 += x4, x12 = rotl32(x12 ^ x0, 8) + vpaddd 0x00(%rsp),%ymm4,%ymm0 + vmovdqa %ymm0,0x00(%rsp) + vpxor %ymm0,%ymm12,%ymm12 + vpshufb %ymm2,%ymm12,%ymm12 + # x1 += x5, x13 = rotl32(x13 ^ x1, 8) + vpaddd 0x20(%rsp),%ymm5,%ymm0 + vmovdqa %ymm0,0x20(%rsp) + vpxor %ymm0,%ymm13,%ymm13 + vpshufb %ymm2,%ymm13,%ymm13 + # x2 += x6, x14 = rotl32(x14 ^ x2, 8) + vpaddd 0x40(%rsp),%ymm6,%ymm0 + vmovdqa %ymm0,0x40(%rsp) + vpxor %ymm0,%ymm14,%ymm14 + vpshufb %ymm2,%ymm14,%ymm14 + # x3 += x7, x15 = rotl32(x15 ^ x3, 8) + vpaddd 0x60(%rsp),%ymm7,%ymm0 + vmovdqa %ymm0,0x60(%rsp) + vpxor %ymm0,%ymm15,%ymm15 + vpshufb %ymm2,%ymm15,%ymm15 + + # x8 += x12, x4 = rotl32(x4 ^ x8, 7) + vpaddd %ymm12,%ymm8,%ymm8 + vpxor %ymm8,%ymm4,%ymm4 + vpslld $7,%ymm4,%ymm0 + vpsrld $25,%ymm4,%ymm4 + vpor %ymm0,%ymm4,%ymm4 + # x9 += x13, x5 = rotl32(x5 ^ x9, 7) + vpaddd %ymm13,%ymm9,%ymm9 + vpxor %ymm9,%ymm5,%ymm5 + vpslld $7,%ymm5,%ymm0 + vpsrld $25,%ymm5,%ymm5 + vpor %ymm0,%ymm5,%ymm5 + # x10 += x14, x6 = rotl32(x6 ^ x10, 7) + vpaddd %ymm14,%ymm10,%ymm10 + vpxor %ymm10,%ymm6,%ymm6 + vpslld $7,%ymm6,%ymm0 + vpsrld $25,%ymm6,%ymm6 + vpor %ymm0,%ymm6,%ymm6 + # x11 += x15, x7 = rotl32(x7 ^ x11, 7) + vpaddd %ymm15,%ymm11,%ymm11 + vpxor %ymm11,%ymm7,%ymm7 + vpslld $7,%ymm7,%ymm0 + vpsrld $25,%ymm7,%ymm7 + vpor %ymm0,%ymm7,%ymm7 + + # x0 += x5, x15 = rotl32(x15 ^ x0, 16) + vpaddd 0x00(%rsp),%ymm5,%ymm0 + vmovdqa %ymm0,0x00(%rsp) + vpxor %ymm0,%ymm15,%ymm15 + vpshufb %ymm3,%ymm15,%ymm15 + # x1 += x6, x12 = rotl32(x12 ^ x1, 16)%ymm0 + vpaddd 0x20(%rsp),%ymm6,%ymm0 + vmovdqa %ymm0,0x20(%rsp) + vpxor %ymm0,%ymm12,%ymm12 + vpshufb %ymm3,%ymm12,%ymm12 + # x2 += x7, x13 = rotl32(x13 ^ x2, 16) + vpaddd 0x40(%rsp),%ymm7,%ymm0 + vmovdqa %ymm0,0x40(%rsp) + vpxor %ymm0,%ymm13,%ymm13 + vpshufb %ymm3,%ymm13,%ymm13 + # x3 += x4, x14 = rotl32(x14 ^ x3, 16) + vpaddd 0x60(%rsp),%ymm4,%ymm0 + vmovdqa %ymm0,0x60(%rsp) + vpxor %ymm0,%ymm14,%ymm14 + vpshufb %ymm3,%ymm14,%ymm14 + + # x10 += x15, x5 = rotl32(x5 ^ x10, 12) + vpaddd %ymm15,%ymm10,%ymm10 + vpxor %ymm10,%ymm5,%ymm5 + vpslld $12,%ymm5,%ymm0 + vpsrld $20,%ymm5,%ymm5 + vpor %ymm0,%ymm5,%ymm5 + # x11 += x12, x6 = rotl32(x6 ^ x11, 12) + vpaddd %ymm12,%ymm11,%ymm11 + vpxor %ymm11,%ymm6,%ymm6 + vpslld $12,%ymm6,%ymm0 + vpsrld $20,%ymm6,%ymm6 + vpor %ymm0,%ymm6,%ymm6 + # x8 += x13, x7 = rotl32(x7 ^ x8, 12) + vpaddd %ymm13,%ymm8,%ymm8 + vpxor %ymm8,%ymm7,%ymm7 + vpslld $12,%ymm7,%ymm0 + vpsrld $20,%ymm7,%ymm7 + vpor %ymm0,%ymm7,%ymm7 + # x9 += x14, x4 = rotl32(x4 ^ x9, 12) + vpaddd %ymm14,%ymm9,%ymm9 + vpxor %ymm9,%ymm4,%ymm4 + vpslld $12,%ymm4,%ymm0 + vpsrld $20,%ymm4,%ymm4 + vpor %ymm0,%ymm4,%ymm4 + + # x0 += x5, x15 = rotl32(x15 ^ x0, 8) + vpaddd 0x00(%rsp),%ymm5,%ymm0 + vmovdqa %ymm0,0x00(%rsp) + vpxor %ymm0,%ymm15,%ymm15 + vpshufb %ymm2,%ymm15,%ymm15 + # x1 += x6, x12 = rotl32(x12 ^ x1, 8) + vpaddd 0x20(%rsp),%ymm6,%ymm0 + vmovdqa %ymm0,0x20(%rsp) + vpxor %ymm0,%ymm12,%ymm12 + vpshufb %ymm2,%ymm12,%ymm12 + # x2 += x7, x13 = rotl32(x13 ^ x2, 8) + vpaddd 0x40(%rsp),%ymm7,%ymm0 + vmovdqa %ymm0,0x40(%rsp) + vpxor %ymm0,%ymm13,%ymm13 + vpshufb %ymm2,%ymm13,%ymm13 + # x3 += x4, x14 = rotl32(x14 ^ x3, 8) + vpaddd 0x60(%rsp),%ymm4,%ymm0 + vmovdqa %ymm0,0x60(%rsp) + vpxor %ymm0,%ymm14,%ymm14 + vpshufb %ymm2,%ymm14,%ymm14 + + # x10 += x15, x5 = rotl32(x5 ^ x10, 7) + vpaddd %ymm15,%ymm10,%ymm10 + vpxor %ymm10,%ymm5,%ymm5 + vpslld $7,%ymm5,%ymm0 + vpsrld $25,%ymm5,%ymm5 + vpor %ymm0,%ymm5,%ymm5 + # x11 += x12, x6 = rotl32(x6 ^ x11, 7) + vpaddd %ymm12,%ymm11,%ymm11 + vpxor %ymm11,%ymm6,%ymm6 + vpslld $7,%ymm6,%ymm0 + vpsrld $25,%ymm6,%ymm6 + vpor %ymm0,%ymm6,%ymm6 + # x8 += x13, x7 = rotl32(x7 ^ x8, 7) + vpaddd %ymm13,%ymm8,%ymm8 + vpxor %ymm8,%ymm7,%ymm7 + vpslld $7,%ymm7,%ymm0 + vpsrld $25,%ymm7,%ymm7 + vpor %ymm0,%ymm7,%ymm7 + # x9 += x14, x4 = rotl32(x4 ^ x9, 7) + vpaddd %ymm14,%ymm9,%ymm9 + vpxor %ymm9,%ymm4,%ymm4 + vpslld $7,%ymm4,%ymm0 + vpsrld $25,%ymm4,%ymm4 + vpor %ymm0,%ymm4,%ymm4 + + dec %ecx + jnz .Ldoubleround8 + + # x0..15[0-3] += s[0..15] + vpbroadcastd 0x00(%rdi),%ymm0 + vpaddd 0x00(%rsp),%ymm0,%ymm0 + vmovdqa %ymm0,0x00(%rsp) + vpbroadcastd 0x04(%rdi),%ymm0 + vpaddd 0x20(%rsp),%ymm0,%ymm0 + vmovdqa %ymm0,0x20(%rsp) + vpbroadcastd 0x08(%rdi),%ymm0 + vpaddd 0x40(%rsp),%ymm0,%ymm0 + vmovdqa %ymm0,0x40(%rsp) + vpbroadcastd 0x0c(%rdi),%ymm0 + vpaddd 0x60(%rsp),%ymm0,%ymm0 + vmovdqa %ymm0,0x60(%rsp) + vpbroadcastd 0x10(%rdi),%ymm0 + vpaddd %ymm0,%ymm4,%ymm4 + vpbroadcastd 0x14(%rdi),%ymm0 + vpaddd %ymm0,%ymm5,%ymm5 + vpbroadcastd 0x18(%rdi),%ymm0 + vpaddd %ymm0,%ymm6,%ymm6 + vpbroadcastd 0x1c(%rdi),%ymm0 + vpaddd %ymm0,%ymm7,%ymm7 + vpbroadcastd 0x20(%rdi),%ymm0 + vpaddd %ymm0,%ymm8,%ymm8 + vpbroadcastd 0x24(%rdi),%ymm0 + vpaddd %ymm0,%ymm9,%ymm9 + vpbroadcastd 0x28(%rdi),%ymm0 + vpaddd %ymm0,%ymm10,%ymm10 + vpbroadcastd 0x2c(%rdi),%ymm0 + vpaddd %ymm0,%ymm11,%ymm11 + vpbroadcastd 0x30(%rdi),%ymm0 + vpaddd %ymm0,%ymm12,%ymm12 + vpbroadcastd 0x34(%rdi),%ymm0 + vpaddd %ymm0,%ymm13,%ymm13 + vpbroadcastd 0x38(%rdi),%ymm0 + vpaddd %ymm0,%ymm14,%ymm14 + vpbroadcastd 0x3c(%rdi),%ymm0 + vpaddd %ymm0,%ymm15,%ymm15 + + # x12 += counter values 0-3 + vpaddd %ymm1,%ymm12,%ymm12 + + # interleave 32-bit words in state n, n+1 + vmovdqa 0x00(%rsp),%ymm0 + vmovdqa 0x20(%rsp),%ymm1 + vpunpckldq %ymm1,%ymm0,%ymm2 + vpunpckhdq %ymm1,%ymm0,%ymm1 + vmovdqa %ymm2,0x00(%rsp) + vmovdqa %ymm1,0x20(%rsp) + vmovdqa 0x40(%rsp),%ymm0 + vmovdqa 0x60(%rsp),%ymm1 + vpunpckldq %ymm1,%ymm0,%ymm2 + vpunpckhdq %ymm1,%ymm0,%ymm1 + vmovdqa %ymm2,0x40(%rsp) + vmovdqa %ymm1,0x60(%rsp) + vmovdqa %ymm4,%ymm0 + vpunpckldq %ymm5,%ymm0,%ymm4 + vpunpckhdq %ymm5,%ymm0,%ymm5 + vmovdqa %ymm6,%ymm0 + vpunpckldq %ymm7,%ymm0,%ymm6 + vpunpckhdq %ymm7,%ymm0,%ymm7 + vmovdqa %ymm8,%ymm0 + vpunpckldq %ymm9,%ymm0,%ymm8 + vpunpckhdq %ymm9,%ymm0,%ymm9 + vmovdqa %ymm10,%ymm0 + vpunpckldq %ymm11,%ymm0,%ymm10 + vpunpckhdq %ymm11,%ymm0,%ymm11 + vmovdqa %ymm12,%ymm0 + vpunpckldq %ymm13,%ymm0,%ymm12 + vpunpckhdq %ymm13,%ymm0,%ymm13 + vmovdqa %ymm14,%ymm0 + vpunpckldq %ymm15,%ymm0,%ymm14 + vpunpckhdq %ymm15,%ymm0,%ymm15 + + # interleave 64-bit words in state n, n+2 + vmovdqa 0x00(%rsp),%ymm0 + vmovdqa 0x40(%rsp),%ymm2 + vpunpcklqdq %ymm2,%ymm0,%ymm1 + vpunpckhqdq %ymm2,%ymm0,%ymm2 + vmovdqa %ymm1,0x00(%rsp) + vmovdqa %ymm2,0x40(%rsp) + vmovdqa 0x20(%rsp),%ymm0 + vmovdqa 0x60(%rsp),%ymm2 + vpunpcklqdq %ymm2,%ymm0,%ymm1 + vpunpckhqdq %ymm2,%ymm0,%ymm2 + vmovdqa %ymm1,0x20(%rsp) + vmovdqa %ymm2,0x60(%rsp) + vmovdqa %ymm4,%ymm0 + vpunpcklqdq %ymm6,%ymm0,%ymm4 + vpunpckhqdq %ymm6,%ymm0,%ymm6 + vmovdqa %ymm5,%ymm0 + vpunpcklqdq %ymm7,%ymm0,%ymm5 + vpunpckhqdq %ymm7,%ymm0,%ymm7 + vmovdqa %ymm8,%ymm0 + vpunpcklqdq %ymm10,%ymm0,%ymm8 + vpunpckhqdq %ymm10,%ymm0,%ymm10 + vmovdqa %ymm9,%ymm0 + vpunpcklqdq %ymm11,%ymm0,%ymm9 + vpunpckhqdq %ymm11,%ymm0,%ymm11 + vmovdqa %ymm12,%ymm0 + vpunpcklqdq %ymm14,%ymm0,%ymm12 + vpunpckhqdq %ymm14,%ymm0,%ymm14 + vmovdqa %ymm13,%ymm0 + vpunpcklqdq %ymm15,%ymm0,%ymm13 + vpunpckhqdq %ymm15,%ymm0,%ymm15 + + # interleave 128-bit words in state n, n+4 + vmovdqa 0x00(%rsp),%ymm0 + vperm2i128 $0x20,%ymm4,%ymm0,%ymm1 + vperm2i128 $0x31,%ymm4,%ymm0,%ymm4 + vmovdqa %ymm1,0x00(%rsp) + vmovdqa 0x20(%rsp),%ymm0 + vperm2i128 $0x20,%ymm5,%ymm0,%ymm1 + vperm2i128 $0x31,%ymm5,%ymm0,%ymm5 + vmovdqa %ymm1,0x20(%rsp) + vmovdqa 0x40(%rsp),%ymm0 + vperm2i128 $0x20,%ymm6,%ymm0,%ymm1 + vperm2i128 $0x31,%ymm6,%ymm0,%ymm6 + vmovdqa %ymm1,0x40(%rsp) + vmovdqa 0x60(%rsp),%ymm0 + vperm2i128 $0x20,%ymm7,%ymm0,%ymm1 + vperm2i128 $0x31,%ymm7,%ymm0,%ymm7 + vmovdqa %ymm1,0x60(%rsp) + vperm2i128 $0x20,%ymm12,%ymm8,%ymm0 + vperm2i128 $0x31,%ymm12,%ymm8,%ymm12 + vmovdqa %ymm0,%ymm8 + vperm2i128 $0x20,%ymm13,%ymm9,%ymm0 + vperm2i128 $0x31,%ymm13,%ymm9,%ymm13 + vmovdqa %ymm0,%ymm9 + vperm2i128 $0x20,%ymm14,%ymm10,%ymm0 + vperm2i128 $0x31,%ymm14,%ymm10,%ymm14 + vmovdqa %ymm0,%ymm10 + vperm2i128 $0x20,%ymm15,%ymm11,%ymm0 + vperm2i128 $0x31,%ymm15,%ymm11,%ymm15 + vmovdqa %ymm0,%ymm11 + + # xor with corresponding input, write to output + vmovdqa 0x00(%rsp),%ymm0 + vpxor 0x0000(%rdx),%ymm0,%ymm0 + vmovdqu %ymm0,0x0000(%rsi) + vmovdqa 0x20(%rsp),%ymm0 + vpxor 0x0080(%rdx),%ymm0,%ymm0 + vmovdqu %ymm0,0x0080(%rsi) + vmovdqa 0x40(%rsp),%ymm0 + vpxor 0x0040(%rdx),%ymm0,%ymm0 + vmovdqu %ymm0,0x0040(%rsi) + vmovdqa 0x60(%rsp),%ymm0 + vpxor 0x00c0(%rdx),%ymm0,%ymm0 + vmovdqu %ymm0,0x00c0(%rsi) + vpxor 0x0100(%rdx),%ymm4,%ymm4 + vmovdqu %ymm4,0x0100(%rsi) + vpxor 0x0180(%rdx),%ymm5,%ymm5 + vmovdqu %ymm5,0x00180(%rsi) + vpxor 0x0140(%rdx),%ymm6,%ymm6 + vmovdqu %ymm6,0x0140(%rsi) + vpxor 0x01c0(%rdx),%ymm7,%ymm7 + vmovdqu %ymm7,0x01c0(%rsi) + vpxor 0x0020(%rdx),%ymm8,%ymm8 + vmovdqu %ymm8,0x0020(%rsi) + vpxor 0x00a0(%rdx),%ymm9,%ymm9 + vmovdqu %ymm9,0x00a0(%rsi) + vpxor 0x0060(%rdx),%ymm10,%ymm10 + vmovdqu %ymm10,0x0060(%rsi) + vpxor 0x00e0(%rdx),%ymm11,%ymm11 + vmovdqu %ymm11,0x00e0(%rsi) + vpxor 0x0120(%rdx),%ymm12,%ymm12 + vmovdqu %ymm12,0x0120(%rsi) + vpxor 0x01a0(%rdx),%ymm13,%ymm13 + vmovdqu %ymm13,0x01a0(%rsi) + vpxor 0x0160(%rdx),%ymm14,%ymm14 + vmovdqu %ymm14,0x0160(%rsi) + vpxor 0x01e0(%rdx),%ymm15,%ymm15 + vmovdqu %ymm15,0x01e0(%rsi) + + vzeroupper + mov %r8,%rsp + ret +ENDPROC(chacha20_8block_xor_avx2) diff --git a/arch/x86/crypto/chacha20-ssse3-x86_64.S b/arch/x86/crypto/chacha20-ssse3-x86_64.S new file mode 100644 index 000000000000..712b13047b41 --- /dev/null +++ b/arch/x86/crypto/chacha20-ssse3-x86_64.S @@ -0,0 +1,625 @@ +/* + * ChaCha20 256-bit cipher algorithm, RFC7539, x64 SSSE3 functions + * + * Copyright (C) 2015 Martin Willi + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + */ + +#include <linux/linkage.h> + +.data +.align 16 + +ROT8: .octa 0x0e0d0c0f0a09080b0605040702010003 +ROT16: .octa 0x0d0c0f0e09080b0a0504070601000302 +CTRINC: .octa 0x00000003000000020000000100000000 + +.text + +ENTRY(chacha20_block_xor_ssse3) + # %rdi: Input state matrix, s + # %rsi: 1 data block output, o + # %rdx: 1 data block input, i + + # This function encrypts one ChaCha20 block by loading the state matrix + # in four SSE registers. It performs matrix operation on four words in + # parallel, but requireds shuffling to rearrange the words after each + # round. 8/16-bit word rotation is done with the slightly better + # performing SSSE3 byte shuffling, 7/12-bit word rotation uses + # traditional shift+OR. + + # x0..3 = s0..3 + movdqa 0x00(%rdi),%xmm0 + movdqa 0x10(%rdi),%xmm1 + movdqa 0x20(%rdi),%xmm2 + movdqa 0x30(%rdi),%xmm3 + movdqa %xmm0,%xmm8 + movdqa %xmm1,%xmm9 + movdqa %xmm2,%xmm10 + movdqa %xmm3,%xmm11 + + movdqa ROT8(%rip),%xmm4 + movdqa ROT16(%rip),%xmm5 + + mov $10,%ecx + +.Ldoubleround: + + # x0 += x1, x3 = rotl32(x3 ^ x0, 16) + paddd %xmm1,%xmm0 + pxor %xmm0,%xmm3 + pshufb %xmm5,%xmm3 + + # x2 += x3, x1 = rotl32(x1 ^ x2, 12) + paddd %xmm3,%xmm2 + pxor %xmm2,%xmm1 + movdqa %xmm1,%xmm6 + pslld $12,%xmm6 + psrld $20,%xmm1 + por %xmm6,%xmm1 + + # x0 += x1, x3 = rotl32(x3 ^ x0, 8) + paddd %xmm1,%xmm0 + pxor %xmm0,%xmm3 + pshufb %xmm4,%xmm3 + + # x2 += x3, x1 = rotl32(x1 ^ x2, 7) + paddd %xmm3,%xmm2 + pxor %xmm2,%xmm1 + movdqa %xmm1,%xmm7 + pslld $7,%xmm7 + psrld $25,%xmm1 + por %xmm7,%xmm1 + + # x1 = shuffle32(x1, MASK(0, 3, 2, 1)) + pshufd $0x39,%xmm1,%xmm1 + # x2 = shuffle32(x2, MASK(1, 0, 3, 2)) + pshufd $0x4e,%xmm2,%xmm2 + # x3 = shuffle32(x3, MASK(2, 1, 0, 3)) + pshufd $0x93,%xmm3,%xmm3 + + # x0 += x1, x3 = rotl32(x3 ^ x0, 16) + paddd %xmm1,%xmm0 + pxor %xmm0,%xmm3 + pshufb %xmm5,%xmm3 + + # x2 += x3, x1 = rotl32(x1 ^ x2, 12) + paddd %xmm3,%xmm2 + pxor %xmm2,%xmm1 + movdqa %xmm1,%xmm6 + pslld $12,%xmm6 + psrld $20,%xmm1 + por %xmm6,%xmm1 + + # x0 += x1, x3 = rotl32(x3 ^ x0, 8) + paddd %xmm1,%xmm0 + pxor %xmm0,%xmm3 + pshufb %xmm4,%xmm3 + + # x2 += x3, x1 = rotl32(x1 ^ x2, 7) + paddd %xmm3,%xmm2 + pxor %xmm2,%xmm1 + movdqa %xmm1,%xmm7 + pslld $7,%xmm7 + psrld $25,%xmm1 + por %xmm7,%xmm1 + + # x1 = shuffle32(x1, MASK(2, 1, 0, 3)) + pshufd $0x93,%xmm1,%xmm1 + # x2 = shuffle32(x2, MASK(1, 0, 3, 2)) + pshufd $0x4e,%xmm2,%xmm2 + # x3 = shuffle32(x3, MASK(0, 3, 2, 1)) + pshufd $0x39,%xmm3,%xmm3 + + dec %ecx + jnz .Ldoubleround + + # o0 = i0 ^ (x0 + s0) + movdqu 0x00(%rdx),%xmm4 + paddd %xmm8,%xmm0 + pxor %xmm4,%xmm0 + movdqu %xmm0,0x00(%rsi) + # o1 = i1 ^ (x1 + s1) + movdqu 0x10(%rdx),%xmm5 + paddd %xmm9,%xmm1 + pxor %xmm5,%xmm1 + movdqu %xmm1,0x10(%rsi) + # o2 = i2 ^ (x2 + s2) + movdqu 0x20(%rdx),%xmm6 + paddd %xmm10,%xmm2 + pxor %xmm6,%xmm2 + movdqu %xmm2,0x20(%rsi) + # o3 = i3 ^ (x3 + s3) + movdqu 0x30(%rdx),%xmm7 + paddd %xmm11,%xmm3 + pxor %xmm7,%xmm3 + movdqu %xmm3,0x30(%rsi) + + ret +ENDPROC(chacha20_block_xor_ssse3) + +ENTRY(chacha20_4block_xor_ssse3) + # %rdi: Input state matrix, s + # %rsi: 4 data blocks output, o + # %rdx: 4 data blocks input, i + + # This function encrypts four consecutive ChaCha20 blocks by loading the + # the state matrix in SSE registers four times. As we need some scratch + # registers, we save the first four registers on the stack. The + # algorithm performs each operation on the corresponding word of each + # state matrix, hence requires no word shuffling. For final XORing step + # we transpose the matrix by interleaving 32- and then 64-bit words, + # which allows us to do XOR in SSE registers. 8/16-bit word rotation is + # done with the slightly better performing SSSE3 byte shuffling, + # 7/12-bit word rotation uses traditional shift+OR. + + sub $0x40,%rsp + + # x0..15[0-3] = s0..3[0..3] + movq 0x00(%rdi),%xmm1 + pshufd $0x00,%xmm1,%xmm0 + pshufd $0x55,%xmm1,%xmm1 + movq 0x08(%rdi),%xmm3 + pshufd $0x00,%xmm3,%xmm2 + pshufd $0x55,%xmm3,%xmm3 + movq 0x10(%rdi),%xmm5 + pshufd $0x00,%xmm5,%xmm4 + pshufd $0x55,%xmm5,%xmm5 + movq 0x18(%rdi),%xmm7 + pshufd $0x00,%xmm7,%xmm6 + pshufd $0x55,%xmm7,%xmm7 + movq 0x20(%rdi),%xmm9 + pshufd $0x00,%xmm9,%xmm8 + pshufd $0x55,%xmm9,%xmm9 + movq 0x28(%rdi),%xmm11 + pshufd $0x00,%xmm11,%xmm10 + pshufd $0x55,%xmm11,%xmm11 + movq 0x30(%rdi),%xmm13 + pshufd $0x00,%xmm13,%xmm12 + pshufd $0x55,%xmm13,%xmm13 + movq 0x38(%rdi),%xmm15 + pshufd $0x00,%xmm15,%xmm14 + pshufd $0x55,%xmm15,%xmm15 + # x0..3 on stack + movdqa %xmm0,0x00(%rsp) + movdqa %xmm1,0x10(%rsp) + movdqa %xmm2,0x20(%rsp) + movdqa %xmm3,0x30(%rsp) + + movdqa CTRINC(%rip),%xmm1 + movdqa ROT8(%rip),%xmm2 + movdqa ROT16(%rip),%xmm3 + + # x12 += counter values 0-3 + paddd %xmm1,%xmm12 + + mov $10,%ecx + +.Ldoubleround4: + # x0 += x4, x12 = rotl32(x12 ^ x0, 16) + movdqa 0x00(%rsp),%xmm0 + paddd %xmm4,%xmm0 + movdqa %xmm0,0x00(%rsp) + pxor %xmm0,%xmm12 + pshufb %xmm3,%xmm12 + # x1 += x5, x13 = rotl32(x13 ^ x1, 16) + movdqa 0x10(%rsp),%xmm0 + paddd %xmm5,%xmm0 + movdqa %xmm0,0x10(%rsp) + pxor %xmm0,%xmm13 + pshufb %xmm3,%xmm13 + # x2 += x6, x14 = rotl32(x14 ^ x2, 16) + movdqa 0x20(%rsp),%xmm0 + paddd %xmm6,%xmm0 + movdqa %xmm0,0x20(%rsp) + pxor %xmm0,%xmm14 + pshufb %xmm3,%xmm14 + # x3 += x7, x15 = rotl32(x15 ^ x3, 16) + movdqa 0x30(%rsp),%xmm0 + paddd %xmm7,%xmm0 + movdqa %xmm0,0x30(%rsp) + pxor %xmm0,%xmm15 + pshufb %xmm3,%xmm15 + + # x8 += x12, x4 = rotl32(x4 ^ x8, 12) + paddd %xmm12,%xmm8 + pxor %xmm8,%xmm4 + movdqa %xmm4,%xmm0 + pslld $12,%xmm0 + psrld $20,%xmm4 + por %xmm0,%xmm4 + # x9 += x13, x5 = rotl32(x5 ^ x9, 12) + paddd %xmm13,%xmm9 + pxor %xmm9,%xmm5 + movdqa %xmm5,%xmm0 + pslld $12,%xmm0 + psrld $20,%xmm5 + por %xmm0,%xmm5 + # x10 += x14, x6 = rotl32(x6 ^ x10, 12) + paddd %xmm14,%xmm10 + pxor %xmm10,%xmm6 + movdqa %xmm6,%xmm0 + pslld $12,%xmm0 + psrld $20,%xmm6 + por %xmm0,%xmm6 + # x11 += x15, x7 = rotl32(x7 ^ x11, 12) + paddd %xmm15,%xmm11 + pxor %xmm11,%xmm7 + movdqa %xmm7,%xmm0 + pslld $12,%xmm0 + psrld $20,%xmm7 + por %xmm0,%xmm7 + + # x0 += x4, x12 = rotl32(x12 ^ x0, 8) + movdqa 0x00(%rsp),%xmm0 + paddd %xmm4,%xmm0 + movdqa %xmm0,0x00(%rsp) + pxor %xmm0,%xmm12 + pshufb %xmm2,%xmm12 + # x1 += x5, x13 = rotl32(x13 ^ x1, 8) + movdqa 0x10(%rsp),%xmm0 + paddd %xmm5,%xmm0 + movdqa %xmm0,0x10(%rsp) + pxor %xmm0,%xmm13 + pshufb %xmm2,%xmm13 + # x2 += x6, x14 = rotl32(x14 ^ x2, 8) + movdqa 0x20(%rsp),%xmm0 + paddd %xmm6,%xmm0 + movdqa %xmm0,0x20(%rsp) + pxor %xmm0,%xmm14 + pshufb %xmm2,%xmm14 + # x3 += x7, x15 = rotl32(x15 ^ x3, 8) + movdqa 0x30(%rsp),%xmm0 + paddd %xmm7,%xmm0 + movdqa %xmm0,0x30(%rsp) + pxor %xmm0,%xmm15 + pshufb %xmm2,%xmm15 + + # x8 += x12, x4 = rotl32(x4 ^ x8, 7) + paddd %xmm12,%xmm8 + pxor %xmm8,%xmm4 + movdqa %xmm4,%xmm0 + pslld $7,%xmm0 + psrld $25,%xmm4 + por %xmm0,%xmm4 + # x9 += x13, x5 = rotl32(x5 ^ x9, 7) + paddd %xmm13,%xmm9 + pxor %xmm9,%xmm5 + movdqa %xmm5,%xmm0 + pslld $7,%xmm0 + psrld $25,%xmm5 + por %xmm0,%xmm5 + # x10 += x14, x6 = rotl32(x6 ^ x10, 7) + paddd %xmm14,%xmm10 + pxor %xmm10,%xmm6 + movdqa %xmm6,%xmm0 + pslld $7,%xmm0 + psrld $25,%xmm6 + por %xmm0,%xmm6 + # x11 += x15, x7 = rotl32(x7 ^ x11, 7) + paddd %xmm15,%xmm11 + pxor %xmm11,%xmm7 + movdqa %xmm7,%xmm0 + pslld $7,%xmm0 + psrld $25,%xmm7 + por %xmm0,%xmm7 + + # x0 += x5, x15 = rotl32(x15 ^ x0, 16) + movdqa 0x00(%rsp),%xmm0 + paddd %xmm5,%xmm0 + movdqa %xmm0,0x00(%rsp) + pxor %xmm0,%xmm15 + pshufb %xmm3,%xmm15 + # x1 += x6, x12 = rotl32(x12 ^ x1, 16) + movdqa 0x10(%rsp),%xmm0 + paddd %xmm6,%xmm0 + movdqa %xmm0,0x10(%rsp) + pxor %xmm0,%xmm12 + pshufb %xmm3,%xmm12 + # x2 += x7, x13 = rotl32(x13 ^ x2, 16) + movdqa 0x20(%rsp),%xmm0 + paddd %xmm7,%xmm0 + movdqa %xmm0,0x20(%rsp) + pxor %xmm0,%xmm13 + pshufb %xmm3,%xmm13 + # x3 += x4, x14 = rotl32(x14 ^ x3, 16) + movdqa 0x30(%rsp),%xmm0 + paddd %xmm4,%xmm0 + movdqa %xmm0,0x30(%rsp) + pxor %xmm0,%xmm14 + pshufb %xmm3,%xmm14 + + # x10 += x15, x5 = rotl32(x5 ^ x10, 12) + paddd %xmm15,%xmm10 + pxor %xmm10,%xmm5 + movdqa %xmm5,%xmm0 + pslld $12,%xmm0 + psrld $20,%xmm5 + por %xmm0,%xmm5 + # x11 += x12, x6 = rotl32(x6 ^ x11, 12) + paddd %xmm12,%xmm11 + pxor %xmm11,%xmm6 + movdqa %xmm6,%xmm0 + pslld $12,%xmm0 + psrld $20,%xmm6 + por %xmm0,%xmm6 + # x8 += x13, x7 = rotl32(x7 ^ x8, 12) + paddd %xmm13,%xmm8 + pxor %xmm8,%xmm7 + movdqa %xmm7,%xmm0 + pslld $12,%xmm0 + psrld $20,%xmm7 + por %xmm0,%xmm7 + # x9 += x14, x4 = rotl32(x4 ^ x9, 12) + paddd %xmm14,%xmm9 + pxor %xmm9,%xmm4 + movdqa %xmm4,%xmm0 + pslld $12,%xmm0 + psrld $20,%xmm4 + por %xmm0,%xmm4 + + # x0 += x5, x15 = rotl32(x15 ^ x0, 8) + movdqa 0x00(%rsp),%xmm0 + paddd %xmm5,%xmm0 + movdqa %xmm0,0x00(%rsp) + pxor %xmm0,%xmm15 + pshufb %xmm2,%xmm15 + # x1 += x6, x12 = rotl32(x12 ^ x1, 8) + movdqa 0x10(%rsp),%xmm0 + paddd %xmm6,%xmm0 + movdqa %xmm0,0x10(%rsp) + pxor %xmm0,%xmm12 + pshufb %xmm2,%xmm12 + # x2 += x7, x13 = rotl32(x13 ^ x2, 8) + movdqa 0x20(%rsp),%xmm0 + paddd %xmm7,%xmm0 + movdqa %xmm0,0x20(%rsp) + pxor %xmm0,%xmm13 + pshufb %xmm2,%xmm13 + # x3 += x4, x14 = rotl32(x14 ^ x3, 8) + movdqa 0x30(%rsp),%xmm0 + paddd %xmm4,%xmm0 + movdqa %xmm0,0x30(%rsp) + pxor %xmm0,%xmm14 + pshufb %xmm2,%xmm14 + + # x10 += x15, x5 = rotl32(x5 ^ x10, 7) + paddd %xmm15,%xmm10 + pxor %xmm10,%xmm5 + movdqa %xmm5,%xmm0 + pslld $7,%xmm0 + psrld $25,%xmm5 + por %xmm0,%xmm5 + # x11 += x12, x6 = rotl32(x6 ^ x11, 7) + paddd %xmm12,%xmm11 + pxor %xmm11,%xmm6 + movdqa %xmm6,%xmm0 + pslld $7,%xmm0 + psrld $25,%xmm6 + por %xmm0,%xmm6 + # x8 += x13, x7 = rotl32(x7 ^ x8, 7) + paddd %xmm13,%xmm8 + pxor %xmm8,%xmm7 + movdqa %xmm7,%xmm0 + pslld $7,%xmm0 + psrld $25,%xmm7 + por %xmm0,%xmm7 + # x9 += x14, x4 = rotl32(x4 ^ x9, 7) + paddd %xmm14,%xmm9 + pxor %xmm9,%xmm4 + movdqa %xmm4,%xmm0 + pslld $7,%xmm0 + psrld $25,%xmm4 + por %xmm0,%xmm4 + + dec %ecx + jnz .Ldoubleround4 + + # x0[0-3] += s0[0] + # x1[0-3] += s0[1] + movq 0x00(%rdi),%xmm3 + pshufd $0x00,%xmm3,%xmm2 + pshufd $0x55,%xmm3,%xmm3 + paddd 0x00(%rsp),%xmm2 + movdqa %xmm2,0x00(%rsp) + paddd 0x10(%rsp),%xmm3 + movdqa %xmm3,0x10(%rsp) + # x2[0-3] += s0[2] + # x3[0-3] += s0[3] + movq 0x08(%rdi),%xmm3 + pshufd $0x00,%xmm3,%xmm2 + pshufd $0x55,%xmm3,%xmm3 + paddd 0x20(%rsp),%xmm2 + movdqa %xmm2,0x20(%rsp) + paddd 0x30(%rsp),%xmm3 + movdqa %xmm3,0x30(%rsp) + + # x4[0-3] += s1[0] + # x5[0-3] += s1[1] + movq 0x10(%rdi),%xmm3 + pshufd $0x00,%xmm3,%xmm2 + pshufd $0x55,%xmm3,%xmm3 + paddd %xmm2,%xmm4 + paddd %xmm3,%xmm5 + # x6[0-3] += s1[2] + # x7[0-3] += s1[3] + movq 0x18(%rdi),%xmm3 + pshufd $0x00,%xmm3,%xmm2 + pshufd $0x55,%xmm3,%xmm3 + paddd %xmm2,%xmm6 + paddd %xmm3,%xmm7 + + # x8[0-3] += s2[0] + # x9[0-3] += s2[1] + movq 0x20(%rdi),%xmm3 + pshufd $0x00,%xmm3,%xmm2 + pshufd $0x55,%xmm3,%xmm3 + paddd %xmm2,%xmm8 + paddd %xmm3,%xmm9 + # x10[0-3] += s2[2] + # x11[0-3] += s2[3] + movq 0x28(%rdi),%xmm3 + pshufd $0x00,%xmm3,%xmm2 + pshufd $0x55,%xmm3,%xmm3 + paddd %xmm2,%xmm10 + paddd %xmm3,%xmm11 + + # x12[0-3] += s3[0] + # x13[0-3] += s3[1] + movq 0x30(%rdi),%xmm3 + pshufd $0x00,%xmm3,%xmm2 + pshufd $0x55,%xmm3,%xmm3 + paddd %xmm2,%xmm12 + paddd %xmm3,%xmm13 + # x14[0-3] += s3[2] + # x15[0-3] += s3[3] + movq 0x38(%rdi),%xmm3 + pshufd $0x00,%xmm3,%xmm2 + pshufd $0x55,%xmm3,%xmm3 + paddd %xmm2,%xmm14 + paddd %xmm3,%xmm15 + + # x12 += counter values 0-3 + paddd %xmm1,%xmm12 + + # interleave 32-bit words in state n, n+1 + movdqa 0x00(%rsp),%xmm0 + movdqa 0x10(%rsp),%xmm1 + movdqa %xmm0,%xmm2 + punpckldq %xmm1,%xmm2 + punpckhdq %xmm1,%xmm0 + movdqa %xmm2,0x00(%rsp) + movdqa %xmm0,0x10(%rsp) + movdqa 0x20(%rsp),%xmm0 + movdqa 0x30(%rsp),%xmm1 + movdqa %xmm0,%xmm2 + punpckldq %xmm1,%xmm2 + punpckhdq %xmm1,%xmm0 + movdqa %xmm2,0x20(%rsp) + movdqa %xmm0,0x30(%rsp) + movdqa %xmm4,%xmm0 + punpckldq %xmm5,%xmm4 + punpckhdq %xmm5,%xmm0 + movdqa %xmm0,%xmm5 + movdqa %xmm6,%xmm0 + punpckldq %xmm7,%xmm6 + punpckhdq %xmm7,%xmm0 + movdqa %xmm0,%xmm7 + movdqa %xmm8,%xmm0 + punpckldq %xmm9,%xmm8 + punpckhdq %xmm9,%xmm0 + movdqa %xmm0,%xmm9 + movdqa %xmm10,%xmm0 + punpckldq %xmm11,%xmm10 + punpckhdq %xmm11,%xmm0 + movdqa %xmm0,%xmm11 + movdqa %xmm12,%xmm0 + punpckldq %xmm13,%xmm12 + punpckhdq %xmm13,%xmm0 + movdqa %xmm0,%xmm13 + movdqa %xmm14,%xmm0 + punpckldq %xmm15,%xmm14 + punpckhdq %xmm15,%xmm0 + movdqa %xmm0,%xmm15 + + # interleave 64-bit words in state n, n+2 + movdqa 0x00(%rsp),%xmm0 + movdqa 0x20(%rsp),%xmm1 + movdqa %xmm0,%xmm2 + punpcklqdq %xmm1,%xmm2 + punpckhqdq %xmm1,%xmm0 + movdqa %xmm2,0x00(%rsp) + movdqa %xmm0,0x20(%rsp) + movdqa 0x10(%rsp),%xmm0 + movdqa 0x30(%rsp),%xmm1 + movdqa %xmm0,%xmm2 + punpcklqdq %xmm1,%xmm2 + punpckhqdq %xmm1,%xmm0 + movdqa %xmm2,0x10(%rsp) + movdqa %xmm0,0x30(%rsp) + movdqa %xmm4,%xmm0 + punpcklqdq %xmm6,%xmm4 + punpckhqdq %xmm6,%xmm0 + movdqa %xmm0,%xmm6 + movdqa %xmm5,%xmm0 + punpcklqdq %xmm7,%xmm5 + punpckhqdq %xmm7,%xmm0 + movdqa %xmm0,%xmm7 + movdqa %xmm8,%xmm0 + punpcklqdq %xmm10,%xmm8 + punpckhqdq %xmm10,%xmm0 + movdqa %xmm0,%xmm10 + movdqa %xmm9,%xmm0 + punpcklqdq %xmm11,%xmm9 + punpckhqdq %xmm11,%xmm0 + movdqa %xmm0,%xmm11 + movdqa %xmm12,%xmm0 + punpcklqdq %xmm14,%xmm12 + punpckhqdq %xmm14,%xmm0 + movdqa %xmm0,%xmm14 + movdqa %xmm13,%xmm0 + punpcklqdq %xmm15,%xmm13 + punpckhqdq %xmm15,%xmm0 + movdqa %xmm0,%xmm15 + + # xor with corresponding input, write to output + movdqa 0x00(%rsp),%xmm0 + movdqu 0x00(%rdx),%xmm1 + pxor %xmm1,%xmm0 + movdqu %xmm0,0x00(%rsi) + movdqa 0x10(%rsp),%xmm0 + movdqu 0x80(%rdx),%xmm1 + pxor %xmm1,%xmm0 + movdqu %xmm0,0x80(%rsi) + movdqa 0x20(%rsp),%xmm0 + movdqu 0x40(%rdx),%xmm1 + pxor %xmm1,%xmm0 + movdqu %xmm0,0x40(%rsi) + movdqa 0x30(%rsp),%xmm0 + movdqu 0xc0(%rdx),%xmm1 + pxor %xmm1,%xmm0 + movdqu %xmm0,0xc0(%rsi) + movdqu 0x10(%rdx),%xmm1 + pxor %xmm1,%xmm4 + movdqu %xmm4,0x10(%rsi) + movdqu 0x90(%rdx),%xmm1 + pxor %xmm1,%xmm5 + movdqu %xmm5,0x90(%rsi) + movdqu 0x50(%rdx),%xmm1 + pxor %xmm1,%xmm6 + movdqu %xmm6,0x50(%rsi) + movdqu 0xd0(%rdx),%xmm1 + pxor %xmm1,%xmm7 + movdqu %xmm7,0xd0(%rsi) + movdqu 0x20(%rdx),%xmm1 + pxor %xmm1,%xmm8 + movdqu %xmm8,0x20(%rsi) + movdqu 0xa0(%rdx),%xmm1 + pxor %xmm1,%xmm9 + movdqu %xmm9,0xa0(%rsi) + movdqu 0x60(%rdx),%xmm1 + pxor %xmm1,%xmm10 + movdqu %xmm10,0x60(%rsi) + movdqu 0xe0(%rdx),%xmm1 + pxor %xmm1,%xmm11 + movdqu %xmm11,0xe0(%rsi) + movdqu 0x30(%rdx),%xmm1 + pxor %xmm1,%xmm12 + movdqu %xmm12,0x30(%rsi) + movdqu 0xb0(%rdx),%xmm1 + pxor %xmm1,%xmm13 + movdqu %xmm13,0xb0(%rsi) + movdqu 0x70(%rdx),%xmm1 + pxor %xmm1,%xmm14 + movdqu %xmm14,0x70(%rsi) + movdqu 0xf0(%rdx),%xmm1 + pxor %xmm1,%xmm15 + movdqu %xmm15,0xf0(%rsi) + + add $0x40,%rsp + ret +ENDPROC(chacha20_4block_xor_ssse3) diff --git a/arch/x86/crypto/chacha20_glue.c b/arch/x86/crypto/chacha20_glue.c new file mode 100644 index 000000000000..effe2160b7c5 --- /dev/null +++ b/arch/x86/crypto/chacha20_glue.c @@ -0,0 +1,150 @@ +/* + * ChaCha20 256-bit cipher algorithm, RFC7539, SIMD glue code + * + * Copyright (C) 2015 Martin Willi + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + */ + +#include <crypto/algapi.h> +#include <crypto/chacha20.h> +#include <linux/crypto.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <asm/fpu/api.h> +#include <asm/simd.h> + +#define CHACHA20_STATE_ALIGN 16 + +asmlinkage void chacha20_block_xor_ssse3(u32 *state, u8 *dst, const u8 *src); +asmlinkage void chacha20_4block_xor_ssse3(u32 *state, u8 *dst, const u8 *src); +#ifdef CONFIG_AS_AVX2 +asmlinkage void chacha20_8block_xor_avx2(u32 *state, u8 *dst, const u8 *src); +static bool chacha20_use_avx2; +#endif + +static void chacha20_dosimd(u32 *state, u8 *dst, const u8 *src, + unsigned int bytes) +{ + u8 buf[CHACHA20_BLOCK_SIZE]; + +#ifdef CONFIG_AS_AVX2 + if (chacha20_use_avx2) { + while (bytes >= CHACHA20_BLOCK_SIZE * 8) { + chacha20_8block_xor_avx2(state, dst, src); + bytes -= CHACHA20_BLOCK_SIZE * 8; + src += CHACHA20_BLOCK_SIZE * 8; + dst += CHACHA20_BLOCK_SIZE * 8; + state[12] += 8; + } + } +#endif + while (bytes >= CHACHA20_BLOCK_SIZE * 4) { + chacha20_4block_xor_ssse3(state, dst, src); + bytes -= CHACHA20_BLOCK_SIZE * 4; + src += CHACHA20_BLOCK_SIZE * 4; + dst += CHACHA20_BLOCK_SIZE * 4; + state[12] += 4; + } + while (bytes >= CHACHA20_BLOCK_SIZE) { + chacha20_block_xor_ssse3(state, dst, src); + bytes -= CHACHA20_BLOCK_SIZE; + src += CHACHA20_BLOCK_SIZE; + dst += CHACHA20_BLOCK_SIZE; + state[12]++; + } + if (bytes) { + memcpy(buf, src, bytes); + chacha20_block_xor_ssse3(state, buf, buf); + memcpy(dst, buf, bytes); + } +} + +static int chacha20_simd(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + u32 *state, state_buf[16 + (CHACHA20_STATE_ALIGN / sizeof(u32)) - 1]; + struct blkcipher_walk walk; + int err; + + if (!may_use_simd()) + return crypto_chacha20_crypt(desc, dst, src, nbytes); + + state = (u32 *)roundup((uintptr_t)state_buf, CHACHA20_STATE_ALIGN); + + blkcipher_walk_init(&walk, dst, src, nbytes); + err = blkcipher_walk_virt_block(desc, &walk, CHACHA20_BLOCK_SIZE); + + crypto_chacha20_init(state, crypto_blkcipher_ctx(desc->tfm), walk.iv); + + kernel_fpu_begin(); + + while (walk.nbytes >= CHACHA20_BLOCK_SIZE) { + chacha20_dosimd(state, walk.dst.virt.addr, walk.src.virt.addr, + rounddown(walk.nbytes, CHACHA20_BLOCK_SIZE)); + err = blkcipher_walk_done(desc, &walk, + walk.nbytes % CHACHA20_BLOCK_SIZE); + } + + if (walk.nbytes) { + chacha20_dosimd(state, walk.dst.virt.addr, walk.src.virt.addr, + walk.nbytes); + err = blkcipher_walk_done(desc, &walk, 0); + } + + kernel_fpu_end(); + + return err; +} + +static struct crypto_alg alg = { + .cra_name = "chacha20", + .cra_driver_name = "chacha20-simd", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_blocksize = 1, + .cra_type = &crypto_blkcipher_type, + .cra_ctxsize = sizeof(struct chacha20_ctx), + .cra_alignmask = sizeof(u32) - 1, + .cra_module = THIS_MODULE, + .cra_u = { + .blkcipher = { + .min_keysize = CHACHA20_KEY_SIZE, + .max_keysize = CHACHA20_KEY_SIZE, + .ivsize = CHACHA20_IV_SIZE, + .geniv = "seqiv", + .setkey = crypto_chacha20_setkey, + .encrypt = chacha20_simd, + .decrypt = chacha20_simd, + }, + }, +}; + +static int __init chacha20_simd_mod_init(void) +{ + if (!cpu_has_ssse3) + return -ENODEV; + +#ifdef CONFIG_AS_AVX2 + chacha20_use_avx2 = cpu_has_avx && cpu_has_avx2 && + cpu_has_xfeatures(XSTATE_SSE | XSTATE_YMM, NULL); +#endif + return crypto_register_alg(&alg); +} + +static void __exit chacha20_simd_mod_fini(void) +{ + crypto_unregister_alg(&alg); +} + +module_init(chacha20_simd_mod_init); +module_exit(chacha20_simd_mod_fini); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Martin Willi <martin@strongswan.org>"); +MODULE_DESCRIPTION("chacha20 cipher algorithm, SIMD accelerated"); +MODULE_ALIAS_CRYPTO("chacha20"); +MODULE_ALIAS_CRYPTO("chacha20-simd"); diff --git a/arch/x86/crypto/poly1305-avx2-x86_64.S b/arch/x86/crypto/poly1305-avx2-x86_64.S new file mode 100644 index 000000000000..eff2f414e22b --- /dev/null +++ b/arch/x86/crypto/poly1305-avx2-x86_64.S @@ -0,0 +1,386 @@ +/* + * Poly1305 authenticator algorithm, RFC7539, x64 AVX2 functions + * + * Copyright (C) 2015 Martin Willi + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + */ + +#include <linux/linkage.h> + +.data +.align 32 + +ANMASK: .octa 0x0000000003ffffff0000000003ffffff + .octa 0x0000000003ffffff0000000003ffffff +ORMASK: .octa 0x00000000010000000000000001000000 + .octa 0x00000000010000000000000001000000 + +.text + +#define h0 0x00(%rdi) +#define h1 0x04(%rdi) +#define h2 0x08(%rdi) +#define h3 0x0c(%rdi) +#define h4 0x10(%rdi) +#define r0 0x00(%rdx) +#define r1 0x04(%rdx) +#define r2 0x08(%rdx) +#define r3 0x0c(%rdx) +#define r4 0x10(%rdx) +#define u0 0x00(%r8) +#define u1 0x04(%r8) +#define u2 0x08(%r8) +#define u3 0x0c(%r8) +#define u4 0x10(%r8) +#define w0 0x14(%r8) +#define w1 0x18(%r8) +#define w2 0x1c(%r8) +#define w3 0x20(%r8) +#define w4 0x24(%r8) +#define y0 0x28(%r8) +#define y1 0x2c(%r8) +#define y2 0x30(%r8) +#define y3 0x34(%r8) +#define y4 0x38(%r8) +#define m %rsi +#define hc0 %ymm0 +#define hc1 %ymm1 +#define hc2 %ymm2 +#define hc3 %ymm3 +#define hc4 %ymm4 +#define hc0x %xmm0 +#define hc1x %xmm1 +#define hc2x %xmm2 +#define hc3x %xmm3 +#define hc4x %xmm4 +#define t1 %ymm5 +#define t2 %ymm6 +#define t1x %xmm5 +#define t2x %xmm6 +#define ruwy0 %ymm7 +#define ruwy1 %ymm8 +#define ruwy2 %ymm9 +#define ruwy3 %ymm10 +#define ruwy4 %ymm11 +#define ruwy0x %xmm7 +#define ruwy1x %xmm8 +#define ruwy2x %xmm9 +#define ruwy3x %xmm10 +#define ruwy4x %xmm11 +#define svxz1 %ymm12 +#define svxz2 %ymm13 +#define svxz3 %ymm14 +#define svxz4 %ymm15 +#define d0 %r9 +#define d1 %r10 +#define d2 %r11 +#define d3 %r12 +#define d4 %r13 + +ENTRY(poly1305_4block_avx2) + # %rdi: Accumulator h[5] + # %rsi: 64 byte input block m + # %rdx: Poly1305 key r[5] + # %rcx: Quadblock count + # %r8: Poly1305 derived key r^2 u[5], r^3 w[5], r^4 y[5], + + # This four-block variant uses loop unrolled block processing. It + # requires 4 Poly1305 keys: r, r^2, r^3 and r^4: + # h = (h + m) * r => h = (h + m1) * r^4 + m2 * r^3 + m3 * r^2 + m4 * r + + vzeroupper + push %rbx + push %r12 + push %r13 + + # combine r0,u0,w0,y0 + vmovd y0,ruwy0x + vmovd w0,t1x + vpunpcklqdq t1,ruwy0,ruwy0 + vmovd u0,t1x + vmovd r0,t2x + vpunpcklqdq t2,t1,t1 + vperm2i128 $0x20,t1,ruwy0,ruwy0 + + # combine r1,u1,w1,y1 and s1=r1*5,v1=u1*5,x1=w1*5,z1=y1*5 + vmovd y1,ruwy1x + vmovd w1,t1x + vpunpcklqdq t1,ruwy1,ruwy1 + vmovd u1,t1x + vmovd r1,t2x + vpunpcklqdq t2,t1,t1 + vperm2i128 $0x20,t1,ruwy1,ruwy1 + vpslld $2,ruwy1,svxz1 + vpaddd ruwy1,svxz1,svxz1 + + # combine r2,u2,w2,y2 and s2=r2*5,v2=u2*5,x2=w2*5,z2=y2*5 + vmovd y2,ruwy2x + vmovd w2,t1x + vpunpcklqdq t1,ruwy2,ruwy2 + vmovd u2,t1x + vmovd r2,t2x + vpunpcklqdq t2,t1,t1 + vperm2i128 $0x20,t1,ruwy2,ruwy2 + vpslld $2,ruwy2,svxz2 + vpaddd ruwy2,svxz2,svxz2 + + # combine r3,u3,w3,y3 and s3=r3*5,v3=u3*5,x3=w3*5,z3=y3*5 + vmovd y3,ruwy3x + vmovd w3,t1x + vpunpcklqdq t1,ruwy3,ruwy3 + vmovd u3,t1x + vmovd r3,t2x + vpunpcklqdq t2,t1,t1 + vperm2i128 $0x20,t1,ruwy3,ruwy3 + vpslld $2,ruwy3,svxz3 + vpaddd ruwy3,svxz3,svxz3 + + # combine r4,u4,w4,y4 and s4=r4*5,v4=u4*5,x4=w4*5,z4=y4*5 + vmovd y4,ruwy4x + vmovd w4,t1x + vpunpcklqdq t1,ruwy4,ruwy4 + vmovd u4,t1x + vmovd r4,t2x + vpunpcklqdq t2,t1,t1 + vperm2i128 $0x20,t1,ruwy4,ruwy4 + vpslld $2,ruwy4,svxz4 + vpaddd ruwy4,svxz4,svxz4 + +.Ldoblock4: + # hc0 = [m[48-51] & 0x3ffffff, m[32-35] & 0x3ffffff, + # m[16-19] & 0x3ffffff, m[ 0- 3] & 0x3ffffff + h0] + vmovd 0x00(m),hc0x + vmovd 0x10(m),t1x + vpunpcklqdq t1,hc0,hc0 + vmovd 0x20(m),t1x + vmovd 0x30(m),t2x + vpunpcklqdq t2,t1,t1 + vperm2i128 $0x20,t1,hc0,hc0 + vpand ANMASK(%rip),hc0,hc0 + vmovd h0,t1x + vpaddd t1,hc0,hc0 + # hc1 = [(m[51-54] >> 2) & 0x3ffffff, (m[35-38] >> 2) & 0x3ffffff, + # (m[19-22] >> 2) & 0x3ffffff, (m[ 3- 6] >> 2) & 0x3ffffff + h1] + vmovd 0x03(m),hc1x + vmovd 0x13(m),t1x + vpunpcklqdq t1,hc1,hc1 + vmovd 0x23(m),t1x + vmovd 0x33(m),t2x + vpunpcklqdq t2,t1,t1 + vperm2i128 $0x20,t1,hc1,hc1 + vpsrld $2,hc1,hc1 + vpand ANMASK(%rip),hc1,hc1 + vmovd h1,t1x + vpaddd t1,hc1,hc1 + # hc2 = [(m[54-57] >> 4) & 0x3ffffff, (m[38-41] >> 4) & 0x3ffffff, + # (m[22-25] >> 4) & 0x3ffffff, (m[ 6- 9] >> 4) & 0x3ffffff + h2] + vmovd 0x06(m),hc2x + vmovd 0x16(m),t1x + vpunpcklqdq t1,hc2,hc2 + vmovd 0x26(m),t1x + vmovd 0x36(m),t2x + vpunpcklqdq t2,t1,t1 + vperm2i128 $0x20,t1,hc2,hc2 + vpsrld $4,hc2,hc2 + vpand ANMASK(%rip),hc2,hc2 + vmovd h2,t1x + vpaddd t1,hc2,hc2 + # hc3 = [(m[57-60] >> 6) & 0x3ffffff, (m[41-44] >> 6) & 0x3ffffff, + # (m[25-28] >> 6) & 0x3ffffff, (m[ 9-12] >> 6) & 0x3ffffff + h3] + vmovd 0x09(m),hc3x + vmovd 0x19(m),t1x + vpunpcklqdq t1,hc3,hc3 + vmovd 0x29(m),t1x + vmovd 0x39(m),t2x + vpunpcklqdq t2,t1,t1 + vperm2i128 $0x20,t1,hc3,hc3 + vpsrld $6,hc3,hc3 + vpand ANMASK(%rip),hc3,hc3 + vmovd h3,t1x + vpaddd t1,hc3,hc3 + # hc4 = [(m[60-63] >> 8) | (1<<24), (m[44-47] >> 8) | (1<<24), + # (m[28-31] >> 8) | (1<<24), (m[12-15] >> 8) | (1<<24) + h4] + vmovd 0x0c(m),hc4x + vmovd 0x1c(m),t1x + vpunpcklqdq t1,hc4,hc4 + vmovd 0x2c(m),t1x + vmovd 0x3c(m),t2x + vpunpcklqdq t2,t1,t1 + vperm2i128 $0x20,t1,hc4,hc4 + vpsrld $8,hc4,hc4 + vpor ORMASK(%rip),hc4,hc4 + vmovd h4,t1x + vpaddd t1,hc4,hc4 + + # t1 = [ hc0[3] * r0, hc0[2] * u0, hc0[1] * w0, hc0[0] * y0 ] + vpmuludq hc0,ruwy0,t1 + # t1 += [ hc1[3] * s4, hc1[2] * v4, hc1[1] * x4, hc1[0] * z4 ] + vpmuludq hc1,svxz4,t2 + vpaddq t2,t1,t1 + # t1 += [ hc2[3] * s3, hc2[2] * v3, hc2[1] * x3, hc2[0] * z3 ] + vpmuludq hc2,svxz3,t2 + vpaddq t2,t1,t1 + # t1 += [ hc3[3] * s2, hc3[2] * v2, hc3[1] * x2, hc3[0] * z2 ] + vpmuludq hc3,svxz2,t2 + vpaddq t2,t1,t1 + # t1 += [ hc4[3] * s1, hc4[2] * v1, hc4[1] * x1, hc4[0] * z1 ] + vpmuludq hc4,svxz1,t2 + vpaddq t2,t1,t1 + # d0 = t1[0] + t1[1] + t[2] + t[3] + vpermq $0xee,t1,t2 + vpaddq t2,t1,t1 + vpsrldq $8,t1,t2 + vpaddq t2,t1,t1 + vmovq t1x,d0 + + # t1 = [ hc0[3] * r1, hc0[2] * u1,hc0[1] * w1, hc0[0] * y1 ] + vpmuludq hc0,ruwy1,t1 + # t1 += [ hc1[3] * r0, hc1[2] * u0, hc1[1] * w0, hc1[0] * y0 ] + vpmuludq hc1,ruwy0,t2 + vpaddq t2,t1,t1 + # t1 += [ hc2[3] * s4, hc2[2] * v4, hc2[1] * x4, hc2[0] * z4 ] + vpmuludq hc2,svxz4,t2 + vpaddq t2,t1,t1 + # t1 += [ hc3[3] * s3, hc3[2] * v3, hc3[1] * x3, hc3[0] * z3 ] + vpmuludq hc3,svxz3,t2 + vpaddq t2,t1,t1 + # t1 += [ hc4[3] * s2, hc4[2] * v2, hc4[1] * x2, hc4[0] * z2 ] + vpmuludq hc4,svxz2,t2 + vpaddq t2,t1,t1 + # d1 = t1[0] + t1[1] + t1[3] + t1[4] + vpermq $0xee,t1,t2 + vpaddq t2,t1,t1 + vpsrldq $8,t1,t2 + vpaddq t2,t1,t1 + vmovq t1x,d1 + + # t1 = [ hc0[3] * r2, hc0[2] * u2, hc0[1] * w2, hc0[0] * y2 ] + vpmuludq hc0,ruwy2,t1 + # t1 += [ hc1[3] * r1, hc1[2] * u1, hc1[1] * w1, hc1[0] * y1 ] + vpmuludq hc1,ruwy1,t2 + vpaddq t2,t1,t1 + # t1 += [ hc2[3] * r0, hc2[2] * u0, hc2[1] * w0, hc2[0] * y0 ] + vpmuludq hc2,ruwy0,t2 + vpaddq t2,t1,t1 + # t1 += [ hc3[3] * s4, hc3[2] * v4, hc3[1] * x4, hc3[0] * z4 ] + vpmuludq hc3,svxz4,t2 + vpaddq t2,t1,t1 + # t1 += [ hc4[3] * s3, hc4[2] * v3, hc4[1] * x3, hc4[0] * z3 ] + vpmuludq hc4,svxz3,t2 + vpaddq t2,t1,t1 + # d2 = t1[0] + t1[1] + t1[2] + t1[3] + vpermq $0xee,t1,t2 + vpaddq t2,t1,t1 + vpsrldq $8,t1,t2 + vpaddq t2,t1,t1 + vmovq t1x,d2 + + # t1 = [ hc0[3] * r3, hc0[2] * u3, hc0[1] * w3, hc0[0] * y3 ] + vpmuludq hc0,ruwy3,t1 + # t1 += [ hc1[3] * r2, hc1[2] * u2, hc1[1] * w2, hc1[0] * y2 ] + vpmuludq hc1,ruwy2,t2 + vpaddq t2,t1,t1 + # t1 += [ hc2[3] * r1, hc2[2] * u1, hc2[1] * w1, hc2[0] * y1 ] + vpmuludq hc2,ruwy1,t2 + vpaddq t2,t1,t1 + # t1 += [ hc3[3] * r0, hc3[2] * u0, hc3[1] * w0, hc3[0] * y0 ] + vpmuludq hc3,ruwy0,t2 + vpaddq t2,t1,t1 + # t1 += [ hc4[3] * s4, hc4[2] * v4, hc4[1] * x4, hc4[0] * z4 ] + vpmuludq hc4,svxz4,t2 + vpaddq t2,t1,t1 + # d3 = t1[0] + t1[1] + t1[2] + t1[3] + vpermq $0xee,t1,t2 + vpaddq t2,t1,t1 + vpsrldq $8,t1,t2 + vpaddq t2,t1,t1 + vmovq t1x,d3 + + # t1 = [ hc0[3] * r4, hc0[2] * u4, hc0[1] * w4, hc0[0] * y4 ] + vpmuludq hc0,ruwy4,t1 + # t1 += [ hc1[3] * r3, hc1[2] * u3, hc1[1] * w3, hc1[0] * y3 ] + vpmuludq hc1,ruwy3,t2 + vpaddq t2,t1,t1 + # t1 += [ hc2[3] * r2, hc2[2] * u2, hc2[1] * w2, hc2[0] * y2 ] + vpmuludq hc2,ruwy2,t2 + vpaddq t2,t1,t1 + # t1 += [ hc3[3] * r1, hc3[2] * u1, hc3[1] * w1, hc3[0] * y1 ] + vpmuludq hc3,ruwy1,t2 + vpaddq t2,t1,t1 + # t1 += [ hc4[3] * r0, hc4[2] * u0, hc4[1] * w0, hc4[0] * y0 ] + vpmuludq hc4,ruwy0,t2 + vpaddq t2,t1,t1 + # d4 = t1[0] + t1[1] + t1[2] + t1[3] + vpermq $0xee,t1,t2 + vpaddq t2,t1,t1 + vpsrldq $8,t1,t2 + vpaddq t2,t1,t1 + vmovq t1x,d4 + + # d1 += d0 >> 26 + mov d0,%rax + shr $26,%rax + add %rax,d1 + # h0 = d0 & 0x3ffffff + mov d0,%rbx + and $0x3ffffff,%ebx + + # d2 += d1 >> 26 + mov d1,%rax + shr $26,%rax + add %rax,d2 + # h1 = d1 & 0x3ffffff + mov d1,%rax + and $0x3ffffff,%eax + mov %eax,h1 + + # d3 += d2 >> 26 + mov d2,%rax + shr $26,%rax + add %rax,d3 + # h2 = d2 & 0x3ffffff + mov d2,%rax + and $0x3ffffff,%eax + mov %eax,h2 + + # d4 += d3 >> 26 + mov d3,%rax + shr $26,%rax + add %rax,d4 + # h3 = d3 & 0x3ffffff + mov d3,%rax + and $0x3ffffff,%eax + mov %eax,h3 + + # h0 += (d4 >> 26) * 5 + mov d4,%rax + shr $26,%rax + lea (%eax,%eax,4),%eax + add %eax,%ebx + # h4 = d4 & 0x3ffffff + mov d4,%rax + and $0x3ffffff,%eax + mov %eax,h4 + + # h1 += h0 >> 26 + mov %ebx,%eax + shr $26,%eax + add %eax,h1 + # h0 = h0 & 0x3ffffff + andl $0x3ffffff,%ebx + mov %ebx,h0 + + add $0x40,m + dec %rcx + jnz .Ldoblock4 + + vzeroupper + pop %r13 + pop %r12 + pop %rbx + ret +ENDPROC(poly1305_4block_avx2) diff --git a/arch/x86/crypto/poly1305-sse2-x86_64.S b/arch/x86/crypto/poly1305-sse2-x86_64.S new file mode 100644 index 000000000000..338c748054ed --- /dev/null +++ b/arch/x86/crypto/poly1305-sse2-x86_64.S @@ -0,0 +1,582 @@ +/* + * Poly1305 authenticator algorithm, RFC7539, x64 SSE2 functions + * + * Copyright (C) 2015 Martin Willi + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + */ + +#include <linux/linkage.h> + +.data +.align 16 + +ANMASK: .octa 0x0000000003ffffff0000000003ffffff +ORMASK: .octa 0x00000000010000000000000001000000 + +.text + +#define h0 0x00(%rdi) +#define h1 0x04(%rdi) +#define h2 0x08(%rdi) +#define h3 0x0c(%rdi) +#define h4 0x10(%rdi) +#define r0 0x00(%rdx) +#define r1 0x04(%rdx) +#define r2 0x08(%rdx) +#define r3 0x0c(%rdx) +#define r4 0x10(%rdx) +#define s1 0x00(%rsp) +#define s2 0x04(%rsp) +#define s3 0x08(%rsp) +#define s4 0x0c(%rsp) +#define m %rsi +#define h01 %xmm0 +#define h23 %xmm1 +#define h44 %xmm2 +#define t1 %xmm3 +#define t2 %xmm4 +#define t3 %xmm5 +#define t4 %xmm6 +#define mask %xmm7 +#define d0 %r8 +#define d1 %r9 +#define d2 %r10 +#define d3 %r11 +#define d4 %r12 + +ENTRY(poly1305_block_sse2) + # %rdi: Accumulator h[5] + # %rsi: 16 byte input block m + # %rdx: Poly1305 key r[5] + # %rcx: Block count + + # This single block variant tries to improve performance by doing two + # multiplications in parallel using SSE instructions. There is quite + # some quardword packing involved, hence the speedup is marginal. + + push %rbx + push %r12 + sub $0x10,%rsp + + # s1..s4 = r1..r4 * 5 + mov r1,%eax + lea (%eax,%eax,4),%eax + mov %eax,s1 + mov r2,%eax + lea (%eax,%eax,4),%eax + mov %eax,s2 + mov r3,%eax + lea (%eax,%eax,4),%eax + mov %eax,s3 + mov r4,%eax + lea (%eax,%eax,4),%eax + mov %eax,s4 + + movdqa ANMASK(%rip),mask + +.Ldoblock: + # h01 = [0, h1, 0, h0] + # h23 = [0, h3, 0, h2] + # h44 = [0, h4, 0, h4] + movd h0,h01 + movd h1,t1 + movd h2,h23 + movd h3,t2 + movd h4,h44 + punpcklqdq t1,h01 + punpcklqdq t2,h23 + punpcklqdq h44,h44 + + # h01 += [ (m[3-6] >> 2) & 0x3ffffff, m[0-3] & 0x3ffffff ] + movd 0x00(m),t1 + movd 0x03(m),t2 + psrld $2,t2 + punpcklqdq t2,t1 + pand mask,t1 + paddd t1,h01 + # h23 += [ (m[9-12] >> 6) & 0x3ffffff, (m[6-9] >> 4) & 0x3ffffff ] + movd 0x06(m),t1 + movd 0x09(m),t2 + psrld $4,t1 + psrld $6,t2 + punpcklqdq t2,t1 + pand mask,t1 + paddd t1,h23 + # h44 += [ (m[12-15] >> 8) | (1 << 24), (m[12-15] >> 8) | (1 << 24) ] + mov 0x0c(m),%eax + shr $8,%eax + or $0x01000000,%eax + movd %eax,t1 + pshufd $0xc4,t1,t1 + paddd t1,h44 + + # t1[0] = h0 * r0 + h2 * s3 + # t1[1] = h1 * s4 + h3 * s2 + movd r0,t1 + movd s4,t2 + punpcklqdq t2,t1 + pmuludq h01,t1 + movd s3,t2 + movd s2,t3 + punpcklqdq t3,t2 + pmuludq h23,t2 + paddq t2,t1 + # t2[0] = h0 * r1 + h2 * s4 + # t2[1] = h1 * r0 + h3 * s3 + movd r1,t2 + movd r0,t3 + punpcklqdq t3,t2 + pmuludq h01,t2 + movd s4,t3 + movd s3,t4 + punpcklqdq t4,t3 + pmuludq h23,t3 + paddq t3,t2 + # t3[0] = h4 * s1 + # t3[1] = h4 * s2 + movd s1,t3 + movd s2,t4 + punpcklqdq t4,t3 + pmuludq h44,t3 + # d0 = t1[0] + t1[1] + t3[0] + # d1 = t2[0] + t2[1] + t3[1] + movdqa t1,t4 + punpcklqdq t2,t4 + punpckhqdq t2,t1 + paddq t4,t1 + paddq t3,t1 + movq t1,d0 + psrldq $8,t1 + movq t1,d1 + + # t1[0] = h0 * r2 + h2 * r0 + # t1[1] = h1 * r1 + h3 * s4 + movd r2,t1 + movd r1,t2 + punpcklqdq t2,t1 + pmuludq h01,t1 + movd r0,t2 + movd s4,t3 + punpcklqdq t3,t2 + pmuludq h23,t2 + paddq t2,t1 + # t2[0] = h0 * r3 + h2 * r1 + # t2[1] = h1 * r2 + h3 * r0 + movd r3,t2 + movd r2,t3 + punpcklqdq t3,t2 + pmuludq h01,t2 + movd r1,t3 + movd r0,t4 + punpcklqdq t4,t3 + pmuludq h23,t3 + paddq t3,t2 + # t3[0] = h4 * s3 + # t3[1] = h4 * s4 + movd s3,t3 + movd s4,t4 + punpcklqdq t4,t3 + pmuludq h44,t3 + # d2 = t1[0] + t1[1] + t3[0] + # d3 = t2[0] + t2[1] + t3[1] + movdqa t1,t4 + punpcklqdq t2,t4 + punpckhqdq t2,t1 + paddq t4,t1 + paddq t3,t1 + movq t1,d2 + psrldq $8,t1 + movq t1,d3 + + # t1[0] = h0 * r4 + h2 * r2 + # t1[1] = h1 * r3 + h3 * r1 + movd r4,t1 + movd r3,t2 + punpcklqdq t2,t1 + pmuludq h01,t1 + movd r2,t2 + movd r1,t3 + punpcklqdq t3,t2 + pmuludq h23,t2 + paddq t2,t1 + # t3[0] = h4 * r0 + movd r0,t3 + pmuludq h44,t3 + # d4 = t1[0] + t1[1] + t3[0] + movdqa t1,t4 + psrldq $8,t4 + paddq t4,t1 + paddq t3,t1 + movq t1,d4 + + # d1 += d0 >> 26 + mov d0,%rax + shr $26,%rax + add %rax,d1 + # h0 = d0 & 0x3ffffff + mov d0,%rbx + and $0x3ffffff,%ebx + + # d2 += d1 >> 26 + mov d1,%rax + shr $26,%rax + add %rax,d2 + # h1 = d1 & 0x3ffffff + mov d1,%rax + and $0x3ffffff,%eax + mov %eax,h1 + + # d3 += d2 >> 26 + mov d2,%rax + shr $26,%rax + add %rax,d3 + # h2 = d2 & 0x3ffffff + mov d2,%rax + and $0x3ffffff,%eax + mov %eax,h2 + + # d4 += d3 >> 26 + mov d3,%rax + shr $26,%rax + add %rax,d4 + # h3 = d3 & 0x3ffffff + mov d3,%rax + and $0x3ffffff,%eax + mov %eax,h3 + + # h0 += (d4 >> 26) * 5 + mov d4,%rax + shr $26,%rax + lea (%eax,%eax,4),%eax + add %eax,%ebx + # h4 = d4 & 0x3ffffff + mov d4,%rax + and $0x3ffffff,%eax + mov %eax,h4 + + # h1 += h0 >> 26 + mov %ebx,%eax + shr $26,%eax + add %eax,h1 + # h0 = h0 & 0x3ffffff + andl $0x3ffffff,%ebx + mov %ebx,h0 + + add $0x10,m + dec %rcx + jnz .Ldoblock + + add $0x10,%rsp + pop %r12 + pop %rbx + ret +ENDPROC(poly1305_block_sse2) + + +#define u0 0x00(%r8) +#define u1 0x04(%r8) +#define u2 0x08(%r8) +#define u3 0x0c(%r8) +#define u4 0x10(%r8) +#define hc0 %xmm0 +#define hc1 %xmm1 +#define hc2 %xmm2 +#define hc3 %xmm5 +#define hc4 %xmm6 +#define ru0 %xmm7 +#define ru1 %xmm8 +#define ru2 %xmm9 +#define ru3 %xmm10 +#define ru4 %xmm11 +#define sv1 %xmm12 +#define sv2 %xmm13 +#define sv3 %xmm14 +#define sv4 %xmm15 +#undef d0 +#define d0 %r13 + +ENTRY(poly1305_2block_sse2) + # %rdi: Accumulator h[5] + # %rsi: 16 byte input block m + # %rdx: Poly1305 key r[5] + # %rcx: Doubleblock count + # %r8: Poly1305 derived key r^2 u[5] + + # This two-block variant further improves performance by using loop + # unrolled block processing. This is more straight forward and does + # less byte shuffling, but requires a second Poly1305 key r^2: + # h = (h + m) * r => h = (h + m1) * r^2 + m2 * r + + push %rbx + push %r12 + push %r13 + + # combine r0,u0 + movd u0,ru0 + movd r0,t1 + punpcklqdq t1,ru0 + + # combine r1,u1 and s1=r1*5,v1=u1*5 + movd u1,ru1 + movd r1,t1 + punpcklqdq t1,ru1 + movdqa ru1,sv1 + pslld $2,sv1 + paddd ru1,sv1 + + # combine r2,u2 and s2=r2*5,v2=u2*5 + movd u2,ru2 + movd r2,t1 + punpcklqdq t1,ru2 + movdqa ru2,sv2 + pslld $2,sv2 + paddd ru2,sv2 + + # combine r3,u3 and s3=r3*5,v3=u3*5 + movd u3,ru3 + movd r3,t1 + punpcklqdq t1,ru3 + movdqa ru3,sv3 + pslld $2,sv3 + paddd ru3,sv3 + + # combine r4,u4 and s4=r4*5,v4=u4*5 + movd u4,ru4 + movd r4,t1 + punpcklqdq t1,ru4 + movdqa ru4,sv4 + pslld $2,sv4 + paddd ru4,sv4 + +.Ldoblock2: + # hc0 = [ m[16-19] & 0x3ffffff, h0 + m[0-3] & 0x3ffffff ] + movd 0x00(m),hc0 + movd 0x10(m),t1 + punpcklqdq t1,hc0 + pand ANMASK(%rip),hc0 + movd h0,t1 + paddd t1,hc0 + # hc1 = [ (m[19-22] >> 2) & 0x3ffffff, h1 + (m[3-6] >> 2) & 0x3ffffff ] + movd 0x03(m),hc1 + movd 0x13(m),t1 + punpcklqdq t1,hc1 + psrld $2,hc1 + pand ANMASK(%rip),hc1 + movd h1,t1 + paddd t1,hc1 + # hc2 = [ (m[22-25] >> 4) & 0x3ffffff, h2 + (m[6-9] >> 4) & 0x3ffffff ] + movd 0x06(m),hc2 + movd 0x16(m),t1 + punpcklqdq t1,hc2 + psrld $4,hc2 + pand ANMASK(%rip),hc2 + movd h2,t1 + paddd t1,hc2 + # hc3 = [ (m[25-28] >> 6) & 0x3ffffff, h3 + (m[9-12] >> 6) & 0x3ffffff ] + movd 0x09(m),hc3 + movd 0x19(m),t1 + punpcklqdq t1,hc3 + psrld $6,hc3 + pand ANMASK(%rip),hc3 + movd h3,t1 + paddd t1,hc3 + # hc4 = [ (m[28-31] >> 8) | (1<<24), h4 + (m[12-15] >> 8) | (1<<24) ] + movd 0x0c(m),hc4 + movd 0x1c(m),t1 + punpcklqdq t1,hc4 + psrld $8,hc4 + por ORMASK(%rip),hc4 + movd h4,t1 + paddd t1,hc4 + + # t1 = [ hc0[1] * r0, hc0[0] * u0 ] + movdqa ru0,t1 + pmuludq hc0,t1 + # t1 += [ hc1[1] * s4, hc1[0] * v4 ] + movdqa sv4,t2 + pmuludq hc1,t2 + paddq t2,t1 + # t1 += [ hc2[1] * s3, hc2[0] * v3 ] + movdqa sv3,t2 + pmuludq hc2,t2 + paddq t2,t1 + # t1 += [ hc3[1] * s2, hc3[0] * v2 ] + movdqa sv2,t2 + pmuludq hc3,t2 + paddq t2,t1 + # t1 += [ hc4[1] * s1, hc4[0] * v1 ] + movdqa sv1,t2 + pmuludq hc4,t2 + paddq t2,t1 + # d0 = t1[0] + t1[1] + movdqa t1,t2 + psrldq $8,t2 + paddq t2,t1 + movq t1,d0 + + # t1 = [ hc0[1] * r1, hc0[0] * u1 ] + movdqa ru1,t1 + pmuludq hc0,t1 + # t1 += [ hc1[1] * r0, hc1[0] * u0 ] + movdqa ru0,t2 + pmuludq hc1,t2 + paddq t2,t1 + # t1 += [ hc2[1] * s4, hc2[0] * v4 ] + movdqa sv4,t2 + pmuludq hc2,t2 + paddq t2,t1 + # t1 += [ hc3[1] * s3, hc3[0] * v3 ] + movdqa sv3,t2 + pmuludq hc3,t2 + paddq t2,t1 + # t1 += [ hc4[1] * s2, hc4[0] * v2 ] + movdqa sv2,t2 + pmuludq hc4,t2 + paddq t2,t1 + # d1 = t1[0] + t1[1] + movdqa t1,t2 + psrldq $8,t2 + paddq t2,t1 + movq t1,d1 + + # t1 = [ hc0[1] * r2, hc0[0] * u2 ] + movdqa ru2,t1 + pmuludq hc0,t1 + # t1 += [ hc1[1] * r1, hc1[0] * u1 ] + movdqa ru1,t2 + pmuludq hc1,t2 + paddq t2,t1 + # t1 += [ hc2[1] * r0, hc2[0] * u0 ] + movdqa ru0,t2 + pmuludq hc2,t2 + paddq t2,t1 + # t1 += [ hc3[1] * s4, hc3[0] * v4 ] + movdqa sv4,t2 + pmuludq hc3,t2 + paddq t2,t1 + # t1 += [ hc4[1] * s3, hc4[0] * v3 ] + movdqa sv3,t2 + pmuludq hc4,t2 + paddq t2,t1 + # d2 = t1[0] + t1[1] + movdqa t1,t2 + psrldq $8,t2 + paddq t2,t1 + movq t1,d2 + + # t1 = [ hc0[1] * r3, hc0[0] * u3 ] + movdqa ru3,t1 + pmuludq hc0,t1 + # t1 += [ hc1[1] * r2, hc1[0] * u2 ] + movdqa ru2,t2 + pmuludq hc1,t2 + paddq t2,t1 + # t1 += [ hc2[1] * r1, hc2[0] * u1 ] + movdqa ru1,t2 + pmuludq hc2,t2 + paddq t2,t1 + # t1 += [ hc3[1] * r0, hc3[0] * u0 ] + movdqa ru0,t2 + pmuludq hc3,t2 + paddq t2,t1 + # t1 += [ hc4[1] * s4, hc4[0] * v4 ] + movdqa sv4,t2 + pmuludq hc4,t2 + paddq t2,t1 + # d3 = t1[0] + t1[1] + movdqa t1,t2 + psrldq $8,t2 + paddq t2,t1 + movq t1,d3 + + # t1 = [ hc0[1] * r4, hc0[0] * u4 ] + movdqa ru4,t1 + pmuludq hc0,t1 + # t1 += [ hc1[1] * r3, hc1[0] * u3 ] + movdqa ru3,t2 + pmuludq hc1,t2 + paddq t2,t1 + # t1 += [ hc2[1] * r2, hc2[0] * u2 ] + movdqa ru2,t2 + pmuludq hc2,t2 + paddq t2,t1 + # t1 += [ hc3[1] * r1, hc3[0] * u1 ] + movdqa ru1,t2 + pmuludq hc3,t2 + paddq t2,t1 + # t1 += [ hc4[1] * r0, hc4[0] * u0 ] + movdqa ru0,t2 + pmuludq hc4,t2 + paddq t2,t1 + # d4 = t1[0] + t1[1] + movdqa t1,t2 + psrldq $8,t2 + paddq t2,t1 + movq t1,d4 + + # d1 += d0 >> 26 + mov d0,%rax + shr $26,%rax + add %rax,d1 + # h0 = d0 & 0x3ffffff + mov d0,%rbx + and $0x3ffffff,%ebx + + # d2 += d1 >> 26 + mov d1,%rax + shr $26,%rax + add %rax,d2 + # h1 = d1 & 0x3ffffff + mov d1,%rax + and $0x3ffffff,%eax + mov %eax,h1 + + # d3 += d2 >> 26 + mov d2,%rax + shr $26,%rax + add %rax,d3 + # h2 = d2 & 0x3ffffff + mov d2,%rax + and $0x3ffffff,%eax + mov %eax,h2 + + # d4 += d3 >> 26 + mov d3,%rax + shr $26,%rax + add %rax,d4 + # h3 = d3 & 0x3ffffff + mov d3,%rax + and $0x3ffffff,%eax + mov %eax,h3 + + # h0 += (d4 >> 26) * 5 + mov d4,%rax + shr $26,%rax + lea (%eax,%eax,4),%eax + add %eax,%ebx + # h4 = d4 & 0x3ffffff + mov d4,%rax + and $0x3ffffff,%eax + mov %eax,h4 + + # h1 += h0 >> 26 + mov %ebx,%eax + shr $26,%eax + add %eax,h1 + # h0 = h0 & 0x3ffffff + andl $0x3ffffff,%ebx + mov %ebx,h0 + + add $0x20,m + dec %rcx + jnz .Ldoblock2 + + pop %r13 + pop %r12 + pop %rbx + ret +ENDPROC(poly1305_2block_sse2) diff --git a/arch/x86/crypto/poly1305_glue.c b/arch/x86/crypto/poly1305_glue.c new file mode 100644 index 000000000000..f7170d764f32 --- /dev/null +++ b/arch/x86/crypto/poly1305_glue.c @@ -0,0 +1,207 @@ +/* + * Poly1305 authenticator algorithm, RFC7539, SIMD glue code + * + * Copyright (C) 2015 Martin Willi + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + */ + +#include <crypto/algapi.h> +#include <crypto/internal/hash.h> +#include <crypto/poly1305.h> +#include <linux/crypto.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <asm/fpu/api.h> +#include <asm/simd.h> + +struct poly1305_simd_desc_ctx { + struct poly1305_desc_ctx base; + /* derived key u set? */ + bool uset; +#ifdef CONFIG_AS_AVX2 + /* derived keys r^3, r^4 set? */ + bool wset; +#endif + /* derived Poly1305 key r^2 */ + u32 u[5]; + /* ... silently appended r^3 and r^4 when using AVX2 */ +}; + +asmlinkage void poly1305_block_sse2(u32 *h, const u8 *src, + const u32 *r, unsigned int blocks); +asmlinkage void poly1305_2block_sse2(u32 *h, const u8 *src, const u32 *r, + unsigned int blocks, const u32 *u); +#ifdef CONFIG_AS_AVX2 +asmlinkage void poly1305_4block_avx2(u32 *h, const u8 *src, const u32 *r, + unsigned int blocks, const u32 *u); +static bool poly1305_use_avx2; +#endif + +static int poly1305_simd_init(struct shash_desc *desc) +{ + struct poly1305_simd_desc_ctx *sctx = shash_desc_ctx(desc); + + sctx->uset = false; +#ifdef CONFIG_AS_AVX2 + sctx->wset = false; +#endif + + return crypto_poly1305_init(desc); +} + +static void poly1305_simd_mult(u32 *a, const u32 *b) +{ + u8 m[POLY1305_BLOCK_SIZE]; + + memset(m, 0, sizeof(m)); + /* The poly1305 block function adds a hi-bit to the accumulator which + * we don't need for key multiplication; compensate for it. */ + a[4] -= 1 << 24; + poly1305_block_sse2(a, m, b, 1); +} + +static unsigned int poly1305_simd_blocks(struct poly1305_desc_ctx *dctx, + const u8 *src, unsigned int srclen) +{ + struct poly1305_simd_desc_ctx *sctx; + unsigned int blocks, datalen; + + BUILD_BUG_ON(offsetof(struct poly1305_simd_desc_ctx, base)); + sctx = container_of(dctx, struct poly1305_simd_desc_ctx, base); + + if (unlikely(!dctx->sset)) { + datalen = crypto_poly1305_setdesckey(dctx, src, srclen); + src += srclen - datalen; + srclen = datalen; + } + +#ifdef CONFIG_AS_AVX2 + if (poly1305_use_avx2 && srclen >= POLY1305_BLOCK_SIZE * 4) { + if (unlikely(!sctx->wset)) { + if (!sctx->uset) { + memcpy(sctx->u, dctx->r, sizeof(sctx->u)); + poly1305_simd_mult(sctx->u, dctx->r); + sctx->uset = true; + } + memcpy(sctx->u + 5, sctx->u, sizeof(sctx->u)); + poly1305_simd_mult(sctx->u + 5, dctx->r); + memcpy(sctx->u + 10, sctx->u + 5, sizeof(sctx->u)); + poly1305_simd_mult(sctx->u + 10, dctx->r); + sctx->wset = true; + } + blocks = srclen / (POLY1305_BLOCK_SIZE * 4); + poly1305_4block_avx2(dctx->h, src, dctx->r, blocks, sctx->u); + src += POLY1305_BLOCK_SIZE * 4 * blocks; + srclen -= POLY1305_BLOCK_SIZE * 4 * blocks; + } +#endif + if (likely(srclen >= POLY1305_BLOCK_SIZE * 2)) { + if (unlikely(!sctx->uset)) { + memcpy(sctx->u, dctx->r, sizeof(sctx->u)); + poly1305_simd_mult(sctx->u, dctx->r); + sctx->uset = true; + } + blocks = srclen / (POLY1305_BLOCK_SIZE * 2); + poly1305_2block_sse2(dctx->h, src, dctx->r, blocks, sctx->u); + src += POLY1305_BLOCK_SIZE * 2 * blocks; + srclen -= POLY1305_BLOCK_SIZE * 2 * blocks; + } + if (srclen >= POLY1305_BLOCK_SIZE) { + poly1305_block_sse2(dctx->h, src, dctx->r, 1); + srclen -= POLY1305_BLOCK_SIZE; + } + return srclen; +} + +static int poly1305_simd_update(struct shash_desc *desc, + const u8 *src, unsigned int srclen) +{ + struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc); + unsigned int bytes; + + /* kernel_fpu_begin/end is costly, use fallback for small updates */ + if (srclen <= 288 || !may_use_simd()) + return crypto_poly1305_update(desc, src, srclen); + + kernel_fpu_begin(); + + if (unlikely(dctx->buflen)) { + bytes = min(srclen, POLY1305_BLOCK_SIZE - dctx->buflen); + memcpy(dctx->buf + dctx->buflen, src, bytes); + src += bytes; + srclen -= bytes; + dctx->buflen += bytes; + + if (dctx->buflen == POLY1305_BLOCK_SIZE) { + poly1305_simd_blocks(dctx, dctx->buf, + POLY1305_BLOCK_SIZE); + dctx->buflen = 0; + } + } + + if (likely(srclen >= POLY1305_BLOCK_SIZE)) { + bytes = poly1305_simd_blocks(dctx, src, srclen); + src += srclen - bytes; + srclen = bytes; + } + + kernel_fpu_end(); + + if (unlikely(srclen)) { + dctx->buflen = srclen; + memcpy(dctx->buf, src, srclen); + } + + return 0; +} + +static struct shash_alg alg = { + .digestsize = POLY1305_DIGEST_SIZE, + .init = poly1305_simd_init, + .update = poly1305_simd_update, + .final = crypto_poly1305_final, + .setkey = crypto_poly1305_setkey, + .descsize = sizeof(struct poly1305_simd_desc_ctx), + .base = { + .cra_name = "poly1305", + .cra_driver_name = "poly1305-simd", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_SHASH, + .cra_alignmask = sizeof(u32) - 1, + .cra_blocksize = POLY1305_BLOCK_SIZE, + .cra_module = THIS_MODULE, + }, +}; + +static int __init poly1305_simd_mod_init(void) +{ + if (!cpu_has_xmm2) + return -ENODEV; + +#ifdef CONFIG_AS_AVX2 + poly1305_use_avx2 = cpu_has_avx && cpu_has_avx2 && + cpu_has_xfeatures(XSTATE_SSE | XSTATE_YMM, NULL); + alg.descsize = sizeof(struct poly1305_simd_desc_ctx); + if (poly1305_use_avx2) + alg.descsize += 10 * sizeof(u32); +#endif + return crypto_register_shash(&alg); +} + +static void __exit poly1305_simd_mod_exit(void) +{ + crypto_unregister_shash(&alg); +} + +module_init(poly1305_simd_mod_init); +module_exit(poly1305_simd_mod_exit); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Martin Willi <martin@strongswan.org>"); +MODULE_DESCRIPTION("Poly1305 authenticator"); +MODULE_ALIAS_CRYPTO("poly1305"); +MODULE_ALIAS_CRYPTO("poly1305-simd"); diff --git a/crypto/Kconfig b/crypto/Kconfig index b4cfc5754033..b582ea7f78d3 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -48,6 +48,8 @@ config CRYPTO_AEAD config CRYPTO_AEAD2 tristate select CRYPTO_ALGAPI2 + select CRYPTO_NULL2 + select CRYPTO_RNG2 config CRYPTO_BLKCIPHER tristate @@ -150,12 +152,16 @@ config CRYPTO_GF128MUL config CRYPTO_NULL tristate "Null algorithms" - select CRYPTO_ALGAPI - select CRYPTO_BLKCIPHER - select CRYPTO_HASH + select CRYPTO_NULL2 help These are 'Null' algorithms, used by IPsec, which do nothing. +config CRYPTO_NULL2 + tristate + select CRYPTO_ALGAPI2 + select CRYPTO_BLKCIPHER2 + select CRYPTO_HASH2 + config CRYPTO_PCRYPT tristate "Parallel crypto engine" depends on SMP @@ -200,6 +206,7 @@ config CRYPTO_AUTHENC select CRYPTO_BLKCIPHER select CRYPTO_MANAGER select CRYPTO_HASH + select CRYPTO_NULL help Authenc: Combined mode wrapper for IPsec. This is required for IPSec. @@ -470,6 +477,18 @@ config CRYPTO_POLY1305 It is used for the ChaCha20-Poly1305 AEAD, specified in RFC7539 for use in IETF protocols. This is the portable C implementation of Poly1305. +config CRYPTO_POLY1305_X86_64 + tristate "Poly1305 authenticator algorithm (x86_64/SSE2/AVX2)" + depends on X86 && 64BIT + select CRYPTO_POLY1305 + help + Poly1305 authenticator algorithm, RFC7539. + + Poly1305 is an authenticator algorithm designed by Daniel J. Bernstein. + It is used for the ChaCha20-Poly1305 AEAD, specified in RFC7539 for use + in IETF protocols. This is the x86_64 assembler implementation using SIMD + instructions. + config CRYPTO_MD4 tristate "MD4 digest algorithm" select CRYPTO_HASH @@ -1213,6 +1232,21 @@ config CRYPTO_CHACHA20 See also: <http://cr.yp.to/chacha/chacha-20080128.pdf> +config CRYPTO_CHACHA20_X86_64 + tristate "ChaCha20 cipher algorithm (x86_64/SSSE3/AVX2)" + depends on X86 && 64BIT + select CRYPTO_BLKCIPHER + select CRYPTO_CHACHA20 + help + ChaCha20 cipher algorithm, RFC7539. + + ChaCha20 is a 256-bit high-speed stream cipher designed by Daniel J. + Bernstein and further specified in RFC7539 for use in IETF protocols. + This is the x86_64 assembler implementation using SIMD instructions. + + See also: + <http://cr.yp.to/chacha/chacha-20080128.pdf> + config CRYPTO_SEED tristate "SEED cipher algorithm" select CRYPTO_ALGAPI diff --git a/crypto/Makefile b/crypto/Makefile index a16a7e7f2d60..e2c59819b236 100644 --- a/crypto/Makefile +++ b/crypto/Makefile @@ -17,6 +17,7 @@ obj-$(CONFIG_CRYPTO_AEAD2) += aead.o crypto_blkcipher-y := ablkcipher.o crypto_blkcipher-y += blkcipher.o +crypto_blkcipher-y += skcipher.o obj-$(CONFIG_CRYPTO_BLKCIPHER2) += crypto_blkcipher.o obj-$(CONFIG_CRYPTO_BLKCIPHER2) += chainiv.o obj-$(CONFIG_CRYPTO_BLKCIPHER2) += eseqiv.o @@ -46,7 +47,7 @@ obj-$(CONFIG_CRYPTO_CMAC) += cmac.o obj-$(CONFIG_CRYPTO_HMAC) += hmac.o obj-$(CONFIG_CRYPTO_VMAC) += vmac.o obj-$(CONFIG_CRYPTO_XCBC) += xcbc.o -obj-$(CONFIG_CRYPTO_NULL) += crypto_null.o +obj-$(CONFIG_CRYPTO_NULL2) += crypto_null.o obj-$(CONFIG_CRYPTO_MD4) += md4.o obj-$(CONFIG_CRYPTO_MD5) += md5.o obj-$(CONFIG_CRYPTO_RMD128) += rmd128.o diff --git a/crypto/aead.c b/crypto/aead.c index 07bf99773548..9b18a1e40d6a 100644 --- a/crypto/aead.c +++ b/crypto/aead.c @@ -3,7 +3,7 @@ * * This file provides API support for AEAD algorithms. * - * Copyright (c) 2007 Herbert Xu <herbert@gondor.apana.org.au> + * Copyright (c) 2007-2015 Herbert Xu <herbert@gondor.apana.org.au> * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the Free @@ -13,13 +13,14 @@ */ #include <crypto/internal/geniv.h> +#include <crypto/internal/rng.h> +#include <crypto/null.h> #include <crypto/scatterwalk.h> #include <linux/err.h> #include <linux/init.h> #include <linux/kernel.h> #include <linux/module.h> #include <linux/rtnetlink.h> -#include <linux/sched.h> #include <linux/slab.h> #include <linux/seq_file.h> #include <linux/cryptouser.h> @@ -27,17 +28,6 @@ #include "internal.h" -struct compat_request_ctx { - struct scatterlist src[2]; - struct scatterlist dst[2]; - struct scatterlist ivbuf[2]; - struct scatterlist *ivsg; - struct aead_givcrypt_request subreq; -}; - -static int aead_null_givencrypt(struct aead_givcrypt_request *req); -static int aead_null_givdecrypt(struct aead_givcrypt_request *req); - static int setkey_unaligned(struct crypto_aead *tfm, const u8 *key, unsigned int keylen) { @@ -53,7 +43,7 @@ static int setkey_unaligned(struct crypto_aead *tfm, const u8 *key, alignbuffer = (u8 *)ALIGN((unsigned long)buffer, alignmask + 1); memcpy(alignbuffer, key, keylen); - ret = tfm->setkey(tfm, alignbuffer, keylen); + ret = crypto_aead_alg(tfm)->setkey(tfm, alignbuffer, keylen); memset(alignbuffer, 0, keylen); kfree(buffer); return ret; @@ -64,12 +54,10 @@ int crypto_aead_setkey(struct crypto_aead *tfm, { unsigned long alignmask = crypto_aead_alignmask(tfm); - tfm = tfm->child; - if ((unsigned long)key & alignmask) return setkey_unaligned(tfm, key, keylen); - return tfm->setkey(tfm, key, keylen); + return crypto_aead_alg(tfm)->setkey(tfm, key, keylen); } EXPORT_SYMBOL_GPL(crypto_aead_setkey); @@ -80,100 +68,17 @@ int crypto_aead_setauthsize(struct crypto_aead *tfm, unsigned int authsize) if (authsize > crypto_aead_maxauthsize(tfm)) return -EINVAL; - if (tfm->setauthsize) { - err = tfm->setauthsize(tfm->child, authsize); + if (crypto_aead_alg(tfm)->setauthsize) { + err = crypto_aead_alg(tfm)->setauthsize(tfm, authsize); if (err) return err; } - tfm->child->authsize = authsize; tfm->authsize = authsize; return 0; } EXPORT_SYMBOL_GPL(crypto_aead_setauthsize); -struct aead_old_request { - struct scatterlist srcbuf[2]; - struct scatterlist dstbuf[2]; - struct aead_request subreq; -}; - -unsigned int crypto_aead_reqsize(struct crypto_aead *tfm) -{ - return tfm->reqsize + sizeof(struct aead_old_request); -} -EXPORT_SYMBOL_GPL(crypto_aead_reqsize); - -static int old_crypt(struct aead_request *req, - int (*crypt)(struct aead_request *req)) -{ - struct aead_old_request *nreq = aead_request_ctx(req); - struct crypto_aead *aead = crypto_aead_reqtfm(req); - struct scatterlist *src, *dst; - - if (req->old) - return crypt(req); - - src = scatterwalk_ffwd(nreq->srcbuf, req->src, req->assoclen); - dst = req->src == req->dst ? - src : scatterwalk_ffwd(nreq->dstbuf, req->dst, req->assoclen); - - aead_request_set_tfm(&nreq->subreq, aead); - aead_request_set_callback(&nreq->subreq, aead_request_flags(req), - req->base.complete, req->base.data); - aead_request_set_crypt(&nreq->subreq, src, dst, req->cryptlen, - req->iv); - aead_request_set_assoc(&nreq->subreq, req->src, req->assoclen); - - return crypt(&nreq->subreq); -} - -static int old_encrypt(struct aead_request *req) -{ - struct crypto_aead *aead = crypto_aead_reqtfm(req); - struct old_aead_alg *alg = crypto_old_aead_alg(aead); - - return old_crypt(req, alg->encrypt); -} - -static int old_decrypt(struct aead_request *req) -{ - struct crypto_aead *aead = crypto_aead_reqtfm(req); - struct old_aead_alg *alg = crypto_old_aead_alg(aead); - - return old_crypt(req, alg->decrypt); -} - -static int no_givcrypt(struct aead_givcrypt_request *req) -{ - return -ENOSYS; -} - -static int crypto_old_aead_init_tfm(struct crypto_tfm *tfm) -{ - struct old_aead_alg *alg = &tfm->__crt_alg->cra_aead; - struct crypto_aead *crt = __crypto_aead_cast(tfm); - - if (max(alg->maxauthsize, alg->ivsize) > PAGE_SIZE / 8) - return -EINVAL; - - crt->setkey = alg->setkey; - crt->setauthsize = alg->setauthsize; - crt->encrypt = old_encrypt; - crt->decrypt = old_decrypt; - if (alg->ivsize) { - crt->givencrypt = alg->givencrypt ?: no_givcrypt; - crt->givdecrypt = alg->givdecrypt ?: no_givcrypt; - } else { - crt->givencrypt = aead_null_givencrypt; - crt->givdecrypt = aead_null_givdecrypt; - } - crt->child = __crypto_aead_cast(tfm); - crt->authsize = alg->maxauthsize; - - return 0; -} - static void crypto_aead_exit_tfm(struct crypto_tfm *tfm) { struct crypto_aead *aead = __crypto_aead_cast(tfm); @@ -187,14 +92,6 @@ static int crypto_aead_init_tfm(struct crypto_tfm *tfm) struct crypto_aead *aead = __crypto_aead_cast(tfm); struct aead_alg *alg = crypto_aead_alg(aead); - if (crypto_old_aead_alg(aead)->encrypt) - return crypto_old_aead_init_tfm(tfm); - - aead->setkey = alg->setkey; - aead->setauthsize = alg->setauthsize; - aead->encrypt = alg->encrypt; - aead->decrypt = alg->decrypt; - aead->child = __crypto_aead_cast(tfm); aead->authsize = alg->maxauthsize; if (alg->exit) @@ -207,64 +104,6 @@ static int crypto_aead_init_tfm(struct crypto_tfm *tfm) } #ifdef CONFIG_NET -static int crypto_old_aead_report(struct sk_buff *skb, struct crypto_alg *alg) -{ - struct crypto_report_aead raead; - struct old_aead_alg *aead = &alg->cra_aead; - - strncpy(raead.type, "aead", sizeof(raead.type)); - strncpy(raead.geniv, aead->geniv ?: "<built-in>", sizeof(raead.geniv)); - - raead.blocksize = alg->cra_blocksize; - raead.maxauthsize = aead->maxauthsize; - raead.ivsize = aead->ivsize; - - if (nla_put(skb, CRYPTOCFGA_REPORT_AEAD, - sizeof(struct crypto_report_aead), &raead)) - goto nla_put_failure; - return 0; - -nla_put_failure: - return -EMSGSIZE; -} -#else -static int crypto_old_aead_report(struct sk_buff *skb, struct crypto_alg *alg) -{ - return -ENOSYS; -} -#endif - -static void crypto_old_aead_show(struct seq_file *m, struct crypto_alg *alg) - __attribute__ ((unused)); -static void crypto_old_aead_show(struct seq_file *m, struct crypto_alg *alg) -{ - struct old_aead_alg *aead = &alg->cra_aead; - - seq_printf(m, "type : aead\n"); - seq_printf(m, "async : %s\n", alg->cra_flags & CRYPTO_ALG_ASYNC ? - "yes" : "no"); - seq_printf(m, "blocksize : %u\n", alg->cra_blocksize); - seq_printf(m, "ivsize : %u\n", aead->ivsize); - seq_printf(m, "maxauthsize : %u\n", aead->maxauthsize); - seq_printf(m, "geniv : %s\n", aead->geniv ?: "<built-in>"); -} - -const struct crypto_type crypto_aead_type = { - .extsize = crypto_alg_extsize, - .init_tfm = crypto_aead_init_tfm, -#ifdef CONFIG_PROC_FS - .show = crypto_old_aead_show, -#endif - .report = crypto_old_aead_report, - .lookup = crypto_lookup_aead, - .maskclear = ~(CRYPTO_ALG_TYPE_MASK | CRYPTO_ALG_GENIV), - .maskset = CRYPTO_ALG_TYPE_MASK, - .type = CRYPTO_ALG_TYPE_AEAD, - .tfmsize = offsetof(struct crypto_aead, base), -}; -EXPORT_SYMBOL_GPL(crypto_aead_type); - -#ifdef CONFIG_NET static int crypto_aead_report(struct sk_buff *skb, struct crypto_alg *alg) { struct crypto_report_aead raead; @@ -307,93 +146,31 @@ static void crypto_aead_show(struct seq_file *m, struct crypto_alg *alg) seq_printf(m, "geniv : <none>\n"); } -static const struct crypto_type crypto_new_aead_type = { - .extsize = crypto_alg_extsize, - .init_tfm = crypto_aead_init_tfm, -#ifdef CONFIG_PROC_FS - .show = crypto_aead_show, -#endif - .report = crypto_aead_report, - .maskclear = ~CRYPTO_ALG_TYPE_MASK, - .maskset = CRYPTO_ALG_TYPE_MASK, - .type = CRYPTO_ALG_TYPE_AEAD, - .tfmsize = offsetof(struct crypto_aead, base), -}; - -static int aead_null_givencrypt(struct aead_givcrypt_request *req) -{ - return crypto_aead_encrypt(&req->areq); -} - -static int aead_null_givdecrypt(struct aead_givcrypt_request *req) -{ - return crypto_aead_decrypt(&req->areq); -} - -#ifdef CONFIG_NET -static int crypto_nivaead_report(struct sk_buff *skb, struct crypto_alg *alg) -{ - struct crypto_report_aead raead; - struct old_aead_alg *aead = &alg->cra_aead; - - strncpy(raead.type, "nivaead", sizeof(raead.type)); - strncpy(raead.geniv, aead->geniv, sizeof(raead.geniv)); - - raead.blocksize = alg->cra_blocksize; - raead.maxauthsize = aead->maxauthsize; - raead.ivsize = aead->ivsize; - - if (nla_put(skb, CRYPTOCFGA_REPORT_AEAD, - sizeof(struct crypto_report_aead), &raead)) - goto nla_put_failure; - return 0; - -nla_put_failure: - return -EMSGSIZE; -} -#else -static int crypto_nivaead_report(struct sk_buff *skb, struct crypto_alg *alg) +static void crypto_aead_free_instance(struct crypto_instance *inst) { - return -ENOSYS; -} -#endif - + struct aead_instance *aead = aead_instance(inst); -static void crypto_nivaead_show(struct seq_file *m, struct crypto_alg *alg) - __attribute__ ((unused)); -static void crypto_nivaead_show(struct seq_file *m, struct crypto_alg *alg) -{ - struct old_aead_alg *aead = &alg->cra_aead; + if (!aead->free) { + inst->tmpl->free(inst); + return; + } - seq_printf(m, "type : nivaead\n"); - seq_printf(m, "async : %s\n", alg->cra_flags & CRYPTO_ALG_ASYNC ? - "yes" : "no"); - seq_printf(m, "blocksize : %u\n", alg->cra_blocksize); - seq_printf(m, "ivsize : %u\n", aead->ivsize); - seq_printf(m, "maxauthsize : %u\n", aead->maxauthsize); - seq_printf(m, "geniv : %s\n", aead->geniv); + aead->free(aead); } -const struct crypto_type crypto_nivaead_type = { +static const struct crypto_type crypto_aead_type = { .extsize = crypto_alg_extsize, .init_tfm = crypto_aead_init_tfm, + .free = crypto_aead_free_instance, #ifdef CONFIG_PROC_FS - .show = crypto_nivaead_show, + .show = crypto_aead_show, #endif - .report = crypto_nivaead_report, - .maskclear = ~(CRYPTO_ALG_TYPE_MASK | CRYPTO_ALG_GENIV), - .maskset = CRYPTO_ALG_TYPE_MASK | CRYPTO_ALG_GENIV, + .report = crypto_aead_report, + .maskclear = ~CRYPTO_ALG_TYPE_MASK, + .maskset = CRYPTO_ALG_TYPE_MASK, .type = CRYPTO_ALG_TYPE_AEAD, .tfmsize = offsetof(struct crypto_aead, base), }; -EXPORT_SYMBOL_GPL(crypto_nivaead_type); - -static int crypto_grab_nivaead(struct crypto_aead_spawn *spawn, - const char *name, u32 type, u32 mask) -{ - spawn->base.frontend = &crypto_nivaead_type; - return crypto_grab_spawn(&spawn->base, name, type, mask); -} static int aead_geniv_setkey(struct crypto_aead *tfm, const u8 *key, unsigned int keylen) @@ -411,169 +188,6 @@ static int aead_geniv_setauthsize(struct crypto_aead *tfm, return crypto_aead_setauthsize(ctx->child, authsize); } -static void compat_encrypt_complete2(struct aead_request *req, int err) -{ - struct compat_request_ctx *rctx = aead_request_ctx(req); - struct aead_givcrypt_request *subreq = &rctx->subreq; - struct crypto_aead *geniv; - - if (err == -EINPROGRESS) - return; - - if (err) - goto out; - - geniv = crypto_aead_reqtfm(req); - scatterwalk_map_and_copy(subreq->giv, rctx->ivsg, 0, - crypto_aead_ivsize(geniv), 1); - -out: - kzfree(subreq->giv); -} - -static void compat_encrypt_complete(struct crypto_async_request *base, int err) -{ - struct aead_request *req = base->data; - - compat_encrypt_complete2(req, err); - aead_request_complete(req, err); -} - -static int compat_encrypt(struct aead_request *req) -{ - struct crypto_aead *geniv = crypto_aead_reqtfm(req); - struct aead_geniv_ctx *ctx = crypto_aead_ctx(geniv); - struct compat_request_ctx *rctx = aead_request_ctx(req); - struct aead_givcrypt_request *subreq = &rctx->subreq; - unsigned int ivsize = crypto_aead_ivsize(geniv); - struct scatterlist *src, *dst; - crypto_completion_t compl; - void *data; - u8 *info; - __be64 seq; - int err; - - if (req->cryptlen < ivsize) - return -EINVAL; - - compl = req->base.complete; - data = req->base.data; - - rctx->ivsg = scatterwalk_ffwd(rctx->ivbuf, req->dst, req->assoclen); - info = PageHighMem(sg_page(rctx->ivsg)) ? NULL : sg_virt(rctx->ivsg); - - if (!info) { - info = kmalloc(ivsize, req->base.flags & - CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL: - GFP_ATOMIC); - if (!info) - return -ENOMEM; - - compl = compat_encrypt_complete; - data = req; - } - - memcpy(&seq, req->iv + ivsize - sizeof(seq), sizeof(seq)); - - src = scatterwalk_ffwd(rctx->src, req->src, req->assoclen + ivsize); - dst = req->src == req->dst ? - src : scatterwalk_ffwd(rctx->dst, rctx->ivsg, ivsize); - - aead_givcrypt_set_tfm(subreq, ctx->child); - aead_givcrypt_set_callback(subreq, req->base.flags, - req->base.complete, req->base.data); - aead_givcrypt_set_crypt(subreq, src, dst, - req->cryptlen - ivsize, req->iv); - aead_givcrypt_set_assoc(subreq, req->src, req->assoclen); - aead_givcrypt_set_giv(subreq, info, be64_to_cpu(seq)); - - err = crypto_aead_givencrypt(subreq); - if (unlikely(PageHighMem(sg_page(rctx->ivsg)))) - compat_encrypt_complete2(req, err); - return err; -} - -static int compat_decrypt(struct aead_request *req) -{ - struct crypto_aead *geniv = crypto_aead_reqtfm(req); - struct aead_geniv_ctx *ctx = crypto_aead_ctx(geniv); - struct compat_request_ctx *rctx = aead_request_ctx(req); - struct aead_request *subreq = &rctx->subreq.areq; - unsigned int ivsize = crypto_aead_ivsize(geniv); - struct scatterlist *src, *dst; - crypto_completion_t compl; - void *data; - - if (req->cryptlen < ivsize) - return -EINVAL; - - aead_request_set_tfm(subreq, ctx->child); - - compl = req->base.complete; - data = req->base.data; - - src = scatterwalk_ffwd(rctx->src, req->src, req->assoclen + ivsize); - dst = req->src == req->dst ? - src : scatterwalk_ffwd(rctx->dst, req->dst, - req->assoclen + ivsize); - - aead_request_set_callback(subreq, req->base.flags, compl, data); - aead_request_set_crypt(subreq, src, dst, - req->cryptlen - ivsize, req->iv); - aead_request_set_assoc(subreq, req->src, req->assoclen); - - scatterwalk_map_and_copy(req->iv, req->src, req->assoclen, ivsize, 0); - - return crypto_aead_decrypt(subreq); -} - -static int compat_encrypt_first(struct aead_request *req) -{ - struct crypto_aead *geniv = crypto_aead_reqtfm(req); - struct aead_geniv_ctx *ctx = crypto_aead_ctx(geniv); - int err = 0; - - spin_lock_bh(&ctx->lock); - if (geniv->encrypt != compat_encrypt_first) - goto unlock; - - geniv->encrypt = compat_encrypt; - -unlock: - spin_unlock_bh(&ctx->lock); - - if (err) - return err; - - return compat_encrypt(req); -} - -static int aead_geniv_init_compat(struct crypto_tfm *tfm) -{ - struct crypto_aead *geniv = __crypto_aead_cast(tfm); - struct aead_geniv_ctx *ctx = crypto_aead_ctx(geniv); - int err; - - spin_lock_init(&ctx->lock); - - crypto_aead_set_reqsize(geniv, sizeof(struct compat_request_ctx)); - - err = aead_geniv_init(tfm); - - ctx->child = geniv->child; - geniv->child = geniv; - - return err; -} - -static void aead_geniv_exit_compat(struct crypto_tfm *tfm) -{ - struct crypto_aead *geniv = __crypto_aead_cast(tfm); - struct aead_geniv_ctx *ctx = crypto_aead_ctx(geniv); - - crypto_free_aead(ctx->child); -} - struct aead_instance *aead_geniv_alloc(struct crypto_template *tmpl, struct rtattr **tb, u32 type, u32 mask) { @@ -590,8 +204,7 @@ struct aead_instance *aead_geniv_alloc(struct crypto_template *tmpl, if (IS_ERR(algt)) return ERR_CAST(algt); - if ((algt->type ^ (CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_GENIV)) & - algt->mask) + if ((algt->type ^ CRYPTO_ALG_TYPE_AEAD) & algt->mask) return ERR_PTR(-EINVAL); name = crypto_attr_alg_name(tb[1]); @@ -608,9 +221,7 @@ struct aead_instance *aead_geniv_alloc(struct crypto_template *tmpl, mask |= crypto_requires_sync(algt->type, algt->mask); crypto_set_aead_spawn(spawn, aead_crypto_instance(inst)); - err = (algt->mask & CRYPTO_ALG_GENIV) ? - crypto_grab_nivaead(spawn, name, type, mask) : - crypto_grab_aead(spawn, name, type, mask); + err = crypto_grab_aead(spawn, name, type, mask); if (err) goto err_free_inst; @@ -623,43 +234,6 @@ struct aead_instance *aead_geniv_alloc(struct crypto_template *tmpl, if (ivsize < sizeof(u64)) goto err_drop_alg; - /* - * This is only true if we're constructing an algorithm with its - * default IV generator. For the default generator we elide the - * template name and double-check the IV generator. - */ - if (algt->mask & CRYPTO_ALG_GENIV) { - if (!alg->base.cra_aead.encrypt) - goto err_drop_alg; - if (strcmp(tmpl->name, alg->base.cra_aead.geniv)) - goto err_drop_alg; - - memcpy(inst->alg.base.cra_name, alg->base.cra_name, - CRYPTO_MAX_ALG_NAME); - memcpy(inst->alg.base.cra_driver_name, - alg->base.cra_driver_name, CRYPTO_MAX_ALG_NAME); - - inst->alg.base.cra_flags = CRYPTO_ALG_TYPE_AEAD | - CRYPTO_ALG_GENIV; - inst->alg.base.cra_flags |= alg->base.cra_flags & - CRYPTO_ALG_ASYNC; - inst->alg.base.cra_priority = alg->base.cra_priority; - inst->alg.base.cra_blocksize = alg->base.cra_blocksize; - inst->alg.base.cra_alignmask = alg->base.cra_alignmask; - inst->alg.base.cra_type = &crypto_aead_type; - - inst->alg.base.cra_aead.ivsize = ivsize; - inst->alg.base.cra_aead.maxauthsize = maxauthsize; - - inst->alg.base.cra_aead.setkey = alg->base.cra_aead.setkey; - inst->alg.base.cra_aead.setauthsize = - alg->base.cra_aead.setauthsize; - inst->alg.base.cra_aead.encrypt = alg->base.cra_aead.encrypt; - inst->alg.base.cra_aead.decrypt = alg->base.cra_aead.decrypt; - - goto out; - } - err = -ENAMETOOLONG; if (snprintf(inst->alg.base.cra_name, CRYPTO_MAX_ALG_NAME, "%s(%s)", tmpl->name, alg->base.cra_name) >= @@ -682,12 +256,6 @@ struct aead_instance *aead_geniv_alloc(struct crypto_template *tmpl, inst->alg.ivsize = ivsize; inst->alg.maxauthsize = maxauthsize; - inst->alg.encrypt = compat_encrypt_first; - inst->alg.decrypt = compat_decrypt; - - inst->alg.base.cra_init = aead_geniv_init_compat; - inst->alg.base.cra_exit = aead_geniv_exit_compat; - out: return inst; @@ -707,147 +275,58 @@ void aead_geniv_free(struct aead_instance *inst) } EXPORT_SYMBOL_GPL(aead_geniv_free); -int aead_geniv_init(struct crypto_tfm *tfm) +int aead_init_geniv(struct crypto_aead *aead) { - struct crypto_instance *inst = (void *)tfm->__crt_alg; + struct aead_geniv_ctx *ctx = crypto_aead_ctx(aead); + struct aead_instance *inst = aead_alg_instance(aead); struct crypto_aead *child; - struct crypto_aead *aead; - - aead = __crypto_aead_cast(tfm); - - child = crypto_spawn_aead(crypto_instance_ctx(inst)); - if (IS_ERR(child)) - return PTR_ERR(child); - - aead->child = child; - aead->reqsize += crypto_aead_reqsize(child); - - return 0; -} -EXPORT_SYMBOL_GPL(aead_geniv_init); - -void aead_geniv_exit(struct crypto_tfm *tfm) -{ - crypto_free_aead(__crypto_aead_cast(tfm)->child); -} -EXPORT_SYMBOL_GPL(aead_geniv_exit); - -static int crypto_nivaead_default(struct crypto_alg *alg, u32 type, u32 mask) -{ - struct rtattr *tb[3]; - struct { - struct rtattr attr; - struct crypto_attr_type data; - } ptype; - struct { - struct rtattr attr; - struct crypto_attr_alg data; - } palg; - struct crypto_template *tmpl; - struct crypto_instance *inst; - struct crypto_alg *larval; - const char *geniv; int err; - larval = crypto_larval_lookup(alg->cra_driver_name, - CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_GENIV, - CRYPTO_ALG_TYPE_MASK | CRYPTO_ALG_GENIV); - err = PTR_ERR(larval); - if (IS_ERR(larval)) - goto out; - - err = -EAGAIN; - if (!crypto_is_larval(larval)) - goto drop_larval; - - ptype.attr.rta_len = sizeof(ptype); - ptype.attr.rta_type = CRYPTOA_TYPE; - ptype.data.type = type | CRYPTO_ALG_GENIV; - /* GENIV tells the template that we're making a default geniv. */ - ptype.data.mask = mask | CRYPTO_ALG_GENIV; - tb[0] = &ptype.attr; - - palg.attr.rta_len = sizeof(palg); - palg.attr.rta_type = CRYPTOA_ALG; - /* Must use the exact name to locate ourselves. */ - memcpy(palg.data.name, alg->cra_driver_name, CRYPTO_MAX_ALG_NAME); - tb[1] = &palg.attr; - - tb[2] = NULL; + spin_lock_init(&ctx->lock); - geniv = alg->cra_aead.geniv; + err = crypto_get_default_rng(); + if (err) + goto out; - tmpl = crypto_lookup_template(geniv); - err = -ENOENT; - if (!tmpl) - goto kill_larval; + err = crypto_rng_get_bytes(crypto_default_rng, ctx->salt, + crypto_aead_ivsize(aead)); + crypto_put_default_rng(); + if (err) + goto out; - if (tmpl->create) { - err = tmpl->create(tmpl, tb); - if (err) - goto put_tmpl; - goto ok; - } + ctx->null = crypto_get_default_null_skcipher(); + err = PTR_ERR(ctx->null); + if (IS_ERR(ctx->null)) + goto out; - inst = tmpl->alloc(tb); - err = PTR_ERR(inst); - if (IS_ERR(inst)) - goto put_tmpl; + child = crypto_spawn_aead(aead_instance_ctx(inst)); + err = PTR_ERR(child); + if (IS_ERR(child)) + goto drop_null; - err = crypto_register_instance(tmpl, inst); - if (err) { - tmpl->free(inst); - goto put_tmpl; - } + ctx->child = child; + crypto_aead_set_reqsize(aead, crypto_aead_reqsize(child) + + sizeof(struct aead_request)); -ok: - /* Redo the lookup to use the instance we just registered. */ - err = -EAGAIN; + err = 0; -put_tmpl: - crypto_tmpl_put(tmpl); -kill_larval: - crypto_larval_kill(larval); -drop_larval: - crypto_mod_put(larval); out: - crypto_mod_put(alg); return err; + +drop_null: + crypto_put_default_null_skcipher(); + goto out; } +EXPORT_SYMBOL_GPL(aead_init_geniv); -struct crypto_alg *crypto_lookup_aead(const char *name, u32 type, u32 mask) +void aead_exit_geniv(struct crypto_aead *tfm) { - struct crypto_alg *alg; - - alg = crypto_alg_mod_lookup(name, type, mask); - if (IS_ERR(alg)) - return alg; - - if (alg->cra_type == &crypto_aead_type) - return alg; - - if (!alg->cra_aead.ivsize) - return alg; - - crypto_mod_put(alg); - alg = crypto_alg_mod_lookup(name, type | CRYPTO_ALG_TESTED, - mask & ~CRYPTO_ALG_TESTED); - if (IS_ERR(alg)) - return alg; - - if (alg->cra_type == &crypto_aead_type) { - if (~alg->cra_flags & (type ^ ~mask) & CRYPTO_ALG_TESTED) { - crypto_mod_put(alg); - alg = ERR_PTR(-ENOENT); - } - return alg; - } - - BUG_ON(!alg->cra_aead.ivsize); + struct aead_geniv_ctx *ctx = crypto_aead_ctx(tfm); - return ERR_PTR(crypto_nivaead_default(alg, type, mask)); + crypto_free_aead(ctx->child); + crypto_put_default_null_skcipher(); } -EXPORT_SYMBOL_GPL(crypto_lookup_aead); +EXPORT_SYMBOL_GPL(aead_exit_geniv); int crypto_grab_aead(struct crypto_aead_spawn *spawn, const char *name, u32 type, u32 mask) @@ -870,7 +349,7 @@ static int aead_prepare_alg(struct aead_alg *alg) if (max(alg->maxauthsize, alg->ivsize) > PAGE_SIZE / 8) return -EINVAL; - base->cra_type = &crypto_new_aead_type; + base->cra_type = &crypto_aead_type; base->cra_flags &= ~CRYPTO_ALG_TYPE_MASK; base->cra_flags |= CRYPTO_ALG_TYPE_AEAD; diff --git a/crypto/algapi.c b/crypto/algapi.c index 3c079b7f23f6..d130b41dbaea 100644 --- a/crypto/algapi.c +++ b/crypto/algapi.c @@ -67,12 +67,22 @@ static int crypto_check_alg(struct crypto_alg *alg) return crypto_set_driver_name(alg); } +static void crypto_free_instance(struct crypto_instance *inst) +{ + if (!inst->alg.cra_type->free) { + inst->tmpl->free(inst); + return; + } + + inst->alg.cra_type->free(inst); +} + static void crypto_destroy_instance(struct crypto_alg *alg) { struct crypto_instance *inst = (void *)alg; struct crypto_template *tmpl = inst->tmpl; - tmpl->free(inst); + crypto_free_instance(inst); crypto_tmpl_put(tmpl); } @@ -481,7 +491,7 @@ void crypto_unregister_template(struct crypto_template *tmpl) hlist_for_each_entry_safe(inst, n, list, list) { BUG_ON(atomic_read(&inst->alg.cra_refcnt) != 1); - tmpl->free(inst); + crypto_free_instance(inst); } crypto_remove_final(&users); } @@ -892,7 +902,7 @@ out: } EXPORT_SYMBOL_GPL(crypto_enqueue_request); -void *__crypto_dequeue_request(struct crypto_queue *queue, unsigned int offset) +struct crypto_async_request *crypto_dequeue_request(struct crypto_queue *queue) { struct list_head *request; @@ -907,14 +917,7 @@ void *__crypto_dequeue_request(struct crypto_queue *queue, unsigned int offset) request = queue->list.next; list_del(request); - return (char *)list_entry(request, struct crypto_async_request, list) - - offset; -} -EXPORT_SYMBOL_GPL(__crypto_dequeue_request); - -struct crypto_async_request *crypto_dequeue_request(struct crypto_queue *queue) -{ - return __crypto_dequeue_request(queue, 0); + return list_entry(request, struct crypto_async_request, list); } EXPORT_SYMBOL_GPL(crypto_dequeue_request); diff --git a/crypto/algboss.c b/crypto/algboss.c index 76fc0b23fc6c..6e39d9c05b98 100644 --- a/crypto/algboss.c +++ b/crypto/algboss.c @@ -248,13 +248,11 @@ static int cryptomgr_schedule_test(struct crypto_alg *alg) type = alg->cra_flags; /* This piece of crap needs to disappear into per-type test hooks. */ - if ((!((type ^ CRYPTO_ALG_TYPE_BLKCIPHER) & - CRYPTO_ALG_TYPE_BLKCIPHER_MASK) && !(type & CRYPTO_ALG_GENIV) && - ((alg->cra_flags & CRYPTO_ALG_TYPE_MASK) == - CRYPTO_ALG_TYPE_BLKCIPHER ? alg->cra_blkcipher.ivsize : - alg->cra_ablkcipher.ivsize)) || - (!((type ^ CRYPTO_ALG_TYPE_AEAD) & CRYPTO_ALG_TYPE_MASK) && - alg->cra_type == &crypto_nivaead_type && alg->cra_aead.ivsize)) + if (!((type ^ CRYPTO_ALG_TYPE_BLKCIPHER) & + CRYPTO_ALG_TYPE_BLKCIPHER_MASK) && !(type & CRYPTO_ALG_GENIV) && + ((alg->cra_flags & CRYPTO_ALG_TYPE_MASK) == + CRYPTO_ALG_TYPE_BLKCIPHER ? alg->cra_blkcipher.ivsize : + alg->cra_ablkcipher.ivsize)) type |= CRYPTO_ALG_TESTED; param->type = type; diff --git a/crypto/algif_aead.c b/crypto/algif_aead.c index e0408a480d2f..0aa6fdfb448a 100644 --- a/crypto/algif_aead.c +++ b/crypto/algif_aead.c @@ -90,6 +90,7 @@ static void aead_put_sgl(struct sock *sk) put_page(sg_page(sg + i)); sg_assign_page(sg + i, NULL); } + sg_init_table(sg, ALG_MAX_PAGES); sgl->cur = 0; ctx->used = 0; ctx->more = 0; @@ -514,8 +515,7 @@ static struct proto_ops algif_aead_ops = { static void *aead_bind(const char *name, u32 type, u32 mask) { - return crypto_alloc_aead(name, type | CRYPTO_ALG_AEAD_NEW, - mask | CRYPTO_ALG_AEAD_NEW); + return crypto_alloc_aead(name, type, mask); } static void aead_release(void *private) diff --git a/crypto/authenc.c b/crypto/authenc.c index 3e852299afb4..55a354d57251 100644 --- a/crypto/authenc.c +++ b/crypto/authenc.c @@ -1,7 +1,7 @@ /* * Authenc: Simple AEAD wrapper for IPsec * - * Copyright (c) 2007 Herbert Xu <herbert@gondor.apana.org.au> + * Copyright (c) 2007-2015 Herbert Xu <herbert@gondor.apana.org.au> * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the Free @@ -14,6 +14,7 @@ #include <crypto/internal/hash.h> #include <crypto/internal/skcipher.h> #include <crypto/authenc.h> +#include <crypto/null.h> #include <crypto/scatterwalk.h> #include <linux/err.h> #include <linux/init.h> @@ -23,26 +24,21 @@ #include <linux/slab.h> #include <linux/spinlock.h> -typedef u8 *(*authenc_ahash_t)(struct aead_request *req, unsigned int flags); - struct authenc_instance_ctx { struct crypto_ahash_spawn auth; struct crypto_skcipher_spawn enc; + unsigned int reqoff; }; struct crypto_authenc_ctx { - unsigned int reqoff; struct crypto_ahash *auth; struct crypto_ablkcipher *enc; + struct crypto_blkcipher *null; }; struct authenc_request_ctx { - unsigned int cryptlen; - struct scatterlist *sg; - struct scatterlist asg[2]; - struct scatterlist cipher[2]; - crypto_completion_t complete; - crypto_completion_t update_complete; + struct scatterlist src[2]; + struct scatterlist dst[2]; char tail[]; }; @@ -119,189 +115,35 @@ badkey: goto out; } -static void authenc_geniv_ahash_update_done(struct crypto_async_request *areq, - int err) -{ - struct aead_request *req = areq->data; - struct crypto_aead *authenc = crypto_aead_reqtfm(req); - struct crypto_authenc_ctx *ctx = crypto_aead_ctx(authenc); - struct authenc_request_ctx *areq_ctx = aead_request_ctx(req); - struct ahash_request *ahreq = (void *)(areq_ctx->tail + ctx->reqoff); - - if (err) - goto out; - - ahash_request_set_crypt(ahreq, areq_ctx->sg, ahreq->result, - areq_ctx->cryptlen); - ahash_request_set_callback(ahreq, aead_request_flags(req) & - CRYPTO_TFM_REQ_MAY_SLEEP, - areq_ctx->complete, req); - - err = crypto_ahash_finup(ahreq); - if (err) - goto out; - - scatterwalk_map_and_copy(ahreq->result, areq_ctx->sg, - areq_ctx->cryptlen, - crypto_aead_authsize(authenc), 1); - -out: - authenc_request_complete(req, err); -} - static void authenc_geniv_ahash_done(struct crypto_async_request *areq, int err) { struct aead_request *req = areq->data; struct crypto_aead *authenc = crypto_aead_reqtfm(req); - struct crypto_authenc_ctx *ctx = crypto_aead_ctx(authenc); + struct aead_instance *inst = aead_alg_instance(authenc); + struct authenc_instance_ctx *ictx = aead_instance_ctx(inst); struct authenc_request_ctx *areq_ctx = aead_request_ctx(req); - struct ahash_request *ahreq = (void *)(areq_ctx->tail + ctx->reqoff); + struct ahash_request *ahreq = (void *)(areq_ctx->tail + ictx->reqoff); if (err) goto out; - scatterwalk_map_and_copy(ahreq->result, areq_ctx->sg, - areq_ctx->cryptlen, + scatterwalk_map_and_copy(ahreq->result, req->dst, + req->assoclen + req->cryptlen, crypto_aead_authsize(authenc), 1); out: aead_request_complete(req, err); } -static void authenc_verify_ahash_update_done(struct crypto_async_request *areq, - int err) -{ - u8 *ihash; - unsigned int authsize; - struct ablkcipher_request *abreq; - struct aead_request *req = areq->data; - struct crypto_aead *authenc = crypto_aead_reqtfm(req); - struct crypto_authenc_ctx *ctx = crypto_aead_ctx(authenc); - struct authenc_request_ctx *areq_ctx = aead_request_ctx(req); - struct ahash_request *ahreq = (void *)(areq_ctx->tail + ctx->reqoff); - unsigned int cryptlen = req->cryptlen; - - if (err) - goto out; - - ahash_request_set_crypt(ahreq, areq_ctx->sg, ahreq->result, - areq_ctx->cryptlen); - ahash_request_set_callback(ahreq, aead_request_flags(req) & - CRYPTO_TFM_REQ_MAY_SLEEP, - areq_ctx->complete, req); - - err = crypto_ahash_finup(ahreq); - if (err) - goto out; - - authsize = crypto_aead_authsize(authenc); - cryptlen -= authsize; - ihash = ahreq->result + authsize; - scatterwalk_map_and_copy(ihash, areq_ctx->sg, areq_ctx->cryptlen, - authsize, 0); - - err = crypto_memneq(ihash, ahreq->result, authsize) ? -EBADMSG : 0; - if (err) - goto out; - - abreq = aead_request_ctx(req); - ablkcipher_request_set_tfm(abreq, ctx->enc); - ablkcipher_request_set_callback(abreq, aead_request_flags(req), - req->base.complete, req->base.data); - ablkcipher_request_set_crypt(abreq, req->src, req->dst, - cryptlen, req->iv); - - err = crypto_ablkcipher_decrypt(abreq); - -out: - authenc_request_complete(req, err); -} - -static void authenc_verify_ahash_done(struct crypto_async_request *areq, - int err) -{ - u8 *ihash; - unsigned int authsize; - struct ablkcipher_request *abreq; - struct aead_request *req = areq->data; - struct crypto_aead *authenc = crypto_aead_reqtfm(req); - struct crypto_authenc_ctx *ctx = crypto_aead_ctx(authenc); - struct authenc_request_ctx *areq_ctx = aead_request_ctx(req); - struct ahash_request *ahreq = (void *)(areq_ctx->tail + ctx->reqoff); - unsigned int cryptlen = req->cryptlen; - - if (err) - goto out; - - authsize = crypto_aead_authsize(authenc); - cryptlen -= authsize; - ihash = ahreq->result + authsize; - scatterwalk_map_and_copy(ihash, areq_ctx->sg, areq_ctx->cryptlen, - authsize, 0); - - err = crypto_memneq(ihash, ahreq->result, authsize) ? -EBADMSG : 0; - if (err) - goto out; - - abreq = aead_request_ctx(req); - ablkcipher_request_set_tfm(abreq, ctx->enc); - ablkcipher_request_set_callback(abreq, aead_request_flags(req), - req->base.complete, req->base.data); - ablkcipher_request_set_crypt(abreq, req->src, req->dst, - cryptlen, req->iv); - - err = crypto_ablkcipher_decrypt(abreq); - -out: - authenc_request_complete(req, err); -} - -static u8 *crypto_authenc_ahash_fb(struct aead_request *req, unsigned int flags) -{ - struct crypto_aead *authenc = crypto_aead_reqtfm(req); - struct crypto_authenc_ctx *ctx = crypto_aead_ctx(authenc); - struct crypto_ahash *auth = ctx->auth; - struct authenc_request_ctx *areq_ctx = aead_request_ctx(req); - struct ahash_request *ahreq = (void *)(areq_ctx->tail + ctx->reqoff); - u8 *hash = areq_ctx->tail; - int err; - - hash = (u8 *)ALIGN((unsigned long)hash + crypto_ahash_alignmask(auth), - crypto_ahash_alignmask(auth) + 1); - - ahash_request_set_tfm(ahreq, auth); - - err = crypto_ahash_init(ahreq); - if (err) - return ERR_PTR(err); - - ahash_request_set_crypt(ahreq, req->assoc, hash, req->assoclen); - ahash_request_set_callback(ahreq, aead_request_flags(req) & flags, - areq_ctx->update_complete, req); - - err = crypto_ahash_update(ahreq); - if (err) - return ERR_PTR(err); - - ahash_request_set_crypt(ahreq, areq_ctx->sg, hash, - areq_ctx->cryptlen); - ahash_request_set_callback(ahreq, aead_request_flags(req) & flags, - areq_ctx->complete, req); - - err = crypto_ahash_finup(ahreq); - if (err) - return ERR_PTR(err); - - return hash; -} - -static u8 *crypto_authenc_ahash(struct aead_request *req, unsigned int flags) +static int crypto_authenc_genicv(struct aead_request *req, unsigned int flags) { struct crypto_aead *authenc = crypto_aead_reqtfm(req); + struct aead_instance *inst = aead_alg_instance(authenc); struct crypto_authenc_ctx *ctx = crypto_aead_ctx(authenc); + struct authenc_instance_ctx *ictx = aead_instance_ctx(inst); struct crypto_ahash *auth = ctx->auth; struct authenc_request_ctx *areq_ctx = aead_request_ctx(req); - struct ahash_request *ahreq = (void *)(areq_ctx->tail + ctx->reqoff); + struct ahash_request *ahreq = (void *)(areq_ctx->tail + ictx->reqoff); u8 *hash = areq_ctx->tail; int err; @@ -309,66 +151,18 @@ static u8 *crypto_authenc_ahash(struct aead_request *req, unsigned int flags) crypto_ahash_alignmask(auth) + 1); ahash_request_set_tfm(ahreq, auth); - ahash_request_set_crypt(ahreq, areq_ctx->sg, hash, - areq_ctx->cryptlen); - ahash_request_set_callback(ahreq, aead_request_flags(req) & flags, - areq_ctx->complete, req); + ahash_request_set_crypt(ahreq, req->dst, hash, + req->assoclen + req->cryptlen); + ahash_request_set_callback(ahreq, flags, + authenc_geniv_ahash_done, req); err = crypto_ahash_digest(ahreq); if (err) - return ERR_PTR(err); - - return hash; -} - -static int crypto_authenc_genicv(struct aead_request *req, u8 *iv, - unsigned int flags) -{ - struct crypto_aead *authenc = crypto_aead_reqtfm(req); - struct authenc_request_ctx *areq_ctx = aead_request_ctx(req); - struct scatterlist *dst = req->dst; - struct scatterlist *assoc = req->assoc; - struct scatterlist *cipher = areq_ctx->cipher; - struct scatterlist *asg = areq_ctx->asg; - unsigned int ivsize = crypto_aead_ivsize(authenc); - unsigned int cryptlen = req->cryptlen; - authenc_ahash_t authenc_ahash_fn = crypto_authenc_ahash_fb; - struct page *dstp; - u8 *vdst; - u8 *hash; - - dstp = sg_page(dst); - vdst = PageHighMem(dstp) ? NULL : page_address(dstp) + dst->offset; - - if (ivsize) { - sg_init_table(cipher, 2); - sg_set_buf(cipher, iv, ivsize); - scatterwalk_crypto_chain(cipher, dst, vdst == iv + ivsize, 2); - dst = cipher; - cryptlen += ivsize; - } - - if (req->assoclen && sg_is_last(assoc)) { - authenc_ahash_fn = crypto_authenc_ahash; - sg_init_table(asg, 2); - sg_set_page(asg, sg_page(assoc), assoc->length, assoc->offset); - scatterwalk_crypto_chain(asg, dst, 0, 2); - dst = asg; - cryptlen += req->assoclen; - } - - areq_ctx->cryptlen = cryptlen; - areq_ctx->sg = dst; - - areq_ctx->complete = authenc_geniv_ahash_done; - areq_ctx->update_complete = authenc_geniv_ahash_update_done; - - hash = authenc_ahash_fn(req, flags); - if (IS_ERR(hash)) - return PTR_ERR(hash); + return err; - scatterwalk_map_and_copy(hash, dst, cryptlen, + scatterwalk_map_and_copy(hash, req->dst, req->assoclen + req->cryptlen, crypto_aead_authsize(authenc), 1); + return 0; } @@ -377,180 +171,155 @@ static void crypto_authenc_encrypt_done(struct crypto_async_request *req, { struct aead_request *areq = req->data; - if (!err) { - struct crypto_aead *authenc = crypto_aead_reqtfm(areq); - struct crypto_authenc_ctx *ctx = crypto_aead_ctx(authenc); - struct authenc_request_ctx *areq_ctx = aead_request_ctx(areq); - struct ablkcipher_request *abreq = (void *)(areq_ctx->tail - + ctx->reqoff); - u8 *iv = (u8 *)abreq - crypto_ablkcipher_ivsize(ctx->enc); + if (err) + goto out; - err = crypto_authenc_genicv(areq, iv, 0); - } + err = crypto_authenc_genicv(areq, 0); +out: authenc_request_complete(areq, err); } +static int crypto_authenc_copy_assoc(struct aead_request *req) +{ + struct crypto_aead *authenc = crypto_aead_reqtfm(req); + struct crypto_authenc_ctx *ctx = crypto_aead_ctx(authenc); + struct blkcipher_desc desc = { + .tfm = ctx->null, + }; + + return crypto_blkcipher_encrypt(&desc, req->dst, req->src, + req->assoclen); +} + static int crypto_authenc_encrypt(struct aead_request *req) { struct crypto_aead *authenc = crypto_aead_reqtfm(req); + struct aead_instance *inst = aead_alg_instance(authenc); struct crypto_authenc_ctx *ctx = crypto_aead_ctx(authenc); + struct authenc_instance_ctx *ictx = aead_instance_ctx(inst); struct authenc_request_ctx *areq_ctx = aead_request_ctx(req); struct crypto_ablkcipher *enc = ctx->enc; - struct scatterlist *dst = req->dst; unsigned int cryptlen = req->cryptlen; - struct ablkcipher_request *abreq = (void *)(areq_ctx->tail - + ctx->reqoff); - u8 *iv = (u8 *)abreq - crypto_ablkcipher_ivsize(enc); + struct ablkcipher_request *abreq = (void *)(areq_ctx->tail + + ictx->reqoff); + struct scatterlist *src, *dst; int err; + sg_init_table(areq_ctx->src, 2); + src = scatterwalk_ffwd(areq_ctx->src, req->src, req->assoclen); + dst = src; + + if (req->src != req->dst) { + err = crypto_authenc_copy_assoc(req); + if (err) + return err; + + sg_init_table(areq_ctx->dst, 2); + dst = scatterwalk_ffwd(areq_ctx->dst, req->dst, req->assoclen); + } + ablkcipher_request_set_tfm(abreq, enc); ablkcipher_request_set_callback(abreq, aead_request_flags(req), crypto_authenc_encrypt_done, req); - ablkcipher_request_set_crypt(abreq, req->src, dst, cryptlen, req->iv); - - memcpy(iv, req->iv, crypto_aead_ivsize(authenc)); + ablkcipher_request_set_crypt(abreq, src, dst, cryptlen, req->iv); err = crypto_ablkcipher_encrypt(abreq); if (err) return err; - return crypto_authenc_genicv(req, iv, CRYPTO_TFM_REQ_MAY_SLEEP); + return crypto_authenc_genicv(req, aead_request_flags(req)); } -static void crypto_authenc_givencrypt_done(struct crypto_async_request *req, - int err) +static int crypto_authenc_decrypt_tail(struct aead_request *req, + unsigned int flags) { - struct aead_request *areq = req->data; - - if (!err) { - struct skcipher_givcrypt_request *greq = aead_request_ctx(areq); - - err = crypto_authenc_genicv(areq, greq->giv, 0); - } + struct crypto_aead *authenc = crypto_aead_reqtfm(req); + struct aead_instance *inst = aead_alg_instance(authenc); + struct crypto_authenc_ctx *ctx = crypto_aead_ctx(authenc); + struct authenc_instance_ctx *ictx = aead_instance_ctx(inst); + struct authenc_request_ctx *areq_ctx = aead_request_ctx(req); + struct ahash_request *ahreq = (void *)(areq_ctx->tail + ictx->reqoff); + struct ablkcipher_request *abreq = (void *)(areq_ctx->tail + + ictx->reqoff); + unsigned int authsize = crypto_aead_authsize(authenc); + u8 *ihash = ahreq->result + authsize; + struct scatterlist *src, *dst; - authenc_request_complete(areq, err); -} + scatterwalk_map_and_copy(ihash, req->src, ahreq->nbytes, authsize, 0); -static int crypto_authenc_givencrypt(struct aead_givcrypt_request *req) -{ - struct crypto_aead *authenc = aead_givcrypt_reqtfm(req); - struct crypto_authenc_ctx *ctx = crypto_aead_ctx(authenc); - struct aead_request *areq = &req->areq; - struct skcipher_givcrypt_request *greq = aead_request_ctx(areq); - u8 *iv = req->giv; - int err; + if (crypto_memneq(ihash, ahreq->result, authsize)) + return -EBADMSG; - skcipher_givcrypt_set_tfm(greq, ctx->enc); - skcipher_givcrypt_set_callback(greq, aead_request_flags(areq), - crypto_authenc_givencrypt_done, areq); - skcipher_givcrypt_set_crypt(greq, areq->src, areq->dst, areq->cryptlen, - areq->iv); - skcipher_givcrypt_set_giv(greq, iv, req->seq); + sg_init_table(areq_ctx->src, 2); + src = scatterwalk_ffwd(areq_ctx->src, req->src, req->assoclen); + dst = src; - err = crypto_skcipher_givencrypt(greq); - if (err) - return err; + if (req->src != req->dst) { + sg_init_table(areq_ctx->dst, 2); + dst = scatterwalk_ffwd(areq_ctx->dst, req->dst, req->assoclen); + } - return crypto_authenc_genicv(areq, iv, CRYPTO_TFM_REQ_MAY_SLEEP); -} + ablkcipher_request_set_tfm(abreq, ctx->enc); + ablkcipher_request_set_callback(abreq, aead_request_flags(req), + req->base.complete, req->base.data); + ablkcipher_request_set_crypt(abreq, src, dst, + req->cryptlen - authsize, req->iv); -static int crypto_authenc_verify(struct aead_request *req, - authenc_ahash_t authenc_ahash_fn) -{ - struct crypto_aead *authenc = crypto_aead_reqtfm(req); - struct authenc_request_ctx *areq_ctx = aead_request_ctx(req); - u8 *ohash; - u8 *ihash; - unsigned int authsize; - - areq_ctx->complete = authenc_verify_ahash_done; - areq_ctx->update_complete = authenc_verify_ahash_update_done; - - ohash = authenc_ahash_fn(req, CRYPTO_TFM_REQ_MAY_SLEEP); - if (IS_ERR(ohash)) - return PTR_ERR(ohash); - - authsize = crypto_aead_authsize(authenc); - ihash = ohash + authsize; - scatterwalk_map_and_copy(ihash, areq_ctx->sg, areq_ctx->cryptlen, - authsize, 0); - return crypto_memneq(ihash, ohash, authsize) ? -EBADMSG : 0; + return crypto_ablkcipher_decrypt(abreq); } -static int crypto_authenc_iverify(struct aead_request *req, u8 *iv, - unsigned int cryptlen) +static void authenc_verify_ahash_done(struct crypto_async_request *areq, + int err) { - struct crypto_aead *authenc = crypto_aead_reqtfm(req); - struct authenc_request_ctx *areq_ctx = aead_request_ctx(req); - struct scatterlist *src = req->src; - struct scatterlist *assoc = req->assoc; - struct scatterlist *cipher = areq_ctx->cipher; - struct scatterlist *asg = areq_ctx->asg; - unsigned int ivsize = crypto_aead_ivsize(authenc); - authenc_ahash_t authenc_ahash_fn = crypto_authenc_ahash_fb; - struct page *srcp; - u8 *vsrc; - - srcp = sg_page(src); - vsrc = PageHighMem(srcp) ? NULL : page_address(srcp) + src->offset; - - if (ivsize) { - sg_init_table(cipher, 2); - sg_set_buf(cipher, iv, ivsize); - scatterwalk_crypto_chain(cipher, src, vsrc == iv + ivsize, 2); - src = cipher; - cryptlen += ivsize; - } + struct aead_request *req = areq->data; - if (req->assoclen && sg_is_last(assoc)) { - authenc_ahash_fn = crypto_authenc_ahash; - sg_init_table(asg, 2); - sg_set_page(asg, sg_page(assoc), assoc->length, assoc->offset); - scatterwalk_crypto_chain(asg, src, 0, 2); - src = asg; - cryptlen += req->assoclen; - } + if (err) + goto out; - areq_ctx->cryptlen = cryptlen; - areq_ctx->sg = src; + err = crypto_authenc_decrypt_tail(req, 0); - return crypto_authenc_verify(req, authenc_ahash_fn); +out: + authenc_request_complete(req, err); } static int crypto_authenc_decrypt(struct aead_request *req) { struct crypto_aead *authenc = crypto_aead_reqtfm(req); - struct crypto_authenc_ctx *ctx = crypto_aead_ctx(authenc); - struct ablkcipher_request *abreq = aead_request_ctx(req); - unsigned int cryptlen = req->cryptlen; unsigned int authsize = crypto_aead_authsize(authenc); - u8 *iv = req->iv; + struct aead_instance *inst = aead_alg_instance(authenc); + struct crypto_authenc_ctx *ctx = crypto_aead_ctx(authenc); + struct authenc_instance_ctx *ictx = aead_instance_ctx(inst); + struct crypto_ahash *auth = ctx->auth; + struct authenc_request_ctx *areq_ctx = aead_request_ctx(req); + struct ahash_request *ahreq = (void *)(areq_ctx->tail + ictx->reqoff); + u8 *hash = areq_ctx->tail; int err; - if (cryptlen < authsize) - return -EINVAL; - cryptlen -= authsize; + hash = (u8 *)ALIGN((unsigned long)hash + crypto_ahash_alignmask(auth), + crypto_ahash_alignmask(auth) + 1); - err = crypto_authenc_iverify(req, iv, cryptlen); + ahash_request_set_tfm(ahreq, auth); + ahash_request_set_crypt(ahreq, req->src, hash, + req->assoclen + req->cryptlen - authsize); + ahash_request_set_callback(ahreq, aead_request_flags(req), + authenc_verify_ahash_done, req); + + err = crypto_ahash_digest(ahreq); if (err) return err; - ablkcipher_request_set_tfm(abreq, ctx->enc); - ablkcipher_request_set_callback(abreq, aead_request_flags(req), - req->base.complete, req->base.data); - ablkcipher_request_set_crypt(abreq, req->src, req->dst, cryptlen, iv); - - return crypto_ablkcipher_decrypt(abreq); + return crypto_authenc_decrypt_tail(req, aead_request_flags(req)); } -static int crypto_authenc_init_tfm(struct crypto_tfm *tfm) +static int crypto_authenc_init_tfm(struct crypto_aead *tfm) { - struct crypto_instance *inst = crypto_tfm_alg_instance(tfm); - struct authenc_instance_ctx *ictx = crypto_instance_ctx(inst); - struct crypto_authenc_ctx *ctx = crypto_tfm_ctx(tfm); + struct aead_instance *inst = aead_alg_instance(tfm); + struct authenc_instance_ctx *ictx = aead_instance_ctx(inst); + struct crypto_authenc_ctx *ctx = crypto_aead_ctx(tfm); struct crypto_ahash *auth; struct crypto_ablkcipher *enc; + struct crypto_blkcipher *null; int err; auth = crypto_spawn_ahash(&ictx->auth); @@ -562,42 +331,57 @@ static int crypto_authenc_init_tfm(struct crypto_tfm *tfm) if (IS_ERR(enc)) goto err_free_ahash; + null = crypto_get_default_null_skcipher(); + err = PTR_ERR(null); + if (IS_ERR(null)) + goto err_free_skcipher; + ctx->auth = auth; ctx->enc = enc; + ctx->null = null; - ctx->reqoff = ALIGN(2 * crypto_ahash_digestsize(auth) + - crypto_ahash_alignmask(auth), - crypto_ahash_alignmask(auth) + 1) + - crypto_ablkcipher_ivsize(enc); - - crypto_aead_set_reqsize(__crypto_aead_cast(tfm), + crypto_aead_set_reqsize( + tfm, sizeof(struct authenc_request_ctx) + - ctx->reqoff + + ictx->reqoff + max_t(unsigned int, - crypto_ahash_reqsize(auth) + - sizeof(struct ahash_request), - sizeof(struct skcipher_givcrypt_request) + - crypto_ablkcipher_reqsize(enc))); + crypto_ahash_reqsize(auth) + + sizeof(struct ahash_request), + sizeof(struct ablkcipher_request) + + crypto_ablkcipher_reqsize(enc))); return 0; +err_free_skcipher: + crypto_free_ablkcipher(enc); err_free_ahash: crypto_free_ahash(auth); return err; } -static void crypto_authenc_exit_tfm(struct crypto_tfm *tfm) +static void crypto_authenc_exit_tfm(struct crypto_aead *tfm) { - struct crypto_authenc_ctx *ctx = crypto_tfm_ctx(tfm); + struct crypto_authenc_ctx *ctx = crypto_aead_ctx(tfm); crypto_free_ahash(ctx->auth); crypto_free_ablkcipher(ctx->enc); + crypto_put_default_null_skcipher(); } -static struct crypto_instance *crypto_authenc_alloc(struct rtattr **tb) +static void crypto_authenc_free(struct aead_instance *inst) +{ + struct authenc_instance_ctx *ctx = aead_instance_ctx(inst); + + crypto_drop_skcipher(&ctx->enc); + crypto_drop_ahash(&ctx->auth); + kfree(inst); +} + +static int crypto_authenc_create(struct crypto_template *tmpl, + struct rtattr **tb) { struct crypto_attr_type *algt; - struct crypto_instance *inst; + struct aead_instance *inst; struct hash_alg_common *auth; struct crypto_alg *auth_base; struct crypto_alg *enc; @@ -607,15 +391,15 @@ static struct crypto_instance *crypto_authenc_alloc(struct rtattr **tb) algt = crypto_get_attr_type(tb); if (IS_ERR(algt)) - return ERR_CAST(algt); + return PTR_ERR(algt); if ((algt->type ^ CRYPTO_ALG_TYPE_AEAD) & algt->mask) - return ERR_PTR(-EINVAL); + return -EINVAL; auth = ahash_attr_alg(tb[1], CRYPTO_ALG_TYPE_HASH, CRYPTO_ALG_TYPE_AHASH_MASK); if (IS_ERR(auth)) - return ERR_CAST(auth); + return PTR_ERR(auth); auth_base = &auth->base; @@ -629,13 +413,14 @@ static struct crypto_instance *crypto_authenc_alloc(struct rtattr **tb) if (!inst) goto out_put_auth; - ctx = crypto_instance_ctx(inst); + ctx = aead_instance_ctx(inst); - err = crypto_init_ahash_spawn(&ctx->auth, auth, inst); + err = crypto_init_ahash_spawn(&ctx->auth, auth, + aead_crypto_instance(inst)); if (err) goto err_free_inst; - crypto_set_skcipher_spawn(&ctx->enc, inst); + crypto_set_skcipher_spawn(&ctx->enc, aead_crypto_instance(inst)); err = crypto_grab_skcipher(&ctx->enc, enc_name, 0, crypto_requires_sync(algt->type, algt->mask)); @@ -644,41 +429,47 @@ static struct crypto_instance *crypto_authenc_alloc(struct rtattr **tb) enc = crypto_skcipher_spawn_alg(&ctx->enc); + ctx->reqoff = ALIGN(2 * auth->digestsize + auth_base->cra_alignmask, + auth_base->cra_alignmask + 1); + err = -ENAMETOOLONG; - if (snprintf(inst->alg.cra_name, CRYPTO_MAX_ALG_NAME, + if (snprintf(inst->alg.base.cra_name, CRYPTO_MAX_ALG_NAME, "authenc(%s,%s)", auth_base->cra_name, enc->cra_name) >= CRYPTO_MAX_ALG_NAME) goto err_drop_enc; - if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME, + if (snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME, "authenc(%s,%s)", auth_base->cra_driver_name, enc->cra_driver_name) >= CRYPTO_MAX_ALG_NAME) goto err_drop_enc; - inst->alg.cra_flags = CRYPTO_ALG_TYPE_AEAD; - inst->alg.cra_flags |= enc->cra_flags & CRYPTO_ALG_ASYNC; - inst->alg.cra_priority = enc->cra_priority * - 10 + auth_base->cra_priority; - inst->alg.cra_blocksize = enc->cra_blocksize; - inst->alg.cra_alignmask = auth_base->cra_alignmask | enc->cra_alignmask; - inst->alg.cra_type = &crypto_aead_type; + inst->alg.base.cra_flags = enc->cra_flags & CRYPTO_ALG_ASYNC; + inst->alg.base.cra_priority = enc->cra_priority * 10 + + auth_base->cra_priority; + inst->alg.base.cra_blocksize = enc->cra_blocksize; + inst->alg.base.cra_alignmask = auth_base->cra_alignmask | + enc->cra_alignmask; + inst->alg.base.cra_ctxsize = sizeof(struct crypto_authenc_ctx); + + inst->alg.ivsize = enc->cra_ablkcipher.ivsize; + inst->alg.maxauthsize = auth->digestsize; - inst->alg.cra_aead.ivsize = enc->cra_ablkcipher.ivsize; - inst->alg.cra_aead.maxauthsize = auth->digestsize; + inst->alg.init = crypto_authenc_init_tfm; + inst->alg.exit = crypto_authenc_exit_tfm; - inst->alg.cra_ctxsize = sizeof(struct crypto_authenc_ctx); + inst->alg.setkey = crypto_authenc_setkey; + inst->alg.encrypt = crypto_authenc_encrypt; + inst->alg.decrypt = crypto_authenc_decrypt; - inst->alg.cra_init = crypto_authenc_init_tfm; - inst->alg.cra_exit = crypto_authenc_exit_tfm; + inst->free = crypto_authenc_free; - inst->alg.cra_aead.setkey = crypto_authenc_setkey; - inst->alg.cra_aead.encrypt = crypto_authenc_encrypt; - inst->alg.cra_aead.decrypt = crypto_authenc_decrypt; - inst->alg.cra_aead.givencrypt = crypto_authenc_givencrypt; + err = aead_register_instance(tmpl, inst); + if (err) + goto err_drop_enc; out: crypto_mod_put(auth_base); - return inst; + return err; err_drop_enc: crypto_drop_skcipher(&ctx->enc); @@ -687,23 +478,12 @@ err_drop_auth: err_free_inst: kfree(inst); out_put_auth: - inst = ERR_PTR(err); goto out; } -static void crypto_authenc_free(struct crypto_instance *inst) -{ - struct authenc_instance_ctx *ctx = crypto_instance_ctx(inst); - - crypto_drop_skcipher(&ctx->enc); - crypto_drop_ahash(&ctx->auth); - kfree(inst); -} - static struct crypto_template crypto_authenc_tmpl = { .name = "authenc", - .alloc = crypto_authenc_alloc, - .free = crypto_authenc_free, + .create = crypto_authenc_create, .module = THIS_MODULE, }; diff --git a/crypto/authencesn.c b/crypto/authencesn.c index b8efe36ce114..0c0468869e25 100644 --- a/crypto/authencesn.c +++ b/crypto/authencesn.c @@ -4,6 +4,7 @@ * * Copyright (C) 2010 secunet Security Networks AG * Copyright (C) 2010 Steffen Klassert <steffen.klassert@secunet.com> + * Copyright (c) 2015 Herbert Xu <herbert@gondor.apana.org.au> * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the Free @@ -16,6 +17,7 @@ #include <crypto/internal/hash.h> #include <crypto/internal/skcipher.h> #include <crypto/authenc.h> +#include <crypto/null.h> #include <crypto/scatterwalk.h> #include <linux/err.h> #include <linux/init.h> @@ -34,19 +36,12 @@ struct crypto_authenc_esn_ctx { unsigned int reqoff; struct crypto_ahash *auth; struct crypto_ablkcipher *enc; + struct crypto_blkcipher *null; }; struct authenc_esn_request_ctx { - unsigned int cryptlen; - unsigned int headlen; - unsigned int trailen; - struct scatterlist *sg; - struct scatterlist hsg[2]; - struct scatterlist tsg[1]; - struct scatterlist cipher[2]; - crypto_completion_t complete; - crypto_completion_t update_complete; - crypto_completion_t update_complete2; + struct scatterlist src[2]; + struct scatterlist dst[2]; char tail[]; }; @@ -56,6 +51,15 @@ static void authenc_esn_request_complete(struct aead_request *req, int err) aead_request_complete(req, err); } +static int crypto_authenc_esn_setauthsize(struct crypto_aead *authenc_esn, + unsigned int authsize) +{ + if (authsize > 0 && authsize < 4) + return -EINVAL; + + return 0; +} + static int crypto_authenc_esn_setkey(struct crypto_aead *authenc_esn, const u8 *key, unsigned int keylen) { @@ -93,536 +97,242 @@ badkey: goto out; } -static void authenc_esn_geniv_ahash_update_done(struct crypto_async_request *areq, - int err) +static int crypto_authenc_esn_genicv_tail(struct aead_request *req, + unsigned int flags) { - struct aead_request *req = areq->data; struct crypto_aead *authenc_esn = crypto_aead_reqtfm(req); struct crypto_authenc_esn_ctx *ctx = crypto_aead_ctx(authenc_esn); struct authenc_esn_request_ctx *areq_ctx = aead_request_ctx(req); - struct ahash_request *ahreq = (void *)(areq_ctx->tail + ctx->reqoff); - - if (err) - goto out; - - ahash_request_set_crypt(ahreq, areq_ctx->sg, ahreq->result, - areq_ctx->cryptlen); - ahash_request_set_callback(ahreq, aead_request_flags(req) & - CRYPTO_TFM_REQ_MAY_SLEEP, - areq_ctx->update_complete2, req); - - err = crypto_ahash_update(ahreq); - if (err) - goto out; - - ahash_request_set_crypt(ahreq, areq_ctx->tsg, ahreq->result, - areq_ctx->trailen); - ahash_request_set_callback(ahreq, aead_request_flags(req) & - CRYPTO_TFM_REQ_MAY_SLEEP, - areq_ctx->complete, req); - - err = crypto_ahash_finup(ahreq); - if (err) - goto out; - - scatterwalk_map_and_copy(ahreq->result, areq_ctx->sg, - areq_ctx->cryptlen, - crypto_aead_authsize(authenc_esn), 1); - -out: - authenc_esn_request_complete(req, err); -} - -static void authenc_esn_geniv_ahash_update_done2(struct crypto_async_request *areq, - int err) -{ - struct aead_request *req = areq->data; - struct crypto_aead *authenc_esn = crypto_aead_reqtfm(req); - struct crypto_authenc_esn_ctx *ctx = crypto_aead_ctx(authenc_esn); - struct authenc_esn_request_ctx *areq_ctx = aead_request_ctx(req); - struct ahash_request *ahreq = (void *)(areq_ctx->tail + ctx->reqoff); - - if (err) - goto out; - - ahash_request_set_crypt(ahreq, areq_ctx->tsg, ahreq->result, - areq_ctx->trailen); - ahash_request_set_callback(ahreq, aead_request_flags(req) & - CRYPTO_TFM_REQ_MAY_SLEEP, - areq_ctx->complete, req); - - err = crypto_ahash_finup(ahreq); - if (err) - goto out; + struct crypto_ahash *auth = ctx->auth; + u8 *hash = PTR_ALIGN((u8 *)areq_ctx->tail, + crypto_ahash_alignmask(auth) + 1); + unsigned int authsize = crypto_aead_authsize(authenc_esn); + unsigned int assoclen = req->assoclen; + unsigned int cryptlen = req->cryptlen; + struct scatterlist *dst = req->dst; + u32 tmp[2]; - scatterwalk_map_and_copy(ahreq->result, areq_ctx->sg, - areq_ctx->cryptlen, - crypto_aead_authsize(authenc_esn), 1); + /* Move high-order bits of sequence number back. */ + scatterwalk_map_and_copy(tmp, dst, 4, 4, 0); + scatterwalk_map_and_copy(tmp + 1, dst, assoclen + cryptlen, 4, 0); + scatterwalk_map_and_copy(tmp, dst, 0, 8, 1); -out: - authenc_esn_request_complete(req, err); + scatterwalk_map_and_copy(hash, dst, assoclen + cryptlen, authsize, 1); + return 0; } - static void authenc_esn_geniv_ahash_done(struct crypto_async_request *areq, int err) { struct aead_request *req = areq->data; - struct crypto_aead *authenc_esn = crypto_aead_reqtfm(req); - struct crypto_authenc_esn_ctx *ctx = crypto_aead_ctx(authenc_esn); - struct authenc_esn_request_ctx *areq_ctx = aead_request_ctx(req); - struct ahash_request *ahreq = (void *)(areq_ctx->tail + ctx->reqoff); - if (err) - goto out; - - scatterwalk_map_and_copy(ahreq->result, areq_ctx->sg, - areq_ctx->cryptlen, - crypto_aead_authsize(authenc_esn), 1); - -out: + err = err ?: crypto_authenc_esn_genicv_tail(req, 0); aead_request_complete(req, err); } - -static void authenc_esn_verify_ahash_update_done(struct crypto_async_request *areq, - int err) +static int crypto_authenc_esn_genicv(struct aead_request *req, + unsigned int flags) { - u8 *ihash; - unsigned int authsize; - struct ablkcipher_request *abreq; - struct aead_request *req = areq->data; struct crypto_aead *authenc_esn = crypto_aead_reqtfm(req); - struct crypto_authenc_esn_ctx *ctx = crypto_aead_ctx(authenc_esn); struct authenc_esn_request_ctx *areq_ctx = aead_request_ctx(req); - struct ahash_request *ahreq = (void *)(areq_ctx->tail + ctx->reqoff); - unsigned int cryptlen = req->cryptlen; - - if (err) - goto out; - - ahash_request_set_crypt(ahreq, areq_ctx->sg, ahreq->result, - areq_ctx->cryptlen); - - ahash_request_set_callback(ahreq, - aead_request_flags(req) & - CRYPTO_TFM_REQ_MAY_SLEEP, - areq_ctx->update_complete2, req); - - err = crypto_ahash_update(ahreq); - if (err) - goto out; - - ahash_request_set_crypt(ahreq, areq_ctx->tsg, ahreq->result, - areq_ctx->trailen); - ahash_request_set_callback(ahreq, aead_request_flags(req) & - CRYPTO_TFM_REQ_MAY_SLEEP, - areq_ctx->complete, req); - - err = crypto_ahash_finup(ahreq); - if (err) - goto out; - - authsize = crypto_aead_authsize(authenc_esn); - cryptlen -= authsize; - ihash = ahreq->result + authsize; - scatterwalk_map_and_copy(ihash, areq_ctx->sg, areq_ctx->cryptlen, - authsize, 0); - - err = crypto_memneq(ihash, ahreq->result, authsize) ? -EBADMSG : 0; - if (err) - goto out; - - abreq = aead_request_ctx(req); - ablkcipher_request_set_tfm(abreq, ctx->enc); - ablkcipher_request_set_callback(abreq, aead_request_flags(req), - req->base.complete, req->base.data); - ablkcipher_request_set_crypt(abreq, req->src, req->dst, - cryptlen, req->iv); - - err = crypto_ablkcipher_decrypt(abreq); - -out: - authenc_esn_request_complete(req, err); -} - -static void authenc_esn_verify_ahash_update_done2(struct crypto_async_request *areq, - int err) -{ - u8 *ihash; - unsigned int authsize; - struct ablkcipher_request *abreq; - struct aead_request *req = areq->data; - struct crypto_aead *authenc_esn = crypto_aead_reqtfm(req); struct crypto_authenc_esn_ctx *ctx = crypto_aead_ctx(authenc_esn); - struct authenc_esn_request_ctx *areq_ctx = aead_request_ctx(req); + struct crypto_ahash *auth = ctx->auth; + u8 *hash = PTR_ALIGN((u8 *)areq_ctx->tail, + crypto_ahash_alignmask(auth) + 1); struct ahash_request *ahreq = (void *)(areq_ctx->tail + ctx->reqoff); + unsigned int authsize = crypto_aead_authsize(authenc_esn); + unsigned int assoclen = req->assoclen; unsigned int cryptlen = req->cryptlen; + struct scatterlist *dst = req->dst; + u32 tmp[2]; - if (err) - goto out; + if (!authsize) + return 0; - ahash_request_set_crypt(ahreq, areq_ctx->tsg, ahreq->result, - areq_ctx->trailen); - ahash_request_set_callback(ahreq, aead_request_flags(req) & - CRYPTO_TFM_REQ_MAY_SLEEP, - areq_ctx->complete, req); + /* Move high-order bits of sequence number to the end. */ + scatterwalk_map_and_copy(tmp, dst, 0, 8, 0); + scatterwalk_map_and_copy(tmp, dst, 4, 4, 1); + scatterwalk_map_and_copy(tmp + 1, dst, assoclen + cryptlen, 4, 1); - err = crypto_ahash_finup(ahreq); - if (err) - goto out; + sg_init_table(areq_ctx->dst, 2); + dst = scatterwalk_ffwd(areq_ctx->dst, dst, 4); - authsize = crypto_aead_authsize(authenc_esn); - cryptlen -= authsize; - ihash = ahreq->result + authsize; - scatterwalk_map_and_copy(ihash, areq_ctx->sg, areq_ctx->cryptlen, - authsize, 0); - - err = crypto_memneq(ihash, ahreq->result, authsize) ? -EBADMSG : 0; - if (err) - goto out; - - abreq = aead_request_ctx(req); - ablkcipher_request_set_tfm(abreq, ctx->enc); - ablkcipher_request_set_callback(abreq, aead_request_flags(req), - req->base.complete, req->base.data); - ablkcipher_request_set_crypt(abreq, req->src, req->dst, - cryptlen, req->iv); - - err = crypto_ablkcipher_decrypt(abreq); + ahash_request_set_tfm(ahreq, auth); + ahash_request_set_crypt(ahreq, dst, hash, assoclen + cryptlen); + ahash_request_set_callback(ahreq, flags, + authenc_esn_geniv_ahash_done, req); -out: - authenc_esn_request_complete(req, err); + return crypto_ahash_digest(ahreq) ?: + crypto_authenc_esn_genicv_tail(req, aead_request_flags(req)); } -static void authenc_esn_verify_ahash_done(struct crypto_async_request *areq, - int err) +static void crypto_authenc_esn_encrypt_done(struct crypto_async_request *req, + int err) { - u8 *ihash; - unsigned int authsize; - struct ablkcipher_request *abreq; - struct aead_request *req = areq->data; - struct crypto_aead *authenc_esn = crypto_aead_reqtfm(req); - struct crypto_authenc_esn_ctx *ctx = crypto_aead_ctx(authenc_esn); - struct authenc_esn_request_ctx *areq_ctx = aead_request_ctx(req); - struct ahash_request *ahreq = (void *)(areq_ctx->tail + ctx->reqoff); - unsigned int cryptlen = req->cryptlen; - - if (err) - goto out; - - authsize = crypto_aead_authsize(authenc_esn); - cryptlen -= authsize; - ihash = ahreq->result + authsize; - scatterwalk_map_and_copy(ihash, areq_ctx->sg, areq_ctx->cryptlen, - authsize, 0); - - err = crypto_memneq(ihash, ahreq->result, authsize) ? -EBADMSG : 0; - if (err) - goto out; - - abreq = aead_request_ctx(req); - ablkcipher_request_set_tfm(abreq, ctx->enc); - ablkcipher_request_set_callback(abreq, aead_request_flags(req), - req->base.complete, req->base.data); - ablkcipher_request_set_crypt(abreq, req->src, req->dst, - cryptlen, req->iv); + struct aead_request *areq = req->data; - err = crypto_ablkcipher_decrypt(abreq); + if (!err) + err = crypto_authenc_esn_genicv(areq, 0); -out: - authenc_esn_request_complete(req, err); + authenc_esn_request_complete(areq, err); } -static u8 *crypto_authenc_esn_ahash(struct aead_request *req, - unsigned int flags) +static int crypto_authenc_esn_copy(struct aead_request *req, unsigned int len) { struct crypto_aead *authenc_esn = crypto_aead_reqtfm(req); struct crypto_authenc_esn_ctx *ctx = crypto_aead_ctx(authenc_esn); - struct crypto_ahash *auth = ctx->auth; - struct authenc_esn_request_ctx *areq_ctx = aead_request_ctx(req); - struct ahash_request *ahreq = (void *)(areq_ctx->tail + ctx->reqoff); - u8 *hash = areq_ctx->tail; - int err; + struct blkcipher_desc desc = { + .tfm = ctx->null, + }; - hash = (u8 *)ALIGN((unsigned long)hash + crypto_ahash_alignmask(auth), - crypto_ahash_alignmask(auth) + 1); - - ahash_request_set_tfm(ahreq, auth); - - err = crypto_ahash_init(ahreq); - if (err) - return ERR_PTR(err); - - ahash_request_set_crypt(ahreq, areq_ctx->hsg, hash, areq_ctx->headlen); - ahash_request_set_callback(ahreq, aead_request_flags(req) & flags, - areq_ctx->update_complete, req); - - err = crypto_ahash_update(ahreq); - if (err) - return ERR_PTR(err); - - ahash_request_set_crypt(ahreq, areq_ctx->sg, hash, areq_ctx->cryptlen); - ahash_request_set_callback(ahreq, aead_request_flags(req) & flags, - areq_ctx->update_complete2, req); - - err = crypto_ahash_update(ahreq); - if (err) - return ERR_PTR(err); - - ahash_request_set_crypt(ahreq, areq_ctx->tsg, hash, - areq_ctx->trailen); - ahash_request_set_callback(ahreq, aead_request_flags(req) & flags, - areq_ctx->complete, req); - - err = crypto_ahash_finup(ahreq); - if (err) - return ERR_PTR(err); - - return hash; + return crypto_blkcipher_encrypt(&desc, req->dst, req->src, len); } -static int crypto_authenc_esn_genicv(struct aead_request *req, u8 *iv, - unsigned int flags) +static int crypto_authenc_esn_encrypt(struct aead_request *req) { struct crypto_aead *authenc_esn = crypto_aead_reqtfm(req); struct authenc_esn_request_ctx *areq_ctx = aead_request_ctx(req); - struct scatterlist *dst = req->dst; - struct scatterlist *assoc = req->assoc; - struct scatterlist *cipher = areq_ctx->cipher; - struct scatterlist *hsg = areq_ctx->hsg; - struct scatterlist *tsg = areq_ctx->tsg; - unsigned int ivsize = crypto_aead_ivsize(authenc_esn); + struct crypto_authenc_esn_ctx *ctx = crypto_aead_ctx(authenc_esn); + struct ablkcipher_request *abreq = (void *)(areq_ctx->tail + + ctx->reqoff); + struct crypto_ablkcipher *enc = ctx->enc; + unsigned int assoclen = req->assoclen; unsigned int cryptlen = req->cryptlen; - struct page *dstp; - u8 *vdst; - u8 *hash; - - dstp = sg_page(dst); - vdst = PageHighMem(dstp) ? NULL : page_address(dstp) + dst->offset; - - if (ivsize) { - sg_init_table(cipher, 2); - sg_set_buf(cipher, iv, ivsize); - scatterwalk_crypto_chain(cipher, dst, vdst == iv + ivsize, 2); - dst = cipher; - cryptlen += ivsize; - } - - if (assoc->length < 12) - return -EINVAL; - - sg_init_table(hsg, 2); - sg_set_page(hsg, sg_page(assoc), 4, assoc->offset); - sg_set_page(hsg + 1, sg_page(assoc), 4, assoc->offset + 8); - - sg_init_table(tsg, 1); - sg_set_page(tsg, sg_page(assoc), 4, assoc->offset + 4); - - areq_ctx->cryptlen = cryptlen; - areq_ctx->headlen = 8; - areq_ctx->trailen = 4; - areq_ctx->sg = dst; - - areq_ctx->complete = authenc_esn_geniv_ahash_done; - areq_ctx->update_complete = authenc_esn_geniv_ahash_update_done; - areq_ctx->update_complete2 = authenc_esn_geniv_ahash_update_done2; - - hash = crypto_authenc_esn_ahash(req, flags); - if (IS_ERR(hash)) - return PTR_ERR(hash); + struct scatterlist *src, *dst; + int err; - scatterwalk_map_and_copy(hash, dst, cryptlen, - crypto_aead_authsize(authenc_esn), 1); - return 0; -} + sg_init_table(areq_ctx->src, 2); + src = scatterwalk_ffwd(areq_ctx->src, req->src, assoclen); + dst = src; + if (req->src != req->dst) { + err = crypto_authenc_esn_copy(req, assoclen); + if (err) + return err; -static void crypto_authenc_esn_encrypt_done(struct crypto_async_request *req, - int err) -{ - struct aead_request *areq = req->data; - - if (!err) { - struct crypto_aead *authenc_esn = crypto_aead_reqtfm(areq); - struct crypto_authenc_esn_ctx *ctx = crypto_aead_ctx(authenc_esn); - struct ablkcipher_request *abreq = aead_request_ctx(areq); - u8 *iv = (u8 *)(abreq + 1) + - crypto_ablkcipher_reqsize(ctx->enc); - - err = crypto_authenc_esn_genicv(areq, iv, 0); + sg_init_table(areq_ctx->dst, 2); + dst = scatterwalk_ffwd(areq_ctx->dst, req->dst, assoclen); } - authenc_esn_request_complete(areq, err); -} - -static int crypto_authenc_esn_encrypt(struct aead_request *req) -{ - struct crypto_aead *authenc_esn = crypto_aead_reqtfm(req); - struct crypto_authenc_esn_ctx *ctx = crypto_aead_ctx(authenc_esn); - struct authenc_esn_request_ctx *areq_ctx = aead_request_ctx(req); - struct crypto_ablkcipher *enc = ctx->enc; - struct scatterlist *dst = req->dst; - unsigned int cryptlen = req->cryptlen; - struct ablkcipher_request *abreq = (void *)(areq_ctx->tail - + ctx->reqoff); - u8 *iv = (u8 *)abreq - crypto_ablkcipher_ivsize(enc); - int err; - ablkcipher_request_set_tfm(abreq, enc); ablkcipher_request_set_callback(abreq, aead_request_flags(req), crypto_authenc_esn_encrypt_done, req); - ablkcipher_request_set_crypt(abreq, req->src, dst, cryptlen, req->iv); - - memcpy(iv, req->iv, crypto_aead_ivsize(authenc_esn)); + ablkcipher_request_set_crypt(abreq, src, dst, cryptlen, req->iv); err = crypto_ablkcipher_encrypt(abreq); if (err) return err; - return crypto_authenc_esn_genicv(req, iv, CRYPTO_TFM_REQ_MAY_SLEEP); + return crypto_authenc_esn_genicv(req, aead_request_flags(req)); } -static void crypto_authenc_esn_givencrypt_done(struct crypto_async_request *req, - int err) +static int crypto_authenc_esn_decrypt_tail(struct aead_request *req, + unsigned int flags) { - struct aead_request *areq = req->data; - - if (!err) { - struct skcipher_givcrypt_request *greq = aead_request_ctx(areq); - - err = crypto_authenc_esn_genicv(areq, greq->giv, 0); - } + struct crypto_aead *authenc_esn = crypto_aead_reqtfm(req); + unsigned int authsize = crypto_aead_authsize(authenc_esn); + struct authenc_esn_request_ctx *areq_ctx = aead_request_ctx(req); + struct crypto_authenc_esn_ctx *ctx = crypto_aead_ctx(authenc_esn); + struct ablkcipher_request *abreq = (void *)(areq_ctx->tail + + ctx->reqoff); + struct crypto_ahash *auth = ctx->auth; + u8 *ohash = PTR_ALIGN((u8 *)areq_ctx->tail, + crypto_ahash_alignmask(auth) + 1); + unsigned int cryptlen = req->cryptlen - authsize; + unsigned int assoclen = req->assoclen; + struct scatterlist *dst = req->dst; + u8 *ihash = ohash + crypto_ahash_digestsize(auth); + u32 tmp[2]; - authenc_esn_request_complete(areq, err); -} + /* Move high-order bits of sequence number back. */ + scatterwalk_map_and_copy(tmp, dst, 4, 4, 0); + scatterwalk_map_and_copy(tmp + 1, dst, assoclen + cryptlen, 4, 0); + scatterwalk_map_and_copy(tmp, dst, 0, 8, 1); -static int crypto_authenc_esn_givencrypt(struct aead_givcrypt_request *req) -{ - struct crypto_aead *authenc_esn = aead_givcrypt_reqtfm(req); - struct crypto_authenc_esn_ctx *ctx = crypto_aead_ctx(authenc_esn); - struct aead_request *areq = &req->areq; - struct skcipher_givcrypt_request *greq = aead_request_ctx(areq); - u8 *iv = req->giv; - int err; + if (crypto_memneq(ihash, ohash, authsize)) + return -EBADMSG; - skcipher_givcrypt_set_tfm(greq, ctx->enc); - skcipher_givcrypt_set_callback(greq, aead_request_flags(areq), - crypto_authenc_esn_givencrypt_done, areq); - skcipher_givcrypt_set_crypt(greq, areq->src, areq->dst, areq->cryptlen, - areq->iv); - skcipher_givcrypt_set_giv(greq, iv, req->seq); + sg_init_table(areq_ctx->dst, 2); + dst = scatterwalk_ffwd(areq_ctx->dst, dst, assoclen); - err = crypto_skcipher_givencrypt(greq); - if (err) - return err; + ablkcipher_request_set_tfm(abreq, ctx->enc); + ablkcipher_request_set_callback(abreq, flags, + req->base.complete, req->base.data); + ablkcipher_request_set_crypt(abreq, dst, dst, cryptlen, req->iv); - return crypto_authenc_esn_genicv(areq, iv, CRYPTO_TFM_REQ_MAY_SLEEP); + return crypto_ablkcipher_decrypt(abreq); } -static int crypto_authenc_esn_verify(struct aead_request *req) +static void authenc_esn_verify_ahash_done(struct crypto_async_request *areq, + int err) { - struct crypto_aead *authenc_esn = crypto_aead_reqtfm(req); - struct authenc_esn_request_ctx *areq_ctx = aead_request_ctx(req); - u8 *ohash; - u8 *ihash; - unsigned int authsize; - - areq_ctx->complete = authenc_esn_verify_ahash_done; - areq_ctx->update_complete = authenc_esn_verify_ahash_update_done; - - ohash = crypto_authenc_esn_ahash(req, CRYPTO_TFM_REQ_MAY_SLEEP); - if (IS_ERR(ohash)) - return PTR_ERR(ohash); + struct aead_request *req = areq->data; - authsize = crypto_aead_authsize(authenc_esn); - ihash = ohash + authsize; - scatterwalk_map_and_copy(ihash, areq_ctx->sg, areq_ctx->cryptlen, - authsize, 0); - return crypto_memneq(ihash, ohash, authsize) ? -EBADMSG : 0; + err = err ?: crypto_authenc_esn_decrypt_tail(req, 0); + aead_request_complete(req, err); } -static int crypto_authenc_esn_iverify(struct aead_request *req, u8 *iv, - unsigned int cryptlen) +static int crypto_authenc_esn_decrypt(struct aead_request *req) { struct crypto_aead *authenc_esn = crypto_aead_reqtfm(req); struct authenc_esn_request_ctx *areq_ctx = aead_request_ctx(req); - struct scatterlist *src = req->src; - struct scatterlist *assoc = req->assoc; - struct scatterlist *cipher = areq_ctx->cipher; - struct scatterlist *hsg = areq_ctx->hsg; - struct scatterlist *tsg = areq_ctx->tsg; - unsigned int ivsize = crypto_aead_ivsize(authenc_esn); - struct page *srcp; - u8 *vsrc; - - srcp = sg_page(src); - vsrc = PageHighMem(srcp) ? NULL : page_address(srcp) + src->offset; - - if (ivsize) { - sg_init_table(cipher, 2); - sg_set_buf(cipher, iv, ivsize); - scatterwalk_crypto_chain(cipher, src, vsrc == iv + ivsize, 2); - src = cipher; - cryptlen += ivsize; - } + struct crypto_authenc_esn_ctx *ctx = crypto_aead_ctx(authenc_esn); + struct ahash_request *ahreq = (void *)(areq_ctx->tail + ctx->reqoff); + unsigned int authsize = crypto_aead_authsize(authenc_esn); + struct crypto_ahash *auth = ctx->auth; + u8 *ohash = PTR_ALIGN((u8 *)areq_ctx->tail, + crypto_ahash_alignmask(auth) + 1); + unsigned int assoclen = req->assoclen; + unsigned int cryptlen = req->cryptlen; + u8 *ihash = ohash + crypto_ahash_digestsize(auth); + struct scatterlist *dst = req->dst; + u32 tmp[2]; + int err; - if (assoc->length < 12) - return -EINVAL; + cryptlen -= authsize; - sg_init_table(hsg, 2); - sg_set_page(hsg, sg_page(assoc), 4, assoc->offset); - sg_set_page(hsg + 1, sg_page(assoc), 4, assoc->offset + 8); + if (req->src != dst) { + err = crypto_authenc_esn_copy(req, assoclen + cryptlen); + if (err) + return err; + } - sg_init_table(tsg, 1); - sg_set_page(tsg, sg_page(assoc), 4, assoc->offset + 4); + scatterwalk_map_and_copy(ihash, req->src, assoclen + cryptlen, + authsize, 0); - areq_ctx->cryptlen = cryptlen; - areq_ctx->headlen = 8; - areq_ctx->trailen = 4; - areq_ctx->sg = src; + if (!authsize) + goto tail; - areq_ctx->complete = authenc_esn_verify_ahash_done; - areq_ctx->update_complete = authenc_esn_verify_ahash_update_done; - areq_ctx->update_complete2 = authenc_esn_verify_ahash_update_done2; + /* Move high-order bits of sequence number to the end. */ + scatterwalk_map_and_copy(tmp, dst, 0, 8, 0); + scatterwalk_map_and_copy(tmp, dst, 4, 4, 1); + scatterwalk_map_and_copy(tmp + 1, dst, assoclen + cryptlen, 4, 1); - return crypto_authenc_esn_verify(req); -} + sg_init_table(areq_ctx->dst, 2); + dst = scatterwalk_ffwd(areq_ctx->dst, dst, 4); -static int crypto_authenc_esn_decrypt(struct aead_request *req) -{ - struct crypto_aead *authenc_esn = crypto_aead_reqtfm(req); - struct crypto_authenc_esn_ctx *ctx = crypto_aead_ctx(authenc_esn); - struct ablkcipher_request *abreq = aead_request_ctx(req); - unsigned int cryptlen = req->cryptlen; - unsigned int authsize = crypto_aead_authsize(authenc_esn); - u8 *iv = req->iv; - int err; - - if (cryptlen < authsize) - return -EINVAL; - cryptlen -= authsize; + ahash_request_set_tfm(ahreq, auth); + ahash_request_set_crypt(ahreq, dst, ohash, assoclen + cryptlen); + ahash_request_set_callback(ahreq, aead_request_flags(req), + authenc_esn_verify_ahash_done, req); - err = crypto_authenc_esn_iverify(req, iv, cryptlen); + err = crypto_ahash_digest(ahreq); if (err) return err; - ablkcipher_request_set_tfm(abreq, ctx->enc); - ablkcipher_request_set_callback(abreq, aead_request_flags(req), - req->base.complete, req->base.data); - ablkcipher_request_set_crypt(abreq, req->src, req->dst, cryptlen, iv); - - return crypto_ablkcipher_decrypt(abreq); +tail: + return crypto_authenc_esn_decrypt_tail(req, aead_request_flags(req)); } -static int crypto_authenc_esn_init_tfm(struct crypto_tfm *tfm) +static int crypto_authenc_esn_init_tfm(struct crypto_aead *tfm) { - struct crypto_instance *inst = crypto_tfm_alg_instance(tfm); - struct authenc_esn_instance_ctx *ictx = crypto_instance_ctx(inst); - struct crypto_authenc_esn_ctx *ctx = crypto_tfm_ctx(tfm); + struct aead_instance *inst = aead_alg_instance(tfm); + struct authenc_esn_instance_ctx *ictx = aead_instance_ctx(inst); + struct crypto_authenc_esn_ctx *ctx = crypto_aead_ctx(tfm); struct crypto_ahash *auth; struct crypto_ablkcipher *enc; + struct crypto_blkcipher *null; int err; auth = crypto_spawn_ahash(&ictx->auth); @@ -634,15 +344,20 @@ static int crypto_authenc_esn_init_tfm(struct crypto_tfm *tfm) if (IS_ERR(enc)) goto err_free_ahash; + null = crypto_get_default_null_skcipher(); + err = PTR_ERR(null); + if (IS_ERR(null)) + goto err_free_skcipher; + ctx->auth = auth; ctx->enc = enc; + ctx->null = null; - ctx->reqoff = ALIGN(2 * crypto_ahash_digestsize(auth) + - crypto_ahash_alignmask(auth), - crypto_ahash_alignmask(auth) + 1) + - crypto_ablkcipher_ivsize(enc); + ctx->reqoff = ALIGN(2 * crypto_ahash_digestsize(auth), + crypto_ahash_alignmask(auth) + 1); - crypto_aead_set_reqsize(__crypto_aead_cast(tfm), + crypto_aead_set_reqsize( + tfm, sizeof(struct authenc_esn_request_ctx) + ctx->reqoff + max_t(unsigned int, @@ -653,23 +368,36 @@ static int crypto_authenc_esn_init_tfm(struct crypto_tfm *tfm) return 0; +err_free_skcipher: + crypto_free_ablkcipher(enc); err_free_ahash: crypto_free_ahash(auth); return err; } -static void crypto_authenc_esn_exit_tfm(struct crypto_tfm *tfm) +static void crypto_authenc_esn_exit_tfm(struct crypto_aead *tfm) { - struct crypto_authenc_esn_ctx *ctx = crypto_tfm_ctx(tfm); + struct crypto_authenc_esn_ctx *ctx = crypto_aead_ctx(tfm); crypto_free_ahash(ctx->auth); crypto_free_ablkcipher(ctx->enc); + crypto_put_default_null_skcipher(); +} + +static void crypto_authenc_esn_free(struct aead_instance *inst) +{ + struct authenc_esn_instance_ctx *ctx = aead_instance_ctx(inst); + + crypto_drop_skcipher(&ctx->enc); + crypto_drop_ahash(&ctx->auth); + kfree(inst); } -static struct crypto_instance *crypto_authenc_esn_alloc(struct rtattr **tb) +static int crypto_authenc_esn_create(struct crypto_template *tmpl, + struct rtattr **tb) { struct crypto_attr_type *algt; - struct crypto_instance *inst; + struct aead_instance *inst; struct hash_alg_common *auth; struct crypto_alg *auth_base; struct crypto_alg *enc; @@ -679,15 +407,15 @@ static struct crypto_instance *crypto_authenc_esn_alloc(struct rtattr **tb) algt = crypto_get_attr_type(tb); if (IS_ERR(algt)) - return ERR_CAST(algt); + return PTR_ERR(algt); if ((algt->type ^ CRYPTO_ALG_TYPE_AEAD) & algt->mask) - return ERR_PTR(-EINVAL); + return -EINVAL; auth = ahash_attr_alg(tb[1], CRYPTO_ALG_TYPE_HASH, CRYPTO_ALG_TYPE_AHASH_MASK); if (IS_ERR(auth)) - return ERR_CAST(auth); + return PTR_ERR(auth); auth_base = &auth->base; @@ -701,13 +429,14 @@ static struct crypto_instance *crypto_authenc_esn_alloc(struct rtattr **tb) if (!inst) goto out_put_auth; - ctx = crypto_instance_ctx(inst); + ctx = aead_instance_ctx(inst); - err = crypto_init_ahash_spawn(&ctx->auth, auth, inst); + err = crypto_init_ahash_spawn(&ctx->auth, auth, + aead_crypto_instance(inst)); if (err) goto err_free_inst; - crypto_set_skcipher_spawn(&ctx->enc, inst); + crypto_set_skcipher_spawn(&ctx->enc, aead_crypto_instance(inst)); err = crypto_grab_skcipher(&ctx->enc, enc_name, 0, crypto_requires_sync(algt->type, algt->mask)); @@ -717,40 +446,44 @@ static struct crypto_instance *crypto_authenc_esn_alloc(struct rtattr **tb) enc = crypto_skcipher_spawn_alg(&ctx->enc); err = -ENAMETOOLONG; - if (snprintf(inst->alg.cra_name, CRYPTO_MAX_ALG_NAME, - "authencesn(%s,%s)", auth_base->cra_name, enc->cra_name) >= - CRYPTO_MAX_ALG_NAME) + if (snprintf(inst->alg.base.cra_name, CRYPTO_MAX_ALG_NAME, + "authencesn(%s,%s)", auth_base->cra_name, + enc->cra_name) >= CRYPTO_MAX_ALG_NAME) goto err_drop_enc; - if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME, + if (snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME, "authencesn(%s,%s)", auth_base->cra_driver_name, enc->cra_driver_name) >= CRYPTO_MAX_ALG_NAME) goto err_drop_enc; - inst->alg.cra_flags = CRYPTO_ALG_TYPE_AEAD; - inst->alg.cra_flags |= enc->cra_flags & CRYPTO_ALG_ASYNC; - inst->alg.cra_priority = enc->cra_priority * - 10 + auth_base->cra_priority; - inst->alg.cra_blocksize = enc->cra_blocksize; - inst->alg.cra_alignmask = auth_base->cra_alignmask | enc->cra_alignmask; - inst->alg.cra_type = &crypto_aead_type; + inst->alg.base.cra_flags = enc->cra_flags & CRYPTO_ALG_ASYNC; + inst->alg.base.cra_priority = enc->cra_priority * 10 + + auth_base->cra_priority; + inst->alg.base.cra_blocksize = enc->cra_blocksize; + inst->alg.base.cra_alignmask = auth_base->cra_alignmask | + enc->cra_alignmask; + inst->alg.base.cra_ctxsize = sizeof(struct crypto_authenc_esn_ctx); + + inst->alg.ivsize = enc->cra_ablkcipher.ivsize; + inst->alg.maxauthsize = auth->digestsize; - inst->alg.cra_aead.ivsize = enc->cra_ablkcipher.ivsize; - inst->alg.cra_aead.maxauthsize = auth->digestsize; + inst->alg.init = crypto_authenc_esn_init_tfm; + inst->alg.exit = crypto_authenc_esn_exit_tfm; - inst->alg.cra_ctxsize = sizeof(struct crypto_authenc_esn_ctx); + inst->alg.setkey = crypto_authenc_esn_setkey; + inst->alg.setauthsize = crypto_authenc_esn_setauthsize; + inst->alg.encrypt = crypto_authenc_esn_encrypt; + inst->alg.decrypt = crypto_authenc_esn_decrypt; - inst->alg.cra_init = crypto_authenc_esn_init_tfm; - inst->alg.cra_exit = crypto_authenc_esn_exit_tfm; + inst->free = crypto_authenc_esn_free, - inst->alg.cra_aead.setkey = crypto_authenc_esn_setkey; - inst->alg.cra_aead.encrypt = crypto_authenc_esn_encrypt; - inst->alg.cra_aead.decrypt = crypto_authenc_esn_decrypt; - inst->alg.cra_aead.givencrypt = crypto_authenc_esn_givencrypt; + err = aead_register_instance(tmpl, inst); + if (err) + goto err_drop_enc; out: crypto_mod_put(auth_base); - return inst; + return err; err_drop_enc: crypto_drop_skcipher(&ctx->enc); @@ -759,23 +492,12 @@ err_drop_auth: err_free_inst: kfree(inst); out_put_auth: - inst = ERR_PTR(err); goto out; } -static void crypto_authenc_esn_free(struct crypto_instance *inst) -{ - struct authenc_esn_instance_ctx *ctx = crypto_instance_ctx(inst); - - crypto_drop_skcipher(&ctx->enc); - crypto_drop_ahash(&ctx->auth); - kfree(inst); -} - static struct crypto_template crypto_authenc_esn_tmpl = { .name = "authencesn", - .alloc = crypto_authenc_esn_alloc, - .free = crypto_authenc_esn_free, + .create = crypto_authenc_esn_create, .module = THIS_MODULE, }; diff --git a/crypto/ccm.c b/crypto/ccm.c index a4d1a5eda18b..cc31ea4335bf 100644 --- a/crypto/ccm.c +++ b/crypto/ccm.c @@ -36,14 +36,20 @@ struct crypto_rfc4309_ctx { u8 nonce[3]; }; +struct crypto_rfc4309_req_ctx { + struct scatterlist src[3]; + struct scatterlist dst[3]; + struct aead_request subreq; +}; + struct crypto_ccm_req_priv_ctx { u8 odata[16]; u8 idata[16]; u8 auth_tag[16]; u32 ilen; u32 flags; - struct scatterlist src[2]; - struct scatterlist dst[2]; + struct scatterlist src[3]; + struct scatterlist dst[3]; struct ablkcipher_request abreq; }; @@ -265,7 +271,7 @@ static int crypto_ccm_auth(struct aead_request *req, struct scatterlist *plain, /* format associated data and compute into mac */ if (assoclen) { pctx->ilen = format_adata(idata, assoclen); - get_data_to_compute(cipher, pctx, req->assoc, req->assoclen); + get_data_to_compute(cipher, pctx, req->src, req->assoclen); } else { pctx->ilen = 0; } @@ -286,7 +292,8 @@ static void crypto_ccm_encrypt_done(struct crypto_async_request *areq, int err) u8 *odata = pctx->odata; if (!err) - scatterwalk_map_and_copy(odata, req->dst, req->cryptlen, + scatterwalk_map_and_copy(odata, req->dst, + req->assoclen + req->cryptlen, crypto_aead_authsize(aead), 1); aead_request_complete(req, err); } @@ -300,6 +307,41 @@ static inline int crypto_ccm_check_iv(const u8 *iv) return 0; } +static int crypto_ccm_init_crypt(struct aead_request *req, u8 *tag) +{ + struct crypto_ccm_req_priv_ctx *pctx = crypto_ccm_reqctx(req); + struct scatterlist *sg; + u8 *iv = req->iv; + int err; + + err = crypto_ccm_check_iv(iv); + if (err) + return err; + + pctx->flags = aead_request_flags(req); + + /* Note: rfc 3610 and NIST 800-38C require counter of + * zero to encrypt auth tag. + */ + memset(iv + 15 - iv[0], 0, iv[0] + 1); + + sg_init_table(pctx->src, 3); + sg_set_buf(pctx->src, tag, 16); + sg = scatterwalk_ffwd(pctx->src + 1, req->src, req->assoclen); + if (sg != pctx->src + 1) + sg_chain(pctx->src, 2, sg); + + if (req->src != req->dst) { + sg_init_table(pctx->dst, 3); + sg_set_buf(pctx->dst, tag, 16); + sg = scatterwalk_ffwd(pctx->dst + 1, req->dst, req->assoclen); + if (sg != pctx->dst + 1) + sg_chain(pctx->dst, 2, sg); + } + + return 0; +} + static int crypto_ccm_encrypt(struct aead_request *req) { struct crypto_aead *aead = crypto_aead_reqtfm(req); @@ -312,32 +354,17 @@ static int crypto_ccm_encrypt(struct aead_request *req) u8 *iv = req->iv; int err; - err = crypto_ccm_check_iv(iv); + err = crypto_ccm_init_crypt(req, odata); if (err) return err; - pctx->flags = aead_request_flags(req); - - err = crypto_ccm_auth(req, req->src, cryptlen); + err = crypto_ccm_auth(req, sg_next(pctx->src), cryptlen); if (err) return err; - /* Note: rfc 3610 and NIST 800-38C require counter of - * zero to encrypt auth tag. - */ - memset(iv + 15 - iv[0], 0, iv[0] + 1); - - sg_init_table(pctx->src, 2); - sg_set_buf(pctx->src, odata, 16); - scatterwalk_sg_chain(pctx->src, 2, req->src); - dst = pctx->src; - if (req->src != req->dst) { - sg_init_table(pctx->dst, 2); - sg_set_buf(pctx->dst, odata, 16); - scatterwalk_sg_chain(pctx->dst, 2, req->dst); + if (req->src != req->dst) dst = pctx->dst; - } ablkcipher_request_set_tfm(abreq, ctx->ctr); ablkcipher_request_set_callback(abreq, pctx->flags, @@ -348,7 +375,7 @@ static int crypto_ccm_encrypt(struct aead_request *req) return err; /* copy authtag to end of dst */ - scatterwalk_map_and_copy(odata, req->dst, cryptlen, + scatterwalk_map_and_copy(odata, sg_next(dst), cryptlen, crypto_aead_authsize(aead), 1); return err; } @@ -361,9 +388,14 @@ static void crypto_ccm_decrypt_done(struct crypto_async_request *areq, struct crypto_aead *aead = crypto_aead_reqtfm(req); unsigned int authsize = crypto_aead_authsize(aead); unsigned int cryptlen = req->cryptlen - authsize; + struct scatterlist *dst; + + pctx->flags = 0; + + dst = sg_next(req->src == req->dst ? pctx->src : pctx->dst); if (!err) { - err = crypto_ccm_auth(req, req->dst, cryptlen); + err = crypto_ccm_auth(req, dst, cryptlen); if (!err && crypto_memneq(pctx->auth_tag, pctx->odata, authsize)) err = -EBADMSG; } @@ -384,31 +416,18 @@ static int crypto_ccm_decrypt(struct aead_request *req) u8 *iv = req->iv; int err; - if (cryptlen < authsize) - return -EINVAL; cryptlen -= authsize; - err = crypto_ccm_check_iv(iv); + err = crypto_ccm_init_crypt(req, authtag); if (err) return err; - pctx->flags = aead_request_flags(req); - - scatterwalk_map_and_copy(authtag, req->src, cryptlen, authsize, 0); - - memset(iv + 15 - iv[0], 0, iv[0] + 1); - - sg_init_table(pctx->src, 2); - sg_set_buf(pctx->src, authtag, 16); - scatterwalk_sg_chain(pctx->src, 2, req->src); + scatterwalk_map_and_copy(authtag, sg_next(pctx->src), cryptlen, + authsize, 0); dst = pctx->src; - if (req->src != req->dst) { - sg_init_table(pctx->dst, 2); - sg_set_buf(pctx->dst, authtag, 16); - scatterwalk_sg_chain(pctx->dst, 2, req->dst); + if (req->src != req->dst) dst = pctx->dst; - } ablkcipher_request_set_tfm(abreq, ctx->ctr); ablkcipher_request_set_callback(abreq, pctx->flags, @@ -418,7 +437,7 @@ static int crypto_ccm_decrypt(struct aead_request *req) if (err) return err; - err = crypto_ccm_auth(req, req->dst, cryptlen); + err = crypto_ccm_auth(req, sg_next(dst), cryptlen); if (err) return err; @@ -429,11 +448,11 @@ static int crypto_ccm_decrypt(struct aead_request *req) return err; } -static int crypto_ccm_init_tfm(struct crypto_tfm *tfm) +static int crypto_ccm_init_tfm(struct crypto_aead *tfm) { - struct crypto_instance *inst = (void *)tfm->__crt_alg; - struct ccm_instance_ctx *ictx = crypto_instance_ctx(inst); - struct crypto_ccm_ctx *ctx = crypto_tfm_ctx(tfm); + struct aead_instance *inst = aead_alg_instance(tfm); + struct ccm_instance_ctx *ictx = aead_instance_ctx(inst); + struct crypto_ccm_ctx *ctx = crypto_aead_ctx(tfm); struct crypto_cipher *cipher; struct crypto_ablkcipher *ctr; unsigned long align; @@ -451,9 +470,10 @@ static int crypto_ccm_init_tfm(struct crypto_tfm *tfm) ctx->cipher = cipher; ctx->ctr = ctr; - align = crypto_tfm_alg_alignmask(tfm); + align = crypto_aead_alignmask(tfm); align &= ~(crypto_tfm_ctx_alignment() - 1); - crypto_aead_set_reqsize(__crypto_aead_cast(tfm), + crypto_aead_set_reqsize( + tfm, align + sizeof(struct crypto_ccm_req_priv_ctx) + crypto_ablkcipher_reqsize(ctr)); @@ -464,21 +484,31 @@ err_free_cipher: return err; } -static void crypto_ccm_exit_tfm(struct crypto_tfm *tfm) +static void crypto_ccm_exit_tfm(struct crypto_aead *tfm) { - struct crypto_ccm_ctx *ctx = crypto_tfm_ctx(tfm); + struct crypto_ccm_ctx *ctx = crypto_aead_ctx(tfm); crypto_free_cipher(ctx->cipher); crypto_free_ablkcipher(ctx->ctr); } -static struct crypto_instance *crypto_ccm_alloc_common(struct rtattr **tb, - const char *full_name, - const char *ctr_name, - const char *cipher_name) +static void crypto_ccm_free(struct aead_instance *inst) +{ + struct ccm_instance_ctx *ctx = aead_instance_ctx(inst); + + crypto_drop_spawn(&ctx->cipher); + crypto_drop_skcipher(&ctx->ctr); + kfree(inst); +} + +static int crypto_ccm_create_common(struct crypto_template *tmpl, + struct rtattr **tb, + const char *full_name, + const char *ctr_name, + const char *cipher_name) { struct crypto_attr_type *algt; - struct crypto_instance *inst; + struct aead_instance *inst; struct crypto_alg *ctr; struct crypto_alg *cipher; struct ccm_instance_ctx *ictx; @@ -486,15 +516,15 @@ static struct crypto_instance *crypto_ccm_alloc_common(struct rtattr **tb, algt = crypto_get_attr_type(tb); if (IS_ERR(algt)) - return ERR_CAST(algt); + return PTR_ERR(algt); if ((algt->type ^ CRYPTO_ALG_TYPE_AEAD) & algt->mask) - return ERR_PTR(-EINVAL); + return -EINVAL; cipher = crypto_alg_mod_lookup(cipher_name, CRYPTO_ALG_TYPE_CIPHER, CRYPTO_ALG_TYPE_MASK); if (IS_ERR(cipher)) - return ERR_CAST(cipher); + return PTR_ERR(cipher); err = -EINVAL; if (cipher->cra_blocksize != 16) @@ -505,14 +535,15 @@ static struct crypto_instance *crypto_ccm_alloc_common(struct rtattr **tb, if (!inst) goto out_put_cipher; - ictx = crypto_instance_ctx(inst); + ictx = aead_instance_ctx(inst); - err = crypto_init_spawn(&ictx->cipher, cipher, inst, + err = crypto_init_spawn(&ictx->cipher, cipher, + aead_crypto_instance(inst), CRYPTO_ALG_TYPE_MASK); if (err) goto err_free_inst; - crypto_set_skcipher_spawn(&ictx->ctr, inst); + crypto_set_skcipher_spawn(&ictx->ctr, aead_crypto_instance(inst)); err = crypto_grab_skcipher(&ictx->ctr, ctr_name, 0, crypto_requires_sync(algt->type, algt->mask)); @@ -531,33 +562,39 @@ static struct crypto_instance *crypto_ccm_alloc_common(struct rtattr **tb, goto err_drop_ctr; err = -ENAMETOOLONG; - if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME, + if (snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME, "ccm_base(%s,%s)", ctr->cra_driver_name, cipher->cra_driver_name) >= CRYPTO_MAX_ALG_NAME) goto err_drop_ctr; - memcpy(inst->alg.cra_name, full_name, CRYPTO_MAX_ALG_NAME); - - inst->alg.cra_flags = CRYPTO_ALG_TYPE_AEAD; - inst->alg.cra_flags |= ctr->cra_flags & CRYPTO_ALG_ASYNC; - inst->alg.cra_priority = cipher->cra_priority + ctr->cra_priority; - inst->alg.cra_blocksize = 1; - inst->alg.cra_alignmask = cipher->cra_alignmask | ctr->cra_alignmask | - (__alignof__(u32) - 1); - inst->alg.cra_type = &crypto_aead_type; - inst->alg.cra_aead.ivsize = 16; - inst->alg.cra_aead.maxauthsize = 16; - inst->alg.cra_ctxsize = sizeof(struct crypto_ccm_ctx); - inst->alg.cra_init = crypto_ccm_init_tfm; - inst->alg.cra_exit = crypto_ccm_exit_tfm; - inst->alg.cra_aead.setkey = crypto_ccm_setkey; - inst->alg.cra_aead.setauthsize = crypto_ccm_setauthsize; - inst->alg.cra_aead.encrypt = crypto_ccm_encrypt; - inst->alg.cra_aead.decrypt = crypto_ccm_decrypt; + memcpy(inst->alg.base.cra_name, full_name, CRYPTO_MAX_ALG_NAME); + + inst->alg.base.cra_flags = ctr->cra_flags & CRYPTO_ALG_ASYNC; + inst->alg.base.cra_priority = (cipher->cra_priority + + ctr->cra_priority) / 2; + inst->alg.base.cra_blocksize = 1; + inst->alg.base.cra_alignmask = cipher->cra_alignmask | + ctr->cra_alignmask | + (__alignof__(u32) - 1); + inst->alg.ivsize = 16; + inst->alg.maxauthsize = 16; + inst->alg.base.cra_ctxsize = sizeof(struct crypto_ccm_ctx); + inst->alg.init = crypto_ccm_init_tfm; + inst->alg.exit = crypto_ccm_exit_tfm; + inst->alg.setkey = crypto_ccm_setkey; + inst->alg.setauthsize = crypto_ccm_setauthsize; + inst->alg.encrypt = crypto_ccm_encrypt; + inst->alg.decrypt = crypto_ccm_decrypt; + + inst->free = crypto_ccm_free; + + err = aead_register_instance(tmpl, inst); + if (err) + goto err_drop_ctr; -out: +out_put_cipher: crypto_mod_put(cipher); - return inst; + return err; err_drop_ctr: crypto_drop_skcipher(&ictx->ctr); @@ -565,12 +602,10 @@ err_drop_cipher: crypto_drop_spawn(&ictx->cipher); err_free_inst: kfree(inst); -out_put_cipher: - inst = ERR_PTR(err); - goto out; + goto out_put_cipher; } -static struct crypto_instance *crypto_ccm_alloc(struct rtattr **tb) +static int crypto_ccm_create(struct crypto_template *tmpl, struct rtattr **tb) { const char *cipher_name; char ctr_name[CRYPTO_MAX_ALG_NAME]; @@ -578,36 +613,28 @@ static struct crypto_instance *crypto_ccm_alloc(struct rtattr **tb) cipher_name = crypto_attr_alg_name(tb[1]); if (IS_ERR(cipher_name)) - return ERR_CAST(cipher_name); + return PTR_ERR(cipher_name); if (snprintf(ctr_name, CRYPTO_MAX_ALG_NAME, "ctr(%s)", cipher_name) >= CRYPTO_MAX_ALG_NAME) - return ERR_PTR(-ENAMETOOLONG); + return -ENAMETOOLONG; if (snprintf(full_name, CRYPTO_MAX_ALG_NAME, "ccm(%s)", cipher_name) >= CRYPTO_MAX_ALG_NAME) - return ERR_PTR(-ENAMETOOLONG); + return -ENAMETOOLONG; - return crypto_ccm_alloc_common(tb, full_name, ctr_name, cipher_name); -} - -static void crypto_ccm_free(struct crypto_instance *inst) -{ - struct ccm_instance_ctx *ctx = crypto_instance_ctx(inst); - - crypto_drop_spawn(&ctx->cipher); - crypto_drop_skcipher(&ctx->ctr); - kfree(inst); + return crypto_ccm_create_common(tmpl, tb, full_name, ctr_name, + cipher_name); } static struct crypto_template crypto_ccm_tmpl = { .name = "ccm", - .alloc = crypto_ccm_alloc, - .free = crypto_ccm_free, + .create = crypto_ccm_create, .module = THIS_MODULE, }; -static struct crypto_instance *crypto_ccm_base_alloc(struct rtattr **tb) +static int crypto_ccm_base_create(struct crypto_template *tmpl, + struct rtattr **tb) { const char *ctr_name; const char *cipher_name; @@ -615,23 +642,23 @@ static struct crypto_instance *crypto_ccm_base_alloc(struct rtattr **tb) ctr_name = crypto_attr_alg_name(tb[1]); if (IS_ERR(ctr_name)) - return ERR_CAST(ctr_name); + return PTR_ERR(ctr_name); cipher_name = crypto_attr_alg_name(tb[2]); if (IS_ERR(cipher_name)) - return ERR_CAST(cipher_name); + return PTR_ERR(cipher_name); if (snprintf(full_name, CRYPTO_MAX_ALG_NAME, "ccm_base(%s,%s)", ctr_name, cipher_name) >= CRYPTO_MAX_ALG_NAME) - return ERR_PTR(-ENAMETOOLONG); + return -ENAMETOOLONG; - return crypto_ccm_alloc_common(tb, full_name, ctr_name, cipher_name); + return crypto_ccm_create_common(tmpl, tb, full_name, ctr_name, + cipher_name); } static struct crypto_template crypto_ccm_base_tmpl = { .name = "ccm_base", - .alloc = crypto_ccm_base_alloc, - .free = crypto_ccm_free, + .create = crypto_ccm_base_create, .module = THIS_MODULE, }; @@ -677,10 +704,12 @@ static int crypto_rfc4309_setauthsize(struct crypto_aead *parent, static struct aead_request *crypto_rfc4309_crypt(struct aead_request *req) { - struct aead_request *subreq = aead_request_ctx(req); + struct crypto_rfc4309_req_ctx *rctx = aead_request_ctx(req); + struct aead_request *subreq = &rctx->subreq; struct crypto_aead *aead = crypto_aead_reqtfm(req); struct crypto_rfc4309_ctx *ctx = crypto_aead_ctx(aead); struct crypto_aead *child = ctx->child; + struct scatterlist *sg; u8 *iv = PTR_ALIGN((u8 *)(subreq + 1) + crypto_aead_reqsize(child), crypto_aead_alignmask(child) + 1); @@ -690,17 +719,38 @@ static struct aead_request *crypto_rfc4309_crypt(struct aead_request *req) memcpy(iv + 1, ctx->nonce, 3); memcpy(iv + 4, req->iv, 8); + scatterwalk_map_and_copy(iv + 16, req->src, 0, req->assoclen - 8, 0); + + sg_init_table(rctx->src, 3); + sg_set_buf(rctx->src, iv + 16, req->assoclen - 8); + sg = scatterwalk_ffwd(rctx->src + 1, req->src, req->assoclen); + if (sg != rctx->src + 1) + sg_chain(rctx->src, 2, sg); + + if (req->src != req->dst) { + sg_init_table(rctx->dst, 3); + sg_set_buf(rctx->dst, iv + 16, req->assoclen - 8); + sg = scatterwalk_ffwd(rctx->dst + 1, req->dst, req->assoclen); + if (sg != rctx->dst + 1) + sg_chain(rctx->dst, 2, sg); + } + aead_request_set_tfm(subreq, child); aead_request_set_callback(subreq, req->base.flags, req->base.complete, req->base.data); - aead_request_set_crypt(subreq, req->src, req->dst, req->cryptlen, iv); - aead_request_set_assoc(subreq, req->assoc, req->assoclen); + aead_request_set_crypt(subreq, rctx->src, + req->src == req->dst ? rctx->src : rctx->dst, + req->cryptlen, iv); + aead_request_set_ad(subreq, req->assoclen - 8); return subreq; } static int crypto_rfc4309_encrypt(struct aead_request *req) { + if (req->assoclen != 16 && req->assoclen != 20) + return -EINVAL; + req = crypto_rfc4309_crypt(req); return crypto_aead_encrypt(req); @@ -708,16 +758,19 @@ static int crypto_rfc4309_encrypt(struct aead_request *req) static int crypto_rfc4309_decrypt(struct aead_request *req) { + if (req->assoclen != 16 && req->assoclen != 20) + return -EINVAL; + req = crypto_rfc4309_crypt(req); return crypto_aead_decrypt(req); } -static int crypto_rfc4309_init_tfm(struct crypto_tfm *tfm) +static int crypto_rfc4309_init_tfm(struct crypto_aead *tfm) { - struct crypto_instance *inst = (void *)tfm->__crt_alg; - struct crypto_aead_spawn *spawn = crypto_instance_ctx(inst); - struct crypto_rfc4309_ctx *ctx = crypto_tfm_ctx(tfm); + struct aead_instance *inst = aead_alg_instance(tfm); + struct crypto_aead_spawn *spawn = aead_instance_ctx(inst); + struct crypto_rfc4309_ctx *ctx = crypto_aead_ctx(tfm); struct crypto_aead *aead; unsigned long align; @@ -729,115 +782,118 @@ static int crypto_rfc4309_init_tfm(struct crypto_tfm *tfm) align = crypto_aead_alignmask(aead); align &= ~(crypto_tfm_ctx_alignment() - 1); - crypto_aead_set_reqsize(__crypto_aead_cast(tfm), - sizeof(struct aead_request) + + crypto_aead_set_reqsize( + tfm, + sizeof(struct crypto_rfc4309_req_ctx) + ALIGN(crypto_aead_reqsize(aead), crypto_tfm_ctx_alignment()) + - align + 16); + align + 32); return 0; } -static void crypto_rfc4309_exit_tfm(struct crypto_tfm *tfm) +static void crypto_rfc4309_exit_tfm(struct crypto_aead *tfm) { - struct crypto_rfc4309_ctx *ctx = crypto_tfm_ctx(tfm); + struct crypto_rfc4309_ctx *ctx = crypto_aead_ctx(tfm); crypto_free_aead(ctx->child); } -static struct crypto_instance *crypto_rfc4309_alloc(struct rtattr **tb) +static void crypto_rfc4309_free(struct aead_instance *inst) +{ + crypto_drop_aead(aead_instance_ctx(inst)); + kfree(inst); +} + +static int crypto_rfc4309_create(struct crypto_template *tmpl, + struct rtattr **tb) { struct crypto_attr_type *algt; - struct crypto_instance *inst; + struct aead_instance *inst; struct crypto_aead_spawn *spawn; - struct crypto_alg *alg; + struct aead_alg *alg; const char *ccm_name; int err; algt = crypto_get_attr_type(tb); if (IS_ERR(algt)) - return ERR_CAST(algt); + return PTR_ERR(algt); if ((algt->type ^ CRYPTO_ALG_TYPE_AEAD) & algt->mask) - return ERR_PTR(-EINVAL); + return -EINVAL; ccm_name = crypto_attr_alg_name(tb[1]); if (IS_ERR(ccm_name)) - return ERR_CAST(ccm_name); + return PTR_ERR(ccm_name); inst = kzalloc(sizeof(*inst) + sizeof(*spawn), GFP_KERNEL); if (!inst) - return ERR_PTR(-ENOMEM); + return -ENOMEM; - spawn = crypto_instance_ctx(inst); - crypto_set_aead_spawn(spawn, inst); + spawn = aead_instance_ctx(inst); + crypto_set_aead_spawn(spawn, aead_crypto_instance(inst)); err = crypto_grab_aead(spawn, ccm_name, 0, crypto_requires_sync(algt->type, algt->mask)); if (err) goto out_free_inst; - alg = crypto_aead_spawn_alg(spawn); + alg = crypto_spawn_aead_alg(spawn); err = -EINVAL; /* We only support 16-byte blocks. */ - if (alg->cra_aead.ivsize != 16) + if (crypto_aead_alg_ivsize(alg) != 16) goto out_drop_alg; /* Not a stream cipher? */ - if (alg->cra_blocksize != 1) + if (alg->base.cra_blocksize != 1) goto out_drop_alg; err = -ENAMETOOLONG; - if (snprintf(inst->alg.cra_name, CRYPTO_MAX_ALG_NAME, - "rfc4309(%s)", alg->cra_name) >= CRYPTO_MAX_ALG_NAME || - snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME, - "rfc4309(%s)", alg->cra_driver_name) >= + if (snprintf(inst->alg.base.cra_name, CRYPTO_MAX_ALG_NAME, + "rfc4309(%s)", alg->base.cra_name) >= + CRYPTO_MAX_ALG_NAME || + snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME, + "rfc4309(%s)", alg->base.cra_driver_name) >= CRYPTO_MAX_ALG_NAME) goto out_drop_alg; - inst->alg.cra_flags = CRYPTO_ALG_TYPE_AEAD; - inst->alg.cra_flags |= alg->cra_flags & CRYPTO_ALG_ASYNC; - inst->alg.cra_priority = alg->cra_priority; - inst->alg.cra_blocksize = 1; - inst->alg.cra_alignmask = alg->cra_alignmask; - inst->alg.cra_type = &crypto_nivaead_type; + inst->alg.base.cra_flags = alg->base.cra_flags & CRYPTO_ALG_ASYNC; + inst->alg.base.cra_priority = alg->base.cra_priority; + inst->alg.base.cra_blocksize = 1; + inst->alg.base.cra_alignmask = alg->base.cra_alignmask; - inst->alg.cra_aead.ivsize = 8; - inst->alg.cra_aead.maxauthsize = 16; + inst->alg.ivsize = 8; + inst->alg.maxauthsize = 16; - inst->alg.cra_ctxsize = sizeof(struct crypto_rfc4309_ctx); + inst->alg.base.cra_ctxsize = sizeof(struct crypto_rfc4309_ctx); - inst->alg.cra_init = crypto_rfc4309_init_tfm; - inst->alg.cra_exit = crypto_rfc4309_exit_tfm; + inst->alg.init = crypto_rfc4309_init_tfm; + inst->alg.exit = crypto_rfc4309_exit_tfm; - inst->alg.cra_aead.setkey = crypto_rfc4309_setkey; - inst->alg.cra_aead.setauthsize = crypto_rfc4309_setauthsize; - inst->alg.cra_aead.encrypt = crypto_rfc4309_encrypt; - inst->alg.cra_aead.decrypt = crypto_rfc4309_decrypt; + inst->alg.setkey = crypto_rfc4309_setkey; + inst->alg.setauthsize = crypto_rfc4309_setauthsize; + inst->alg.encrypt = crypto_rfc4309_encrypt; + inst->alg.decrypt = crypto_rfc4309_decrypt; - inst->alg.cra_aead.geniv = "seqiv"; + inst->free = crypto_rfc4309_free; + + err = aead_register_instance(tmpl, inst); + if (err) + goto out_drop_alg; out: - return inst; + return err; out_drop_alg: crypto_drop_aead(spawn); out_free_inst: kfree(inst); - inst = ERR_PTR(err); goto out; } -static void crypto_rfc4309_free(struct crypto_instance *inst) -{ - crypto_drop_spawn(crypto_instance_ctx(inst)); - kfree(inst); -} - static struct crypto_template crypto_rfc4309_tmpl = { .name = "rfc4309", - .alloc = crypto_rfc4309_alloc, - .free = crypto_rfc4309_free, + .create = crypto_rfc4309_create, .module = THIS_MODULE, }; diff --git a/crypto/chacha20_generic.c b/crypto/chacha20_generic.c index fa42e708aa96..da9c89968223 100644 --- a/crypto/chacha20_generic.c +++ b/crypto/chacha20_generic.c @@ -13,14 +13,7 @@ #include <linux/crypto.h> #include <linux/kernel.h> #include <linux/module.h> - -#define CHACHA20_NONCE_SIZE 16 -#define CHACHA20_KEY_SIZE 32 -#define CHACHA20_BLOCK_SIZE 64 - -struct chacha20_ctx { - u32 key[8]; -}; +#include <crypto/chacha20.h> static inline u32 rotl32(u32 v, u8 n) { @@ -108,7 +101,7 @@ static void chacha20_docrypt(u32 *state, u8 *dst, const u8 *src, } } -static void chacha20_init(u32 *state, struct chacha20_ctx *ctx, u8 *iv) +void crypto_chacha20_init(u32 *state, struct chacha20_ctx *ctx, u8 *iv) { static const char constant[16] = "expand 32-byte k"; @@ -129,8 +122,9 @@ static void chacha20_init(u32 *state, struct chacha20_ctx *ctx, u8 *iv) state[14] = le32_to_cpuvp(iv + 8); state[15] = le32_to_cpuvp(iv + 12); } +EXPORT_SYMBOL_GPL(crypto_chacha20_init); -static int chacha20_setkey(struct crypto_tfm *tfm, const u8 *key, +int crypto_chacha20_setkey(struct crypto_tfm *tfm, const u8 *key, unsigned int keysize) { struct chacha20_ctx *ctx = crypto_tfm_ctx(tfm); @@ -144,8 +138,9 @@ static int chacha20_setkey(struct crypto_tfm *tfm, const u8 *key, return 0; } +EXPORT_SYMBOL_GPL(crypto_chacha20_setkey); -static int chacha20_crypt(struct blkcipher_desc *desc, struct scatterlist *dst, +int crypto_chacha20_crypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { struct blkcipher_walk walk; @@ -155,7 +150,7 @@ static int chacha20_crypt(struct blkcipher_desc *desc, struct scatterlist *dst, blkcipher_walk_init(&walk, dst, src, nbytes); err = blkcipher_walk_virt_block(desc, &walk, CHACHA20_BLOCK_SIZE); - chacha20_init(state, crypto_blkcipher_ctx(desc->tfm), walk.iv); + crypto_chacha20_init(state, crypto_blkcipher_ctx(desc->tfm), walk.iv); while (walk.nbytes >= CHACHA20_BLOCK_SIZE) { chacha20_docrypt(state, walk.dst.virt.addr, walk.src.virt.addr, @@ -172,6 +167,7 @@ static int chacha20_crypt(struct blkcipher_desc *desc, struct scatterlist *dst, return err; } +EXPORT_SYMBOL_GPL(crypto_chacha20_crypt); static struct crypto_alg alg = { .cra_name = "chacha20", @@ -187,11 +183,11 @@ static struct crypto_alg alg = { .blkcipher = { .min_keysize = CHACHA20_KEY_SIZE, .max_keysize = CHACHA20_KEY_SIZE, - .ivsize = CHACHA20_NONCE_SIZE, + .ivsize = CHACHA20_IV_SIZE, .geniv = "seqiv", - .setkey = chacha20_setkey, - .encrypt = chacha20_crypt, - .decrypt = chacha20_crypt, + .setkey = crypto_chacha20_setkey, + .encrypt = crypto_chacha20_crypt, + .decrypt = crypto_chacha20_crypt, }, }, }; diff --git a/crypto/chacha20poly1305.c b/crypto/chacha20poly1305.c index 7b46ed799a64..99c3cce01290 100644 --- a/crypto/chacha20poly1305.c +++ b/crypto/chacha20poly1305.c @@ -13,6 +13,8 @@ #include <crypto/internal/hash.h> #include <crypto/internal/skcipher.h> #include <crypto/scatterwalk.h> +#include <crypto/chacha20.h> +#include <crypto/poly1305.h> #include <linux/err.h> #include <linux/init.h> #include <linux/kernel.h> @@ -20,11 +22,6 @@ #include "internal.h" -#define POLY1305_BLOCK_SIZE 16 -#define POLY1305_DIGEST_SIZE 16 -#define POLY1305_KEY_SIZE 32 -#define CHACHA20_KEY_SIZE 32 -#define CHACHA20_IV_SIZE 16 #define CHACHAPOLY_IV_SIZE 12 struct chachapoly_instance_ctx { @@ -60,12 +57,16 @@ struct chacha_req { }; struct chachapoly_req_ctx { + struct scatterlist src[2]; + struct scatterlist dst[2]; /* the key we generate for Poly1305 using Chacha20 */ u8 key[POLY1305_KEY_SIZE]; /* calculated Poly1305 tag */ u8 tag[POLY1305_DIGEST_SIZE]; /* length of data to en/decrypt, without ICV */ unsigned int cryptlen; + /* Actual AD, excluding IV */ + unsigned int assoclen; union { struct poly_req poly; struct chacha_req chacha; @@ -98,7 +99,9 @@ static int poly_verify_tag(struct aead_request *req) struct chachapoly_req_ctx *rctx = aead_request_ctx(req); u8 tag[sizeof(rctx->tag)]; - scatterwalk_map_and_copy(tag, req->src, rctx->cryptlen, sizeof(tag), 0); + scatterwalk_map_and_copy(tag, req->src, + req->assoclen + rctx->cryptlen, + sizeof(tag), 0); if (crypto_memneq(tag, rctx->tag, sizeof(tag))) return -EBADMSG; return 0; @@ -108,7 +111,8 @@ static int poly_copy_tag(struct aead_request *req) { struct chachapoly_req_ctx *rctx = aead_request_ctx(req); - scatterwalk_map_and_copy(rctx->tag, req->dst, rctx->cryptlen, + scatterwalk_map_and_copy(rctx->tag, req->dst, + req->assoclen + rctx->cryptlen, sizeof(rctx->tag), 1); return 0; } @@ -123,14 +127,24 @@ static int chacha_decrypt(struct aead_request *req) struct chachapoly_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); struct chachapoly_req_ctx *rctx = aead_request_ctx(req); struct chacha_req *creq = &rctx->u.chacha; + struct scatterlist *src, *dst; int err; chacha_iv(creq->iv, req, 1); + sg_init_table(rctx->src, 2); + src = scatterwalk_ffwd(rctx->src, req->src, req->assoclen); + dst = src; + + if (req->src != req->dst) { + sg_init_table(rctx->dst, 2); + dst = scatterwalk_ffwd(rctx->dst, req->dst, req->assoclen); + } + ablkcipher_request_set_callback(&creq->req, aead_request_flags(req), chacha_decrypt_done, req); ablkcipher_request_set_tfm(&creq->req, ctx->chacha); - ablkcipher_request_set_crypt(&creq->req, req->src, req->dst, + ablkcipher_request_set_crypt(&creq->req, src, dst, rctx->cryptlen, creq->iv); err = crypto_ablkcipher_decrypt(&creq->req); if (err) @@ -156,14 +170,15 @@ static void poly_tail_done(struct crypto_async_request *areq, int err) static int poly_tail(struct aead_request *req) { - struct chachapoly_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct chachapoly_ctx *ctx = crypto_aead_ctx(tfm); struct chachapoly_req_ctx *rctx = aead_request_ctx(req); struct poly_req *preq = &rctx->u.poly; __le64 len; int err; sg_init_table(preq->src, 1); - len = cpu_to_le64(req->assoclen); + len = cpu_to_le64(rctx->assoclen); memcpy(&preq->tail.assoclen, &len, sizeof(len)); len = cpu_to_le64(rctx->cryptlen); memcpy(&preq->tail.cryptlen, &len, sizeof(len)); @@ -228,6 +243,9 @@ static int poly_cipher(struct aead_request *req) if (rctx->cryptlen == req->cryptlen) /* encrypting */ crypt = req->dst; + sg_init_table(rctx->src, 2); + crypt = scatterwalk_ffwd(rctx->src, crypt, req->assoclen); + ahash_request_set_callback(&preq->req, aead_request_flags(req), poly_cipher_done, req); ahash_request_set_tfm(&preq->req, ctx->poly); @@ -253,7 +271,7 @@ static int poly_adpad(struct aead_request *req) unsigned int padlen, bs = POLY1305_BLOCK_SIZE; int err; - padlen = (bs - (req->assoclen % bs)) % bs; + padlen = (bs - (rctx->assoclen % bs)) % bs; memset(preq->pad, 0, sizeof(preq->pad)); sg_init_table(preq->src, 1); sg_set_buf(preq->src, preq->pad, padlen); @@ -285,7 +303,7 @@ static int poly_ad(struct aead_request *req) ahash_request_set_callback(&preq->req, aead_request_flags(req), poly_ad_done, req); ahash_request_set_tfm(&preq->req, ctx->poly); - ahash_request_set_crypt(&preq->req, req->assoc, NULL, req->assoclen); + ahash_request_set_crypt(&preq->req, req->src, NULL, rctx->assoclen); err = crypto_ahash_update(&preq->req); if (err) @@ -351,11 +369,20 @@ static void poly_genkey_done(struct crypto_async_request *areq, int err) static int poly_genkey(struct aead_request *req) { - struct chachapoly_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct chachapoly_ctx *ctx = crypto_aead_ctx(tfm); struct chachapoly_req_ctx *rctx = aead_request_ctx(req); struct chacha_req *creq = &rctx->u.chacha; int err; + rctx->assoclen = req->assoclen; + + if (crypto_aead_ivsize(tfm) == 8) { + if (rctx->assoclen < 8) + return -EINVAL; + rctx->assoclen -= 8; + } + sg_init_table(creq->src, 1); memset(rctx->key, 0, sizeof(rctx->key)); sg_set_buf(creq->src, rctx->key, sizeof(rctx->key)); @@ -385,14 +412,24 @@ static int chacha_encrypt(struct aead_request *req) struct chachapoly_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); struct chachapoly_req_ctx *rctx = aead_request_ctx(req); struct chacha_req *creq = &rctx->u.chacha; + struct scatterlist *src, *dst; int err; chacha_iv(creq->iv, req, 1); + sg_init_table(rctx->src, 2); + src = scatterwalk_ffwd(rctx->src, req->src, req->assoclen); + dst = src; + + if (req->src != req->dst) { + sg_init_table(rctx->dst, 2); + dst = scatterwalk_ffwd(rctx->dst, req->dst, req->assoclen); + } + ablkcipher_request_set_callback(&creq->req, aead_request_flags(req), chacha_encrypt_done, req); ablkcipher_request_set_tfm(&creq->req, ctx->chacha); - ablkcipher_request_set_crypt(&creq->req, req->src, req->dst, + ablkcipher_request_set_crypt(&creq->req, src, dst, req->cryptlen, creq->iv); err = crypto_ablkcipher_encrypt(&creq->req); if (err) @@ -426,8 +463,6 @@ static int chachapoly_decrypt(struct aead_request *req) { struct chachapoly_req_ctx *rctx = aead_request_ctx(req); - if (req->cryptlen < POLY1305_DIGEST_SIZE) - return -EINVAL; rctx->cryptlen = req->cryptlen - POLY1305_DIGEST_SIZE; /* decrypt call chain: @@ -476,11 +511,11 @@ static int chachapoly_setauthsize(struct crypto_aead *tfm, return 0; } -static int chachapoly_init(struct crypto_tfm *tfm) +static int chachapoly_init(struct crypto_aead *tfm) { - struct crypto_instance *inst = (void *)tfm->__crt_alg; - struct chachapoly_instance_ctx *ictx = crypto_instance_ctx(inst); - struct chachapoly_ctx *ctx = crypto_tfm_ctx(tfm); + struct aead_instance *inst = aead_alg_instance(tfm); + struct chachapoly_instance_ctx *ictx = aead_instance_ctx(inst); + struct chachapoly_ctx *ctx = crypto_aead_ctx(tfm); struct crypto_ablkcipher *chacha; struct crypto_ahash *poly; unsigned long align; @@ -499,77 +534,87 @@ static int chachapoly_init(struct crypto_tfm *tfm) ctx->poly = poly; ctx->saltlen = ictx->saltlen; - align = crypto_tfm_alg_alignmask(tfm); + align = crypto_aead_alignmask(tfm); align &= ~(crypto_tfm_ctx_alignment() - 1); - crypto_aead_set_reqsize(__crypto_aead_cast(tfm), - align + offsetof(struct chachapoly_req_ctx, u) + - max(offsetof(struct chacha_req, req) + - sizeof(struct ablkcipher_request) + - crypto_ablkcipher_reqsize(chacha), - offsetof(struct poly_req, req) + - sizeof(struct ahash_request) + - crypto_ahash_reqsize(poly))); + crypto_aead_set_reqsize( + tfm, + align + offsetof(struct chachapoly_req_ctx, u) + + max(offsetof(struct chacha_req, req) + + sizeof(struct ablkcipher_request) + + crypto_ablkcipher_reqsize(chacha), + offsetof(struct poly_req, req) + + sizeof(struct ahash_request) + + crypto_ahash_reqsize(poly))); return 0; } -static void chachapoly_exit(struct crypto_tfm *tfm) +static void chachapoly_exit(struct crypto_aead *tfm) { - struct chachapoly_ctx *ctx = crypto_tfm_ctx(tfm); + struct chachapoly_ctx *ctx = crypto_aead_ctx(tfm); crypto_free_ahash(ctx->poly); crypto_free_ablkcipher(ctx->chacha); } -static struct crypto_instance *chachapoly_alloc(struct rtattr **tb, - const char *name, - unsigned int ivsize) +static void chachapoly_free(struct aead_instance *inst) +{ + struct chachapoly_instance_ctx *ctx = aead_instance_ctx(inst); + + crypto_drop_skcipher(&ctx->chacha); + crypto_drop_ahash(&ctx->poly); + kfree(inst); +} + +static int chachapoly_create(struct crypto_template *tmpl, struct rtattr **tb, + const char *name, unsigned int ivsize) { struct crypto_attr_type *algt; - struct crypto_instance *inst; + struct aead_instance *inst; struct crypto_alg *chacha; struct crypto_alg *poly; - struct ahash_alg *poly_ahash; + struct hash_alg_common *poly_hash; struct chachapoly_instance_ctx *ctx; const char *chacha_name, *poly_name; int err; if (ivsize > CHACHAPOLY_IV_SIZE) - return ERR_PTR(-EINVAL); + return -EINVAL; algt = crypto_get_attr_type(tb); if (IS_ERR(algt)) - return ERR_CAST(algt); + return PTR_ERR(algt); if ((algt->type ^ CRYPTO_ALG_TYPE_AEAD) & algt->mask) - return ERR_PTR(-EINVAL); + return -EINVAL; chacha_name = crypto_attr_alg_name(tb[1]); if (IS_ERR(chacha_name)) - return ERR_CAST(chacha_name); + return PTR_ERR(chacha_name); poly_name = crypto_attr_alg_name(tb[2]); if (IS_ERR(poly_name)) - return ERR_CAST(poly_name); + return PTR_ERR(poly_name); poly = crypto_find_alg(poly_name, &crypto_ahash_type, CRYPTO_ALG_TYPE_HASH, CRYPTO_ALG_TYPE_AHASH_MASK); if (IS_ERR(poly)) - return ERR_CAST(poly); + return PTR_ERR(poly); err = -ENOMEM; inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL); if (!inst) goto out_put_poly; - ctx = crypto_instance_ctx(inst); + ctx = aead_instance_ctx(inst); ctx->saltlen = CHACHAPOLY_IV_SIZE - ivsize; - poly_ahash = container_of(poly, struct ahash_alg, halg.base); - err = crypto_init_ahash_spawn(&ctx->poly, &poly_ahash->halg, inst); + poly_hash = __crypto_hash_alg_common(poly); + err = crypto_init_ahash_spawn(&ctx->poly, poly_hash, + aead_crypto_instance(inst)); if (err) goto err_free_inst; - crypto_set_skcipher_spawn(&ctx->chacha, inst); + crypto_set_skcipher_spawn(&ctx->chacha, aead_crypto_instance(inst)); err = crypto_grab_skcipher(&ctx->chacha, chacha_name, 0, crypto_requires_sync(algt->type, algt->mask)); @@ -587,37 +632,42 @@ static struct crypto_instance *chachapoly_alloc(struct rtattr **tb, goto out_drop_chacha; err = -ENAMETOOLONG; - if (snprintf(inst->alg.cra_name, CRYPTO_MAX_ALG_NAME, + if (snprintf(inst->alg.base.cra_name, CRYPTO_MAX_ALG_NAME, "%s(%s,%s)", name, chacha_name, poly_name) >= CRYPTO_MAX_ALG_NAME) goto out_drop_chacha; - if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME, + if (snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s(%s,%s)", name, chacha->cra_driver_name, poly->cra_driver_name) >= CRYPTO_MAX_ALG_NAME) goto out_drop_chacha; - inst->alg.cra_flags = CRYPTO_ALG_TYPE_AEAD; - inst->alg.cra_flags |= (chacha->cra_flags | - poly->cra_flags) & CRYPTO_ALG_ASYNC; - inst->alg.cra_priority = (chacha->cra_priority + - poly->cra_priority) / 2; - inst->alg.cra_blocksize = 1; - inst->alg.cra_alignmask = chacha->cra_alignmask | poly->cra_alignmask; - inst->alg.cra_type = &crypto_nivaead_type; - inst->alg.cra_aead.ivsize = ivsize; - inst->alg.cra_aead.maxauthsize = POLY1305_DIGEST_SIZE; - inst->alg.cra_ctxsize = sizeof(struct chachapoly_ctx) + ctx->saltlen; - inst->alg.cra_init = chachapoly_init; - inst->alg.cra_exit = chachapoly_exit; - inst->alg.cra_aead.encrypt = chachapoly_encrypt; - inst->alg.cra_aead.decrypt = chachapoly_decrypt; - inst->alg.cra_aead.setkey = chachapoly_setkey; - inst->alg.cra_aead.setauthsize = chachapoly_setauthsize; - inst->alg.cra_aead.geniv = "seqiv"; - -out: + inst->alg.base.cra_flags = (chacha->cra_flags | poly->cra_flags) & + CRYPTO_ALG_ASYNC; + inst->alg.base.cra_priority = (chacha->cra_priority + + poly->cra_priority) / 2; + inst->alg.base.cra_blocksize = 1; + inst->alg.base.cra_alignmask = chacha->cra_alignmask | + poly->cra_alignmask; + inst->alg.base.cra_ctxsize = sizeof(struct chachapoly_ctx) + + ctx->saltlen; + inst->alg.ivsize = ivsize; + inst->alg.maxauthsize = POLY1305_DIGEST_SIZE; + inst->alg.init = chachapoly_init; + inst->alg.exit = chachapoly_exit; + inst->alg.encrypt = chachapoly_encrypt; + inst->alg.decrypt = chachapoly_decrypt; + inst->alg.setkey = chachapoly_setkey; + inst->alg.setauthsize = chachapoly_setauthsize; + + inst->free = chachapoly_free; + + err = aead_register_instance(tmpl, inst); + if (err) + goto out_drop_chacha; + +out_put_poly: crypto_mod_put(poly); - return inst; + return err; out_drop_chacha: crypto_drop_skcipher(&ctx->chacha); @@ -625,41 +675,28 @@ err_drop_poly: crypto_drop_ahash(&ctx->poly); err_free_inst: kfree(inst); -out_put_poly: - inst = ERR_PTR(err); - goto out; -} - -static struct crypto_instance *rfc7539_alloc(struct rtattr **tb) -{ - return chachapoly_alloc(tb, "rfc7539", 12); + goto out_put_poly; } -static struct crypto_instance *rfc7539esp_alloc(struct rtattr **tb) +static int rfc7539_create(struct crypto_template *tmpl, struct rtattr **tb) { - return chachapoly_alloc(tb, "rfc7539esp", 8); + return chachapoly_create(tmpl, tb, "rfc7539", 12); } -static void chachapoly_free(struct crypto_instance *inst) +static int rfc7539esp_create(struct crypto_template *tmpl, struct rtattr **tb) { - struct chachapoly_instance_ctx *ctx = crypto_instance_ctx(inst); - - crypto_drop_skcipher(&ctx->chacha); - crypto_drop_ahash(&ctx->poly); - kfree(inst); + return chachapoly_create(tmpl, tb, "rfc7539esp", 8); } static struct crypto_template rfc7539_tmpl = { .name = "rfc7539", - .alloc = rfc7539_alloc, - .free = chachapoly_free, + .create = rfc7539_create, .module = THIS_MODULE, }; static struct crypto_template rfc7539esp_tmpl = { .name = "rfc7539esp", - .alloc = rfc7539esp_alloc, - .free = chachapoly_free, + .create = rfc7539esp_create, .module = THIS_MODULE, }; @@ -690,6 +727,5 @@ module_exit(chacha20poly1305_module_exit); MODULE_LICENSE("GPL"); MODULE_AUTHOR("Martin Willi <martin@strongswan.org>"); MODULE_DESCRIPTION("ChaCha20-Poly1305 AEAD"); -MODULE_ALIAS_CRYPTO("chacha20poly1305"); MODULE_ALIAS_CRYPTO("rfc7539"); MODULE_ALIAS_CRYPTO("rfc7539esp"); diff --git a/crypto/cryptd.c b/crypto/cryptd.c index 22ba81f76764..c81861b1350b 100644 --- a/crypto/cryptd.c +++ b/crypto/cryptd.c @@ -176,10 +176,9 @@ static inline void cryptd_check_internal(struct rtattr **tb, u32 *type, algt = crypto_get_attr_type(tb); if (IS_ERR(algt)) return; - if ((algt->type & CRYPTO_ALG_INTERNAL)) - *type |= CRYPTO_ALG_INTERNAL; - if ((algt->mask & CRYPTO_ALG_INTERNAL)) - *mask |= CRYPTO_ALG_INTERNAL; + + *type |= algt->type & CRYPTO_ALG_INTERNAL; + *mask |= algt->mask & CRYPTO_ALG_INTERNAL; } static int cryptd_blkcipher_setkey(struct crypto_ablkcipher *parent, @@ -688,16 +687,18 @@ static void cryptd_aead_crypt(struct aead_request *req, int (*crypt)(struct aead_request *req)) { struct cryptd_aead_request_ctx *rctx; + crypto_completion_t compl; + rctx = aead_request_ctx(req); + compl = rctx->complete; if (unlikely(err == -EINPROGRESS)) goto out; aead_request_set_tfm(req, child); err = crypt( req ); - req->base.complete = rctx->complete; out: local_bh_disable(); - rctx->complete(&req->base, err); + compl(&req->base, err); local_bh_enable(); } @@ -708,7 +709,7 @@ static void cryptd_aead_encrypt(struct crypto_async_request *areq, int err) struct aead_request *req; req = container_of(areq, struct aead_request, base); - cryptd_aead_crypt(req, child, err, crypto_aead_crt(child)->encrypt); + cryptd_aead_crypt(req, child, err, crypto_aead_alg(child)->encrypt); } static void cryptd_aead_decrypt(struct crypto_async_request *areq, int err) @@ -718,7 +719,7 @@ static void cryptd_aead_decrypt(struct crypto_async_request *areq, int err) struct aead_request *req; req = container_of(areq, struct aead_request, base); - cryptd_aead_crypt(req, child, err, crypto_aead_crt(child)->decrypt); + cryptd_aead_crypt(req, child, err, crypto_aead_alg(child)->decrypt); } static int cryptd_aead_enqueue(struct aead_request *req, @@ -756,7 +757,9 @@ static int cryptd_aead_init_tfm(struct crypto_aead *tfm) return PTR_ERR(cipher); ctx->child = cipher; - crypto_aead_set_reqsize(tfm, sizeof(struct cryptd_aead_request_ctx)); + crypto_aead_set_reqsize( + tfm, max((unsigned)sizeof(struct cryptd_aead_request_ctx), + crypto_aead_reqsize(cipher))); return 0; } @@ -775,7 +778,7 @@ static int cryptd_create_aead(struct crypto_template *tmpl, struct aead_alg *alg; const char *name; u32 type = 0; - u32 mask = 0; + u32 mask = CRYPTO_ALG_ASYNC; int err; cryptd_check_internal(tb, &type, &mask); diff --git a/crypto/crypto_user.c b/crypto/crypto_user.c index 08ea2867fc8a..d94d99ffe8b9 100644 --- a/crypto/crypto_user.c +++ b/crypto/crypto_user.c @@ -25,7 +25,6 @@ #include <net/netlink.h> #include <linux/security.h> #include <net/net_namespace.h> -#include <crypto/internal/aead.h> #include <crypto/internal/skcipher.h> #include <crypto/internal/rng.h> #include <crypto/akcipher.h> @@ -385,34 +384,6 @@ static struct crypto_alg *crypto_user_skcipher_alg(const char *name, u32 type, return ERR_PTR(err); } -static struct crypto_alg *crypto_user_aead_alg(const char *name, u32 type, - u32 mask) -{ - int err; - struct crypto_alg *alg; - - type &= ~(CRYPTO_ALG_TYPE_MASK | CRYPTO_ALG_GENIV); - type |= CRYPTO_ALG_TYPE_AEAD; - mask &= ~(CRYPTO_ALG_TYPE_MASK | CRYPTO_ALG_GENIV); - mask |= CRYPTO_ALG_TYPE_MASK; - - for (;;) { - alg = crypto_lookup_aead(name, type, mask); - if (!IS_ERR(alg)) - return alg; - - err = PTR_ERR(alg); - if (err != -EAGAIN) - break; - if (signal_pending(current)) { - err = -EINTR; - break; - } - } - - return ERR_PTR(err); -} - static int crypto_add_alg(struct sk_buff *skb, struct nlmsghdr *nlh, struct nlattr **attrs) { @@ -446,9 +417,6 @@ static int crypto_add_alg(struct sk_buff *skb, struct nlmsghdr *nlh, name = p->cru_name; switch (p->cru_type & p->cru_mask & CRYPTO_ALG_TYPE_MASK) { - case CRYPTO_ALG_TYPE_AEAD: - alg = crypto_user_aead_alg(name, p->cru_type, p->cru_mask); - break; case CRYPTO_ALG_TYPE_GIVCIPHER: case CRYPTO_ALG_TYPE_BLKCIPHER: case CRYPTO_ALG_TYPE_ABLKCIPHER: diff --git a/crypto/echainiv.c b/crypto/echainiv.c index b6e43dc61356..b96a84560b67 100644 --- a/crypto/echainiv.c +++ b/crypto/echainiv.c @@ -19,8 +19,6 @@ */ #include <crypto/internal/geniv.h> -#include <crypto/null.h> -#include <crypto/rng.h> #include <crypto/scatterwalk.h> #include <linux/err.h> #include <linux/init.h> @@ -33,13 +31,6 @@ #define MAX_IV_SIZE 16 -struct echainiv_ctx { - /* aead_geniv_ctx must be first the element */ - struct aead_geniv_ctx geniv; - struct crypto_blkcipher *null; - u8 salt[] __attribute__ ((aligned(__alignof__(u32)))); -}; - static DEFINE_PER_CPU(u32 [MAX_IV_SIZE / sizeof(u32)], echainiv_iv); /* We don't care if we get preempted and read/write IVs from the next CPU. */ @@ -103,7 +94,7 @@ static void echainiv_encrypt_complete(struct crypto_async_request *base, static int echainiv_encrypt(struct aead_request *req) { struct crypto_aead *geniv = crypto_aead_reqtfm(req); - struct echainiv_ctx *ctx = crypto_aead_ctx(geniv); + struct aead_geniv_ctx *ctx = crypto_aead_ctx(geniv); struct aead_request *subreq = aead_request_ctx(req); crypto_completion_t compl; void *data; @@ -114,7 +105,7 @@ static int echainiv_encrypt(struct aead_request *req) if (req->cryptlen < ivsize) return -EINVAL; - aead_request_set_tfm(subreq, ctx->geniv.child); + aead_request_set_tfm(subreq, ctx->child); compl = echainiv_encrypt_complete; data = req; @@ -145,8 +136,8 @@ static int echainiv_encrypt(struct aead_request *req) aead_request_set_callback(subreq, req->base.flags, compl, data); aead_request_set_crypt(subreq, req->dst, req->dst, - req->cryptlen - ivsize, info); - aead_request_set_ad(subreq, req->assoclen + ivsize); + req->cryptlen, info); + aead_request_set_ad(subreq, req->assoclen); crypto_xor(info, ctx->salt, ivsize); scatterwalk_map_and_copy(info, req->dst, req->assoclen, ivsize, 1); @@ -160,16 +151,16 @@ static int echainiv_encrypt(struct aead_request *req) static int echainiv_decrypt(struct aead_request *req) { struct crypto_aead *geniv = crypto_aead_reqtfm(req); - struct echainiv_ctx *ctx = crypto_aead_ctx(geniv); + struct aead_geniv_ctx *ctx = crypto_aead_ctx(geniv); struct aead_request *subreq = aead_request_ctx(req); crypto_completion_t compl; void *data; unsigned int ivsize = crypto_aead_ivsize(geniv); - if (req->cryptlen < ivsize + crypto_aead_authsize(geniv)) + if (req->cryptlen < ivsize) return -EINVAL; - aead_request_set_tfm(subreq, ctx->geniv.child); + aead_request_set_tfm(subreq, ctx->child); compl = req->base.complete; data = req->base.data; @@ -180,61 +171,10 @@ static int echainiv_decrypt(struct aead_request *req) aead_request_set_ad(subreq, req->assoclen + ivsize); scatterwalk_map_and_copy(req->iv, req->src, req->assoclen, ivsize, 0); - if (req->src != req->dst) - scatterwalk_map_and_copy(req->iv, req->dst, - req->assoclen, ivsize, 1); return crypto_aead_decrypt(subreq); } -static int echainiv_init(struct crypto_tfm *tfm) -{ - struct crypto_aead *geniv = __crypto_aead_cast(tfm); - struct echainiv_ctx *ctx = crypto_aead_ctx(geniv); - int err; - - spin_lock_init(&ctx->geniv.lock); - - crypto_aead_set_reqsize(geniv, sizeof(struct aead_request)); - - err = crypto_get_default_rng(); - if (err) - goto out; - - err = crypto_rng_get_bytes(crypto_default_rng, ctx->salt, - crypto_aead_ivsize(geniv)); - crypto_put_default_rng(); - if (err) - goto out; - - ctx->null = crypto_get_default_null_skcipher(); - err = PTR_ERR(ctx->null); - if (IS_ERR(ctx->null)) - goto out; - - err = aead_geniv_init(tfm); - if (err) - goto drop_null; - - ctx->geniv.child = geniv->child; - geniv->child = geniv; - -out: - return err; - -drop_null: - crypto_put_default_null_skcipher(); - goto out; -} - -static void echainiv_exit(struct crypto_tfm *tfm) -{ - struct echainiv_ctx *ctx = crypto_tfm_ctx(tfm); - - crypto_free_aead(ctx->geniv.child); - crypto_put_default_null_skcipher(); -} - static int echainiv_aead_create(struct crypto_template *tmpl, struct rtattr **tb) { @@ -251,9 +191,6 @@ static int echainiv_aead_create(struct crypto_template *tmpl, spawn = aead_instance_ctx(inst); alg = crypto_spawn_aead_alg(spawn); - if (alg->base.cra_aead.encrypt) - goto done; - err = -EINVAL; if (inst->alg.ivsize & (sizeof(u32) - 1) || inst->alg.ivsize > MAX_IV_SIZE) @@ -262,14 +199,15 @@ static int echainiv_aead_create(struct crypto_template *tmpl, inst->alg.encrypt = echainiv_encrypt; inst->alg.decrypt = echainiv_decrypt; - inst->alg.base.cra_init = echainiv_init; - inst->alg.base.cra_exit = echainiv_exit; + inst->alg.init = aead_init_geniv; + inst->alg.exit = aead_exit_geniv; inst->alg.base.cra_alignmask |= __alignof__(u32) - 1; - inst->alg.base.cra_ctxsize = sizeof(struct echainiv_ctx); + inst->alg.base.cra_ctxsize = sizeof(struct aead_geniv_ctx); inst->alg.base.cra_ctxsize += inst->alg.ivsize; -done: + inst->free = aead_geniv_free; + err = aead_register_instance(tmpl, inst); if (err) goto free_inst; diff --git a/crypto/gcm.c b/crypto/gcm.c index 7d32d4720564..ddb4f29b2fe6 100644 --- a/crypto/gcm.c +++ b/crypto/gcm.c @@ -38,6 +38,12 @@ struct crypto_rfc4106_ctx { u8 nonce[4]; }; +struct crypto_rfc4106_req_ctx { + struct scatterlist src[3]; + struct scatterlist dst[3]; + struct aead_request subreq; +}; + struct crypto_rfc4543_instance_ctx { struct crypto_aead_spawn aead; }; @@ -601,6 +607,15 @@ static void crypto_gcm_exit_tfm(struct crypto_aead *tfm) crypto_free_ablkcipher(ctx->ctr); } +static void crypto_gcm_free(struct aead_instance *inst) +{ + struct gcm_instance_ctx *ctx = aead_instance_ctx(inst); + + crypto_drop_skcipher(&ctx->ctr); + crypto_drop_ahash(&ctx->ghash); + kfree(inst); +} + static int crypto_gcm_create_common(struct crypto_template *tmpl, struct rtattr **tb, const char *full_name, @@ -689,6 +704,8 @@ static int crypto_gcm_create_common(struct crypto_template *tmpl, inst->alg.encrypt = crypto_gcm_encrypt; inst->alg.decrypt = crypto_gcm_decrypt; + inst->free = crypto_gcm_free; + err = aead_register_instance(tmpl, inst); if (err) goto out_put_ctr; @@ -728,19 +745,9 @@ static int crypto_gcm_create(struct crypto_template *tmpl, struct rtattr **tb) ctr_name, "ghash"); } -static void crypto_gcm_free(struct crypto_instance *inst) -{ - struct gcm_instance_ctx *ctx = crypto_instance_ctx(inst); - - crypto_drop_skcipher(&ctx->ctr); - crypto_drop_ahash(&ctx->ghash); - kfree(aead_instance(inst)); -} - static struct crypto_template crypto_gcm_tmpl = { .name = "gcm", .create = crypto_gcm_create, - .free = crypto_gcm_free, .module = THIS_MODULE, }; @@ -770,7 +777,6 @@ static int crypto_gcm_base_create(struct crypto_template *tmpl, static struct crypto_template crypto_gcm_base_tmpl = { .name = "gcm_base", .create = crypto_gcm_base_create, - .free = crypto_gcm_free, .module = THIS_MODULE, }; @@ -816,27 +822,50 @@ static int crypto_rfc4106_setauthsize(struct crypto_aead *parent, static struct aead_request *crypto_rfc4106_crypt(struct aead_request *req) { - struct aead_request *subreq = aead_request_ctx(req); + struct crypto_rfc4106_req_ctx *rctx = aead_request_ctx(req); struct crypto_aead *aead = crypto_aead_reqtfm(req); struct crypto_rfc4106_ctx *ctx = crypto_aead_ctx(aead); + struct aead_request *subreq = &rctx->subreq; struct crypto_aead *child = ctx->child; + struct scatterlist *sg; u8 *iv = PTR_ALIGN((u8 *)(subreq + 1) + crypto_aead_reqsize(child), crypto_aead_alignmask(child) + 1); + scatterwalk_map_and_copy(iv + 12, req->src, 0, req->assoclen - 8, 0); + memcpy(iv, ctx->nonce, 4); memcpy(iv + 4, req->iv, 8); + sg_init_table(rctx->src, 3); + sg_set_buf(rctx->src, iv + 12, req->assoclen - 8); + sg = scatterwalk_ffwd(rctx->src + 1, req->src, req->assoclen); + if (sg != rctx->src + 1) + sg_chain(rctx->src, 2, sg); + + if (req->src != req->dst) { + sg_init_table(rctx->dst, 3); + sg_set_buf(rctx->dst, iv + 12, req->assoclen - 8); + sg = scatterwalk_ffwd(rctx->dst + 1, req->dst, req->assoclen); + if (sg != rctx->dst + 1) + sg_chain(rctx->dst, 2, sg); + } + aead_request_set_tfm(subreq, child); aead_request_set_callback(subreq, req->base.flags, req->base.complete, req->base.data); - aead_request_set_crypt(subreq, req->src, req->dst, req->cryptlen, iv); - aead_request_set_ad(subreq, req->assoclen); + aead_request_set_crypt(subreq, rctx->src, + req->src == req->dst ? rctx->src : rctx->dst, + req->cryptlen, iv); + aead_request_set_ad(subreq, req->assoclen - 8); return subreq; } static int crypto_rfc4106_encrypt(struct aead_request *req) { + if (req->assoclen != 16 && req->assoclen != 20) + return -EINVAL; + req = crypto_rfc4106_crypt(req); return crypto_aead_encrypt(req); @@ -844,6 +873,9 @@ static int crypto_rfc4106_encrypt(struct aead_request *req) static int crypto_rfc4106_decrypt(struct aead_request *req) { + if (req->assoclen != 16 && req->assoclen != 20) + return -EINVAL; + req = crypto_rfc4106_crypt(req); return crypto_aead_decrypt(req); @@ -867,9 +899,9 @@ static int crypto_rfc4106_init_tfm(struct crypto_aead *tfm) align &= ~(crypto_tfm_ctx_alignment() - 1); crypto_aead_set_reqsize( tfm, - sizeof(struct aead_request) + + sizeof(struct crypto_rfc4106_req_ctx) + ALIGN(crypto_aead_reqsize(aead), crypto_tfm_ctx_alignment()) + - align + 12); + align + 24); return 0; } @@ -881,6 +913,12 @@ static void crypto_rfc4106_exit_tfm(struct crypto_aead *tfm) crypto_free_aead(ctx->child); } +static void crypto_rfc4106_free(struct aead_instance *inst) +{ + crypto_drop_aead(aead_instance_ctx(inst)); + kfree(inst); +} + static int crypto_rfc4106_create(struct crypto_template *tmpl, struct rtattr **tb) { @@ -934,7 +972,7 @@ static int crypto_rfc4106_create(struct crypto_template *tmpl, CRYPTO_MAX_ALG_NAME) goto out_drop_alg; - inst->alg.base.cra_flags |= alg->base.cra_flags & CRYPTO_ALG_ASYNC; + inst->alg.base.cra_flags = alg->base.cra_flags & CRYPTO_ALG_ASYNC; inst->alg.base.cra_priority = alg->base.cra_priority; inst->alg.base.cra_blocksize = 1; inst->alg.base.cra_alignmask = alg->base.cra_alignmask; @@ -952,6 +990,8 @@ static int crypto_rfc4106_create(struct crypto_template *tmpl, inst->alg.encrypt = crypto_rfc4106_encrypt; inst->alg.decrypt = crypto_rfc4106_decrypt; + inst->free = crypto_rfc4106_free; + err = aead_register_instance(tmpl, inst); if (err) goto out_drop_alg; @@ -966,16 +1006,9 @@ out_free_inst: goto out; } -static void crypto_rfc4106_free(struct crypto_instance *inst) -{ - crypto_drop_aead(crypto_instance_ctx(inst)); - kfree(aead_instance(inst)); -} - static struct crypto_template crypto_rfc4106_tmpl = { .name = "rfc4106", .create = crypto_rfc4106_create, - .free = crypto_rfc4106_free, .module = THIS_MODULE, }; @@ -1114,6 +1147,15 @@ static void crypto_rfc4543_exit_tfm(struct crypto_aead *tfm) crypto_put_default_null_skcipher(); } +static void crypto_rfc4543_free(struct aead_instance *inst) +{ + struct crypto_rfc4543_instance_ctx *ctx = aead_instance_ctx(inst); + + crypto_drop_aead(&ctx->aead); + + kfree(inst); +} + static int crypto_rfc4543_create(struct crypto_template *tmpl, struct rtattr **tb) { @@ -1187,6 +1229,8 @@ static int crypto_rfc4543_create(struct crypto_template *tmpl, inst->alg.encrypt = crypto_rfc4543_encrypt; inst->alg.decrypt = crypto_rfc4543_decrypt; + inst->free = crypto_rfc4543_free, + err = aead_register_instance(tmpl, inst); if (err) goto out_drop_alg; @@ -1201,19 +1245,9 @@ out_free_inst: goto out; } -static void crypto_rfc4543_free(struct crypto_instance *inst) -{ - struct crypto_rfc4543_instance_ctx *ctx = crypto_instance_ctx(inst); - - crypto_drop_aead(&ctx->aead); - - kfree(aead_instance(inst)); -} - static struct crypto_template crypto_rfc4543_tmpl = { .name = "rfc4543", .create = crypto_rfc4543_create, - .free = crypto_rfc4543_free, .module = THIS_MODULE, }; diff --git a/crypto/jitterentropy-kcapi.c b/crypto/jitterentropy-kcapi.c index b32d834144cd..ceea83d13168 100644 --- a/crypto/jitterentropy-kcapi.c +++ b/crypto/jitterentropy-kcapi.c @@ -79,7 +79,7 @@ int jent_fips_enabled(void) void jent_panic(char *s) { - panic(s); + panic("%s", s); } void jent_memcpy(void *dest, const void *src, unsigned int n) diff --git a/crypto/pcrypt.c b/crypto/pcrypt.c index 45e7d5155672..ee9cfb99fe25 100644 --- a/crypto/pcrypt.c +++ b/crypto/pcrypt.c @@ -274,11 +274,16 @@ static int pcrypt_create_aead(struct crypto_template *tmpl, struct rtattr **tb, u32 type, u32 mask) { struct pcrypt_instance_ctx *ctx; + struct crypto_attr_type *algt; struct aead_instance *inst; struct aead_alg *alg; const char *name; int err; + algt = crypto_get_attr_type(tb); + if (IS_ERR(algt)) + return PTR_ERR(algt); + name = crypto_attr_alg_name(tb[1]); if (IS_ERR(name)) return PTR_ERR(name); @@ -299,6 +304,8 @@ static int pcrypt_create_aead(struct crypto_template *tmpl, struct rtattr **tb, if (err) goto out_drop_aead; + inst->alg.base.cra_flags = CRYPTO_ALG_ASYNC; + inst->alg.ivsize = crypto_aead_alg_ivsize(alg); inst->alg.maxauthsize = crypto_aead_alg_maxauthsize(alg); diff --git a/crypto/poly1305_generic.c b/crypto/poly1305_generic.c index 387b5c887a80..2df9835dfbc0 100644 --- a/crypto/poly1305_generic.c +++ b/crypto/poly1305_generic.c @@ -13,31 +13,11 @@ #include <crypto/algapi.h> #include <crypto/internal/hash.h> +#include <crypto/poly1305.h> #include <linux/crypto.h> #include <linux/kernel.h> #include <linux/module.h> -#define POLY1305_BLOCK_SIZE 16 -#define POLY1305_KEY_SIZE 32 -#define POLY1305_DIGEST_SIZE 16 - -struct poly1305_desc_ctx { - /* key */ - u32 r[5]; - /* finalize key */ - u32 s[4]; - /* accumulator */ - u32 h[5]; - /* partial buffer */ - u8 buf[POLY1305_BLOCK_SIZE]; - /* bytes used in partial buffer */ - unsigned int buflen; - /* r key has been set */ - bool rset; - /* s key has been set */ - bool sset; -}; - static inline u64 mlt(u64 a, u64 b) { return a * b; @@ -58,7 +38,7 @@ static inline u32 le32_to_cpuvp(const void *p) return le32_to_cpup(p); } -static int poly1305_init(struct shash_desc *desc) +int crypto_poly1305_init(struct shash_desc *desc) { struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc); @@ -69,8 +49,9 @@ static int poly1305_init(struct shash_desc *desc) return 0; } +EXPORT_SYMBOL_GPL(crypto_poly1305_init); -static int poly1305_setkey(struct crypto_shash *tfm, +int crypto_poly1305_setkey(struct crypto_shash *tfm, const u8 *key, unsigned int keylen) { /* Poly1305 requires a unique key for each tag, which implies that @@ -79,6 +60,7 @@ static int poly1305_setkey(struct crypto_shash *tfm, * the update() call. */ return -ENOTSUPP; } +EXPORT_SYMBOL_GPL(crypto_poly1305_setkey); static void poly1305_setrkey(struct poly1305_desc_ctx *dctx, const u8 *key) { @@ -98,16 +80,10 @@ static void poly1305_setskey(struct poly1305_desc_ctx *dctx, const u8 *key) dctx->s[3] = le32_to_cpuvp(key + 12); } -static unsigned int poly1305_blocks(struct poly1305_desc_ctx *dctx, - const u8 *src, unsigned int srclen, - u32 hibit) +unsigned int crypto_poly1305_setdesckey(struct poly1305_desc_ctx *dctx, + const u8 *src, unsigned int srclen) { - u32 r0, r1, r2, r3, r4; - u32 s1, s2, s3, s4; - u32 h0, h1, h2, h3, h4; - u64 d0, d1, d2, d3, d4; - - if (unlikely(!dctx->sset)) { + if (!dctx->sset) { if (!dctx->rset && srclen >= POLY1305_BLOCK_SIZE) { poly1305_setrkey(dctx, src); src += POLY1305_BLOCK_SIZE; @@ -121,6 +97,25 @@ static unsigned int poly1305_blocks(struct poly1305_desc_ctx *dctx, dctx->sset = true; } } + return srclen; +} +EXPORT_SYMBOL_GPL(crypto_poly1305_setdesckey); + +static unsigned int poly1305_blocks(struct poly1305_desc_ctx *dctx, + const u8 *src, unsigned int srclen, + u32 hibit) +{ + u32 r0, r1, r2, r3, r4; + u32 s1, s2, s3, s4; + u32 h0, h1, h2, h3, h4; + u64 d0, d1, d2, d3, d4; + unsigned int datalen; + + if (unlikely(!dctx->sset)) { + datalen = crypto_poly1305_setdesckey(dctx, src, srclen); + src += srclen - datalen; + srclen = datalen; + } r0 = dctx->r[0]; r1 = dctx->r[1]; @@ -181,7 +176,7 @@ static unsigned int poly1305_blocks(struct poly1305_desc_ctx *dctx, return srclen; } -static int poly1305_update(struct shash_desc *desc, +int crypto_poly1305_update(struct shash_desc *desc, const u8 *src, unsigned int srclen) { struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc); @@ -214,8 +209,9 @@ static int poly1305_update(struct shash_desc *desc, return 0; } +EXPORT_SYMBOL_GPL(crypto_poly1305_update); -static int poly1305_final(struct shash_desc *desc, u8 *dst) +int crypto_poly1305_final(struct shash_desc *desc, u8 *dst) { struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc); __le32 *mac = (__le32 *)dst; @@ -282,13 +278,14 @@ static int poly1305_final(struct shash_desc *desc, u8 *dst) return 0; } +EXPORT_SYMBOL_GPL(crypto_poly1305_final); static struct shash_alg poly1305_alg = { .digestsize = POLY1305_DIGEST_SIZE, - .init = poly1305_init, - .update = poly1305_update, - .final = poly1305_final, - .setkey = poly1305_setkey, + .init = crypto_poly1305_init, + .update = crypto_poly1305_update, + .final = crypto_poly1305_final, + .setkey = crypto_poly1305_setkey, .descsize = sizeof(struct poly1305_desc_ctx), .base = { .cra_name = "poly1305", diff --git a/crypto/rsa.c b/crypto/rsa.c index 752af0656f2e..466003e1a8cf 100644 --- a/crypto/rsa.c +++ b/crypto/rsa.c @@ -267,12 +267,36 @@ err_free_m: return ret; } +static int rsa_check_key_length(unsigned int len) +{ + switch (len) { + case 512: + case 1024: + case 1536: + case 2048: + case 3072: + case 4096: + return 0; + } + + return -EINVAL; +} + static int rsa_setkey(struct crypto_akcipher *tfm, const void *key, unsigned int keylen) { struct rsa_key *pkey = akcipher_tfm_ctx(tfm); + int ret; - return rsa_parse_key(pkey, key, keylen); + ret = rsa_parse_key(pkey, key, keylen); + if (ret) + return ret; + + if (rsa_check_key_length(mpi_get_size(pkey->n) << 3)) { + rsa_free_key(pkey); + ret = -EINVAL; + } + return ret; } static void rsa_exit_tfm(struct crypto_akcipher *tfm) diff --git a/crypto/rsa_helper.c b/crypto/rsa_helper.c index 3e8e0a9e5a8e..8d96ce969b44 100644 --- a/crypto/rsa_helper.c +++ b/crypto/rsa_helper.c @@ -28,7 +28,7 @@ int rsa_get_n(void *context, size_t hdrlen, unsigned char tag, return -ENOMEM; /* In FIPS mode only allow key size 2K & 3K */ - if (fips_enabled && (mpi_get_size(key->n) != 256 || + if (fips_enabled && (mpi_get_size(key->n) != 256 && mpi_get_size(key->n) != 384)) { pr_err("RSA: key size not allowed in FIPS mode\n"); mpi_free(key->n); @@ -62,7 +62,7 @@ int rsa_get_d(void *context, size_t hdrlen, unsigned char tag, return -ENOMEM; /* In FIPS mode only allow key size 2K & 3K */ - if (fips_enabled && (mpi_get_size(key->d) != 256 || + if (fips_enabled && (mpi_get_size(key->d) != 256 && mpi_get_size(key->d) != 384)) { pr_err("RSA: key size not allowed in FIPS mode\n"); mpi_free(key->d); diff --git a/crypto/seqiv.c b/crypto/seqiv.c index 122c56e3491b..15a749a5cab7 100644 --- a/crypto/seqiv.c +++ b/crypto/seqiv.c @@ -15,7 +15,6 @@ #include <crypto/internal/geniv.h> #include <crypto/internal/skcipher.h> -#include <crypto/null.h> #include <crypto/rng.h> #include <crypto/scatterwalk.h> #include <linux/err.h> @@ -26,23 +25,11 @@ #include <linux/spinlock.h> #include <linux/string.h> -struct seqniv_request_ctx { - struct scatterlist dst[2]; - struct aead_request subreq; -}; - struct seqiv_ctx { spinlock_t lock; u8 salt[] __attribute__ ((aligned(__alignof__(u32)))); }; -struct seqiv_aead_ctx { - /* aead_geniv_ctx must be first the element */ - struct aead_geniv_ctx geniv; - struct crypto_blkcipher *null; - u8 salt[] __attribute__ ((aligned(__alignof__(u32)))); -}; - static void seqiv_free(struct crypto_instance *inst); static void seqiv_complete2(struct skcipher_givcrypt_request *req, int err) @@ -71,32 +58,6 @@ static void seqiv_complete(struct crypto_async_request *base, int err) skcipher_givcrypt_complete(req, err); } -static void seqiv_aead_complete2(struct aead_givcrypt_request *req, int err) -{ - struct aead_request *subreq = aead_givcrypt_reqctx(req); - struct crypto_aead *geniv; - - if (err == -EINPROGRESS) - return; - - if (err) - goto out; - - geniv = aead_givcrypt_reqtfm(req); - memcpy(req->areq.iv, subreq->iv, crypto_aead_ivsize(geniv)); - -out: - kfree(subreq->iv); -} - -static void seqiv_aead_complete(struct crypto_async_request *base, int err) -{ - struct aead_givcrypt_request *req = base->data; - - seqiv_aead_complete2(req, err); - aead_givcrypt_complete(req, err); -} - static void seqiv_aead_encrypt_complete2(struct aead_request *req, int err) { struct aead_request *subreq = aead_request_ctx(req); @@ -124,50 +85,6 @@ static void seqiv_aead_encrypt_complete(struct crypto_async_request *base, aead_request_complete(req, err); } -static void seqniv_aead_encrypt_complete2(struct aead_request *req, int err) -{ - unsigned int ivsize = 8; - u8 data[20]; - - if (err == -EINPROGRESS) - return; - - /* Swap IV and ESP header back to correct order. */ - scatterwalk_map_and_copy(data, req->dst, 0, req->assoclen + ivsize, 0); - scatterwalk_map_and_copy(data + ivsize, req->dst, 0, req->assoclen, 1); - scatterwalk_map_and_copy(data, req->dst, req->assoclen, ivsize, 1); -} - -static void seqniv_aead_encrypt_complete(struct crypto_async_request *base, - int err) -{ - struct aead_request *req = base->data; - - seqniv_aead_encrypt_complete2(req, err); - aead_request_complete(req, err); -} - -static void seqniv_aead_decrypt_complete2(struct aead_request *req, int err) -{ - u8 data[4]; - - if (err == -EINPROGRESS) - return; - - /* Move ESP header back to correct location. */ - scatterwalk_map_and_copy(data, req->dst, 16, req->assoclen - 8, 0); - scatterwalk_map_and_copy(data, req->dst, 8, req->assoclen - 8, 1); -} - -static void seqniv_aead_decrypt_complete(struct crypto_async_request *base, - int err) -{ - struct aead_request *req = base->data; - - seqniv_aead_decrypt_complete2(req, err); - aead_request_complete(req, err); -} - static void seqiv_geniv(struct seqiv_ctx *ctx, u8 *info, u64 seq, unsigned int ivsize) { @@ -227,112 +144,10 @@ static int seqiv_givencrypt(struct skcipher_givcrypt_request *req) return err; } -static int seqiv_aead_givencrypt(struct aead_givcrypt_request *req) -{ - struct crypto_aead *geniv = aead_givcrypt_reqtfm(req); - struct seqiv_ctx *ctx = crypto_aead_ctx(geniv); - struct aead_request *areq = &req->areq; - struct aead_request *subreq = aead_givcrypt_reqctx(req); - crypto_completion_t compl; - void *data; - u8 *info; - unsigned int ivsize; - int err; - - aead_request_set_tfm(subreq, aead_geniv_base(geniv)); - - compl = areq->base.complete; - data = areq->base.data; - info = areq->iv; - - ivsize = crypto_aead_ivsize(geniv); - - if (unlikely(!IS_ALIGNED((unsigned long)info, - crypto_aead_alignmask(geniv) + 1))) { - info = kmalloc(ivsize, areq->base.flags & - CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL: - GFP_ATOMIC); - if (!info) - return -ENOMEM; - - compl = seqiv_aead_complete; - data = req; - } - - aead_request_set_callback(subreq, areq->base.flags, compl, data); - aead_request_set_crypt(subreq, areq->src, areq->dst, areq->cryptlen, - info); - aead_request_set_assoc(subreq, areq->assoc, areq->assoclen); - - seqiv_geniv(ctx, info, req->seq, ivsize); - memcpy(req->giv, info, ivsize); - - err = crypto_aead_encrypt(subreq); - if (unlikely(info != areq->iv)) - seqiv_aead_complete2(req, err); - return err; -} - -static int seqniv_aead_encrypt(struct aead_request *req) -{ - struct crypto_aead *geniv = crypto_aead_reqtfm(req); - struct seqiv_aead_ctx *ctx = crypto_aead_ctx(geniv); - struct seqniv_request_ctx *rctx = aead_request_ctx(req); - struct aead_request *subreq = &rctx->subreq; - struct scatterlist *dst; - crypto_completion_t compl; - void *data; - unsigned int ivsize = 8; - u8 buf[20] __attribute__ ((aligned(__alignof__(u32)))); - int err; - - if (req->cryptlen < ivsize) - return -EINVAL; - - /* ESP AD is at most 12 bytes (ESN). */ - if (req->assoclen > 12) - return -EINVAL; - - aead_request_set_tfm(subreq, ctx->geniv.child); - - compl = seqniv_aead_encrypt_complete; - data = req; - - if (req->src != req->dst) { - struct blkcipher_desc desc = { - .tfm = ctx->null, - }; - - err = crypto_blkcipher_encrypt(&desc, req->dst, req->src, - req->assoclen + req->cryptlen); - if (err) - return err; - } - - dst = scatterwalk_ffwd(rctx->dst, req->dst, ivsize); - - aead_request_set_callback(subreq, req->base.flags, compl, data); - aead_request_set_crypt(subreq, dst, dst, - req->cryptlen - ivsize, req->iv); - aead_request_set_ad(subreq, req->assoclen); - - memcpy(buf, req->iv, ivsize); - crypto_xor(buf, ctx->salt, ivsize); - memcpy(req->iv, buf, ivsize); - - /* Swap order of IV and ESP AD for ICV generation. */ - scatterwalk_map_and_copy(buf + ivsize, req->dst, 0, req->assoclen, 0); - scatterwalk_map_and_copy(buf, req->dst, 0, req->assoclen + ivsize, 1); - - err = crypto_aead_encrypt(subreq); - seqniv_aead_encrypt_complete2(req, err); - return err; -} - static int seqiv_aead_encrypt(struct aead_request *req) { struct crypto_aead *geniv = crypto_aead_reqtfm(req); - struct seqiv_aead_ctx *ctx = crypto_aead_ctx(geniv); + struct aead_geniv_ctx *ctx = crypto_aead_ctx(geniv); struct aead_request *subreq = aead_request_ctx(req); crypto_completion_t compl; void *data; @@ -343,7 +158,7 @@ static int seqiv_aead_encrypt(struct aead_request *req) if (req->cryptlen < ivsize) return -EINVAL; - aead_request_set_tfm(subreq, ctx->geniv.child); + aead_request_set_tfm(subreq, ctx->child); compl = req->base.complete; data = req->base.data; @@ -387,67 +202,10 @@ static int seqiv_aead_encrypt(struct aead_request *req) return err; } -static int seqniv_aead_decrypt(struct aead_request *req) -{ - struct crypto_aead *geniv = crypto_aead_reqtfm(req); - struct seqiv_aead_ctx *ctx = crypto_aead_ctx(geniv); - struct seqniv_request_ctx *rctx = aead_request_ctx(req); - struct aead_request *subreq = &rctx->subreq; - struct scatterlist *dst; - crypto_completion_t compl; - void *data; - unsigned int ivsize = 8; - u8 buf[20]; - int err; - - if (req->cryptlen < ivsize + crypto_aead_authsize(geniv)) - return -EINVAL; - - aead_request_set_tfm(subreq, ctx->geniv.child); - - compl = req->base.complete; - data = req->base.data; - - if (req->assoclen > 12) - return -EINVAL; - else if (req->assoclen > 8) { - compl = seqniv_aead_decrypt_complete; - data = req; - } - - if (req->src != req->dst) { - struct blkcipher_desc desc = { - .tfm = ctx->null, - }; - - err = crypto_blkcipher_encrypt(&desc, req->dst, req->src, - req->assoclen + req->cryptlen); - if (err) - return err; - } - - /* Move ESP AD forward for ICV generation. */ - scatterwalk_map_and_copy(buf, req->dst, 0, req->assoclen + ivsize, 0); - memcpy(req->iv, buf + req->assoclen, ivsize); - scatterwalk_map_and_copy(buf, req->dst, ivsize, req->assoclen, 1); - - dst = scatterwalk_ffwd(rctx->dst, req->dst, ivsize); - - aead_request_set_callback(subreq, req->base.flags, compl, data); - aead_request_set_crypt(subreq, dst, dst, - req->cryptlen - ivsize, req->iv); - aead_request_set_ad(subreq, req->assoclen); - - err = crypto_aead_decrypt(subreq); - if (req->assoclen > 8) - seqniv_aead_decrypt_complete2(req, err); - return err; -} - static int seqiv_aead_decrypt(struct aead_request *req) { struct crypto_aead *geniv = crypto_aead_reqtfm(req); - struct seqiv_aead_ctx *ctx = crypto_aead_ctx(geniv); + struct aead_geniv_ctx *ctx = crypto_aead_ctx(geniv); struct aead_request *subreq = aead_request_ctx(req); crypto_completion_t compl; void *data; @@ -456,7 +214,7 @@ static int seqiv_aead_decrypt(struct aead_request *req) if (req->cryptlen < ivsize + crypto_aead_authsize(geniv)) return -EINVAL; - aead_request_set_tfm(subreq, ctx->geniv.child); + aead_request_set_tfm(subreq, ctx->child); compl = req->base.complete; data = req->base.data; @@ -467,9 +225,6 @@ static int seqiv_aead_decrypt(struct aead_request *req) aead_request_set_ad(subreq, req->assoclen + ivsize); scatterwalk_map_and_copy(req->iv, req->src, req->assoclen, ivsize, 0); - if (req->src != req->dst) - scatterwalk_map_and_copy(req->iv, req->dst, - req->assoclen, ivsize, 1); return crypto_aead_decrypt(subreq); } @@ -495,85 +250,6 @@ static int seqiv_init(struct crypto_tfm *tfm) return err ?: skcipher_geniv_init(tfm); } -static int seqiv_old_aead_init(struct crypto_tfm *tfm) -{ - struct crypto_aead *geniv = __crypto_aead_cast(tfm); - struct seqiv_ctx *ctx = crypto_aead_ctx(geniv); - int err; - - spin_lock_init(&ctx->lock); - - crypto_aead_set_reqsize(__crypto_aead_cast(tfm), - sizeof(struct aead_request)); - err = 0; - if (!crypto_get_default_rng()) { - geniv->givencrypt = seqiv_aead_givencrypt; - err = crypto_rng_get_bytes(crypto_default_rng, ctx->salt, - crypto_aead_ivsize(geniv)); - crypto_put_default_rng(); - } - - return err ?: aead_geniv_init(tfm); -} - -static int seqiv_aead_init_common(struct crypto_tfm *tfm, unsigned int reqsize) -{ - struct crypto_aead *geniv = __crypto_aead_cast(tfm); - struct seqiv_aead_ctx *ctx = crypto_aead_ctx(geniv); - int err; - - spin_lock_init(&ctx->geniv.lock); - - crypto_aead_set_reqsize(geniv, sizeof(struct aead_request)); - - err = crypto_get_default_rng(); - if (err) - goto out; - - err = crypto_rng_get_bytes(crypto_default_rng, ctx->salt, - crypto_aead_ivsize(geniv)); - crypto_put_default_rng(); - if (err) - goto out; - - ctx->null = crypto_get_default_null_skcipher(); - err = PTR_ERR(ctx->null); - if (IS_ERR(ctx->null)) - goto out; - - err = aead_geniv_init(tfm); - if (err) - goto drop_null; - - ctx->geniv.child = geniv->child; - geniv->child = geniv; - -out: - return err; - -drop_null: - crypto_put_default_null_skcipher(); - goto out; -} - -static int seqiv_aead_init(struct crypto_tfm *tfm) -{ - return seqiv_aead_init_common(tfm, sizeof(struct aead_request)); -} - -static int seqniv_aead_init(struct crypto_tfm *tfm) -{ - return seqiv_aead_init_common(tfm, sizeof(struct seqniv_request_ctx)); -} - -static void seqiv_aead_exit(struct crypto_tfm *tfm) -{ - struct seqiv_aead_ctx *ctx = crypto_tfm_ctx(tfm); - - crypto_free_aead(ctx->geniv.child); - crypto_put_default_null_skcipher(); -} - static int seqiv_ablkcipher_create(struct crypto_template *tmpl, struct rtattr **tb) { @@ -609,33 +285,6 @@ free_inst: goto out; } -static int seqiv_old_aead_create(struct crypto_template *tmpl, - struct aead_instance *aead) -{ - struct crypto_instance *inst = aead_crypto_instance(aead); - int err = -EINVAL; - - if (inst->alg.cra_aead.ivsize < sizeof(u64)) - goto free_inst; - - inst->alg.cra_init = seqiv_old_aead_init; - inst->alg.cra_exit = aead_geniv_exit; - - inst->alg.cra_ctxsize = inst->alg.cra_aead.ivsize; - inst->alg.cra_ctxsize += sizeof(struct seqiv_ctx); - - err = crypto_register_instance(tmpl, inst); - if (err) - goto free_inst; - -out: - return err; - -free_inst: - aead_geniv_free(aead); - goto out; -} - static int seqiv_aead_create(struct crypto_template *tmpl, struct rtattr **tb) { struct aead_instance *inst; @@ -650,15 +299,9 @@ static int seqiv_aead_create(struct crypto_template *tmpl, struct rtattr **tb) inst->alg.base.cra_alignmask |= __alignof__(u32) - 1; - if (inst->alg.base.cra_aead.encrypt) - return seqiv_old_aead_create(tmpl, inst); - spawn = aead_instance_ctx(inst); alg = crypto_spawn_aead_alg(spawn); - if (alg->base.cra_aead.encrypt) - goto done; - err = -EINVAL; if (inst->alg.ivsize != sizeof(u64)) goto free_inst; @@ -666,13 +309,12 @@ static int seqiv_aead_create(struct crypto_template *tmpl, struct rtattr **tb) inst->alg.encrypt = seqiv_aead_encrypt; inst->alg.decrypt = seqiv_aead_decrypt; - inst->alg.base.cra_init = seqiv_aead_init; - inst->alg.base.cra_exit = seqiv_aead_exit; + inst->alg.init = aead_init_geniv; + inst->alg.exit = aead_exit_geniv; - inst->alg.base.cra_ctxsize = sizeof(struct seqiv_aead_ctx); - inst->alg.base.cra_ctxsize += inst->alg.base.cra_aead.ivsize; + inst->alg.base.cra_ctxsize = sizeof(struct aead_geniv_ctx); + inst->alg.base.cra_ctxsize += inst->alg.ivsize; -done: err = aead_register_instance(tmpl, inst); if (err) goto free_inst; @@ -702,51 +344,6 @@ static int seqiv_create(struct crypto_template *tmpl, struct rtattr **tb) return err; } -static int seqniv_create(struct crypto_template *tmpl, struct rtattr **tb) -{ - struct aead_instance *inst; - struct crypto_aead_spawn *spawn; - struct aead_alg *alg; - int err; - - inst = aead_geniv_alloc(tmpl, tb, 0, 0); - err = PTR_ERR(inst); - if (IS_ERR(inst)) - goto out; - - spawn = aead_instance_ctx(inst); - alg = crypto_spawn_aead_alg(spawn); - - if (alg->base.cra_aead.encrypt) - goto done; - - err = -EINVAL; - if (inst->alg.ivsize != sizeof(u64)) - goto free_inst; - - inst->alg.encrypt = seqniv_aead_encrypt; - inst->alg.decrypt = seqniv_aead_decrypt; - - inst->alg.base.cra_init = seqniv_aead_init; - inst->alg.base.cra_exit = seqiv_aead_exit; - - inst->alg.base.cra_alignmask |= __alignof__(u32) - 1; - inst->alg.base.cra_ctxsize = sizeof(struct seqiv_aead_ctx); - inst->alg.base.cra_ctxsize += inst->alg.ivsize; - -done: - err = aead_register_instance(tmpl, inst); - if (err) - goto free_inst; - -out: - return err; - -free_inst: - aead_geniv_free(inst); - goto out; -} - static void seqiv_free(struct crypto_instance *inst) { if ((inst->alg.cra_flags ^ CRYPTO_ALG_TYPE_AEAD) & CRYPTO_ALG_TYPE_MASK) @@ -762,36 +359,13 @@ static struct crypto_template seqiv_tmpl = { .module = THIS_MODULE, }; -static struct crypto_template seqniv_tmpl = { - .name = "seqniv", - .create = seqniv_create, - .free = seqiv_free, - .module = THIS_MODULE, -}; - static int __init seqiv_module_init(void) { - int err; - - err = crypto_register_template(&seqiv_tmpl); - if (err) - goto out; - - err = crypto_register_template(&seqniv_tmpl); - if (err) - goto out_undo_niv; - -out: - return err; - -out_undo_niv: - crypto_unregister_template(&seqiv_tmpl); - goto out; + return crypto_register_template(&seqiv_tmpl); } static void __exit seqiv_module_exit(void) { - crypto_unregister_template(&seqniv_tmpl); crypto_unregister_template(&seqiv_tmpl); } @@ -801,4 +375,3 @@ module_exit(seqiv_module_exit); MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("Sequence Number IV Generator"); MODULE_ALIAS_CRYPTO("seqiv"); -MODULE_ALIAS_CRYPTO("seqniv"); diff --git a/crypto/skcipher.c b/crypto/skcipher.c new file mode 100644 index 000000000000..dd5fc1bf6447 --- /dev/null +++ b/crypto/skcipher.c @@ -0,0 +1,245 @@ +/* + * Symmetric key cipher operations. + * + * Generic encrypt/decrypt wrapper for ciphers, handles operations across + * multiple page boundaries by using temporary blocks. In user context, + * the kernel is given a chance to schedule us once per page. + * + * Copyright (c) 2015 Herbert Xu <herbert@gondor.apana.org.au> + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + * + */ + +#include <crypto/internal/skcipher.h> +#include <linux/bug.h> +#include <linux/module.h> + +#include "internal.h" + +static unsigned int crypto_skcipher_extsize(struct crypto_alg *alg) +{ + if (alg->cra_type == &crypto_blkcipher_type) + return sizeof(struct crypto_blkcipher *); + + BUG_ON(alg->cra_type != &crypto_ablkcipher_type && + alg->cra_type != &crypto_givcipher_type); + + return sizeof(struct crypto_ablkcipher *); +} + +static int skcipher_setkey_blkcipher(struct crypto_skcipher *tfm, + const u8 *key, unsigned int keylen) +{ + struct crypto_blkcipher **ctx = crypto_skcipher_ctx(tfm); + struct crypto_blkcipher *blkcipher = *ctx; + int err; + + crypto_blkcipher_clear_flags(blkcipher, ~0); + crypto_blkcipher_set_flags(blkcipher, crypto_skcipher_get_flags(tfm) & + CRYPTO_TFM_REQ_MASK); + err = crypto_blkcipher_setkey(blkcipher, key, keylen); + crypto_skcipher_set_flags(tfm, crypto_blkcipher_get_flags(blkcipher) & + CRYPTO_TFM_RES_MASK); + + return err; +} + +static int skcipher_crypt_blkcipher(struct skcipher_request *req, + int (*crypt)(struct blkcipher_desc *, + struct scatterlist *, + struct scatterlist *, + unsigned int)) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct crypto_blkcipher **ctx = crypto_skcipher_ctx(tfm); + struct blkcipher_desc desc = { + .tfm = *ctx, + .info = req->iv, + .flags = req->base.flags, + }; + + + return crypt(&desc, req->dst, req->src, req->cryptlen); +} + +static int skcipher_encrypt_blkcipher(struct skcipher_request *req) +{ + struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); + struct crypto_tfm *tfm = crypto_skcipher_tfm(skcipher); + struct blkcipher_alg *alg = &tfm->__crt_alg->cra_blkcipher; + + return skcipher_crypt_blkcipher(req, alg->encrypt); +} + +static int skcipher_decrypt_blkcipher(struct skcipher_request *req) +{ + struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); + struct crypto_tfm *tfm = crypto_skcipher_tfm(skcipher); + struct blkcipher_alg *alg = &tfm->__crt_alg->cra_blkcipher; + + return skcipher_crypt_blkcipher(req, alg->decrypt); +} + +static void crypto_exit_skcipher_ops_blkcipher(struct crypto_tfm *tfm) +{ + struct crypto_blkcipher **ctx = crypto_tfm_ctx(tfm); + + crypto_free_blkcipher(*ctx); +} + +int crypto_init_skcipher_ops_blkcipher(struct crypto_tfm *tfm) +{ + struct crypto_alg *calg = tfm->__crt_alg; + struct crypto_skcipher *skcipher = __crypto_skcipher_cast(tfm); + struct crypto_blkcipher **ctx = crypto_tfm_ctx(tfm); + struct crypto_blkcipher *blkcipher; + struct crypto_tfm *btfm; + + if (!crypto_mod_get(calg)) + return -EAGAIN; + + btfm = __crypto_alloc_tfm(calg, CRYPTO_ALG_TYPE_BLKCIPHER, + CRYPTO_ALG_TYPE_MASK); + if (IS_ERR(btfm)) { + crypto_mod_put(calg); + return PTR_ERR(btfm); + } + + blkcipher = __crypto_blkcipher_cast(btfm); + *ctx = blkcipher; + tfm->exit = crypto_exit_skcipher_ops_blkcipher; + + skcipher->setkey = skcipher_setkey_blkcipher; + skcipher->encrypt = skcipher_encrypt_blkcipher; + skcipher->decrypt = skcipher_decrypt_blkcipher; + + skcipher->ivsize = crypto_blkcipher_ivsize(blkcipher); + + return 0; +} + +static int skcipher_setkey_ablkcipher(struct crypto_skcipher *tfm, + const u8 *key, unsigned int keylen) +{ + struct crypto_ablkcipher **ctx = crypto_skcipher_ctx(tfm); + struct crypto_ablkcipher *ablkcipher = *ctx; + int err; + + crypto_ablkcipher_clear_flags(ablkcipher, ~0); + crypto_ablkcipher_set_flags(ablkcipher, + crypto_skcipher_get_flags(tfm) & + CRYPTO_TFM_REQ_MASK); + err = crypto_ablkcipher_setkey(ablkcipher, key, keylen); + crypto_skcipher_set_flags(tfm, + crypto_ablkcipher_get_flags(ablkcipher) & + CRYPTO_TFM_RES_MASK); + + return err; +} + +static int skcipher_crypt_ablkcipher(struct skcipher_request *req, + int (*crypt)(struct ablkcipher_request *)) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct crypto_ablkcipher **ctx = crypto_skcipher_ctx(tfm); + struct ablkcipher_request *subreq = skcipher_request_ctx(req); + + ablkcipher_request_set_tfm(subreq, *ctx); + ablkcipher_request_set_callback(subreq, skcipher_request_flags(req), + req->base.complete, req->base.data); + ablkcipher_request_set_crypt(subreq, req->src, req->dst, req->cryptlen, + req->iv); + + return crypt(subreq); +} + +static int skcipher_encrypt_ablkcipher(struct skcipher_request *req) +{ + struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); + struct crypto_tfm *tfm = crypto_skcipher_tfm(skcipher); + struct ablkcipher_alg *alg = &tfm->__crt_alg->cra_ablkcipher; + + return skcipher_crypt_ablkcipher(req, alg->encrypt); +} + +static int skcipher_decrypt_ablkcipher(struct skcipher_request *req) +{ + struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); + struct crypto_tfm *tfm = crypto_skcipher_tfm(skcipher); + struct ablkcipher_alg *alg = &tfm->__crt_alg->cra_ablkcipher; + + return skcipher_crypt_ablkcipher(req, alg->decrypt); +} + +static void crypto_exit_skcipher_ops_ablkcipher(struct crypto_tfm *tfm) +{ + struct crypto_ablkcipher **ctx = crypto_tfm_ctx(tfm); + + crypto_free_ablkcipher(*ctx); +} + +int crypto_init_skcipher_ops_ablkcipher(struct crypto_tfm *tfm) +{ + struct crypto_alg *calg = tfm->__crt_alg; + struct crypto_skcipher *skcipher = __crypto_skcipher_cast(tfm); + struct crypto_ablkcipher **ctx = crypto_tfm_ctx(tfm); + struct crypto_ablkcipher *ablkcipher; + struct crypto_tfm *abtfm; + + if (!crypto_mod_get(calg)) + return -EAGAIN; + + abtfm = __crypto_alloc_tfm(calg, 0, 0); + if (IS_ERR(abtfm)) { + crypto_mod_put(calg); + return PTR_ERR(abtfm); + } + + ablkcipher = __crypto_ablkcipher_cast(abtfm); + *ctx = ablkcipher; + tfm->exit = crypto_exit_skcipher_ops_ablkcipher; + + skcipher->setkey = skcipher_setkey_ablkcipher; + skcipher->encrypt = skcipher_encrypt_ablkcipher; + skcipher->decrypt = skcipher_decrypt_ablkcipher; + + skcipher->ivsize = crypto_ablkcipher_ivsize(ablkcipher); + skcipher->reqsize = crypto_ablkcipher_reqsize(ablkcipher) + + sizeof(struct ablkcipher_request); + + return 0; +} + +static int crypto_skcipher_init_tfm(struct crypto_tfm *tfm) +{ + if (tfm->__crt_alg->cra_type == &crypto_blkcipher_type) + return crypto_init_skcipher_ops_blkcipher(tfm); + + BUG_ON(tfm->__crt_alg->cra_type != &crypto_ablkcipher_type && + tfm->__crt_alg->cra_type != &crypto_givcipher_type); + + return crypto_init_skcipher_ops_ablkcipher(tfm); +} + +static const struct crypto_type crypto_skcipher_type2 = { + .extsize = crypto_skcipher_extsize, + .init_tfm = crypto_skcipher_init_tfm, + .maskclear = ~CRYPTO_ALG_TYPE_MASK, + .maskset = CRYPTO_ALG_TYPE_BLKCIPHER_MASK, + .type = CRYPTO_ALG_TYPE_BLKCIPHER, + .tfmsize = offsetof(struct crypto_skcipher, base), +}; + +struct crypto_skcipher *crypto_alloc_skcipher(const char *alg_name, + u32 type, u32 mask) +{ + return crypto_alloc_tfm(alg_name, &crypto_skcipher_type2, type, mask); +} +EXPORT_SYMBOL_GPL(crypto_alloc_skcipher); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("Symmetric key cipher type"); diff --git a/crypto/tcrypt.c b/crypto/tcrypt.c index 9f6f10b498ba..2b00b617daab 100644 --- a/crypto/tcrypt.c +++ b/crypto/tcrypt.c @@ -73,6 +73,22 @@ static char *check[] = { "lzo", "cts", "zlib", NULL }; +struct tcrypt_result { + struct completion completion; + int err; +}; + +static void tcrypt_complete(struct crypto_async_request *req, int err) +{ + struct tcrypt_result *res = req->data; + + if (err == -EINPROGRESS) + return; + + res->err = err; + complete(&res->completion); +} + static int test_cipher_jiffies(struct blkcipher_desc *desc, int enc, struct scatterlist *sg, int blen, int secs) { @@ -143,6 +159,20 @@ out: return ret; } +static inline int do_one_aead_op(struct aead_request *req, int ret) +{ + if (ret == -EINPROGRESS || ret == -EBUSY) { + struct tcrypt_result *tr = req->base.data; + + ret = wait_for_completion_interruptible(&tr->completion); + if (!ret) + ret = tr->err; + reinit_completion(&tr->completion); + } + + return ret; +} + static int test_aead_jiffies(struct aead_request *req, int enc, int blen, int secs) { @@ -153,9 +183,9 @@ static int test_aead_jiffies(struct aead_request *req, int enc, for (start = jiffies, end = start + secs * HZ, bcount = 0; time_before(jiffies, end); bcount++) { if (enc) - ret = crypto_aead_encrypt(req); + ret = do_one_aead_op(req, crypto_aead_encrypt(req)); else - ret = crypto_aead_decrypt(req); + ret = do_one_aead_op(req, crypto_aead_decrypt(req)); if (ret) return ret; @@ -177,9 +207,9 @@ static int test_aead_cycles(struct aead_request *req, int enc, int blen) /* Warm-up run. */ for (i = 0; i < 4; i++) { if (enc) - ret = crypto_aead_encrypt(req); + ret = do_one_aead_op(req, crypto_aead_encrypt(req)); else - ret = crypto_aead_decrypt(req); + ret = do_one_aead_op(req, crypto_aead_decrypt(req)); if (ret) goto out; @@ -191,9 +221,9 @@ static int test_aead_cycles(struct aead_request *req, int enc, int blen) start = get_cycles(); if (enc) - ret = crypto_aead_encrypt(req); + ret = do_one_aead_op(req, crypto_aead_encrypt(req)); else - ret = crypto_aead_decrypt(req); + ret = do_one_aead_op(req, crypto_aead_decrypt(req)); end = get_cycles(); if (ret) @@ -286,6 +316,7 @@ static void test_aead_speed(const char *algo, int enc, unsigned int secs, char *axbuf[XBUFSIZE]; unsigned int *b_size; unsigned int iv_len; + struct tcrypt_result result; iv = kzalloc(MAX_IVLEN, GFP_KERNEL); if (!iv) @@ -321,6 +352,7 @@ static void test_aead_speed(const char *algo, int enc, unsigned int secs, goto out_notfm; } + init_completion(&result.completion); printk(KERN_INFO "\ntesting speed of %s (%s) %s\n", algo, get_driver_name(crypto_aead, tfm), e); @@ -331,6 +363,9 @@ static void test_aead_speed(const char *algo, int enc, unsigned int secs, goto out_noreq; } + aead_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, + tcrypt_complete, &result); + i = 0; do { b_size = aead_sizes; @@ -749,22 +784,6 @@ out: crypto_free_hash(tfm); } -struct tcrypt_result { - struct completion completion; - int err; -}; - -static void tcrypt_complete(struct crypto_async_request *req, int err) -{ - struct tcrypt_result *res = req->data; - - if (err == -EINPROGRESS) - return; - - res->err = err; - complete(&res->completion); -} - static inline int do_one_ahash_op(struct ahash_request *req, int ret) { if (ret == -EINPROGRESS || ret == -EBUSY) { @@ -1759,14 +1778,27 @@ static int do_test(const char *alg, u32 type, u32 mask, int m) case 211: test_aead_speed("rfc4106(gcm(aes))", ENCRYPT, sec, + NULL, 0, 16, 16, aead_speed_template_20); + test_aead_speed("gcm(aes)", ENCRYPT, sec, NULL, 0, 16, 8, aead_speed_template_20); break; case 212: test_aead_speed("rfc4309(ccm(aes))", ENCRYPT, sec, - NULL, 0, 16, 8, aead_speed_template_19); + NULL, 0, 16, 16, aead_speed_template_19); + break; + + case 213: + test_aead_speed("rfc7539esp(chacha20,poly1305)", ENCRYPT, sec, + NULL, 0, 16, 8, aead_speed_template_36); + break; + + case 214: + test_cipher_speed("chacha20", ENCRYPT, sec, NULL, 0, + speed_template_32); break; + case 300: if (alg) { test_hash_speed(alg, sec, generic_hash_speed_template); @@ -1855,6 +1887,10 @@ static int do_test(const char *alg, u32 type, u32 mask, int m) test_hash_speed("crct10dif", sec, generic_hash_speed_template); if (mode > 300 && mode < 400) break; + case 321: + test_hash_speed("poly1305", sec, poly1305_speed_template); + if (mode > 300 && mode < 400) break; + case 399: break; diff --git a/crypto/tcrypt.h b/crypto/tcrypt.h index 6cc1b856871b..f0bfee1bb293 100644 --- a/crypto/tcrypt.h +++ b/crypto/tcrypt.h @@ -61,12 +61,14 @@ static u8 speed_template_32_40_48[] = {32, 40, 48, 0}; static u8 speed_template_32_48[] = {32, 48, 0}; static u8 speed_template_32_48_64[] = {32, 48, 64, 0}; static u8 speed_template_32_64[] = {32, 64, 0}; +static u8 speed_template_32[] = {32, 0}; /* * AEAD speed tests */ static u8 aead_speed_template_19[] = {19, 0}; static u8 aead_speed_template_20[] = {20, 0}; +static u8 aead_speed_template_36[] = {36, 0}; /* * Digest speed tests @@ -127,4 +129,22 @@ static struct hash_speed hash_speed_template_16[] = { { .blen = 0, .plen = 0, .klen = 0, } }; +static struct hash_speed poly1305_speed_template[] = { + { .blen = 96, .plen = 16, }, + { .blen = 96, .plen = 32, }, + { .blen = 96, .plen = 96, }, + { .blen = 288, .plen = 16, }, + { .blen = 288, .plen = 32, }, + { .blen = 288, .plen = 288, }, + { .blen = 1056, .plen = 32, }, + { .blen = 1056, .plen = 1056, }, + { .blen = 2080, .plen = 32, }, + { .blen = 2080, .plen = 2080, }, + { .blen = 4128, .plen = 4128, }, + { .blen = 8224, .plen = 8224, }, + + /* End marker */ + { .blen = 0, .plen = 0, } +}; + #endif /* _CRYPTO_TCRYPT_H */ diff --git a/crypto/testmgr.c b/crypto/testmgr.c index d0a42bd3aae9..35c2de136971 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -22,6 +22,7 @@ #include <crypto/aead.h> #include <crypto/hash.h> +#include <crypto/skcipher.h> #include <linux/err.h> #include <linux/fips.h> #include <linux/module.h> @@ -921,15 +922,15 @@ out_nobuf: return ret; } -static int __test_skcipher(struct crypto_ablkcipher *tfm, int enc, +static int __test_skcipher(struct crypto_skcipher *tfm, int enc, struct cipher_testvec *template, unsigned int tcount, const bool diff_dst, const int align_offset) { const char *algo = - crypto_tfm_alg_driver_name(crypto_ablkcipher_tfm(tfm)); + crypto_tfm_alg_driver_name(crypto_skcipher_tfm(tfm)); unsigned int i, j, k, n, temp; char *q; - struct ablkcipher_request *req; + struct skcipher_request *req; struct scatterlist sg[8]; struct scatterlist sgout[8]; const char *e, *d; @@ -958,15 +959,15 @@ static int __test_skcipher(struct crypto_ablkcipher *tfm, int enc, init_completion(&result.completion); - req = ablkcipher_request_alloc(tfm, GFP_KERNEL); + req = skcipher_request_alloc(tfm, GFP_KERNEL); if (!req) { pr_err("alg: skcipher%s: Failed to allocate request for %s\n", d, algo); goto out; } - ablkcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, - tcrypt_complete, &result); + skcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, + tcrypt_complete, &result); j = 0; for (i = 0; i < tcount; i++) { @@ -987,15 +988,16 @@ static int __test_skcipher(struct crypto_ablkcipher *tfm, int enc, data += align_offset; memcpy(data, template[i].input, template[i].ilen); - crypto_ablkcipher_clear_flags(tfm, ~0); + crypto_skcipher_clear_flags(tfm, ~0); if (template[i].wk) - crypto_ablkcipher_set_flags(tfm, CRYPTO_TFM_REQ_WEAK_KEY); + crypto_skcipher_set_flags(tfm, + CRYPTO_TFM_REQ_WEAK_KEY); - ret = crypto_ablkcipher_setkey(tfm, template[i].key, - template[i].klen); + ret = crypto_skcipher_setkey(tfm, template[i].key, + template[i].klen); if (!ret == template[i].fail) { pr_err("alg: skcipher%s: setkey failed on test %d for %s: flags=%x\n", - d, j, algo, crypto_ablkcipher_get_flags(tfm)); + d, j, algo, crypto_skcipher_get_flags(tfm)); goto out; } else if (ret) continue; @@ -1007,10 +1009,10 @@ static int __test_skcipher(struct crypto_ablkcipher *tfm, int enc, sg_init_one(&sgout[0], data, template[i].ilen); } - ablkcipher_request_set_crypt(req, sg, (diff_dst) ? sgout : sg, - template[i].ilen, iv); - ret = enc ? crypto_ablkcipher_encrypt(req) : - crypto_ablkcipher_decrypt(req); + skcipher_request_set_crypt(req, sg, (diff_dst) ? sgout : sg, + template[i].ilen, iv); + ret = enc ? crypto_skcipher_encrypt(req) : + crypto_skcipher_decrypt(req); switch (ret) { case 0: @@ -1054,15 +1056,16 @@ static int __test_skcipher(struct crypto_ablkcipher *tfm, int enc, memset(iv, 0, MAX_IVLEN); j++; - crypto_ablkcipher_clear_flags(tfm, ~0); + crypto_skcipher_clear_flags(tfm, ~0); if (template[i].wk) - crypto_ablkcipher_set_flags(tfm, CRYPTO_TFM_REQ_WEAK_KEY); + crypto_skcipher_set_flags(tfm, + CRYPTO_TFM_REQ_WEAK_KEY); - ret = crypto_ablkcipher_setkey(tfm, template[i].key, - template[i].klen); + ret = crypto_skcipher_setkey(tfm, template[i].key, + template[i].klen); if (!ret == template[i].fail) { pr_err("alg: skcipher%s: setkey failed on chunk test %d for %s: flags=%x\n", - d, j, algo, crypto_ablkcipher_get_flags(tfm)); + d, j, algo, crypto_skcipher_get_flags(tfm)); goto out; } else if (ret) continue; @@ -1100,11 +1103,11 @@ static int __test_skcipher(struct crypto_ablkcipher *tfm, int enc, temp += template[i].tap[k]; } - ablkcipher_request_set_crypt(req, sg, (diff_dst) ? sgout : sg, - template[i].ilen, iv); + skcipher_request_set_crypt(req, sg, (diff_dst) ? sgout : sg, + template[i].ilen, iv); - ret = enc ? crypto_ablkcipher_encrypt(req) : - crypto_ablkcipher_decrypt(req); + ret = enc ? crypto_skcipher_encrypt(req) : + crypto_skcipher_decrypt(req); switch (ret) { case 0: @@ -1157,7 +1160,7 @@ static int __test_skcipher(struct crypto_ablkcipher *tfm, int enc, ret = 0; out: - ablkcipher_request_free(req); + skcipher_request_free(req); if (diff_dst) testmgr_free_buf(xoutbuf); out_nooutbuf: @@ -1166,7 +1169,7 @@ out_nobuf: return ret; } -static int test_skcipher(struct crypto_ablkcipher *tfm, int enc, +static int test_skcipher(struct crypto_skcipher *tfm, int enc, struct cipher_testvec *template, unsigned int tcount) { unsigned int alignmask; @@ -1578,10 +1581,10 @@ out: static int alg_test_skcipher(const struct alg_test_desc *desc, const char *driver, u32 type, u32 mask) { - struct crypto_ablkcipher *tfm; + struct crypto_skcipher *tfm; int err = 0; - tfm = crypto_alloc_ablkcipher(driver, type | CRYPTO_ALG_INTERNAL, mask); + tfm = crypto_alloc_skcipher(driver, type | CRYPTO_ALG_INTERNAL, mask); if (IS_ERR(tfm)) { printk(KERN_ERR "alg: skcipher: Failed to load transform for " "%s: %ld\n", driver, PTR_ERR(tfm)); @@ -1600,7 +1603,7 @@ static int alg_test_skcipher(const struct alg_test_desc *desc, desc->suite.cipher.dec.count); out: - crypto_free_ablkcipher(tfm); + crypto_free_skcipher(tfm); return err; } @@ -2476,6 +2479,7 @@ static const struct alg_test_desc alg_test_descs[] = { } }, { .alg = "cmac(aes)", + .fips_allowed = 1, .test = alg_test_hash, .suite = { .hash = { @@ -2485,6 +2489,7 @@ static const struct alg_test_desc alg_test_descs[] = { } }, { .alg = "cmac(des3_ede)", + .fips_allowed = 1, .test = alg_test_hash, .suite = { .hash = { diff --git a/crypto/testmgr.h b/crypto/testmgr.h index 868edf117041..64b8a8082645 100644 --- a/crypto/testmgr.h +++ b/crypto/testmgr.h @@ -14504,6 +14504,9 @@ static struct cipher_testvec aes_cbc_enc_tv_template[] = { .result = "\xe3\x53\x77\x9c\x10\x79\xae\xb8" "\x27\x08\x94\x2d\xbe\x77\x18\x1a", .rlen = 16, + .also_non_np = 1, + .np = 8, + .tap = { 3, 2, 3, 2, 3, 1, 1, 1 }, }, { .key = "\xc2\x86\x69\x6d\x88\x7c\x9a\xa0" "\x61\x1b\xbb\x3e\x20\x25\xa4\x5a", @@ -14723,6 +14726,9 @@ static struct cipher_testvec aes_cbc_dec_tv_template[] = { .ilen = 16, .result = "Single block msg", .rlen = 16, + .also_non_np = 1, + .np = 8, + .tap = { 3, 2, 3, 2, 3, 1, 1, 1 }, }, { .key = "\xc2\x86\x69\x6d\x88\x7c\x9a\xa0" "\x61\x1b\xbb\x3e\x20\x25\xa4\x5a", @@ -15032,6 +15038,9 @@ static struct aead_testvec hmac_sha1_aes_cbc_enc_tv_temp[] = { .klen = 8 + 20 + 16, .iv = "\x3d\xaf\xba\x42\x9d\x9e\xb4\x30" "\xb4\x22\xda\x80\x2c\x9f\xac\x41", + .assoc = "\x3d\xaf\xba\x42\x9d\x9e\xb4\x30" + "\xb4\x22\xda\x80\x2c\x9f\xac\x41", + .alen = 16, .input = "Single block msg", .ilen = 16, .result = "\xe3\x53\x77\x9c\x10\x79\xae\xb8" @@ -15057,6 +15066,9 @@ static struct aead_testvec hmac_sha1_aes_cbc_enc_tv_temp[] = { .klen = 8 + 20 + 16, .iv = "\x56\x2e\x17\x99\x6d\x09\x3d\x28" "\xdd\xb3\xba\x69\x5a\x2e\x6f\x58", + .assoc = "\x56\x2e\x17\x99\x6d\x09\x3d\x28" + "\xdd\xb3\xba\x69\x5a\x2e\x6f\x58", + .alen = 16, .input = "\x00\x01\x02\x03\x04\x05\x06\x07" "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" "\x10\x11\x12\x13\x14\x15\x16\x17" @@ -15087,6 +15099,9 @@ static struct aead_testvec hmac_sha1_aes_cbc_enc_tv_temp[] = { .klen = 8 + 20 + 16, .iv = "\xc7\x82\xdc\x4c\x09\x8c\x66\xcb" "\xd9\xcd\x27\xd8\x25\x68\x2c\x81", + .assoc = "\xc7\x82\xdc\x4c\x09\x8c\x66\xcb" + "\xd9\xcd\x27\xd8\x25\x68\x2c\x81", + .alen = 16, .input = "This is a 48-byte message (exactly 3 AES blocks)", .ilen = 48, .result = "\xd0\xa0\x2b\x38\x36\x45\x17\x53" @@ -15116,6 +15131,9 @@ static struct aead_testvec hmac_sha1_aes_cbc_enc_tv_temp[] = { .klen = 8 + 20 + 16, .iv = "\x8c\xe8\x2e\xef\xbe\xa0\xda\x3c" "\x44\x69\x9e\xd7\xdb\x51\xb7\xd9", + .assoc = "\x8c\xe8\x2e\xef\xbe\xa0\xda\x3c" + "\x44\x69\x9e\xd7\xdb\x51\xb7\xd9", + .alen = 16, .input = "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7" "\xa8\xa9\xaa\xab\xac\xad\xae\xaf" "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7" @@ -15154,8 +15172,10 @@ static struct aead_testvec hmac_sha1_aes_cbc_enc_tv_temp[] = { .klen = 8 + 20 + 16, .iv = "\xe9\x6e\x8c\x08\xab\x46\x57\x63" "\xfd\x09\x8d\x45\xdd\x3f\xf8\x93", - .assoc = "\x00\x00\x43\x21\x00\x00\x00\x01", - .alen = 8, + .assoc = "\x00\x00\x43\x21\x00\x00\x00\x01" + "\xe9\x6e\x8c\x08\xab\x46\x57\x63" + "\xfd\x09\x8d\x45\xdd\x3f\xf8\x93", + .alen = 24, .input = "\x08\x00\x0e\xbd\xa7\x0a\x00\x00" "\x8e\x9c\x08\x3d\xb9\x5b\x07\x00" "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" @@ -15199,6 +15219,9 @@ static struct aead_testvec hmac_sha1_aes_cbc_enc_tv_temp[] = { .klen = 8 + 20 + 24, .iv = "\x00\x01\x02\x03\x04\x05\x06\x07" "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + .assoc = "\x00\x01\x02\x03\x04\x05\x06\x07" + "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + .alen = 16, .input = "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96" "\xe9\x3d\x7e\x11\x73\x93\x17\x2a" "\xae\x2d\x8a\x57\x1e\x03\xac\x9c" @@ -15239,6 +15262,9 @@ static struct aead_testvec hmac_sha1_aes_cbc_enc_tv_temp[] = { .klen = 8 + 20 + 32, .iv = "\x00\x01\x02\x03\x04\x05\x06\x07" "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + .assoc = "\x00\x01\x02\x03\x04\x05\x06\x07" + "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + .alen = 16, .input = "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96" "\xe9\x3d\x7e\x11\x73\x93\x17\x2a" "\xae\x2d\x8a\x57\x1e\x03\xac\x9c" @@ -15374,6 +15400,9 @@ static struct aead_testvec hmac_sha256_aes_cbc_enc_tv_temp[] = { .klen = 8 + 32 + 16, .iv = "\x3d\xaf\xba\x42\x9d\x9e\xb4\x30" "\xb4\x22\xda\x80\x2c\x9f\xac\x41", + .assoc = "\x3d\xaf\xba\x42\x9d\x9e\xb4\x30" + "\xb4\x22\xda\x80\x2c\x9f\xac\x41", + .alen = 16, .input = "Single block msg", .ilen = 16, .result = "\xe3\x53\x77\x9c\x10\x79\xae\xb8" @@ -15401,6 +15430,9 @@ static struct aead_testvec hmac_sha256_aes_cbc_enc_tv_temp[] = { .klen = 8 + 32 + 16, .iv = "\x56\x2e\x17\x99\x6d\x09\x3d\x28" "\xdd\xb3\xba\x69\x5a\x2e\x6f\x58", + .assoc = "\x56\x2e\x17\x99\x6d\x09\x3d\x28" + "\xdd\xb3\xba\x69\x5a\x2e\x6f\x58", + .alen = 16, .input = "\x00\x01\x02\x03\x04\x05\x06\x07" "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" "\x10\x11\x12\x13\x14\x15\x16\x17" @@ -15433,6 +15465,9 @@ static struct aead_testvec hmac_sha256_aes_cbc_enc_tv_temp[] = { .klen = 8 + 32 + 16, .iv = "\xc7\x82\xdc\x4c\x09\x8c\x66\xcb" "\xd9\xcd\x27\xd8\x25\x68\x2c\x81", + .assoc = "\xc7\x82\xdc\x4c\x09\x8c\x66\xcb" + "\xd9\xcd\x27\xd8\x25\x68\x2c\x81", + .alen = 16, .input = "This is a 48-byte message (exactly 3 AES blocks)", .ilen = 48, .result = "\xd0\xa0\x2b\x38\x36\x45\x17\x53" @@ -15464,6 +15499,9 @@ static struct aead_testvec hmac_sha256_aes_cbc_enc_tv_temp[] = { .klen = 8 + 32 + 16, .iv = "\x8c\xe8\x2e\xef\xbe\xa0\xda\x3c" "\x44\x69\x9e\xd7\xdb\x51\xb7\xd9", + .assoc = "\x8c\xe8\x2e\xef\xbe\xa0\xda\x3c" + "\x44\x69\x9e\xd7\xdb\x51\xb7\xd9", + .alen = 16, .input = "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7" "\xa8\xa9\xaa\xab\xac\xad\xae\xaf" "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7" @@ -15504,8 +15542,10 @@ static struct aead_testvec hmac_sha256_aes_cbc_enc_tv_temp[] = { .klen = 8 + 32 + 16, .iv = "\xe9\x6e\x8c\x08\xab\x46\x57\x63" "\xfd\x09\x8d\x45\xdd\x3f\xf8\x93", - .assoc = "\x00\x00\x43\x21\x00\x00\x00\x01", - .alen = 8, + .assoc = "\x00\x00\x43\x21\x00\x00\x00\x01" + "\xe9\x6e\x8c\x08\xab\x46\x57\x63" + "\xfd\x09\x8d\x45\xdd\x3f\xf8\x93", + .alen = 24, .input = "\x08\x00\x0e\xbd\xa7\x0a\x00\x00" "\x8e\x9c\x08\x3d\xb9\x5b\x07\x00" "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" @@ -15551,6 +15591,9 @@ static struct aead_testvec hmac_sha256_aes_cbc_enc_tv_temp[] = { .klen = 8 + 32 + 24, .iv = "\x00\x01\x02\x03\x04\x05\x06\x07" "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + .assoc = "\x00\x01\x02\x03\x04\x05\x06\x07" + "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + .alen = 16, .input = "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96" "\xe9\x3d\x7e\x11\x73\x93\x17\x2a" "\xae\x2d\x8a\x57\x1e\x03\xac\x9c" @@ -15593,6 +15636,9 @@ static struct aead_testvec hmac_sha256_aes_cbc_enc_tv_temp[] = { .klen = 8 + 32 + 32, .iv = "\x00\x01\x02\x03\x04\x05\x06\x07" "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + .assoc = "\x00\x01\x02\x03\x04\x05\x06\x07" + "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + .alen = 16, .input = "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96" "\xe9\x3d\x7e\x11\x73\x93\x17\x2a" "\xae\x2d\x8a\x57\x1e\x03\xac\x9c" @@ -15641,6 +15687,9 @@ static struct aead_testvec hmac_sha512_aes_cbc_enc_tv_temp[] = { .klen = 8 + 64 + 16, .iv = "\x3d\xaf\xba\x42\x9d\x9e\xb4\x30" "\xb4\x22\xda\x80\x2c\x9f\xac\x41", + .assoc = "\x3d\xaf\xba\x42\x9d\x9e\xb4\x30" + "\xb4\x22\xda\x80\x2c\x9f\xac\x41", + .alen = 16, .input = "Single block msg", .ilen = 16, .result = "\xe3\x53\x77\x9c\x10\x79\xae\xb8" @@ -15676,6 +15725,9 @@ static struct aead_testvec hmac_sha512_aes_cbc_enc_tv_temp[] = { .klen = 8 + 64 + 16, .iv = "\x56\x2e\x17\x99\x6d\x09\x3d\x28" "\xdd\xb3\xba\x69\x5a\x2e\x6f\x58", + .assoc = "\x56\x2e\x17\x99\x6d\x09\x3d\x28" + "\xdd\xb3\xba\x69\x5a\x2e\x6f\x58", + .alen = 16, .input = "\x00\x01\x02\x03\x04\x05\x06\x07" "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" "\x10\x11\x12\x13\x14\x15\x16\x17" @@ -15716,6 +15768,9 @@ static struct aead_testvec hmac_sha512_aes_cbc_enc_tv_temp[] = { .klen = 8 + 64 + 16, .iv = "\xc7\x82\xdc\x4c\x09\x8c\x66\xcb" "\xd9\xcd\x27\xd8\x25\x68\x2c\x81", + .assoc = "\xc7\x82\xdc\x4c\x09\x8c\x66\xcb" + "\xd9\xcd\x27\xd8\x25\x68\x2c\x81", + .alen = 16, .input = "This is a 48-byte message (exactly 3 AES blocks)", .ilen = 48, .result = "\xd0\xa0\x2b\x38\x36\x45\x17\x53" @@ -15755,6 +15810,9 @@ static struct aead_testvec hmac_sha512_aes_cbc_enc_tv_temp[] = { .klen = 8 + 64 + 16, .iv = "\x8c\xe8\x2e\xef\xbe\xa0\xda\x3c" "\x44\x69\x9e\xd7\xdb\x51\xb7\xd9", + .assoc = "\x8c\xe8\x2e\xef\xbe\xa0\xda\x3c" + "\x44\x69\x9e\xd7\xdb\x51\xb7\xd9", + .alen = 16, .input = "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7" "\xa8\xa9\xaa\xab\xac\xad\xae\xaf" "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7" @@ -15803,8 +15861,10 @@ static struct aead_testvec hmac_sha512_aes_cbc_enc_tv_temp[] = { .klen = 8 + 64 + 16, .iv = "\xe9\x6e\x8c\x08\xab\x46\x57\x63" "\xfd\x09\x8d\x45\xdd\x3f\xf8\x93", - .assoc = "\x00\x00\x43\x21\x00\x00\x00\x01", - .alen = 8, + .assoc = "\x00\x00\x43\x21\x00\x00\x00\x01" + "\xe9\x6e\x8c\x08\xab\x46\x57\x63" + "\xfd\x09\x8d\x45\xdd\x3f\xf8\x93", + .alen = 24, .input = "\x08\x00\x0e\xbd\xa7\x0a\x00\x00" "\x8e\x9c\x08\x3d\xb9\x5b\x07\x00" "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" @@ -15858,6 +15918,9 @@ static struct aead_testvec hmac_sha512_aes_cbc_enc_tv_temp[] = { .klen = 8 + 64 + 24, .iv = "\x00\x01\x02\x03\x04\x05\x06\x07" "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + .assoc = "\x00\x01\x02\x03\x04\x05\x06\x07" + "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + .alen = 16, .input = "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96" "\xe9\x3d\x7e\x11\x73\x93\x17\x2a" "\xae\x2d\x8a\x57\x1e\x03\xac\x9c" @@ -15908,6 +15971,9 @@ static struct aead_testvec hmac_sha512_aes_cbc_enc_tv_temp[] = { .klen = 8 + 64 + 32, .iv = "\x00\x01\x02\x03\x04\x05\x06\x07" "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + .assoc = "\x00\x01\x02\x03\x04\x05\x06\x07" + "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + .alen = 16, .input = "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96" "\xe9\x3d\x7e\x11\x73\x93\x17\x2a" "\xae\x2d\x8a\x57\x1e\x03\xac\x9c" @@ -15955,8 +16021,9 @@ static struct aead_testvec hmac_sha1_des_cbc_enc_tv_temp[] = { "\xE9\xC0\xFF\x2E\x76\x0B\x64\x24", .klen = 8 + 20 + 8, .iv = "\x7D\x33\x88\x93\x0F\x93\xB2\x42", - .assoc = "\x00\x00\x43\x21\x00\x00\x00\x01", - .alen = 8, + .assoc = "\x00\x00\x43\x21\x00\x00\x00\x01" + "\x7D\x33\x88\x93\x0F\x93\xB2\x42", + .alen = 16, .input = "\x6f\x54\x20\x6f\x61\x4d\x79\x6e" "\x53\x20\x63\x65\x65\x72\x73\x74" "\x54\x20\x6f\x6f\x4d\x20\x6e\x61" @@ -16015,8 +16082,9 @@ static struct aead_testvec hmac_sha224_des_cbc_enc_tv_temp[] = { "\xE9\xC0\xFF\x2E\x76\x0B\x64\x24", .klen = 8 + 24 + 8, .iv = "\x7D\x33\x88\x93\x0F\x93\xB2\x42", - .assoc = "\x00\x00\x43\x21\x00\x00\x00\x01", - .alen = 8, + .assoc = "\x00\x00\x43\x21\x00\x00\x00\x01" + "\x7D\x33\x88\x93\x0F\x93\xB2\x42", + .alen = 16, .input = "\x6f\x54\x20\x6f\x61\x4d\x79\x6e" "\x53\x20\x63\x65\x65\x72\x73\x74" "\x54\x20\x6f\x6f\x4d\x20\x6e\x61" @@ -16076,8 +16144,9 @@ static struct aead_testvec hmac_sha256_des_cbc_enc_tv_temp[] = { "\xE9\xC0\xFF\x2E\x76\x0B\x64\x24", .klen = 8 + 32 + 8, .iv = "\x7D\x33\x88\x93\x0F\x93\xB2\x42", - .assoc = "\x00\x00\x43\x21\x00\x00\x00\x01", - .alen = 8, + .assoc = "\x00\x00\x43\x21\x00\x00\x00\x01" + "\x7D\x33\x88\x93\x0F\x93\xB2\x42", + .alen = 16, .input = "\x6f\x54\x20\x6f\x61\x4d\x79\x6e" "\x53\x20\x63\x65\x65\x72\x73\x74" "\x54\x20\x6f\x6f\x4d\x20\x6e\x61" @@ -16140,8 +16209,9 @@ static struct aead_testvec hmac_sha384_des_cbc_enc_tv_temp[] = { "\xE9\xC0\xFF\x2E\x76\x0B\x64\x24", .klen = 8 + 48 + 8, .iv = "\x7D\x33\x88\x93\x0F\x93\xB2\x42", - .assoc = "\x00\x00\x43\x21\x00\x00\x00\x01", - .alen = 8, + .assoc = "\x00\x00\x43\x21\x00\x00\x00\x01" + "\x7D\x33\x88\x93\x0F\x93\xB2\x42", + .alen = 16, .input = "\x6f\x54\x20\x6f\x61\x4d\x79\x6e" "\x53\x20\x63\x65\x65\x72\x73\x74" "\x54\x20\x6f\x6f\x4d\x20\x6e\x61" @@ -16208,8 +16278,9 @@ static struct aead_testvec hmac_sha512_des_cbc_enc_tv_temp[] = { "\xE9\xC0\xFF\x2E\x76\x0B\x64\x24", .klen = 8 + 64 + 8, .iv = "\x7D\x33\x88\x93\x0F\x93\xB2\x42", - .assoc = "\x00\x00\x43\x21\x00\x00\x00\x01", - .alen = 8, + .assoc = "\x00\x00\x43\x21\x00\x00\x00\x01" + "\x7D\x33\x88\x93\x0F\x93\xB2\x42", + .alen = 16, .input = "\x6f\x54\x20\x6f\x61\x4d\x79\x6e" "\x53\x20\x63\x65\x65\x72\x73\x74" "\x54\x20\x6f\x6f\x4d\x20\x6e\x61" @@ -16275,8 +16346,9 @@ static struct aead_testvec hmac_sha1_des3_ede_cbc_enc_tv_temp[] = { "\xEA\xC2\x84\xE8\x14\x95\xDB\xE8", .klen = 8 + 20 + 24, .iv = "\x7D\x33\x88\x93\x0F\x93\xB2\x42", - .assoc = "\x00\x00\x43\x21\x00\x00\x00\x01", - .alen = 8, + .assoc = "\x00\x00\x43\x21\x00\x00\x00\x01" + "\x7D\x33\x88\x93\x0F\x93\xB2\x42", + .alen = 16, .input = "\x6f\x54\x20\x6f\x61\x4d\x79\x6e" "\x53\x20\x63\x65\x65\x72\x73\x74" "\x54\x20\x6f\x6f\x4d\x20\x6e\x61" @@ -16337,8 +16409,9 @@ static struct aead_testvec hmac_sha224_des3_ede_cbc_enc_tv_temp[] = { "\xEA\xC2\x84\xE8\x14\x95\xDB\xE8", .klen = 8 + 24 + 24, .iv = "\x7D\x33\x88\x93\x0F\x93\xB2\x42", - .assoc = "\x00\x00\x43\x21\x00\x00\x00\x01", - .alen = 8, + .assoc = "\x00\x00\x43\x21\x00\x00\x00\x01" + "\x7D\x33\x88\x93\x0F\x93\xB2\x42", + .alen = 16, .input = "\x6f\x54\x20\x6f\x61\x4d\x79\x6e" "\x53\x20\x63\x65\x65\x72\x73\x74" "\x54\x20\x6f\x6f\x4d\x20\x6e\x61" @@ -16400,8 +16473,9 @@ static struct aead_testvec hmac_sha256_des3_ede_cbc_enc_tv_temp[] = { "\xEA\xC2\x84\xE8\x14\x95\xDB\xE8", .klen = 8 + 32 + 24, .iv = "\x7D\x33\x88\x93\x0F\x93\xB2\x42", - .assoc = "\x00\x00\x43\x21\x00\x00\x00\x01", - .alen = 8, + .assoc = "\x00\x00\x43\x21\x00\x00\x00\x01" + "\x7D\x33\x88\x93\x0F\x93\xB2\x42", + .alen = 16, .input = "\x6f\x54\x20\x6f\x61\x4d\x79\x6e" "\x53\x20\x63\x65\x65\x72\x73\x74" "\x54\x20\x6f\x6f\x4d\x20\x6e\x61" @@ -16466,8 +16540,9 @@ static struct aead_testvec hmac_sha384_des3_ede_cbc_enc_tv_temp[] = { "\xEA\xC2\x84\xE8\x14\x95\xDB\xE8", .klen = 8 + 48 + 24, .iv = "\x7D\x33\x88\x93\x0F\x93\xB2\x42", - .assoc = "\x00\x00\x43\x21\x00\x00\x00\x01", - .alen = 8, + .assoc = "\x00\x00\x43\x21\x00\x00\x00\x01" + "\x7D\x33\x88\x93\x0F\x93\xB2\x42", + .alen = 16, .input = "\x6f\x54\x20\x6f\x61\x4d\x79\x6e" "\x53\x20\x63\x65\x65\x72\x73\x74" "\x54\x20\x6f\x6f\x4d\x20\x6e\x61" @@ -16536,8 +16611,9 @@ static struct aead_testvec hmac_sha512_des3_ede_cbc_enc_tv_temp[] = { "\xEA\xC2\x84\xE8\x14\x95\xDB\xE8", .klen = 8 + 64 + 24, .iv = "\x7D\x33\x88\x93\x0F\x93\xB2\x42", - .assoc = "\x00\x00\x43\x21\x00\x00\x00\x01", - .alen = 8, + .assoc = "\x00\x00\x43\x21\x00\x00\x00\x01" + "\x7D\x33\x88\x93\x0F\x93\xB2\x42", + .alen = 16, .input = "\x6f\x54\x20\x6f\x61\x4d\x79\x6e" "\x53\x20\x63\x65\x65\x72\x73\x74" "\x54\x20\x6f\x6f\x4d\x20\x6e\x61" @@ -20129,149 +20205,150 @@ static struct aead_testvec aes_gcm_dec_tv_template[] = { }; static struct aead_testvec aes_gcm_rfc4106_enc_tv_template[] = { - { /* Generated using Crypto++ */ + { /* Generated using Crypto++ */ .key = zeroed_string, .klen = 20, - .iv = zeroed_string, - .input = zeroed_string, - .ilen = 16, - .assoc = zeroed_string, - .alen = 8, + .iv = zeroed_string, + .input = zeroed_string, + .ilen = 16, + .assoc = zeroed_string, + .alen = 16, .result = "\x03\x88\xDA\xCE\x60\xB6\xA3\x92" - "\xF3\x28\xC2\xB9\x71\xB2\xFE\x78" - "\x97\xFE\x4C\x23\x37\x42\x01\xE0" - "\x81\x9F\x8D\xC5\xD7\x41\xA0\x1B", + "\xF3\x28\xC2\xB9\x71\xB2\xFE\x78" + "\x97\xFE\x4C\x23\x37\x42\x01\xE0" + "\x81\x9F\x8D\xC5\xD7\x41\xA0\x1B", .rlen = 32, - },{ + },{ .key = "\xfe\xff\xe9\x92\x86\x65\x73\x1c" "\x6d\x6a\x8f\x94\x67\x30\x83\x08" - "\x00\x00\x00\x00", + "\x00\x00\x00\x00", .klen = 20, - .iv = "\x00\x00\x00\x00\x00\x00\x00\x01" - "\x00\x00\x00\x00", - .input = zeroed_string, - .ilen = 16, - .assoc = zeroed_string, - .alen = 8, + .iv = "\x00\x00\x00\x00\x00\x00\x00\x01", + .input = zeroed_string, + .ilen = 16, + .assoc = "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x01", + .alen = 16, .result = "\xC0\x0D\x8B\x42\x0F\x8F\x34\x18" - "\x88\xB1\xC5\xBC\xC5\xB6\xD6\x28" - "\x6A\x9D\xDF\x11\x5E\xFE\x5E\x9D" - "\x2F\x70\x44\x92\xF7\xF2\xE3\xEF", + "\x88\xB1\xC5\xBC\xC5\xB6\xD6\x28" + "\x6A\x9D\xDF\x11\x5E\xFE\x5E\x9D" + "\x2F\x70\x44\x92\xF7\xF2\xE3\xEF", .rlen = 32, - }, { + }, { .key = "\xfe\xff\xe9\x92\x86\x65\x73\x1c" "\x6d\x6a\x8f\x94\x67\x30\x83\x08" - "\x00\x00\x00\x00", + "\x00\x00\x00\x00", .klen = 20, - .iv = zeroed_string, - .input = "\x01\x01\x01\x01\x01\x01\x01\x01" - "\x01\x01\x01\x01\x01\x01\x01\x01", - .ilen = 16, - .assoc = zeroed_string, - .alen = 8, + .iv = zeroed_string, + .input = "\x01\x01\x01\x01\x01\x01\x01\x01" + "\x01\x01\x01\x01\x01\x01\x01\x01", + .ilen = 16, + .assoc = zeroed_string, + .alen = 16, .result = "\x4B\xB1\xB5\xE3\x25\x71\x70\xDE" - "\x7F\xC9\x9C\xA5\x14\x19\xF2\xAC" - "\x0B\x8F\x88\x69\x17\xE6\xB4\x3C" - "\xB1\x68\xFD\x14\x52\x64\x61\xB2", + "\x7F\xC9\x9C\xA5\x14\x19\xF2\xAC" + "\x0B\x8F\x88\x69\x17\xE6\xB4\x3C" + "\xB1\x68\xFD\x14\x52\x64\x61\xB2", .rlen = 32, - }, { + }, { .key = "\xfe\xff\xe9\x92\x86\x65\x73\x1c" "\x6d\x6a\x8f\x94\x67\x30\x83\x08" - "\x00\x00\x00\x00", + "\x00\x00\x00\x00", .klen = 20, - .iv = zeroed_string, - .input = "\x01\x01\x01\x01\x01\x01\x01\x01" - "\x01\x01\x01\x01\x01\x01\x01\x01", - .ilen = 16, - .assoc = "\x01\x01\x01\x01\x01\x01\x01\x01", - .alen = 8, + .iv = zeroed_string, + .input = "\x01\x01\x01\x01\x01\x01\x01\x01" + "\x01\x01\x01\x01\x01\x01\x01\x01", + .ilen = 16, + .assoc = "\x01\x01\x01\x01\x01\x01\x01\x01" + "\x00\x00\x00\x00\x00\x00\x00\x00", + .alen = 16, .result = "\x4B\xB1\xB5\xE3\x25\x71\x70\xDE" - "\x7F\xC9\x9C\xA5\x14\x19\xF2\xAC" - "\x90\x92\xB7\xE3\x5F\xA3\x9A\x63" - "\x7E\xD7\x1F\xD8\xD3\x7C\x4B\xF5", + "\x7F\xC9\x9C\xA5\x14\x19\xF2\xAC" + "\x90\x92\xB7\xE3\x5F\xA3\x9A\x63" + "\x7E\xD7\x1F\xD8\xD3\x7C\x4B\xF5", .rlen = 32, - }, { + }, { .key = "\xfe\xff\xe9\x92\x86\x65\x73\x1c" "\x6d\x6a\x8f\x94\x67\x30\x83\x08" - "\x00\x00\x00\x00", + "\x00\x00\x00\x00", .klen = 20, - .iv = "\x00\x00\x00\x00\x00\x00\x00\x01" - "\x00\x00\x00\x00", - .input = "\x01\x01\x01\x01\x01\x01\x01\x01" - "\x01\x01\x01\x01\x01\x01\x01\x01", - .ilen = 16, - .assoc = "\x01\x01\x01\x01\x01\x01\x01\x01", - .alen = 8, + .iv = "\x00\x00\x00\x00\x00\x00\x00\x01", + .input = "\x01\x01\x01\x01\x01\x01\x01\x01" + "\x01\x01\x01\x01\x01\x01\x01\x01", + .ilen = 16, + .assoc = "\x01\x01\x01\x01\x01\x01\x01\x01" + "\x00\x00\x00\x00\x00\x00\x00\x01", + .alen = 16, .result = "\xC1\x0C\x8A\x43\x0E\x8E\x35\x19" - "\x89\xB0\xC4\xBD\xC4\xB7\xD7\x29" - "\x64\x50\xF9\x32\x13\xFB\x74\x61" - "\xF4\xED\x52\xD3\xC5\x10\x55\x3C", + "\x89\xB0\xC4\xBD\xC4\xB7\xD7\x29" + "\x64\x50\xF9\x32\x13\xFB\x74\x61" + "\xF4\xED\x52\xD3\xC5\x10\x55\x3C", .rlen = 32, - }, { + }, { .key = "\xfe\xff\xe9\x92\x86\x65\x73\x1c" "\x6d\x6a\x8f\x94\x67\x30\x83\x08" - "\x00\x00\x00\x00", + "\x00\x00\x00\x00", .klen = 20, - .iv = "\x00\x00\x00\x00\x00\x00\x00\x01" - "\x00\x00\x00\x00", - .input = "\x01\x01\x01\x01\x01\x01\x01\x01" - "\x01\x01\x01\x01\x01\x01\x01\x01" - "\x01\x01\x01\x01\x01\x01\x01\x01" - "\x01\x01\x01\x01\x01\x01\x01\x01" - "\x01\x01\x01\x01\x01\x01\x01\x01" - "\x01\x01\x01\x01\x01\x01\x01\x01" - "\x01\x01\x01\x01\x01\x01\x01\x01" - "\x01\x01\x01\x01\x01\x01\x01\x01", - .ilen = 64, - .assoc = "\x01\x01\x01\x01\x01\x01\x01\x01", - .alen = 8, + .iv = "\x00\x00\x00\x00\x00\x00\x00\x01", + .input = "\x01\x01\x01\x01\x01\x01\x01\x01" + "\x01\x01\x01\x01\x01\x01\x01\x01" + "\x01\x01\x01\x01\x01\x01\x01\x01" + "\x01\x01\x01\x01\x01\x01\x01\x01" + "\x01\x01\x01\x01\x01\x01\x01\x01" + "\x01\x01\x01\x01\x01\x01\x01\x01" + "\x01\x01\x01\x01\x01\x01\x01\x01" + "\x01\x01\x01\x01\x01\x01\x01\x01", + .ilen = 64, + .assoc = "\x01\x01\x01\x01\x01\x01\x01\x01" + "\x00\x00\x00\x00\x00\x00\x00\x01", + .alen = 16, .result = "\xC1\x0C\x8A\x43\x0E\x8E\x35\x19" - "\x89\xB0\xC4\xBD\xC4\xB7\xD7\x29" - "\x98\x14\xA1\x42\x37\x80\xFD\x90" - "\x68\x12\x01\xA8\x91\x89\xB9\x83" - "\x5B\x11\x77\x12\x9B\xFF\x24\x89" - "\x94\x5F\x18\x12\xBA\x27\x09\x39" - "\x99\x96\x76\x42\x15\x1C\xCD\xCB" - "\xDC\xD3\xDA\x65\x73\xAF\x80\xCD" - "\xD2\xB6\xC2\x4A\x76\xC2\x92\x85" - "\xBD\xCF\x62\x98\x58\x14\xE5\xBD", + "\x89\xB0\xC4\xBD\xC4\xB7\xD7\x29" + "\x98\x14\xA1\x42\x37\x80\xFD\x90" + "\x68\x12\x01\xA8\x91\x89\xB9\x83" + "\x5B\x11\x77\x12\x9B\xFF\x24\x89" + "\x94\x5F\x18\x12\xBA\x27\x09\x39" + "\x99\x96\x76\x42\x15\x1C\xCD\xCB" + "\xDC\xD3\xDA\x65\x73\xAF\x80\xCD" + "\xD2\xB6\xC2\x4A\x76\xC2\x92\x85" + "\xBD\xCF\x62\x98\x58\x14\xE5\xBD", .rlen = 80, - }, { + }, { .key = "\x00\x01\x02\x03\x04\x05\x06\x07" "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" - "\x00\x00\x00\x00", + "\x00\x00\x00\x00", .klen = 20, - .iv = "\x00\x00\x45\x67\x89\xab\xcd\xef" - "\x00\x00\x00\x00", - .input = "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff", - .ilen = 192, - .assoc = "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" - "\xaa\xaa\xaa\xaa", - .alen = 12, + .iv = "\x00\x00\x45\x67\x89\xab\xcd\xef", + .input = "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff", + .ilen = 192, + .assoc = "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\x00\x00\x45\x67" + "\x89\xab\xcd\xef", + .alen = 20, .result = "\xC1\x76\x33\x85\xE2\x9B\x5F\xDE" "\xDE\x89\x3D\x42\xE7\xC9\x69\x8A" "\x44\x6D\xC3\x88\x46\x2E\xC2\x01" @@ -20316,8 +20393,9 @@ static struct aead_testvec aes_gcm_rfc4106_enc_tv_template[] = { "\x00\x21\x00\x01\x01\x02\x02\x01", .ilen = 72, .assoc = "\x00\x00\x43\x21\x87\x65\x43\x21" - "\x00\x00\x00\x00", - .alen = 12, + "\x00\x00\x00\x00\x49\x56\xED\x7E" + "\x3B\x24\x4C\xFE", + .alen = 20, .result = "\xFE\xCF\x53\x7E\x72\x9D\x5B\x07" "\xDC\x30\xDF\x52\x8D\xD2\x2B\x76" "\x8D\x1B\x98\x73\x66\x96\xA6\xFD" @@ -20345,8 +20423,9 @@ static struct aead_testvec aes_gcm_rfc4106_enc_tv_template[] = { "\x65\x72\x63\x69\x74\x79\x02\x64" "\x6B\x00\x00\x01\x00\x01\x00\x01", .ilen = 64, - .assoc = "\x00\x00\xA5\xF8\x00\x00\x00\x0A", - .alen = 8, + .assoc = "\x00\x00\xA5\xF8\x00\x00\x00\x0A" + "\xFA\xCE\xDB\xAD\xDE\xCA\xF8\x88", + .alen = 16, .result = "\xDE\xB2\x2C\xD9\xB0\x7C\x72\xC1" "\x6E\x3A\x65\xBE\xEB\x8D\xF3\x04" "\xA5\xA5\x89\x7D\x33\xAE\x53\x0F" @@ -20374,8 +20453,9 @@ static struct aead_testvec aes_gcm_rfc4106_enc_tv_template[] = { "\x02\x04\x05\xB4\x01\x01\x04\x02" "\x01\x02\x02\x01", .ilen = 52, - .assoc = "\x4A\x2C\xBF\xE3\x00\x00\x00\x02", - .alen = 8, + .assoc = "\x4A\x2C\xBF\xE3\x00\x00\x00\x02" + "\x01\x02\x03\x04\x05\x06\x07\x08", + .alen = 16, .result = "\xFF\x42\x5C\x9B\x72\x45\x99\xDF" "\x7A\x3B\xCD\x51\x01\x94\xE0\x0D" "\x6A\x78\x10\x7F\x1B\x0B\x1C\xBF" @@ -20401,8 +20481,9 @@ static struct aead_testvec aes_gcm_rfc4106_enc_tv_template[] = { "\x75\x76\x77\x61\x62\x63\x64\x65" "\x66\x67\x68\x69\x01\x02\x02\x01", .ilen = 64, - .assoc = "\x00\x00\x00\x00\x00\x00\x00\x01", - .alen = 8, + .assoc = "\x00\x00\x00\x00\x00\x00\x00\x01" + "\x00\x00\x00\x00\x00\x00\x00\x00", + .alen = 16, .result = "\x46\x88\xDA\xF2\xF9\x73\xA3\x92" "\x73\x29\x09\xC3\x31\xD5\x6D\x60" "\xF6\x94\xAB\xAA\x41\x4B\x5E\x7F" @@ -20430,8 +20511,9 @@ static struct aead_testvec aes_gcm_rfc4106_enc_tv_template[] = { "\x66\x67\x68\x69\x01\x02\x02\x01", .ilen = 64, .assoc = "\x42\xF6\x7E\x3F\x10\x10\x10\x10" - "\x10\x10\x10\x10", - .alen = 12, + "\x10\x10\x10\x10\x4E\x28\x00\x00" + "\xA2\xFC\xA1\xA3", + .alen = 20, .result = "\xFB\xA2\xCA\xA4\x85\x3C\xF9\xF0" "\xF2\x2C\xB1\x0D\x86\xDD\x83\xB0" "\xFE\xC7\x56\x91\xCF\x1A\x04\xB0" @@ -20455,8 +20537,9 @@ static struct aead_testvec aes_gcm_rfc4106_enc_tv_template[] = { "\x01\x02\x02\x01", .ilen = 28, .assoc = "\x42\xF6\x7E\x3F\x10\x10\x10\x10" - "\x10\x10\x10\x10", - .alen = 12, + "\x10\x10\x10\x10\x4E\x28\x00\x00" + "\xA2\xFC\xA1\xA3", + .alen = 20, .result = "\xFB\xA2\xCA\x84\x5E\x5D\xF9\xF0" "\xF2\x2C\x3E\x6E\x86\xDD\x83\x1E" "\x1F\xC6\x57\x92\xCD\x1A\xF9\x13" @@ -20477,8 +20560,9 @@ static struct aead_testvec aes_gcm_rfc4106_enc_tv_template[] = { "\xCB\x71\x26\x02\xDD\x6B\xB0\x3E" "\x50\x10\x16\xD0\x75\x68\x00\x01", .ilen = 40, - .assoc = "\x00\x00\xA5\xF8\x00\x00\x00\x0A", - .alen = 8, + .assoc = "\x00\x00\xA5\xF8\x00\x00\x00\x0A" + "\xFA\xCE\xDB\xAD\xDE\xCA\xF8\x88", + .alen = 16, .result = "\xA5\xB1\xF8\x06\x60\x29\xAE\xA4" "\x0E\x59\x8B\x81\x22\xDE\x02\x42" "\x09\x38\xB3\xAB\x33\xF8\x28\xE6" @@ -20505,8 +20589,9 @@ static struct aead_testvec aes_gcm_rfc4106_enc_tv_template[] = { "\x23\x01\x01\x01", .ilen = 76, .assoc = "\x00\x00\x01\x00\x00\x00\x00\x00" - "\x00\x00\x00\x01", - .alen = 12, + "\x00\x00\x00\x01\xCA\xFE\xDE\xBA" + "\xCE\xFA\xCE\x74", + .alen = 20, .result = "\x18\xA6\xFD\x42\xF7\x2C\xBF\x4A" "\xB2\xA2\xEA\x90\x1F\x73\xD8\x14" "\xE3\xE7\xF2\x43\xD9\x54\x12\xE1" @@ -20535,8 +20620,9 @@ static struct aead_testvec aes_gcm_rfc4106_enc_tv_template[] = { "\x50\x10\x1F\x64\x6D\x54\x00\x01", .ilen = 40, .assoc = "\x17\x40\x5E\x67\x15\x6F\x31\x26" - "\xDD\x0D\xB9\x9B", - .alen = 12, + "\xDD\x0D\xB9\x9B\x61\x6E\x64\x01" + "\x69\x76\x65\x63", + .alen = 20, .result = "\xF2\xD6\x9E\xCD\xBD\x5A\x0D\x5B" "\x8D\x5E\xF3\x8B\xAD\x4D\xA5\x8D" "\x1F\x27\x8F\xDE\x98\xEF\x67\x54" @@ -20563,8 +20649,9 @@ static struct aead_testvec aes_gcm_rfc4106_enc_tv_template[] = { "\x15\x01\x01\x01", .ilen = 76, .assoc = "\x42\xF6\x7E\x3F\x10\x10\x10\x10" - "\x10\x10\x10\x10", - .alen = 12, + "\x10\x10\x10\x10\x4E\x28\x00\x00" + "\xA2\xFC\xA1\xA3", + .alen = 20, .result = "\xFB\xA2\xCA\xD1\x2F\xC1\xF9\xF0" "\x0D\x3C\xEB\xF3\x05\x41\x0D\xB8" "\x3D\x77\x84\xB6\x07\x32\x3D\x22" @@ -20597,8 +20684,9 @@ static struct aead_testvec aes_gcm_rfc4106_enc_tv_template[] = { "\x72\x72\x6F\x77\x01\x02\x02\x01", .ilen = 72, .assoc = "\x17\x40\x5E\x67\x15\x6F\x31\x26" - "\xDD\x0D\xB9\x9B", - .alen = 12, + "\xDD\x0D\xB9\x9B\x61\x6E\x64\x01" + "\x69\x76\x65\x63", + .alen = 20, .result = "\xD4\xB7\xED\x86\xA1\x77\x7F\x2E" "\xA1\x3D\x69\x73\xD3\x24\xC6\x9E" "\x7B\x43\xF8\x26\xFB\x56\x83\x12" @@ -20619,8 +20707,9 @@ static struct aead_testvec aes_gcm_rfc4106_enc_tv_template[] = { .iv = "\x43\x45\x7E\x91\x82\x44\x3B\xC6", .input = "\x01\x02\x02\x01", .ilen = 4, - .assoc = "\x33\x54\x67\xAE\xFF\xFF\xFF\xFF", - .alen = 8, + .assoc = "\x33\x54\x67\xAE\xFF\xFF\xFF\xFF" + "\x43\x45\x7E\x91\x82\x44\x3B\xC6", + .alen = 16, .result = "\x43\x7F\x86\x6B\xCB\x3F\x69\x9F" "\xE9\xB0\x82\x2B\xAC\x96\x1C\x45" "\x04\xBE\xF2\x70", @@ -20636,8 +20725,9 @@ static struct aead_testvec aes_gcm_rfc4106_enc_tv_template[] = { "\x62\x65\x00\x01", .ilen = 20, .assoc = "\x00\x00\x01\x00\x00\x00\x00\x00" - "\x00\x00\x00\x01", - .alen = 12, + "\x00\x00\x00\x01\xCA\xFE\xDE\xBA" + "\xCE\xFA\xCE\x74", + .alen = 20, .result = "\x29\xC9\xFC\x69\xA1\x97\xD0\x38" "\xCC\xDD\x14\xE2\xDD\xFC\xAA\x05" "\x43\x33\x21\x64\x41\x25\x03\x52" @@ -20661,8 +20751,9 @@ static struct aead_testvec aes_gcm_rfc4106_enc_tv_template[] = { "\x01\x02\x02\x01", .ilen = 52, .assoc = "\x79\x6B\x69\x63\xFF\xFF\xFF\xFF" - "\xFF\xFF\xFF\xFF", - .alen = 12, + "\xFF\xFF\xFF\xFF\x33\x30\x21\x69" + "\x67\x65\x74\x6D", + .alen = 20, .result = "\xF9\x7A\xB2\xAA\x35\x6D\x8E\xDC" "\xE1\x76\x44\xAC\x8C\x78\xE2\x5D" "\xD2\x4D\xED\xBB\x29\xEB\xF1\xB6" @@ -20688,8 +20779,9 @@ static struct aead_testvec aes_gcm_rfc4106_enc_tv_template[] = { "\x01\x02\x02\x01", .ilen = 52, .assoc = "\x3F\x7E\xF6\x42\x10\x10\x10\x10" - "\x10\x10\x10\x10", - .alen = 12, + "\x10\x10\x10\x10\x4E\x28\x00\x00" + "\xA2\xFC\xA1\xA3", + .alen = 20, .result = "\xFB\xA2\xCA\xA8\xC6\xC5\xF9\xF0" "\xF2\x2C\xA5\x4A\x06\x12\x10\xAD" "\x3F\x6E\x57\x91\xCF\x1A\xCA\x21" @@ -20712,8 +20804,9 @@ static struct aead_testvec aes_gcm_rfc4106_enc_tv_template[] = { "\x71\x72\x73\x74\x01\x02\x02\x01", .ilen = 32, .assoc = "\x00\x00\x43\x21\x87\x65\x43\x21" - "\x00\x00\x00\x07", - .alen = 12, + "\x00\x00\x00\x07\x48\x55\xEC\x7D" + "\x3A\x23\x4B\xFD", + .alen = 20, .result = "\x74\x75\x2E\x8A\xEB\x5D\x87\x3C" "\xD7\xC0\xF4\xAC\xC3\x6C\x4B\xFF" "\x84\xB7\xD7\xB9\x8F\x0C\xA8\xB6" @@ -20725,122 +20818,122 @@ static struct aead_testvec aes_gcm_rfc4106_enc_tv_template[] = { }; static struct aead_testvec aes_gcm_rfc4106_dec_tv_template[] = { - { /* Generated using Crypto++ */ + { /* Generated using Crypto++ */ .key = zeroed_string, .klen = 20, - .iv = zeroed_string, + .iv = zeroed_string, .input = "\x03\x88\xDA\xCE\x60\xB6\xA3\x92" - "\xF3\x28\xC2\xB9\x71\xB2\xFE\x78" - "\x97\xFE\x4C\x23\x37\x42\x01\xE0" - "\x81\x9F\x8D\xC5\xD7\x41\xA0\x1B", + "\xF3\x28\xC2\xB9\x71\xB2\xFE\x78" + "\x97\xFE\x4C\x23\x37\x42\x01\xE0" + "\x81\x9F\x8D\xC5\xD7\x41\xA0\x1B", .ilen = 32, - .assoc = zeroed_string, - .alen = 8, - .result = zeroed_string, - .rlen = 16, + .assoc = zeroed_string, + .alen = 16, + .result = zeroed_string, + .rlen = 16, - },{ + },{ .key = "\xfe\xff\xe9\x92\x86\x65\x73\x1c" "\x6d\x6a\x8f\x94\x67\x30\x83\x08" - "\x00\x00\x00\x00", + "\x00\x00\x00\x00", .klen = 20, - .iv = "\x00\x00\x00\x00\x00\x00\x00\x01" - "\x00\x00\x00\x00", + .iv = "\x00\x00\x00\x00\x00\x00\x00\x01", .input = "\xC0\x0D\x8B\x42\x0F\x8F\x34\x18" - "\x88\xB1\xC5\xBC\xC5\xB6\xD6\x28" - "\x6A\x9D\xDF\x11\x5E\xFE\x5E\x9D" - "\x2F\x70\x44\x92\xF7\xF2\xE3\xEF", + "\x88\xB1\xC5\xBC\xC5\xB6\xD6\x28" + "\x6A\x9D\xDF\x11\x5E\xFE\x5E\x9D" + "\x2F\x70\x44\x92\xF7\xF2\xE3\xEF", .ilen = 32, - .assoc = zeroed_string, - .alen = 8, - .result = zeroed_string, - .rlen = 16, - }, { + .assoc = "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x01", + .alen = 16, + .result = zeroed_string, + .rlen = 16, + }, { .key = "\xfe\xff\xe9\x92\x86\x65\x73\x1c" "\x6d\x6a\x8f\x94\x67\x30\x83\x08" - "\x00\x00\x00\x00", + "\x00\x00\x00\x00", .klen = 20, - .iv = zeroed_string, + .iv = zeroed_string, .input = "\x4B\xB1\xB5\xE3\x25\x71\x70\xDE" - "\x7F\xC9\x9C\xA5\x14\x19\xF2\xAC" - "\x0B\x8F\x88\x69\x17\xE6\xB4\x3C" - "\xB1\x68\xFD\x14\x52\x64\x61\xB2", + "\x7F\xC9\x9C\xA5\x14\x19\xF2\xAC" + "\x0B\x8F\x88\x69\x17\xE6\xB4\x3C" + "\xB1\x68\xFD\x14\x52\x64\x61\xB2", .ilen = 32, - .assoc = zeroed_string, - .alen = 8, - .result = "\x01\x01\x01\x01\x01\x01\x01\x01" - "\x01\x01\x01\x01\x01\x01\x01\x01", - .rlen = 16, - }, { + .assoc = zeroed_string, + .alen = 16, + .result = "\x01\x01\x01\x01\x01\x01\x01\x01" + "\x01\x01\x01\x01\x01\x01\x01\x01", + .rlen = 16, + }, { .key = "\xfe\xff\xe9\x92\x86\x65\x73\x1c" "\x6d\x6a\x8f\x94\x67\x30\x83\x08" - "\x00\x00\x00\x00", + "\x00\x00\x00\x00", .klen = 20, - .iv = zeroed_string, + .iv = zeroed_string, .input = "\x4B\xB1\xB5\xE3\x25\x71\x70\xDE" - "\x7F\xC9\x9C\xA5\x14\x19\xF2\xAC" - "\x90\x92\xB7\xE3\x5F\xA3\x9A\x63" - "\x7E\xD7\x1F\xD8\xD3\x7C\x4B\xF5", + "\x7F\xC9\x9C\xA5\x14\x19\xF2\xAC" + "\x90\x92\xB7\xE3\x5F\xA3\x9A\x63" + "\x7E\xD7\x1F\xD8\xD3\x7C\x4B\xF5", .ilen = 32, - .assoc = "\x01\x01\x01\x01\x01\x01\x01\x01", - .alen = 8, - .result = "\x01\x01\x01\x01\x01\x01\x01\x01" - "\x01\x01\x01\x01\x01\x01\x01\x01", - .rlen = 16, + .assoc = "\x01\x01\x01\x01\x01\x01\x01\x01" + "\x00\x00\x00\x00\x00\x00\x00\x00", + .alen = 16, + .result = "\x01\x01\x01\x01\x01\x01\x01\x01" + "\x01\x01\x01\x01\x01\x01\x01\x01", + .rlen = 16, - }, { + }, { .key = "\xfe\xff\xe9\x92\x86\x65\x73\x1c" "\x6d\x6a\x8f\x94\x67\x30\x83\x08" - "\x00\x00\x00\x00", + "\x00\x00\x00\x00", .klen = 20, - .iv = "\x00\x00\x00\x00\x00\x00\x00\x01" - "\x00\x00\x00\x00", + .iv = "\x00\x00\x00\x00\x00\x00\x00\x01", .input = "\xC1\x0C\x8A\x43\x0E\x8E\x35\x19" - "\x89\xB0\xC4\xBD\xC4\xB7\xD7\x29" - "\x64\x50\xF9\x32\x13\xFB\x74\x61" - "\xF4\xED\x52\xD3\xC5\x10\x55\x3C", + "\x89\xB0\xC4\xBD\xC4\xB7\xD7\x29" + "\x64\x50\xF9\x32\x13\xFB\x74\x61" + "\xF4\xED\x52\xD3\xC5\x10\x55\x3C", .ilen = 32, - .assoc = "\x01\x01\x01\x01\x01\x01\x01\x01", - .alen = 8, - .result = "\x01\x01\x01\x01\x01\x01\x01\x01" - "\x01\x01\x01\x01\x01\x01\x01\x01", - .rlen = 16, - }, { + .assoc = "\x01\x01\x01\x01\x01\x01\x01\x01" + "\x00\x00\x00\x00\x00\x00\x00\x01", + .alen = 16, + .result = "\x01\x01\x01\x01\x01\x01\x01\x01" + "\x01\x01\x01\x01\x01\x01\x01\x01", + .rlen = 16, + }, { .key = "\xfe\xff\xe9\x92\x86\x65\x73\x1c" "\x6d\x6a\x8f\x94\x67\x30\x83\x08" - "\x00\x00\x00\x00", + "\x00\x00\x00\x00", .klen = 20, - .iv = "\x00\x00\x00\x00\x00\x00\x00\x01" - "\x00\x00\x00\x00", + .iv = "\x00\x00\x00\x00\x00\x00\x00\x01", .input = "\xC1\x0C\x8A\x43\x0E\x8E\x35\x19" - "\x89\xB0\xC4\xBD\xC4\xB7\xD7\x29" - "\x98\x14\xA1\x42\x37\x80\xFD\x90" - "\x68\x12\x01\xA8\x91\x89\xB9\x83" - "\x5B\x11\x77\x12\x9B\xFF\x24\x89" - "\x94\x5F\x18\x12\xBA\x27\x09\x39" - "\x99\x96\x76\x42\x15\x1C\xCD\xCB" - "\xDC\xD3\xDA\x65\x73\xAF\x80\xCD" - "\xD2\xB6\xC2\x4A\x76\xC2\x92\x85" - "\xBD\xCF\x62\x98\x58\x14\xE5\xBD", + "\x89\xB0\xC4\xBD\xC4\xB7\xD7\x29" + "\x98\x14\xA1\x42\x37\x80\xFD\x90" + "\x68\x12\x01\xA8\x91\x89\xB9\x83" + "\x5B\x11\x77\x12\x9B\xFF\x24\x89" + "\x94\x5F\x18\x12\xBA\x27\x09\x39" + "\x99\x96\x76\x42\x15\x1C\xCD\xCB" + "\xDC\xD3\xDA\x65\x73\xAF\x80\xCD" + "\xD2\xB6\xC2\x4A\x76\xC2\x92\x85" + "\xBD\xCF\x62\x98\x58\x14\xE5\xBD", .ilen = 80, - .assoc = "\x01\x01\x01\x01\x01\x01\x01\x01", - .alen = 8, - .result = "\x01\x01\x01\x01\x01\x01\x01\x01" - "\x01\x01\x01\x01\x01\x01\x01\x01" - "\x01\x01\x01\x01\x01\x01\x01\x01" - "\x01\x01\x01\x01\x01\x01\x01\x01" - "\x01\x01\x01\x01\x01\x01\x01\x01" - "\x01\x01\x01\x01\x01\x01\x01\x01" - "\x01\x01\x01\x01\x01\x01\x01\x01" - "\x01\x01\x01\x01\x01\x01\x01\x01", - .rlen = 64, - }, { + .assoc = "\x01\x01\x01\x01\x01\x01\x01\x01" + "\x00\x00\x00\x00\x00\x00\x00\x01", + .alen = 16, + .result = "\x01\x01\x01\x01\x01\x01\x01\x01" + "\x01\x01\x01\x01\x01\x01\x01\x01" + "\x01\x01\x01\x01\x01\x01\x01\x01" + "\x01\x01\x01\x01\x01\x01\x01\x01" + "\x01\x01\x01\x01\x01\x01\x01\x01" + "\x01\x01\x01\x01\x01\x01\x01\x01" + "\x01\x01\x01\x01\x01\x01\x01\x01" + "\x01\x01\x01\x01\x01\x01\x01\x01", + .rlen = 64, + }, { .key = "\x00\x01\x02\x03\x04\x05\x06\x07" "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" - "\x00\x00\x00\x00", + "\x00\x00\x00\x00", .klen = 20, - .iv = "\x00\x00\x45\x67\x89\xab\xcd\xef" - "\x00\x00\x00\x00", + .iv = "\x00\x00\x45\x67\x89\xab\xcd\xef", .input = "\xC1\x76\x33\x85\xE2\x9B\x5F\xDE" "\xDE\x89\x3D\x42\xE7\xC9\x69\x8A" "\x44\x6D\xC3\x88\x46\x2E\xC2\x01" @@ -20868,34 +20961,35 @@ static struct aead_testvec aes_gcm_rfc4106_dec_tv_template[] = { "\x37\x08\x1C\xCF\xBA\x5D\x71\x46" "\x80\x72\xB0\x4C\x82\x0D\x60\x3C", .ilen = 208, - .assoc = "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" - "\xaa\xaa\xaa\xaa", - .alen = 12, - .result = "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff", - .rlen = 192, + .assoc = "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\x00\x00\x45\x67" + "\x89\xab\xcd\xef", + .alen = 20, + .result = "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff", + .rlen = 192, }, { .key = "\x4C\x80\xCD\xEF\xBB\x5D\x10\xDA" "\x90\x6A\xC7\x3C\x36\x13\xA6\x34" @@ -20913,8 +21007,9 @@ static struct aead_testvec aes_gcm_rfc4106_dec_tv_template[] = { "\x00\x21\x00\x01\x01\x02\x02\x01", .rlen = 72, .assoc = "\x00\x00\x43\x21\x87\x65\x43\x21" - "\x00\x00\x00\x00", - .alen = 12, + "\x00\x00\x00\x00\x49\x56\xED\x7E" + "\x3B\x24\x4C\xFE", + .alen = 20, .input = "\xFE\xCF\x53\x7E\x72\x9D\x5B\x07" "\xDC\x30\xDF\x52\x8D\xD2\x2B\x76" "\x8D\x1B\x98\x73\x66\x96\xA6\xFD" @@ -20942,8 +21037,9 @@ static struct aead_testvec aes_gcm_rfc4106_dec_tv_template[] = { "\x65\x72\x63\x69\x74\x79\x02\x64" "\x6B\x00\x00\x01\x00\x01\x00\x01", .rlen = 64, - .assoc = "\x00\x00\xA5\xF8\x00\x00\x00\x0A", - .alen = 8, + .assoc = "\x00\x00\xA5\xF8\x00\x00\x00\x0A" + "\xFA\xCE\xDB\xAD\xDE\xCA\xF8\x88", + .alen = 16, .input = "\xDE\xB2\x2C\xD9\xB0\x7C\x72\xC1" "\x6E\x3A\x65\xBE\xEB\x8D\xF3\x04" "\xA5\xA5\x89\x7D\x33\xAE\x53\x0F" @@ -20971,8 +21067,9 @@ static struct aead_testvec aes_gcm_rfc4106_dec_tv_template[] = { "\x02\x04\x05\xB4\x01\x01\x04\x02" "\x01\x02\x02\x01", .rlen = 52, - .assoc = "\x4A\x2C\xBF\xE3\x00\x00\x00\x02", - .alen = 8, + .assoc = "\x4A\x2C\xBF\xE3\x00\x00\x00\x02" + "\x01\x02\x03\x04\x05\x06\x07\x08", + .alen = 16, .input = "\xFF\x42\x5C\x9B\x72\x45\x99\xDF" "\x7A\x3B\xCD\x51\x01\x94\xE0\x0D" "\x6A\x78\x10\x7F\x1B\x0B\x1C\xBF" @@ -20998,8 +21095,9 @@ static struct aead_testvec aes_gcm_rfc4106_dec_tv_template[] = { "\x75\x76\x77\x61\x62\x63\x64\x65" "\x66\x67\x68\x69\x01\x02\x02\x01", .rlen = 64, - .assoc = "\x00\x00\x00\x00\x00\x00\x00\x01", - .alen = 8, + .assoc = "\x00\x00\x00\x00\x00\x00\x00\x01" + "\x00\x00\x00\x00\x00\x00\x00\x00", + .alen = 16, .input = "\x46\x88\xDA\xF2\xF9\x73\xA3\x92" "\x73\x29\x09\xC3\x31\xD5\x6D\x60" "\xF6\x94\xAB\xAA\x41\x4B\x5E\x7F" @@ -21027,8 +21125,9 @@ static struct aead_testvec aes_gcm_rfc4106_dec_tv_template[] = { "\x66\x67\x68\x69\x01\x02\x02\x01", .rlen = 64, .assoc = "\x42\xF6\x7E\x3F\x10\x10\x10\x10" - "\x10\x10\x10\x10", - .alen = 12, + "\x10\x10\x10\x10\x4E\x28\x00\x00" + "\xA2\xFC\xA1\xA3", + .alen = 20, .input = "\xFB\xA2\xCA\xA4\x85\x3C\xF9\xF0" "\xF2\x2C\xB1\x0D\x86\xDD\x83\xB0" "\xFE\xC7\x56\x91\xCF\x1A\x04\xB0" @@ -21052,8 +21151,9 @@ static struct aead_testvec aes_gcm_rfc4106_dec_tv_template[] = { "\x01\x02\x02\x01", .rlen = 28, .assoc = "\x42\xF6\x7E\x3F\x10\x10\x10\x10" - "\x10\x10\x10\x10", - .alen = 12, + "\x10\x10\x10\x10\x4E\x28\x00\x00" + "\xA2\xFC\xA1\xA3", + .alen = 20, .input = "\xFB\xA2\xCA\x84\x5E\x5D\xF9\xF0" "\xF2\x2C\x3E\x6E\x86\xDD\x83\x1E" "\x1F\xC6\x57\x92\xCD\x1A\xF9\x13" @@ -21074,8 +21174,9 @@ static struct aead_testvec aes_gcm_rfc4106_dec_tv_template[] = { "\xCB\x71\x26\x02\xDD\x6B\xB0\x3E" "\x50\x10\x16\xD0\x75\x68\x00\x01", .rlen = 40, - .assoc = "\x00\x00\xA5\xF8\x00\x00\x00\x0A", - .alen = 8, + .assoc = "\x00\x00\xA5\xF8\x00\x00\x00\x0A" + "\xFA\xCE\xDB\xAD\xDE\xCA\xF8\x88", + .alen = 16, .input = "\xA5\xB1\xF8\x06\x60\x29\xAE\xA4" "\x0E\x59\x8B\x81\x22\xDE\x02\x42" "\x09\x38\xB3\xAB\x33\xF8\x28\xE6" @@ -21102,8 +21203,9 @@ static struct aead_testvec aes_gcm_rfc4106_dec_tv_template[] = { "\x23\x01\x01\x01", .rlen = 76, .assoc = "\x00\x00\x01\x00\x00\x00\x00\x00" - "\x00\x00\x00\x01", - .alen = 12, + "\x00\x00\x00\x01\xCA\xFE\xDE\xBA" + "\xCE\xFA\xCE\x74", + .alen = 20, .input = "\x18\xA6\xFD\x42\xF7\x2C\xBF\x4A" "\xB2\xA2\xEA\x90\x1F\x73\xD8\x14" "\xE3\xE7\xF2\x43\xD9\x54\x12\xE1" @@ -21132,8 +21234,9 @@ static struct aead_testvec aes_gcm_rfc4106_dec_tv_template[] = { "\x50\x10\x1F\x64\x6D\x54\x00\x01", .rlen = 40, .assoc = "\x17\x40\x5E\x67\x15\x6F\x31\x26" - "\xDD\x0D\xB9\x9B", - .alen = 12, + "\xDD\x0D\xB9\x9B\x61\x6E\x64\x01" + "\x69\x76\x65\x63", + .alen = 20, .input = "\xF2\xD6\x9E\xCD\xBD\x5A\x0D\x5B" "\x8D\x5E\xF3\x8B\xAD\x4D\xA5\x8D" "\x1F\x27\x8F\xDE\x98\xEF\x67\x54" @@ -21160,8 +21263,9 @@ static struct aead_testvec aes_gcm_rfc4106_dec_tv_template[] = { "\x15\x01\x01\x01", .rlen = 76, .assoc = "\x42\xF6\x7E\x3F\x10\x10\x10\x10" - "\x10\x10\x10\x10", - .alen = 12, + "\x10\x10\x10\x10\x4E\x28\x00\x00" + "\xA2\xFC\xA1\xA3", + .alen = 20, .input = "\xFB\xA2\xCA\xD1\x2F\xC1\xF9\xF0" "\x0D\x3C\xEB\xF3\x05\x41\x0D\xB8" "\x3D\x77\x84\xB6\x07\x32\x3D\x22" @@ -21194,8 +21298,9 @@ static struct aead_testvec aes_gcm_rfc4106_dec_tv_template[] = { "\x72\x72\x6F\x77\x01\x02\x02\x01", .rlen = 72, .assoc = "\x17\x40\x5E\x67\x15\x6F\x31\x26" - "\xDD\x0D\xB9\x9B", - .alen = 12, + "\xDD\x0D\xB9\x9B\x61\x6E\x64\x01" + "\x69\x76\x65\x63", + .alen = 20, .input = "\xD4\xB7\xED\x86\xA1\x77\x7F\x2E" "\xA1\x3D\x69\x73\xD3\x24\xC6\x9E" "\x7B\x43\xF8\x26\xFB\x56\x83\x12" @@ -21216,8 +21321,9 @@ static struct aead_testvec aes_gcm_rfc4106_dec_tv_template[] = { .iv = "\x43\x45\x7E\x91\x82\x44\x3B\xC6", .result = "\x01\x02\x02\x01", .rlen = 4, - .assoc = "\x33\x54\x67\xAE\xFF\xFF\xFF\xFF", - .alen = 8, + .assoc = "\x33\x54\x67\xAE\xFF\xFF\xFF\xFF" + "\x43\x45\x7E\x91\x82\x44\x3B\xC6", + .alen = 16, .input = "\x43\x7F\x86\x6B\xCB\x3F\x69\x9F" "\xE9\xB0\x82\x2B\xAC\x96\x1C\x45" "\x04\xBE\xF2\x70", @@ -21233,8 +21339,9 @@ static struct aead_testvec aes_gcm_rfc4106_dec_tv_template[] = { "\x62\x65\x00\x01", .rlen = 20, .assoc = "\x00\x00\x01\x00\x00\x00\x00\x00" - "\x00\x00\x00\x01", - .alen = 12, + "\x00\x00\x00\x01\xCA\xFE\xDE\xBA" + "\xCE\xFA\xCE\x74", + .alen = 20, .input = "\x29\xC9\xFC\x69\xA1\x97\xD0\x38" "\xCC\xDD\x14\xE2\xDD\xFC\xAA\x05" "\x43\x33\x21\x64\x41\x25\x03\x52" @@ -21258,8 +21365,9 @@ static struct aead_testvec aes_gcm_rfc4106_dec_tv_template[] = { "\x01\x02\x02\x01", .rlen = 52, .assoc = "\x79\x6B\x69\x63\xFF\xFF\xFF\xFF" - "\xFF\xFF\xFF\xFF", - .alen = 12, + "\xFF\xFF\xFF\xFF\x33\x30\x21\x69" + "\x67\x65\x74\x6D", + .alen = 20, .input = "\xF9\x7A\xB2\xAA\x35\x6D\x8E\xDC" "\xE1\x76\x44\xAC\x8C\x78\xE2\x5D" "\xD2\x4D\xED\xBB\x29\xEB\xF1\xB6" @@ -21285,8 +21393,9 @@ static struct aead_testvec aes_gcm_rfc4106_dec_tv_template[] = { "\x01\x02\x02\x01", .rlen = 52, .assoc = "\x3F\x7E\xF6\x42\x10\x10\x10\x10" - "\x10\x10\x10\x10", - .alen = 12, + "\x10\x10\x10\x10\x4E\x28\x00\x00" + "\xA2\xFC\xA1\xA3", + .alen = 20, .input = "\xFB\xA2\xCA\xA8\xC6\xC5\xF9\xF0" "\xF2\x2C\xA5\x4A\x06\x12\x10\xAD" "\x3F\x6E\x57\x91\xCF\x1A\xCA\x21" @@ -21309,8 +21418,9 @@ static struct aead_testvec aes_gcm_rfc4106_dec_tv_template[] = { "\x71\x72\x73\x74\x01\x02\x02\x01", .rlen = 32, .assoc = "\x00\x00\x43\x21\x87\x65\x43\x21" - "\x00\x00\x00\x07", - .alen = 12, + "\x00\x00\x00\x07\x48\x55\xEC\x7D" + "\x3A\x23\x4B\xFD", + .alen = 20, .input = "\x74\x75\x2E\x8A\xEB\x5D\x87\x3C" "\xD7\xC0\xF4\xAC\xC3\x6C\x4B\xFF" "\x84\xB7\xD7\xB9\x8F\x0C\xA8\xB6" @@ -21538,10 +21648,7 @@ static struct aead_testvec aes_ccm_enc_tv_template[] = { "\xba", .rlen = 33, }, { - /* - * This is the same vector as aes_ccm_rfc4309_enc_tv_template[0] - * below but rewritten to use the ccm algorithm directly. - */ + /* This is taken from FIPS CAVS. */ .key = "\x83\xac\x54\x66\xc2\xeb\xe5\x05" "\x2e\x01\xd1\xfc\x5d\x82\x66\x2e", .klen = 16, @@ -21559,6 +21666,142 @@ static struct aead_testvec aes_ccm_enc_tv_template[] = { "\xda\x24\xea\xd9\xa1\x39\x98\xfd" "\xa4\xbe\xd9\xf2\x1a\x6d\x22\xa8", .rlen = 48, + }, { + .key = "\x1e\x2c\x7e\x01\x41\x9a\xef\xc0" + "\x0d\x58\x96\x6e\x5c\xa2\x4b\xd3", + .klen = 16, + .iv = "\x03\x4f\xa3\x19\xd3\x01\x5a\xd8" + "\x30\x60\x15\x56\x00\x00\x00\x00", + .assoc = "\xda\xe6\x28\x9c\x45\x2d\xfd\x63" + "\x5e\xda\x4c\xb6\xe6\xfc\xf9\xb7" + "\x0c\x56\xcb\xe4\xe0\x05\x7a\xe1" + "\x0a\x63\x09\x78\xbc\x2c\x55\xde", + .alen = 32, + .input = "\x87\xa3\x36\xfd\x96\xb3\x93\x78" + "\xa9\x28\x63\xba\x12\xa3\x14\x85" + "\x57\x1e\x06\xc9\x7b\x21\xef\x76" + "\x7f\x38\x7e\x8e\x29\xa4\x3e\x7e", + .ilen = 32, + .result = "\x8a\x1e\x11\xf0\x02\x6b\xe2\x19" + "\xfc\x70\xc4\x6d\x8e\xb7\x99\xab" + "\xc5\x4b\xa2\xac\xd3\xf3\x48\xff" + "\x3b\xb5\xce\x53\xef\xde\xbb\x02" + "\xa9\x86\x15\x6c\x13\xfe\xda\x0a" + "\x22\xb8\x29\x3d\xd8\x39\x9a\x23", + .rlen = 48, + }, { + .key = "\xf4\x6b\xc2\x75\x62\xfe\xb4\xe1" + "\xa3\xf0\xff\xdd\x4e\x4b\x12\x75" + "\x53\x14\x73\x66\x8d\x88\xf6\x80", + .klen = 24, + .iv = "\x03\xa0\x20\x35\x26\xf2\x21\x8d" + "\x50\x20\xda\xe2\x00\x00\x00\x00", + .assoc = "\x5b\x9e\x13\x67\x02\x5e\xef\xc1" + "\x6c\xf9\xd7\x1e\x52\x8f\x7a\x47" + "\xe9\xd4\xcf\x20\x14\x6e\xf0\x2d" + "\xd8\x9e\x2b\x56\x10\x23\x56\xe7", + .alen = 32, + .result = "\x36\xea\x7a\x70\x08\xdc\x6a\xbc" + "\xad\x0c\x7a\x63\xf6\x61\xfd\x9b", + .rlen = 16, + }, { + .key = "\x56\xdf\x5c\x8f\x26\x3f\x0e\x42" + "\xef\x7a\xd3\xce\xfc\x84\x60\x62" + "\xca\xb4\x40\xaf\x5f\xc9\xc9\x01", + .klen = 24, + .iv = "\x03\xd6\x3c\x8c\x86\x84\xb6\xcd" + "\xef\x09\x2e\x94\x00\x00\x00\x00", + .assoc = "\x02\x65\x78\x3c\xe9\x21\x30\x91" + "\xb1\xb9\xda\x76\x9a\x78\x6d\x95" + "\xf2\x88\x32\xa3\xf2\x50\xcb\x4c" + "\xe3\x00\x73\x69\x84\x69\x87\x79", + .alen = 32, + .input = "\x9f\xd2\x02\x4b\x52\x49\x31\x3c" + "\x43\x69\x3a\x2d\x8e\x70\xad\x7e" + "\xe0\xe5\x46\x09\x80\x89\x13\xb2" + "\x8c\x8b\xd9\x3f\x86\xfb\xb5\x6b", + .ilen = 32, + .result = "\x39\xdf\x7c\x3c\x5a\x29\xb9\x62" + "\x5d\x51\xc2\x16\xd8\xbd\x06\x9f" + "\x9b\x6a\x09\x70\xc1\x51\x83\xc2" + "\x66\x88\x1d\x4f\x9a\xda\xe0\x1e" + "\xc7\x79\x11\x58\xe5\x6b\x20\x40" + "\x7a\xea\x46\x42\x8b\xe4\x6f\xe1", + .rlen = 48, + }, { + .key = "\xe0\x8d\x99\x71\x60\xd7\x97\x1a" + "\xbd\x01\x99\xd5\x8a\xdf\x71\x3a" + "\xd3\xdf\x24\x4b\x5e\x3d\x4b\x4e" + "\x30\x7a\xb9\xd8\x53\x0a\x5e\x2b", + .klen = 32, + .iv = "\x03\x1e\x29\x91\xad\x8e\xc1\x53" + "\x0a\xcf\x2d\xbe\x00\x00\x00\x00", + .assoc = "\x19\xb6\x1f\x57\xc4\xf3\xf0\x8b" + "\x78\x2b\x94\x02\x29\x0f\x42\x27" + "\x6b\x75\xcb\x98\x34\x08\x7e\x79" + "\xe4\x3e\x49\x0d\x84\x8b\x22\x87", + .alen = 32, + .input = "\xe1\xd9\xd8\x13\xeb\x3a\x75\x3f" + "\x9d\xbd\x5f\x66\xbe\xdc\xbb\x66" + "\xbf\x17\x99\x62\x4a\x39\x27\x1f" + "\x1d\xdc\x24\xae\x19\x2f\x98\x4c", + .ilen = 32, + .result = "\x19\xb8\x61\x33\x45\x2b\x43\x96" + "\x6f\x51\xd0\x20\x30\x7d\x9b\xc6" + "\x26\x3d\xf8\xc9\x65\x16\xa8\x9f" + "\xf0\x62\x17\x34\xf2\x1e\x8d\x75" + "\x4e\x13\xcc\xc0\xc3\x2a\x54\x2d", + .rlen = 40, + }, { + .key = "\x7c\xc8\x18\x3b\x8d\x99\xe0\x7c" + "\x45\x41\xb8\xbd\x5c\xa7\xc2\x32" + "\x8a\xb8\x02\x59\xa4\xfe\xa9\x2c" + "\x09\x75\x9a\x9b\x3c\x9b\x27\x39", + .klen = 32, + .iv = "\x03\xf9\xd9\x4e\x63\xb5\x3d\x9d" + "\x43\xf6\x1e\x50", + .assoc = "\x57\xf5\x6b\x8b\x57\x5c\x3d\x3b" + "\x13\x02\x01\x0c\x83\x4c\x96\x35" + "\x8e\xd6\x39\xcf\x7d\x14\x9b\x94" + "\xb0\x39\x36\xe6\x8f\x57\xe0\x13", + .alen = 32, + .input = "\x3b\x6c\x29\x36\xb6\xef\x07\xa6" + "\x83\x72\x07\x4f\xcf\xfa\x66\x89" + "\x5f\xca\xb1\xba\xd5\x8f\x2c\x27" + "\x30\xdb\x75\x09\x93\xd4\x65\xe4", + .ilen = 32, + .result = "\xb0\x88\x5a\x33\xaa\xe5\xc7\x1d" + "\x85\x23\xc7\xc6\x2f\xf4\x1e\x3d" + "\xcc\x63\x44\x25\x07\x78\x4f\x9e" + "\x96\xb8\x88\xeb\xbc\x48\x1f\x06" + "\x39\xaf\x39\xac\xd8\x4a\x80\x39" + "\x7b\x72\x8a\xf7", + .rlen = 44, + }, { + .key = "\xab\xd0\xe9\x33\x07\x26\xe5\x83" + "\x8c\x76\x95\xd4\xb6\xdc\xf3\x46" + "\xf9\x8f\xad\xe3\x02\x13\x83\x77" + "\x3f\xb0\xf1\xa1\xa1\x22\x0f\x2b", + .klen = 32, + .iv = "\x03\x24\xa7\x8b\x07\xcb\xcc\x0e" + "\xe6\x33\xbf\xf5\x00\x00\x00\x00", + .assoc = "\xd4\xdb\x30\x1d\x03\xfe\xfd\x5f" + "\x87\xd4\x8c\xb6\xb6\xf1\x7a\x5d" + "\xab\x90\x65\x8d\x8e\xca\x4d\x4f" + "\x16\x0c\x40\x90\x4b\xc7\x36\x73", + .alen = 32, + .input = "\xf5\xc6\x7d\x48\xc1\xb7\xe6\x92" + "\x97\x5a\xca\xc4\xa9\x6d\xf9\x3d" + "\x6c\xde\xbc\xf1\x90\xea\x6a\xb2" + "\x35\x86\x36\xaf\x5c\xfe\x4b\x3a", + .ilen = 32, + .result = "\x83\x6f\x40\x87\x72\xcf\xc1\x13" + "\xef\xbb\x80\x21\x04\x6c\x58\x09" + "\x07\x1b\xfc\xdf\xc0\x3f\x5b\xc7" + "\xe0\x79\xa8\x6e\x71\x7c\x3f\xcf" + "\x5c\xda\xb2\x33\xe5\x13\xe2\x0d" + "\x74\xd1\xef\xb5\x0f\x3a\xb5\xf8", + .rlen = 48, } }; @@ -21688,186 +21931,13 @@ static struct aead_testvec aes_ccm_dec_tv_template[] = { "\x8e\x5e\x67\x01\xc9\x17\x87\x65" "\x98\x09\xd6\x7d\xbe\xdd\x18", .rlen = 23, - }, -}; - -/* - * rfc4309 refers to section 8 of rfc3610 for test vectors, but they all - * use a 13-byte nonce, we only support an 11-byte nonce. Similarly, all of - * Special Publication 800-38C's test vectors also use nonce lengths our - * implementation doesn't support. The following are taken from fips cavs - * fax files on hand at Red Hat. - * - * nb: actual key lengths are (klen - 3), the last 3 bytes are actually - * part of the nonce which combine w/the iv, but need to be input this way. - */ -static struct aead_testvec aes_ccm_rfc4309_enc_tv_template[] = { - { - .key = "\x83\xac\x54\x66\xc2\xeb\xe5\x05" - "\x2e\x01\xd1\xfc\x5d\x82\x66\x2e" - "\x96\xac\x59", - .klen = 19, - .iv = "\x30\x07\xa1\xe2\xa2\xc7\x55\x24", - .alen = 0, - .input = "\x19\xc8\x81\xf6\xe9\x86\xff\x93" - "\x0b\x78\x67\xe5\xbb\xb7\xfc\x6e" - "\x83\x77\xb3\xa6\x0c\x8c\x9f\x9c" - "\x35\x2e\xad\xe0\x62\xf9\x91\xa1", - .ilen = 32, - .result = "\xab\x6f\xe1\x69\x1d\x19\x99\xa8" - "\x92\xa0\xc4\x6f\x7e\xe2\x8b\xb1" - "\x70\xbb\x8c\xa6\x4c\x6e\x97\x8a" - "\x57\x2b\xbe\x5d\x98\xa6\xb1\x32" - "\xda\x24\xea\xd9\xa1\x39\x98\xfd" - "\xa4\xbe\xd9\xf2\x1a\x6d\x22\xa8", - .rlen = 48, - }, { - .key = "\x1e\x2c\x7e\x01\x41\x9a\xef\xc0" - "\x0d\x58\x96\x6e\x5c\xa2\x4b\xd3" - "\x4f\xa3\x19", - .klen = 19, - .iv = "\xd3\x01\x5a\xd8\x30\x60\x15\x56", - .assoc = "\xda\xe6\x28\x9c\x45\x2d\xfd\x63" - "\x5e\xda\x4c\xb6\xe6\xfc\xf9\xb7" - "\x0c\x56\xcb\xe4\xe0\x05\x7a\xe1" - "\x0a\x63\x09\x78\xbc\x2c\x55\xde", - .alen = 32, - .input = "\x87\xa3\x36\xfd\x96\xb3\x93\x78" - "\xa9\x28\x63\xba\x12\xa3\x14\x85" - "\x57\x1e\x06\xc9\x7b\x21\xef\x76" - "\x7f\x38\x7e\x8e\x29\xa4\x3e\x7e", - .ilen = 32, - .result = "\x8a\x1e\x11\xf0\x02\x6b\xe2\x19" - "\xfc\x70\xc4\x6d\x8e\xb7\x99\xab" - "\xc5\x4b\xa2\xac\xd3\xf3\x48\xff" - "\x3b\xb5\xce\x53\xef\xde\xbb\x02" - "\xa9\x86\x15\x6c\x13\xfe\xda\x0a" - "\x22\xb8\x29\x3d\xd8\x39\x9a\x23", - .rlen = 48, - }, { - .key = "\xf4\x6b\xc2\x75\x62\xfe\xb4\xe1" - "\xa3\xf0\xff\xdd\x4e\x4b\x12\x75" - "\x53\x14\x73\x66\x8d\x88\xf6\x80" - "\xa0\x20\x35", - .klen = 27, - .iv = "\x26\xf2\x21\x8d\x50\x20\xda\xe2", - .assoc = "\x5b\x9e\x13\x67\x02\x5e\xef\xc1" - "\x6c\xf9\xd7\x1e\x52\x8f\x7a\x47" - "\xe9\xd4\xcf\x20\x14\x6e\xf0\x2d" - "\xd8\x9e\x2b\x56\x10\x23\x56\xe7", - .alen = 32, - .ilen = 0, - .result = "\x36\xea\x7a\x70\x08\xdc\x6a\xbc" - "\xad\x0c\x7a\x63\xf6\x61\xfd\x9b", - .rlen = 16, - }, { - .key = "\x56\xdf\x5c\x8f\x26\x3f\x0e\x42" - "\xef\x7a\xd3\xce\xfc\x84\x60\x62" - "\xca\xb4\x40\xaf\x5f\xc9\xc9\x01" - "\xd6\x3c\x8c", - .klen = 27, - .iv = "\x86\x84\xb6\xcd\xef\x09\x2e\x94", - .assoc = "\x02\x65\x78\x3c\xe9\x21\x30\x91" - "\xb1\xb9\xda\x76\x9a\x78\x6d\x95" - "\xf2\x88\x32\xa3\xf2\x50\xcb\x4c" - "\xe3\x00\x73\x69\x84\x69\x87\x79", - .alen = 32, - .input = "\x9f\xd2\x02\x4b\x52\x49\x31\x3c" - "\x43\x69\x3a\x2d\x8e\x70\xad\x7e" - "\xe0\xe5\x46\x09\x80\x89\x13\xb2" - "\x8c\x8b\xd9\x3f\x86\xfb\xb5\x6b", - .ilen = 32, - .result = "\x39\xdf\x7c\x3c\x5a\x29\xb9\x62" - "\x5d\x51\xc2\x16\xd8\xbd\x06\x9f" - "\x9b\x6a\x09\x70\xc1\x51\x83\xc2" - "\x66\x88\x1d\x4f\x9a\xda\xe0\x1e" - "\xc7\x79\x11\x58\xe5\x6b\x20\x40" - "\x7a\xea\x46\x42\x8b\xe4\x6f\xe1", - .rlen = 48, - }, { - .key = "\xe0\x8d\x99\x71\x60\xd7\x97\x1a" - "\xbd\x01\x99\xd5\x8a\xdf\x71\x3a" - "\xd3\xdf\x24\x4b\x5e\x3d\x4b\x4e" - "\x30\x7a\xb9\xd8\x53\x0a\x5e\x2b" - "\x1e\x29\x91", - .klen = 35, - .iv = "\xad\x8e\xc1\x53\x0a\xcf\x2d\xbe", - .assoc = "\x19\xb6\x1f\x57\xc4\xf3\xf0\x8b" - "\x78\x2b\x94\x02\x29\x0f\x42\x27" - "\x6b\x75\xcb\x98\x34\x08\x7e\x79" - "\xe4\x3e\x49\x0d\x84\x8b\x22\x87", - .alen = 32, - .input = "\xe1\xd9\xd8\x13\xeb\x3a\x75\x3f" - "\x9d\xbd\x5f\x66\xbe\xdc\xbb\x66" - "\xbf\x17\x99\x62\x4a\x39\x27\x1f" - "\x1d\xdc\x24\xae\x19\x2f\x98\x4c", - .ilen = 32, - .result = "\x19\xb8\x61\x33\x45\x2b\x43\x96" - "\x6f\x51\xd0\x20\x30\x7d\x9b\xc6" - "\x26\x3d\xf8\xc9\x65\x16\xa8\x9f" - "\xf0\x62\x17\x34\xf2\x1e\x8d\x75" - "\x4e\x13\xcc\xc0\xc3\x2a\x54\x2d", - .rlen = 40, - }, { - .key = "\x7c\xc8\x18\x3b\x8d\x99\xe0\x7c" - "\x45\x41\xb8\xbd\x5c\xa7\xc2\x32" - "\x8a\xb8\x02\x59\xa4\xfe\xa9\x2c" - "\x09\x75\x9a\x9b\x3c\x9b\x27\x39" - "\xf9\xd9\x4e", - .klen = 35, - .iv = "\x63\xb5\x3d\x9d\x43\xf6\x1e\x50", - .assoc = "\x57\xf5\x6b\x8b\x57\x5c\x3d\x3b" - "\x13\x02\x01\x0c\x83\x4c\x96\x35" - "\x8e\xd6\x39\xcf\x7d\x14\x9b\x94" - "\xb0\x39\x36\xe6\x8f\x57\xe0\x13", - .alen = 32, - .input = "\x3b\x6c\x29\x36\xb6\xef\x07\xa6" - "\x83\x72\x07\x4f\xcf\xfa\x66\x89" - "\x5f\xca\xb1\xba\xd5\x8f\x2c\x27" - "\x30\xdb\x75\x09\x93\xd4\x65\xe4", - .ilen = 32, - .result = "\xb0\x88\x5a\x33\xaa\xe5\xc7\x1d" - "\x85\x23\xc7\xc6\x2f\xf4\x1e\x3d" - "\xcc\x63\x44\x25\x07\x78\x4f\x9e" - "\x96\xb8\x88\xeb\xbc\x48\x1f\x06" - "\x39\xaf\x39\xac\xd8\x4a\x80\x39" - "\x7b\x72\x8a\xf7", - .rlen = 44, }, { - .key = "\xab\xd0\xe9\x33\x07\x26\xe5\x83" - "\x8c\x76\x95\xd4\xb6\xdc\xf3\x46" - "\xf9\x8f\xad\xe3\x02\x13\x83\x77" - "\x3f\xb0\xf1\xa1\xa1\x22\x0f\x2b" - "\x24\xa7\x8b", - .klen = 35, - .iv = "\x07\xcb\xcc\x0e\xe6\x33\xbf\xf5", - .assoc = "\xd4\xdb\x30\x1d\x03\xfe\xfd\x5f" - "\x87\xd4\x8c\xb6\xb6\xf1\x7a\x5d" - "\xab\x90\x65\x8d\x8e\xca\x4d\x4f" - "\x16\x0c\x40\x90\x4b\xc7\x36\x73", - .alen = 32, - .input = "\xf5\xc6\x7d\x48\xc1\xb7\xe6\x92" - "\x97\x5a\xca\xc4\xa9\x6d\xf9\x3d" - "\x6c\xde\xbc\xf1\x90\xea\x6a\xb2" - "\x35\x86\x36\xaf\x5c\xfe\x4b\x3a", - .ilen = 32, - .result = "\x83\x6f\x40\x87\x72\xcf\xc1\x13" - "\xef\xbb\x80\x21\x04\x6c\x58\x09" - "\x07\x1b\xfc\xdf\xc0\x3f\x5b\xc7" - "\xe0\x79\xa8\x6e\x71\x7c\x3f\xcf" - "\x5c\xda\xb2\x33\xe5\x13\xe2\x0d" - "\x74\xd1\xef\xb5\x0f\x3a\xb5\xf8", - .rlen = 48, - }, -}; - -static struct aead_testvec aes_ccm_rfc4309_dec_tv_template[] = { - { + /* This is taken from FIPS CAVS. */ .key = "\xab\x2f\x8a\x74\xb7\x1c\xd2\xb1" - "\xff\x80\x2e\x48\x7d\x82\xf8\xb9" - "\xc6\xfb\x7d", - .klen = 19, - .iv = "\x80\x0d\x13\xab\xd8\xa6\xb2\xd8", + "\xff\x80\x2e\x48\x7d\x82\xf8\xb9", + .klen = 16, + .iv = "\x03\xc6\xfb\x7d\x80\x0d\x13\xab" + "\xd8\xa6\xb2\xd8\x00\x00\x00\x00", .alen = 0, .input = "\xd5\xe8\x93\x9f\xc7\x89\x2e\x2b", .ilen = 8, @@ -21876,10 +21946,10 @@ static struct aead_testvec aes_ccm_rfc4309_dec_tv_template[] = { .novrfy = 1, }, { .key = "\xab\x2f\x8a\x74\xb7\x1c\xd2\xb1" - "\xff\x80\x2e\x48\x7d\x82\xf8\xb9" - "\xaf\x94\x87", - .klen = 19, - .iv = "\x78\x35\x82\x81\x7f\x88\x94\x68", + "\xff\x80\x2e\x48\x7d\x82\xf8\xb9", + .klen = 16, + .iv = "\x03\xaf\x94\x87\x78\x35\x82\x81" + "\x7f\x88\x94\x68\x00\x00\x00\x00", .alen = 0, .input = "\x41\x3c\xb8\x87\x73\xcb\xf3\xf3", .ilen = 8, @@ -21887,10 +21957,10 @@ static struct aead_testvec aes_ccm_rfc4309_dec_tv_template[] = { .rlen = 0, }, { .key = "\x61\x0e\x8c\xae\xe3\x23\xb6\x38" - "\x76\x1c\xf6\x3a\x67\xa3\x9c\xd8" - "\xc6\xfb\x7d", - .klen = 19, - .iv = "\x80\x0d\x13\xab\xd8\xa6\xb2\xd8", + "\x76\x1c\xf6\x3a\x67\xa3\x9c\xd8", + .klen = 16, + .iv = "\x03\xc6\xfb\x7d\x80\x0d\x13\xab" + "\xd8\xa6\xb2\xd8\x00\x00\x00\x00", .assoc = "\xf3\x94\x87\x78\x35\x82\x81\x7f" "\x88\x94\x68\xb1\x78\x6b\x2b\xd6" "\x04\x1f\x4e\xed\x78\xd5\x33\x66" @@ -21911,10 +21981,10 @@ static struct aead_testvec aes_ccm_rfc4309_dec_tv_template[] = { .novrfy = 1, }, { .key = "\x61\x0e\x8c\xae\xe3\x23\xb6\x38" - "\x76\x1c\xf6\x3a\x67\xa3\x9c\xd8" - "\x05\xe0\xc9", - .klen = 19, - .iv = "\x0f\xed\x34\xea\x97\xd4\x3b\xdf", + "\x76\x1c\xf6\x3a\x67\xa3\x9c\xd8", + .klen = 16, + .iv = "\x03\x05\xe0\xc9\x0f\xed\x34\xea" + "\x97\xd4\x3b\xdf\x00\x00\x00\x00", .assoc = "\x49\x5c\x50\x1f\x1d\x94\xcc\x81" "\xba\xb7\xb6\x03\xaf\xa5\xc1\xa1" "\xd8\x5c\x42\x68\xe0\x6c\xda\x89" @@ -21935,10 +22005,10 @@ static struct aead_testvec aes_ccm_rfc4309_dec_tv_template[] = { }, { .key = "\x39\xbb\xa7\xbe\x59\x97\x9e\x73" "\xa2\xbc\x6b\x98\xd7\x75\x7f\xe3" - "\xa4\x48\x93\x39\x26\x71\x4a\xc6" - "\xee\x49\x83", - .klen = 27, - .iv = "\xe9\xa9\xff\xe9\x57\xba\xfd\x9e", + "\xa4\x48\x93\x39\x26\x71\x4a\xc6", + .klen = 24, + .iv = "\x03\xee\x49\x83\xe9\xa9\xff\xe9" + "\x57\xba\xfd\x9e\x00\x00\x00\x00", .assoc = "\x44\xa6\x2c\x05\xe9\xe1\x43\xb1" "\x58\x7c\xf2\x5c\x6d\x39\x0a\x64" "\xa4\xf0\x13\x05\xd1\x77\x99\x67" @@ -21951,10 +22021,10 @@ static struct aead_testvec aes_ccm_rfc4309_dec_tv_template[] = { }, { .key = "\x58\x5d\xa0\x96\x65\x1a\x04\xd7" "\x96\xe5\xc5\x68\xaa\x95\x35\xe0" - "\x29\xa0\xba\x9e\x48\x78\xd1\xba" - "\xee\x49\x83", - .klen = 27, - .iv = "\xe9\xa9\xff\xe9\x57\xba\xfd\x9e", + "\x29\xa0\xba\x9e\x48\x78\xd1\xba", + .klen = 24, + .iv = "\x03\xee\x49\x83\xe9\xa9\xff\xe9" + "\x57\xba\xfd\x9e\x00\x00\x00\x00", .assoc = "\x44\xa6\x2c\x05\xe9\xe1\x43\xb1" "\x58\x7c\xf2\x5c\x6d\x39\x0a\x64" "\xa4\xf0\x13\x05\xd1\x77\x99\x67" @@ -21974,10 +22044,10 @@ static struct aead_testvec aes_ccm_rfc4309_dec_tv_template[] = { }, { .key = "\x58\x5d\xa0\x96\x65\x1a\x04\xd7" "\x96\xe5\xc5\x68\xaa\x95\x35\xe0" - "\x29\xa0\xba\x9e\x48\x78\xd1\xba" - "\xd1\xfc\x57", - .klen = 27, - .iv = "\x9c\xfe\xb8\x9c\xad\x71\xaa\x1f", + "\x29\xa0\xba\x9e\x48\x78\xd1\xba", + .klen = 24, + .iv = "\x03\xd1\xfc\x57\x9c\xfe\xb8\x9c" + "\xad\x71\xaa\x1f\x00\x00\x00\x00", .assoc = "\x86\x67\xa5\xa9\x14\x5f\x0d\xc6" "\xff\x14\xc7\x44\xbf\x6c\x3a\xc3" "\xff\xb6\x81\xbd\xe2\xd5\x06\xc7" @@ -22000,10 +22070,10 @@ static struct aead_testvec aes_ccm_rfc4309_dec_tv_template[] = { .key = "\xa4\x4b\x54\x29\x0a\xb8\x6d\x01" "\x5b\x80\x2a\xcf\x25\xc4\xb7\x5c" "\x20\x2c\xad\x30\xc2\x2b\x41\xfb" - "\x0e\x85\xbc\x33\xad\x0f\x2b\xff" - "\xee\x49\x83", - .klen = 35, - .iv = "\xe9\xa9\xff\xe9\x57\xba\xfd\x9e", + "\x0e\x85\xbc\x33\xad\x0f\x2b\xff", + .klen = 32, + .iv = "\x03\xee\x49\x83\xe9\xa9\xff\xe9" + "\x57\xba\xfd\x9e\x00\x00\x00\x00", .alen = 0, .input = "\x1f\xb8\x8f\xa3\xdd\x54\x00\xf2", .ilen = 8, @@ -22013,10 +22083,10 @@ static struct aead_testvec aes_ccm_rfc4309_dec_tv_template[] = { .key = "\x39\xbb\xa7\xbe\x59\x97\x9e\x73" "\xa2\xbc\x6b\x98\xd7\x75\x7f\xe3" "\xa4\x48\x93\x39\x26\x71\x4a\xc6" - "\xae\x8f\x11\x4c\xc2\x9c\x4a\xbb" - "\x85\x34\x66", - .klen = 35, - .iv = "\x42\xc8\x92\x0f\x36\x58\xe0\x6b", + "\xae\x8f\x11\x4c\xc2\x9c\x4a\xbb", + .klen = 32, + .iv = "\x03\x85\x34\x66\x42\xc8\x92\x0f" + "\x36\x58\xe0\x6b\x00\x00\x00\x00", .alen = 0, .input = "\x48\x01\x5e\x02\x24\x04\x66\x47" "\xa1\xea\x6f\xaf\xe8\xfc\xfb\xdd" @@ -22035,10 +22105,10 @@ static struct aead_testvec aes_ccm_rfc4309_dec_tv_template[] = { .key = "\x58\x5d\xa0\x96\x65\x1a\x04\xd7" "\x96\xe5\xc5\x68\xaa\x95\x35\xe0" "\x29\xa0\xba\x9e\x48\x78\xd1\xba" - "\x0d\x1a\x53\x3b\xb5\xe3\xf8\x8b" - "\xcf\x76\x3f", - .klen = 35, - .iv = "\xd9\x95\x75\x8f\x44\x89\x40\x7b", + "\x0d\x1a\x53\x3b\xb5\xe3\xf8\x8b", + .klen = 32, + .iv = "\x03\xcf\x76\x3f\xd9\x95\x75\x8f" + "\x44\x89\x40\x7b\x00\x00\x00\x00", .assoc = "\x8f\x86\x6c\x4d\x1d\xc5\x39\x88" "\xc8\xf3\x5c\x52\x10\x63\x6f\x2b" "\x8a\x2a\xc5\x6f\x30\x23\x58\x7b" @@ -22060,6 +22130,1240 @@ static struct aead_testvec aes_ccm_rfc4309_dec_tv_template[] = { }; /* + * rfc4309 refers to section 8 of rfc3610 for test vectors, but they all + * use a 13-byte nonce, we only support an 11-byte nonce. Worse, + * they use AD lengths which are not valid ESP header lengths. + * + * These vectors are copied/generated from the ones for rfc4106 with + * the key truncated by one byte.. + */ +static struct aead_testvec aes_ccm_rfc4309_enc_tv_template[] = { + { /* Generated using Crypto++ */ + .key = zeroed_string, + .klen = 19, + .iv = zeroed_string, + .input = zeroed_string, + .ilen = 16, + .assoc = zeroed_string, + .alen = 16, + .result = "\x2E\x9A\xCA\x6B\xDA\x54\xFC\x6F" + "\x12\x50\xE8\xDE\x81\x3C\x63\x08" + "\x1A\x22\xBA\x75\xEE\xD4\xD5\xB5" + "\x27\x50\x01\xAC\x03\x33\x39\xFB", + .rlen = 32, + },{ + .key = "\xfe\xff\xe9\x92\x86\x65\x73\x1c" + "\x6d\x6a\x8f\x94\x67\x30\x83\x08" + "\x00\x00\x00", + .klen = 19, + .iv = "\x00\x00\x00\x00\x00\x00\x00\x01", + .input = zeroed_string, + .ilen = 16, + .assoc = "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x01", + .alen = 16, + .result = "\xCF\xB9\x99\x17\xC8\x86\x0E\x7F" + "\x7E\x76\xF8\xE6\xF8\xCC\x1F\x17" + "\x6A\xE0\x53\x9F\x4B\x73\x7E\xDA" + "\x08\x09\x4E\xC4\x1E\xAD\xC6\xB0", + .rlen = 32, + + }, { + .key = "\xfe\xff\xe9\x92\x86\x65\x73\x1c" + "\x6d\x6a\x8f\x94\x67\x30\x83\x08" + "\x00\x00\x00", + .klen = 19, + .iv = zeroed_string, + .input = "\x01\x01\x01\x01\x01\x01\x01\x01" + "\x01\x01\x01\x01\x01\x01\x01\x01", + .ilen = 16, + .assoc = zeroed_string, + .alen = 16, + .result = "\x33\xDE\x73\xBC\xA6\xCE\x4E\xA6" + "\x61\xF4\xF5\x41\x03\x4A\xE3\x86" + "\xA1\xE2\xC2\x42\x2B\x81\x70\x40" + "\xFD\x7F\x76\xD1\x03\x07\xBB\x0C", + .rlen = 32, + }, { + .key = "\xfe\xff\xe9\x92\x86\x65\x73\x1c" + "\x6d\x6a\x8f\x94\x67\x30\x83\x08" + "\x00\x00\x00", + .klen = 19, + .iv = zeroed_string, + .input = "\x01\x01\x01\x01\x01\x01\x01\x01" + "\x01\x01\x01\x01\x01\x01\x01\x01", + .ilen = 16, + .assoc = "\x01\x01\x01\x01\x01\x01\x01\x01" + "\x00\x00\x00\x00\x00\x00\x00\x00", + .alen = 16, + .result = "\x33\xDE\x73\xBC\xA6\xCE\x4E\xA6" + "\x61\xF4\xF5\x41\x03\x4A\xE3\x86" + "\x5B\xC0\x73\xE0\x2B\x73\x68\xC9" + "\x2D\x8C\x58\xC2\x90\x3D\xB0\x3E", + .rlen = 32, + }, { + .key = "\xfe\xff\xe9\x92\x86\x65\x73\x1c" + "\x6d\x6a\x8f\x94\x67\x30\x83\x08" + "\x00\x00\x00", + .klen = 19, + .iv = "\x00\x00\x00\x00\x00\x00\x00\x01", + .input = "\x01\x01\x01\x01\x01\x01\x01\x01" + "\x01\x01\x01\x01\x01\x01\x01\x01", + .ilen = 16, + .assoc = "\x01\x01\x01\x01\x01\x01\x01\x01" + "\x00\x00\x00\x00\x00\x00\x00\x01", + .alen = 16, + .result = "\xCE\xB8\x98\x16\xC9\x87\x0F\x7E" + "\x7F\x77\xF9\xE7\xF9\xCD\x1E\x16" + "\x43\x8E\x76\x57\x3B\xB4\x05\xE8" + "\xA9\x9B\xBF\x25\xE0\x4F\xC0\xED", + .rlen = 32, + }, { + .key = "\xfe\xff\xe9\x92\x86\x65\x73\x1c" + "\x6d\x6a\x8f\x94\x67\x30\x83\x08" + "\x00\x00\x00", + .klen = 19, + .iv = "\x00\x00\x00\x00\x00\x00\x00\x01", + .input = "\x01\x01\x01\x01\x01\x01\x01\x01" + "\x01\x01\x01\x01\x01\x01\x01\x01" + "\x01\x01\x01\x01\x01\x01\x01\x01" + "\x01\x01\x01\x01\x01\x01\x01\x01" + "\x01\x01\x01\x01\x01\x01\x01\x01" + "\x01\x01\x01\x01\x01\x01\x01\x01" + "\x01\x01\x01\x01\x01\x01\x01\x01" + "\x01\x01\x01\x01\x01\x01\x01\x01", + .ilen = 64, + .assoc = "\x01\x01\x01\x01\x01\x01\x01\x01" + "\x00\x00\x00\x00\x00\x00\x00\x01", + .alen = 16, + .result = "\xCE\xB8\x98\x16\xC9\x87\x0F\x7E" + "\x7F\x77\xF9\xE7\xF9\xCD\x1E\x16" + "\x9C\xA4\x97\x83\x3F\x01\xA5\xF4" + "\x43\x09\xE7\xB8\xE9\xD1\xD7\x02" + "\x9B\xAB\x39\x18\xEB\x94\x34\x36" + "\xE6\xC5\xC8\x9B\x00\x81\x9E\x49" + "\x1D\x78\xE1\x48\xE3\xE9\xEA\x8E" + "\x3A\x2B\x67\x5D\x35\x6A\x0F\xDB" + "\x02\x73\xDD\xE7\x30\x4A\x30\x54" + "\x1A\x9D\x09\xCA\xC8\x1C\x32\x5F", + .rlen = 80, + }, { + .key = "\x00\x01\x02\x03\x04\x05\x06\x07" + "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" + "\x00\x00\x00", + .klen = 19, + .iv = "\x00\x00\x45\x67\x89\xab\xcd\xef", + .input = "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff", + .ilen = 192, + .assoc = "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\x00\x00\x45\x67" + "\x89\xab\xcd\xef", + .alen = 20, + .result = "\x64\x17\xDC\x24\x9D\x92\xBA\x5E" + "\x7C\x64\x6D\x33\x46\x77\xAC\xB1" + "\x5C\x9E\xE2\xC7\x27\x11\x3E\x95" + "\x7D\xBE\x28\xC8\xC1\xCA\x5E\x8C" + "\xB4\xE2\xDE\x9F\x53\x59\x26\xDB" + "\x0C\xD4\xE4\x07\x9A\xE6\x3E\x01" + "\x58\x0D\x3E\x3D\xD5\x21\xEB\x04" + "\x06\x9D\x5F\xB9\x02\x49\x1A\x2B" + "\xBA\xF0\x4E\x3B\x85\x50\x5B\x09" + "\xFE\xEC\xFC\x54\xEC\x0C\xE2\x79" + "\x8A\x2F\x5F\xD7\x05\x5D\xF1\x6D" + "\x22\xEB\xD1\x09\x80\x3F\x5A\x70" + "\xB2\xB9\xD3\x63\x99\xC2\x4D\x1B" + "\x36\x12\x00\x89\xAA\x5D\x55\xDA" + "\x1D\x5B\xD8\x3C\x5F\x09\xD2\xE6" + "\x39\x41\x5C\xF0\xBE\x26\x4E\x5F" + "\x2B\x50\x44\x52\xC2\x10\x7D\x38" + "\x82\x64\x83\x0C\xAE\x49\xD0\xE5" + "\x4F\xE5\x66\x4C\x58\x7A\xEE\x43" + "\x3B\x51\xFE\xBA\x24\x8A\xFE\xDC" + "\x19\x6D\x60\x66\x61\xF9\x9A\x3F" + "\x75\xFC\x38\x53\x5B\xB5\xCD\x52" + "\x4F\xE5\xE4\xC9\xFE\x10\xCB\x98" + "\xF0\x06\x5B\x07\xAB\xBB\xF4\x0E" + "\x2D\xC2\xDD\x5D\xDD\x22\x9A\xCC" + "\x39\xAB\x63\xA5\x3D\x9C\x51\x8A", + .rlen = 208, + }, { /* From draft-mcgrew-gcm-test-01 */ + .key = "\x4C\x80\xCD\xEF\xBB\x5D\x10\xDA" + "\x90\x6A\xC7\x3C\x36\x13\xA6\x34" + "\x2E\x44\x3B", + .klen = 19, + .iv = "\x49\x56\xED\x7E\x3B\x24\x4C\xFE", + .input = "\x45\x00\x00\x48\x69\x9A\x00\x00" + "\x80\x11\x4D\xB7\xC0\xA8\x01\x02" + "\xC0\xA8\x01\x01\x0A\x9B\xF1\x56" + "\x38\xD3\x01\x00\x00\x01\x00\x00" + "\x00\x00\x00\x00\x04\x5F\x73\x69" + "\x70\x04\x5F\x75\x64\x70\x03\x73" + "\x69\x70\x09\x63\x79\x62\x65\x72" + "\x63\x69\x74\x79\x02\x64\x6B\x00" + "\x00\x21\x00\x01\x01\x02\x02\x01", + .ilen = 72, + .assoc = "\x00\x00\x43\x21\x87\x65\x43\x21" + "\x00\x00\x00\x00\x49\x56\xED\x7E" + "\x3B\x24\x4C\xFE", + .alen = 20, + .result = "\x89\xBA\x3E\xEF\xE6\xD6\xCF\xDB" + "\x83\x60\xF5\xBA\x3A\x56\x79\xE6" + "\x7E\x0C\x53\xCF\x9E\x87\xE0\x4E" + "\x1A\x26\x01\x24\xC7\x2E\x3D\xBF" + "\x29\x2C\x91\xC1\xB8\xA8\xCF\xE0" + "\x39\xF8\x53\x6D\x31\x22\x2B\xBF" + "\x98\x81\xFC\x34\xEE\x85\x36\xCD" + "\x26\xDB\x6C\x7A\x0C\x77\x8A\x35" + "\x18\x85\x54\xB2\xBC\xDD\x3F\x43" + "\x61\x06\x8A\xDF\x86\x3F\xB4\xAC" + "\x97\xDC\xBD\xFD\x92\x10\xC5\xFF", + .rlen = 88, + }, { + .key = "\xFE\xFF\xE9\x92\x86\x65\x73\x1C" + "\x6D\x6A\x8F\x94\x67\x30\x83\x08" + "\xCA\xFE\xBA", + .klen = 19, + .iv = "\xFA\xCE\xDB\xAD\xDE\xCA\xF8\x88", + .input = "\x45\x00\x00\x3E\x69\x8F\x00\x00" + "\x80\x11\x4D\xCC\xC0\xA8\x01\x02" + "\xC0\xA8\x01\x01\x0A\x98\x00\x35" + "\x00\x2A\x23\x43\xB2\xD0\x01\x00" + "\x00\x01\x00\x00\x00\x00\x00\x00" + "\x03\x73\x69\x70\x09\x63\x79\x62" + "\x65\x72\x63\x69\x74\x79\x02\x64" + "\x6B\x00\x00\x01\x00\x01\x00\x01", + .ilen = 64, + .assoc = "\x00\x00\xA5\xF8\x00\x00\x00\x0A" + "\xFA\xCE\xDB\xAD\xDE\xCA\xF8\x88", + .alen = 16, + .result = "\x4B\xC2\x70\x60\x64\xD2\xF3\xC8" + "\xE5\x26\x8A\xDE\xB8\x7E\x7D\x16" + "\x56\xC7\xD2\x88\xBA\x8D\x58\xAF" + "\xF5\x71\xB6\x37\x84\xA7\xB1\x99" + "\x51\x5C\x0D\xA0\x27\xDE\xE7\x2D" + "\xEF\x25\x88\x1F\x1D\x77\x11\xFF" + "\xDB\xED\xEE\x56\x16\xC5\x5C\x9B" + "\x00\x62\x1F\x68\x4E\x7C\xA0\x97" + "\x10\x72\x7E\x53\x13\x3B\x68\xE4" + "\x30\x99\x91\x79\x09\xEA\xFF\x6A", + .rlen = 80, + }, { + .key = "\xAB\xBC\xCD\xDE\xF0\x01\x12\x23" + "\x34\x45\x56\x67\x78\x89\x9A\xAB" + "\xAB\xBC\xCD\xDE\xF0\x01\x12\x23" + "\x34\x45\x56\x67\x78\x89\x9A\xAB" + "\x11\x22\x33", + .klen = 35, + .iv = "\x01\x02\x03\x04\x05\x06\x07\x08", + .input = "\x45\x00\x00\x30\x69\xA6\x40\x00" + "\x80\x06\x26\x90\xC0\xA8\x01\x02" + "\x93\x89\x15\x5E\x0A\x9E\x00\x8B" + "\x2D\xC5\x7E\xE0\x00\x00\x00\x00" + "\x70\x02\x40\x00\x20\xBF\x00\x00" + "\x02\x04\x05\xB4\x01\x01\x04\x02" + "\x01\x02\x02\x01", + .ilen = 52, + .assoc = "\x4A\x2C\xBF\xE3\x00\x00\x00\x02" + "\x01\x02\x03\x04\x05\x06\x07\x08", + .alen = 16, + .result = "\xD6\x31\x0D\x2B\x3D\x6F\xBD\x2F" + "\x58\x41\x7E\xFF\x9A\x9E\x09\xB4" + "\x1A\xF7\xF6\x42\x31\xCD\xBF\xAD" + "\x27\x0E\x2C\xF2\xDB\x10\xDF\x55" + "\x8F\x0D\xD7\xAC\x23\xBD\x42\x10" + "\xD0\xB2\xAF\xD8\x37\xAC\x6B\x0B" + "\x11\xD4\x0B\x12\xEC\xB4\xB1\x92" + "\x23\xA6\x10\xB0\x26\xD6\xD9\x26" + "\x5A\x48\x6A\x3E", + .rlen = 68, + }, { + .key = "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00", + .klen = 19, + .iv = "\x00\x00\x00\x00\x00\x00\x00\x00", + .input = "\x45\x00\x00\x3C\x99\xC5\x00\x00" + "\x80\x01\xCB\x7A\x40\x67\x93\x18" + "\x01\x01\x01\x01\x08\x00\x07\x5C" + "\x02\x00\x44\x00\x61\x62\x63\x64" + "\x65\x66\x67\x68\x69\x6A\x6B\x6C" + "\x6D\x6E\x6F\x70\x71\x72\x73\x74" + "\x75\x76\x77\x61\x62\x63\x64\x65" + "\x66\x67\x68\x69\x01\x02\x02\x01", + .ilen = 64, + .assoc = "\x00\x00\x00\x00\x00\x00\x00\x01" + "\x00\x00\x00\x00\x00\x00\x00\x00", + .alen = 16, + .result = "\x6B\x9A\xCA\x57\x43\x91\xFC\x6F" + "\x92\x51\x23\xA4\xC1\x5B\xF0\x10" + "\xF3\x13\xF4\xF8\xA1\x9A\xB4\xDC" + "\x89\xC8\xF8\x42\x62\x95\xB7\xCB" + "\xB8\xF5\x0F\x1B\x2E\x94\xA2\xA7" + "\xBF\xFB\x8A\x92\x13\x63\xD1\x3C" + "\x08\xF5\xE8\xA6\xAA\xF6\x34\xF9" + "\x42\x05\xAF\xB3\xE7\x9A\xFC\xEE" + "\x36\x25\xC1\x10\x12\x1C\xCA\x82" + "\xEA\xE6\x63\x5A\x57\x28\xA9\x9A", + .rlen = 80, + }, { + .key = "\x3D\xE0\x98\x74\xB3\x88\xE6\x49" + "\x19\x88\xD0\xC3\x60\x7E\xAE\x1F" + "\x57\x69\x0E", + .klen = 19, + .iv = "\x4E\x28\x00\x00\xA2\xFC\xA1\xA3", + .input = "\x45\x00\x00\x3C\x99\xC3\x00\x00" + "\x80\x01\xCB\x7C\x40\x67\x93\x18" + "\x01\x01\x01\x01\x08\x00\x08\x5C" + "\x02\x00\x43\x00\x61\x62\x63\x64" + "\x65\x66\x67\x68\x69\x6A\x6B\x6C" + "\x6D\x6E\x6F\x70\x71\x72\x73\x74" + "\x75\x76\x77\x61\x62\x63\x64\x65" + "\x66\x67\x68\x69\x01\x02\x02\x01", + .ilen = 64, + .assoc = "\x42\xF6\x7E\x3F\x10\x10\x10\x10" + "\x10\x10\x10\x10\x4E\x28\x00\x00" + "\xA2\xFC\xA1\xA3", + .alen = 20, + .result = "\x6A\x6B\x45\x2B\x7C\x67\x52\xF6" + "\x10\x60\x40\x62\x6B\x4F\x97\x8E" + "\x0B\xB2\x22\x97\xCB\x21\xE0\x90" + "\xA2\xE7\xD1\x41\x30\xE4\x4B\x1B" + "\x79\x01\x58\x50\x01\x06\xE1\xE0" + "\x2C\x83\x79\xD3\xDE\x46\x97\x1A" + "\x30\xB8\xE5\xDF\xD7\x12\x56\x75" + "\xD0\x95\xB7\xB8\x91\x42\xF7\xFD" + "\x97\x57\xCA\xC1\x20\xD0\x86\xB9" + "\x66\x9D\xB4\x2B\x96\x22\xAC\x67", + .rlen = 80, + }, { + .key = "\x3D\xE0\x98\x74\xB3\x88\xE6\x49" + "\x19\x88\xD0\xC3\x60\x7E\xAE\x1F" + "\x57\x69\x0E", + .klen = 19, + .iv = "\x4E\x28\x00\x00\xA2\xFC\xA1\xA3", + .input = "\x45\x00\x00\x1C\x42\xA2\x00\x00" + "\x80\x01\x44\x1F\x40\x67\x93\xB6" + "\xE0\x00\x00\x02\x0A\x00\xF5\xFF" + "\x01\x02\x02\x01", + .ilen = 28, + .assoc = "\x42\xF6\x7E\x3F\x10\x10\x10\x10" + "\x10\x10\x10\x10\x4E\x28\x00\x00" + "\xA2\xFC\xA1\xA3", + .alen = 20, + .result = "\x6A\x6B\x45\x0B\xA7\x06\x52\xF6" + "\x10\x60\xCF\x01\x6B\x4F\x97\x20" + "\xEA\xB3\x23\x94\xC9\x21\x1D\x33" + "\xA1\xE5\x90\x40\x05\x37\x45\x70" + "\xB5\xD6\x09\x0A\x23\x73\x33\xF9" + "\x08\xB4\x22\xE4", + .rlen = 44, + }, { + .key = "\xFE\xFF\xE9\x92\x86\x65\x73\x1C" + "\x6D\x6A\x8F\x94\x67\x30\x83\x08" + "\xFE\xFF\xE9\x92\x86\x65\x73\x1C" + "\xCA\xFE\xBA", + .klen = 27, + .iv = "\xFA\xCE\xDB\xAD\xDE\xCA\xF8\x88", + .input = "\x45\x00\x00\x28\xA4\xAD\x40\x00" + "\x40\x06\x78\x80\x0A\x01\x03\x8F" + "\x0A\x01\x06\x12\x80\x23\x06\xB8" + "\xCB\x71\x26\x02\xDD\x6B\xB0\x3E" + "\x50\x10\x16\xD0\x75\x68\x00\x01", + .ilen = 40, + .assoc = "\x00\x00\xA5\xF8\x00\x00\x00\x0A" + "\xFA\xCE\xDB\xAD\xDE\xCA\xF8\x88", + .alen = 16, + .result = "\x05\x22\x15\xD1\x52\x56\x85\x04" + "\xA8\x5C\x5D\x6D\x7E\x6E\xF5\xFA" + "\xEA\x16\x37\x50\xF3\xDF\x84\x3B" + "\x2F\x32\x18\x57\x34\x2A\x8C\x23" + "\x67\xDF\x6D\x35\x7B\x54\x0D\xFB" + "\x34\xA5\x9F\x6C\x48\x30\x1E\x22" + "\xFE\xB1\x22\x17\x17\x8A\xB9\x5B", + .rlen = 56, + }, { + .key = "\xAB\xBC\xCD\xDE\xF0\x01\x12\x23" + "\x34\x45\x56\x67\x78\x89\x9A\xAB" + "\xDE\xCA\xF8", + .klen = 19, + .iv = "\xCA\xFE\xDE\xBA\xCE\xFA\xCE\x74", + .input = "\x45\x00\x00\x49\x33\xBA\x00\x00" + "\x7F\x11\x91\x06\xC3\xFB\x1D\x10" + "\xC2\xB1\xD3\x26\xC0\x28\x31\xCE" + "\x00\x35\xDD\x7B\x80\x03\x02\xD5" + "\x00\x00\x4E\x20\x00\x1E\x8C\x18" + "\xD7\x5B\x81\xDC\x91\xBA\xA0\x47" + "\x6B\x91\xB9\x24\xB2\x80\x38\x9D" + "\x92\xC9\x63\xBA\xC0\x46\xEC\x95" + "\x9B\x62\x66\xC0\x47\x22\xB1\x49" + "\x23\x01\x01\x01", + .ilen = 76, + .assoc = "\x00\x00\x01\x00\x00\x00\x00\x00" + "\x00\x00\x00\x01\xCA\xFE\xDE\xBA" + "\xCE\xFA\xCE\x74", + .alen = 20, + .result = "\x92\xD0\x53\x79\x33\x38\xD5\xF3" + "\x7D\xE4\x7A\x8E\x86\x03\xC9\x90" + "\x96\x35\xAB\x9C\xFB\xE8\xA3\x76" + "\xE9\xE9\xE2\xD1\x2E\x11\x0E\x00" + "\xFA\xCE\xB5\x9E\x02\xA7\x7B\xEA" + "\x71\x9A\x58\xFB\xA5\x8A\xE1\xB7" + "\x9C\x39\x9D\xE3\xB5\x6E\x69\xE6" + "\x63\xC9\xDB\x05\x69\x51\x12\xAD" + "\x3E\x00\x32\x73\x86\xF2\xEE\xF5" + "\x0F\xE8\x81\x7E\x84\xD3\xC0\x0D" + "\x76\xD6\x55\xC6\xB4\xC2\x34\xC7" + "\x12\x25\x0B\xF9", + .rlen = 92, + }, { + .key = "\xAB\xBC\xCD\xDE\xF0\x01\x12\x23" + "\x34\x45\x56\x67\x78\x89\x9A\xAB" + "\xAB\xBC\xCD\xDE\xF0\x01\x12\x23" + "\x34\x45\x56\x67\x78\x89\x9A\xAB" + "\x73\x61\x6C", + .klen = 35, + .iv = "\x61\x6E\x64\x01\x69\x76\x65\x63", + .input = "\x45\x08\x00\x28\x73\x2C\x00\x00" + "\x40\x06\xE9\xF9\x0A\x01\x06\x12" + "\x0A\x01\x03\x8F\x06\xB8\x80\x23" + "\xDD\x6B\xAF\xBE\xCB\x71\x26\x02" + "\x50\x10\x1F\x64\x6D\x54\x00\x01", + .ilen = 40, + .assoc = "\x17\x40\x5E\x67\x15\x6F\x31\x26" + "\xDD\x0D\xB9\x9B\x61\x6E\x64\x01" + "\x69\x76\x65\x63", + .alen = 20, + .result = "\xCC\x74\xB7\xD3\xB0\x38\x50\x42" + "\x2C\x64\x87\x46\x1E\x34\x10\x05" + "\x29\x6B\xBB\x36\xE9\x69\xAD\x92" + "\x82\xA1\x10\x6A\xEB\x0F\xDC\x7D" + "\x08\xBA\xF3\x91\xCA\xAA\x61\xDA" + "\x62\xF4\x14\x61\x5C\x9D\xB5\xA7" + "\xEE\xD7\xB9\x7E\x87\x99\x9B\x7D", + .rlen = 56, + }, { + .key = "\x3D\xE0\x98\x74\xB3\x88\xE6\x49" + "\x19\x88\xD0\xC3\x60\x7E\xAE\x1F" + "\x57\x69\x0E", + .klen = 19, + .iv = "\x4E\x28\x00\x00\xA2\xFC\xA1\xA3", + .input = "\x45\x00\x00\x49\x33\x3E\x00\x00" + "\x7F\x11\x91\x82\xC3\xFB\x1D\x10" + "\xC2\xB1\xD3\x26\xC0\x28\x31\xCE" + "\x00\x35\xCB\x45\x80\x03\x02\x5B" + "\x00\x00\x01\xE0\x00\x1E\x8C\x18" + "\xD6\x57\x59\xD5\x22\x84\xA0\x35" + "\x2C\x71\x47\x5C\x88\x80\x39\x1C" + "\x76\x4D\x6E\x5E\xE0\x49\x6B\x32" + "\x5A\xE2\x70\xC0\x38\x99\x49\x39" + "\x15\x01\x01\x01", + .ilen = 76, + .assoc = "\x42\xF6\x7E\x3F\x10\x10\x10\x10" + "\x10\x10\x10\x10\x4E\x28\x00\x00" + "\xA2\xFC\xA1\xA3", + .alen = 20, + .result = "\x6A\x6B\x45\x5E\xD6\x9A\x52\xF6" + "\xEF\x70\x1A\x9C\xE8\xD3\x19\x86" + "\xC8\x02\xF0\xB0\x03\x09\xD9\x02" + "\xA0\xD2\x59\x04\xD1\x85\x2A\x24" + "\x1C\x67\x3E\xD8\x68\x72\x06\x94" + "\x97\xBA\x4F\x76\x8D\xB0\x44\x5B" + "\x69\xBF\xD5\xE2\x3D\xF1\x0B\x0C" + "\xC0\xBF\xB1\x8F\x70\x09\x9E\xCE" + "\xA5\xF2\x55\x58\x84\xFA\xF9\xB5" + "\x23\xF4\x84\x40\x74\x14\x8A\x6B" + "\xDB\xD7\x67\xED\xA4\x93\xF3\x47" + "\xCC\xF7\x46\x6F", + .rlen = 92, + }, { + .key = "\xAB\xBC\xCD\xDE\xF0\x01\x12\x23" + "\x34\x45\x56\x67\x78\x89\x9A\xAB" + "\xAB\xBC\xCD\xDE\xF0\x01\x12\x23" + "\x34\x45\x56\x67\x78\x89\x9A\xAB" + "\x73\x61\x6C", + .klen = 35, + .iv = "\x61\x6E\x64\x01\x69\x76\x65\x63", + .input = "\x63\x69\x73\x63\x6F\x01\x72\x75" + "\x6C\x65\x73\x01\x74\x68\x65\x01" + "\x6E\x65\x74\x77\x65\x01\x64\x65" + "\x66\x69\x6E\x65\x01\x74\x68\x65" + "\x74\x65\x63\x68\x6E\x6F\x6C\x6F" + "\x67\x69\x65\x73\x01\x74\x68\x61" + "\x74\x77\x69\x6C\x6C\x01\x64\x65" + "\x66\x69\x6E\x65\x74\x6F\x6D\x6F" + "\x72\x72\x6F\x77\x01\x02\x02\x01", + .ilen = 72, + .assoc = "\x17\x40\x5E\x67\x15\x6F\x31\x26" + "\xDD\x0D\xB9\x9B\x61\x6E\x64\x01" + "\x69\x76\x65\x63", + .alen = 20, + .result = "\xEA\x15\xC4\x98\xAC\x15\x22\x37" + "\x00\x07\x1D\xBE\x60\x5D\x73\x16" + "\x4D\x0F\xCC\xCE\x8A\xD0\x49\xD4" + "\x39\xA3\xD1\xB1\x21\x0A\x92\x1A" + "\x2C\xCF\x8F\x9D\xC9\x91\x0D\xB4" + "\x15\xFC\xBC\xA5\xC5\xBF\x54\xE5" + "\x1C\xC7\x32\x41\x07\x7B\x2C\xB6" + "\x5C\x23\x7C\x93\xEA\xEF\x23\x1C" + "\x73\xF4\xE7\x12\x84\x4C\x37\x0A" + "\x4A\x8F\x06\x37\x48\xF9\xF9\x05" + "\x55\x13\x40\xC3\xD5\x55\x3A\x3D", + .rlen = 88, + }, { + .key = "\x7D\x77\x3D\x00\xC1\x44\xC5\x25" + "\xAC\x61\x9D\x18\xC8\x4A\x3F\x47" + "\xD9\x66\x42", + .klen = 19, + .iv = "\x43\x45\x7E\x91\x82\x44\x3B\xC6", + .input = "\x01\x02\x02\x01", + .ilen = 4, + .assoc = "\x33\x54\x67\xAE\xFF\xFF\xFF\xFF" + "\x43\x45\x7E\x91\x82\x44\x3B\xC6", + .alen = 16, + .result = "\x4C\x72\x63\x30\x2F\xE6\x56\xDD" + "\xD0\xD8\x60\x9D\x8B\xEF\x85\x90" + "\xF7\x61\x24\x62", + .rlen = 20, + }, { + .key = "\xAB\xBC\xCD\xDE\xF0\x01\x12\x23" + "\x34\x45\x56\x67\x78\x89\x9A\xAB" + "\xDE\xCA\xF8", + .klen = 19, + .iv = "\xCA\xFE\xDE\xBA\xCE\xFA\xCE\x74", + .input = "\x74\x6F\x01\x62\x65\x01\x6F\x72" + "\x01\x6E\x6F\x74\x01\x74\x6F\x01" + "\x62\x65\x00\x01", + .ilen = 20, + .assoc = "\x00\x00\x01\x00\x00\x00\x00\x00" + "\x00\x00\x00\x01\xCA\xFE\xDE\xBA" + "\xCE\xFA\xCE\x74", + .alen = 20, + .result = "\xA3\xBF\x52\x52\x65\x83\xBA\x81" + "\x03\x9B\x84\xFC\x44\x8C\xBB\x81" + "\x36\xE1\x78\xBB\xA5\x49\x3A\xD0" + "\xF0\x6B\x21\xAF\x98\xC0\x34\xDC" + "\x17\x17\x65\xAD", + .rlen = 36, + }, { + .key = "\x6C\x65\x67\x61\x6C\x69\x7A\x65" + "\x6D\x61\x72\x69\x6A\x75\x61\x6E" + "\x61\x61\x6E\x64\x64\x6F\x69\x74" + "\x62\x65\x66\x6F\x72\x65\x69\x61" + "\x74\x75\x72", + .klen = 35, + .iv = "\x33\x30\x21\x69\x67\x65\x74\x6D", + .input = "\x45\x00\x00\x30\xDA\x3A\x00\x00" + "\x80\x01\xDF\x3B\xC0\xA8\x00\x05" + "\xC0\xA8\x00\x01\x08\x00\xC6\xCD" + "\x02\x00\x07\x00\x61\x62\x63\x64" + "\x65\x66\x67\x68\x69\x6A\x6B\x6C" + "\x6D\x6E\x6F\x70\x71\x72\x73\x74" + "\x01\x02\x02\x01", + .ilen = 52, + .assoc = "\x79\x6B\x69\x63\xFF\xFF\xFF\xFF" + "\xFF\xFF\xFF\xFF\x33\x30\x21\x69" + "\x67\x65\x74\x6D", + .alen = 20, + .result = "\x96\xFD\x86\xF8\xD1\x98\xFF\x10" + "\xAB\x8C\xDA\x8A\x5A\x08\x38\x1A" + "\x48\x59\x80\x18\x1A\x18\x1A\x04" + "\xC9\x0D\xE3\xE7\x0E\xA4\x0B\x75" + "\x92\x9C\x52\x5C\x0B\xFB\xF8\xAF" + "\x16\xC3\x35\xA8\xE7\xCE\x84\x04" + "\xEB\x40\x6B\x7A\x8E\x75\xBB\x42" + "\xE0\x63\x4B\x21\x44\xA2\x2B\x2B" + "\x39\xDB\xC8\xDC", + .rlen = 68, + }, { + .key = "\x3D\xE0\x98\x74\xB3\x88\xE6\x49" + "\x19\x88\xD0\xC3\x60\x7E\xAE\x1F" + "\x57\x69\x0E", + .klen = 19, + .iv = "\x4E\x28\x00\x00\xA2\xFC\xA1\xA3", + .input = "\x45\x00\x00\x30\xDA\x3A\x00\x00" + "\x80\x01\xDF\x3B\xC0\xA8\x00\x05" + "\xC0\xA8\x00\x01\x08\x00\xC6\xCD" + "\x02\x00\x07\x00\x61\x62\x63\x64" + "\x65\x66\x67\x68\x69\x6A\x6B\x6C" + "\x6D\x6E\x6F\x70\x71\x72\x73\x74" + "\x01\x02\x02\x01", + .ilen = 52, + .assoc = "\x3F\x7E\xF6\x42\x10\x10\x10\x10" + "\x10\x10\x10\x10\x4E\x28\x00\x00" + "\xA2\xFC\xA1\xA3", + .alen = 20, + .result = "\x6A\x6B\x45\x27\x3F\x9E\x52\xF6" + "\x10\x60\x54\x25\xEB\x80\x04\x93" + "\xCA\x1B\x23\x97\xCB\x21\x2E\x01" + "\xA2\xE7\x95\x41\x30\xE4\x4B\x1B" + "\x79\x01\x58\x50\x01\x06\xE1\xE0" + "\x2C\x83\x79\xD3\xDE\x46\x97\x1A" + "\x44\xCC\x90\xBF\x00\x94\x94\x92" + "\x20\x17\x0C\x1B\x55\xDE\x7E\x68" + "\xF4\x95\x5D\x4F", + .rlen = 68, + }, { + .key = "\x4C\x80\xCD\xEF\xBB\x5D\x10\xDA" + "\x90\x6A\xC7\x3C\x36\x13\xA6\x34" + "\x22\x43\x3C", + .klen = 19, + .iv = "\x48\x55\xEC\x7D\x3A\x23\x4B\xFD", + .input = "\x08\x00\xC6\xCD\x02\x00\x07\x00" + "\x61\x62\x63\x64\x65\x66\x67\x68" + "\x69\x6A\x6B\x6C\x6D\x6E\x6F\x70" + "\x71\x72\x73\x74\x01\x02\x02\x01", + .ilen = 32, + .assoc = "\x00\x00\x43\x21\x87\x65\x43\x21" + "\x00\x00\x00\x07\x48\x55\xEC\x7D" + "\x3A\x23\x4B\xFD", + .alen = 20, + .result = "\x67\xE9\x28\xB3\x1C\xA4\x6D\x02" + "\xF0\xB5\x37\xB6\x6B\x2F\xF5\x4F" + "\xF8\xA3\x4C\x53\xB8\x12\x09\xBF" + "\x58\x7D\xCF\x29\xA3\x41\x68\x6B" + "\xCE\xE8\x79\x85\x3C\xB0\x3A\x8F" + "\x16\xB0\xA1\x26\xC9\xBC\xBC\xA6", + .rlen = 48, + } +}; + +static struct aead_testvec aes_ccm_rfc4309_dec_tv_template[] = { + { /* Generated using Crypto++ */ + .key = zeroed_string, + .klen = 19, + .iv = zeroed_string, + .result = zeroed_string, + .rlen = 16, + .assoc = zeroed_string, + .alen = 16, + .input = "\x2E\x9A\xCA\x6B\xDA\x54\xFC\x6F" + "\x12\x50\xE8\xDE\x81\x3C\x63\x08" + "\x1A\x22\xBA\x75\xEE\xD4\xD5\xB5" + "\x27\x50\x01\xAC\x03\x33\x39\xFB", + .ilen = 32, + },{ + .key = "\xfe\xff\xe9\x92\x86\x65\x73\x1c" + "\x6d\x6a\x8f\x94\x67\x30\x83\x08" + "\x00\x00\x00", + .klen = 19, + .iv = "\x00\x00\x00\x00\x00\x00\x00\x01", + .result = zeroed_string, + .rlen = 16, + .assoc = "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x01", + .alen = 16, + .input = "\xCF\xB9\x99\x17\xC8\x86\x0E\x7F" + "\x7E\x76\xF8\xE6\xF8\xCC\x1F\x17" + "\x6A\xE0\x53\x9F\x4B\x73\x7E\xDA" + "\x08\x09\x4E\xC4\x1E\xAD\xC6\xB0", + .ilen = 32, + + }, { + .key = "\xfe\xff\xe9\x92\x86\x65\x73\x1c" + "\x6d\x6a\x8f\x94\x67\x30\x83\x08" + "\x00\x00\x00", + .klen = 19, + .iv = zeroed_string, + .result = "\x01\x01\x01\x01\x01\x01\x01\x01" + "\x01\x01\x01\x01\x01\x01\x01\x01", + .rlen = 16, + .assoc = zeroed_string, + .alen = 16, + .input = "\x33\xDE\x73\xBC\xA6\xCE\x4E\xA6" + "\x61\xF4\xF5\x41\x03\x4A\xE3\x86" + "\xA1\xE2\xC2\x42\x2B\x81\x70\x40" + "\xFD\x7F\x76\xD1\x03\x07\xBB\x0C", + .ilen = 32, + }, { + .key = "\xfe\xff\xe9\x92\x86\x65\x73\x1c" + "\x6d\x6a\x8f\x94\x67\x30\x83\x08" + "\x00\x00\x00", + .klen = 19, + .iv = zeroed_string, + .result = "\x01\x01\x01\x01\x01\x01\x01\x01" + "\x01\x01\x01\x01\x01\x01\x01\x01", + .rlen = 16, + .assoc = "\x01\x01\x01\x01\x01\x01\x01\x01" + "\x00\x00\x00\x00\x00\x00\x00\x00", + .alen = 16, + .input = "\x33\xDE\x73\xBC\xA6\xCE\x4E\xA6" + "\x61\xF4\xF5\x41\x03\x4A\xE3\x86" + "\x5B\xC0\x73\xE0\x2B\x73\x68\xC9" + "\x2D\x8C\x58\xC2\x90\x3D\xB0\x3E", + .ilen = 32, + }, { + .key = "\xfe\xff\xe9\x92\x86\x65\x73\x1c" + "\x6d\x6a\x8f\x94\x67\x30\x83\x08" + "\x00\x00\x00", + .klen = 19, + .iv = "\x00\x00\x00\x00\x00\x00\x00\x01", + .result = "\x01\x01\x01\x01\x01\x01\x01\x01" + "\x01\x01\x01\x01\x01\x01\x01\x01", + .rlen = 16, + .assoc = "\x01\x01\x01\x01\x01\x01\x01\x01" + "\x00\x00\x00\x00\x00\x00\x00\x01", + .alen = 16, + .input = "\xCE\xB8\x98\x16\xC9\x87\x0F\x7E" + "\x7F\x77\xF9\xE7\xF9\xCD\x1E\x16" + "\x43\x8E\x76\x57\x3B\xB4\x05\xE8" + "\xA9\x9B\xBF\x25\xE0\x4F\xC0\xED", + .ilen = 32, + }, { + .key = "\xfe\xff\xe9\x92\x86\x65\x73\x1c" + "\x6d\x6a\x8f\x94\x67\x30\x83\x08" + "\x00\x00\x00", + .klen = 19, + .iv = "\x00\x00\x00\x00\x00\x00\x00\x01", + .result = "\x01\x01\x01\x01\x01\x01\x01\x01" + "\x01\x01\x01\x01\x01\x01\x01\x01" + "\x01\x01\x01\x01\x01\x01\x01\x01" + "\x01\x01\x01\x01\x01\x01\x01\x01" + "\x01\x01\x01\x01\x01\x01\x01\x01" + "\x01\x01\x01\x01\x01\x01\x01\x01" + "\x01\x01\x01\x01\x01\x01\x01\x01" + "\x01\x01\x01\x01\x01\x01\x01\x01", + .rlen = 64, + .assoc = "\x01\x01\x01\x01\x01\x01\x01\x01" + "\x00\x00\x00\x00\x00\x00\x00\x01", + .alen = 16, + .input = "\xCE\xB8\x98\x16\xC9\x87\x0F\x7E" + "\x7F\x77\xF9\xE7\xF9\xCD\x1E\x16" + "\x9C\xA4\x97\x83\x3F\x01\xA5\xF4" + "\x43\x09\xE7\xB8\xE9\xD1\xD7\x02" + "\x9B\xAB\x39\x18\xEB\x94\x34\x36" + "\xE6\xC5\xC8\x9B\x00\x81\x9E\x49" + "\x1D\x78\xE1\x48\xE3\xE9\xEA\x8E" + "\x3A\x2B\x67\x5D\x35\x6A\x0F\xDB" + "\x02\x73\xDD\xE7\x30\x4A\x30\x54" + "\x1A\x9D\x09\xCA\xC8\x1C\x32\x5F", + .ilen = 80, + }, { + .key = "\x00\x01\x02\x03\x04\x05\x06\x07" + "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" + "\x00\x00\x00", + .klen = 19, + .iv = "\x00\x00\x45\x67\x89\xab\xcd\xef", + .result = "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff", + .rlen = 192, + .assoc = "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\x00\x00\x45\x67" + "\x89\xab\xcd\xef", + .alen = 20, + .input = "\x64\x17\xDC\x24\x9D\x92\xBA\x5E" + "\x7C\x64\x6D\x33\x46\x77\xAC\xB1" + "\x5C\x9E\xE2\xC7\x27\x11\x3E\x95" + "\x7D\xBE\x28\xC8\xC1\xCA\x5E\x8C" + "\xB4\xE2\xDE\x9F\x53\x59\x26\xDB" + "\x0C\xD4\xE4\x07\x9A\xE6\x3E\x01" + "\x58\x0D\x3E\x3D\xD5\x21\xEB\x04" + "\x06\x9D\x5F\xB9\x02\x49\x1A\x2B" + "\xBA\xF0\x4E\x3B\x85\x50\x5B\x09" + "\xFE\xEC\xFC\x54\xEC\x0C\xE2\x79" + "\x8A\x2F\x5F\xD7\x05\x5D\xF1\x6D" + "\x22\xEB\xD1\x09\x80\x3F\x5A\x70" + "\xB2\xB9\xD3\x63\x99\xC2\x4D\x1B" + "\x36\x12\x00\x89\xAA\x5D\x55\xDA" + "\x1D\x5B\xD8\x3C\x5F\x09\xD2\xE6" + "\x39\x41\x5C\xF0\xBE\x26\x4E\x5F" + "\x2B\x50\x44\x52\xC2\x10\x7D\x38" + "\x82\x64\x83\x0C\xAE\x49\xD0\xE5" + "\x4F\xE5\x66\x4C\x58\x7A\xEE\x43" + "\x3B\x51\xFE\xBA\x24\x8A\xFE\xDC" + "\x19\x6D\x60\x66\x61\xF9\x9A\x3F" + "\x75\xFC\x38\x53\x5B\xB5\xCD\x52" + "\x4F\xE5\xE4\xC9\xFE\x10\xCB\x98" + "\xF0\x06\x5B\x07\xAB\xBB\xF4\x0E" + "\x2D\xC2\xDD\x5D\xDD\x22\x9A\xCC" + "\x39\xAB\x63\xA5\x3D\x9C\x51\x8A", + .ilen = 208, + }, { /* From draft-mcgrew-gcm-test-01 */ + .key = "\x4C\x80\xCD\xEF\xBB\x5D\x10\xDA" + "\x90\x6A\xC7\x3C\x36\x13\xA6\x34" + "\x2E\x44\x3B", + .klen = 19, + .iv = "\x49\x56\xED\x7E\x3B\x24\x4C\xFE", + .result = "\x45\x00\x00\x48\x69\x9A\x00\x00" + "\x80\x11\x4D\xB7\xC0\xA8\x01\x02" + "\xC0\xA8\x01\x01\x0A\x9B\xF1\x56" + "\x38\xD3\x01\x00\x00\x01\x00\x00" + "\x00\x00\x00\x00\x04\x5F\x73\x69" + "\x70\x04\x5F\x75\x64\x70\x03\x73" + "\x69\x70\x09\x63\x79\x62\x65\x72" + "\x63\x69\x74\x79\x02\x64\x6B\x00" + "\x00\x21\x00\x01\x01\x02\x02\x01", + .rlen = 72, + .assoc = "\x00\x00\x43\x21\x87\x65\x43\x21" + "\x00\x00\x00\x00\x49\x56\xED\x7E" + "\x3B\x24\x4C\xFE", + .alen = 20, + .input = "\x89\xBA\x3E\xEF\xE6\xD6\xCF\xDB" + "\x83\x60\xF5\xBA\x3A\x56\x79\xE6" + "\x7E\x0C\x53\xCF\x9E\x87\xE0\x4E" + "\x1A\x26\x01\x24\xC7\x2E\x3D\xBF" + "\x29\x2C\x91\xC1\xB8\xA8\xCF\xE0" + "\x39\xF8\x53\x6D\x31\x22\x2B\xBF" + "\x98\x81\xFC\x34\xEE\x85\x36\xCD" + "\x26\xDB\x6C\x7A\x0C\x77\x8A\x35" + "\x18\x85\x54\xB2\xBC\xDD\x3F\x43" + "\x61\x06\x8A\xDF\x86\x3F\xB4\xAC" + "\x97\xDC\xBD\xFD\x92\x10\xC5\xFF", + .ilen = 88, + }, { + .key = "\xFE\xFF\xE9\x92\x86\x65\x73\x1C" + "\x6D\x6A\x8F\x94\x67\x30\x83\x08" + "\xCA\xFE\xBA", + .klen = 19, + .iv = "\xFA\xCE\xDB\xAD\xDE\xCA\xF8\x88", + .result = "\x45\x00\x00\x3E\x69\x8F\x00\x00" + "\x80\x11\x4D\xCC\xC0\xA8\x01\x02" + "\xC0\xA8\x01\x01\x0A\x98\x00\x35" + "\x00\x2A\x23\x43\xB2\xD0\x01\x00" + "\x00\x01\x00\x00\x00\x00\x00\x00" + "\x03\x73\x69\x70\x09\x63\x79\x62" + "\x65\x72\x63\x69\x74\x79\x02\x64" + "\x6B\x00\x00\x01\x00\x01\x00\x01", + .rlen = 64, + .assoc = "\x00\x00\xA5\xF8\x00\x00\x00\x0A" + "\xFA\xCE\xDB\xAD\xDE\xCA\xF8\x88", + .alen = 16, + .input = "\x4B\xC2\x70\x60\x64\xD2\xF3\xC8" + "\xE5\x26\x8A\xDE\xB8\x7E\x7D\x16" + "\x56\xC7\xD2\x88\xBA\x8D\x58\xAF" + "\xF5\x71\xB6\x37\x84\xA7\xB1\x99" + "\x51\x5C\x0D\xA0\x27\xDE\xE7\x2D" + "\xEF\x25\x88\x1F\x1D\x77\x11\xFF" + "\xDB\xED\xEE\x56\x16\xC5\x5C\x9B" + "\x00\x62\x1F\x68\x4E\x7C\xA0\x97" + "\x10\x72\x7E\x53\x13\x3B\x68\xE4" + "\x30\x99\x91\x79\x09\xEA\xFF\x6A", + .ilen = 80, + }, { + .key = "\xAB\xBC\xCD\xDE\xF0\x01\x12\x23" + "\x34\x45\x56\x67\x78\x89\x9A\xAB" + "\xAB\xBC\xCD\xDE\xF0\x01\x12\x23" + "\x34\x45\x56\x67\x78\x89\x9A\xAB" + "\x11\x22\x33", + .klen = 35, + .iv = "\x01\x02\x03\x04\x05\x06\x07\x08", + .result = "\x45\x00\x00\x30\x69\xA6\x40\x00" + "\x80\x06\x26\x90\xC0\xA8\x01\x02" + "\x93\x89\x15\x5E\x0A\x9E\x00\x8B" + "\x2D\xC5\x7E\xE0\x00\x00\x00\x00" + "\x70\x02\x40\x00\x20\xBF\x00\x00" + "\x02\x04\x05\xB4\x01\x01\x04\x02" + "\x01\x02\x02\x01", + .rlen = 52, + .assoc = "\x4A\x2C\xBF\xE3\x00\x00\x00\x02" + "\x01\x02\x03\x04\x05\x06\x07\x08", + .alen = 16, + .input = "\xD6\x31\x0D\x2B\x3D\x6F\xBD\x2F" + "\x58\x41\x7E\xFF\x9A\x9E\x09\xB4" + "\x1A\xF7\xF6\x42\x31\xCD\xBF\xAD" + "\x27\x0E\x2C\xF2\xDB\x10\xDF\x55" + "\x8F\x0D\xD7\xAC\x23\xBD\x42\x10" + "\xD0\xB2\xAF\xD8\x37\xAC\x6B\x0B" + "\x11\xD4\x0B\x12\xEC\xB4\xB1\x92" + "\x23\xA6\x10\xB0\x26\xD6\xD9\x26" + "\x5A\x48\x6A\x3E", + .ilen = 68, + }, { + .key = "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00", + .klen = 19, + .iv = "\x00\x00\x00\x00\x00\x00\x00\x00", + .result = "\x45\x00\x00\x3C\x99\xC5\x00\x00" + "\x80\x01\xCB\x7A\x40\x67\x93\x18" + "\x01\x01\x01\x01\x08\x00\x07\x5C" + "\x02\x00\x44\x00\x61\x62\x63\x64" + "\x65\x66\x67\x68\x69\x6A\x6B\x6C" + "\x6D\x6E\x6F\x70\x71\x72\x73\x74" + "\x75\x76\x77\x61\x62\x63\x64\x65" + "\x66\x67\x68\x69\x01\x02\x02\x01", + .rlen = 64, + .assoc = "\x00\x00\x00\x00\x00\x00\x00\x01" + "\x00\x00\x00\x00\x00\x00\x00\x00", + .alen = 16, + .input = "\x6B\x9A\xCA\x57\x43\x91\xFC\x6F" + "\x92\x51\x23\xA4\xC1\x5B\xF0\x10" + "\xF3\x13\xF4\xF8\xA1\x9A\xB4\xDC" + "\x89\xC8\xF8\x42\x62\x95\xB7\xCB" + "\xB8\xF5\x0F\x1B\x2E\x94\xA2\xA7" + "\xBF\xFB\x8A\x92\x13\x63\xD1\x3C" + "\x08\xF5\xE8\xA6\xAA\xF6\x34\xF9" + "\x42\x05\xAF\xB3\xE7\x9A\xFC\xEE" + "\x36\x25\xC1\x10\x12\x1C\xCA\x82" + "\xEA\xE6\x63\x5A\x57\x28\xA9\x9A", + .ilen = 80, + }, { + .key = "\x3D\xE0\x98\x74\xB3\x88\xE6\x49" + "\x19\x88\xD0\xC3\x60\x7E\xAE\x1F" + "\x57\x69\x0E", + .klen = 19, + .iv = "\x4E\x28\x00\x00\xA2\xFC\xA1\xA3", + .result = "\x45\x00\x00\x3C\x99\xC3\x00\x00" + "\x80\x01\xCB\x7C\x40\x67\x93\x18" + "\x01\x01\x01\x01\x08\x00\x08\x5C" + "\x02\x00\x43\x00\x61\x62\x63\x64" + "\x65\x66\x67\x68\x69\x6A\x6B\x6C" + "\x6D\x6E\x6F\x70\x71\x72\x73\x74" + "\x75\x76\x77\x61\x62\x63\x64\x65" + "\x66\x67\x68\x69\x01\x02\x02\x01", + .rlen = 64, + .assoc = "\x42\xF6\x7E\x3F\x10\x10\x10\x10" + "\x10\x10\x10\x10\x4E\x28\x00\x00" + "\xA2\xFC\xA1\xA3", + .alen = 20, + .input = "\x6A\x6B\x45\x2B\x7C\x67\x52\xF6" + "\x10\x60\x40\x62\x6B\x4F\x97\x8E" + "\x0B\xB2\x22\x97\xCB\x21\xE0\x90" + "\xA2\xE7\xD1\x41\x30\xE4\x4B\x1B" + "\x79\x01\x58\x50\x01\x06\xE1\xE0" + "\x2C\x83\x79\xD3\xDE\x46\x97\x1A" + "\x30\xB8\xE5\xDF\xD7\x12\x56\x75" + "\xD0\x95\xB7\xB8\x91\x42\xF7\xFD" + "\x97\x57\xCA\xC1\x20\xD0\x86\xB9" + "\x66\x9D\xB4\x2B\x96\x22\xAC\x67", + .ilen = 80, + }, { + .key = "\x3D\xE0\x98\x74\xB3\x88\xE6\x49" + "\x19\x88\xD0\xC3\x60\x7E\xAE\x1F" + "\x57\x69\x0E", + .klen = 19, + .iv = "\x4E\x28\x00\x00\xA2\xFC\xA1\xA3", + .result = "\x45\x00\x00\x1C\x42\xA2\x00\x00" + "\x80\x01\x44\x1F\x40\x67\x93\xB6" + "\xE0\x00\x00\x02\x0A\x00\xF5\xFF" + "\x01\x02\x02\x01", + .rlen = 28, + .assoc = "\x42\xF6\x7E\x3F\x10\x10\x10\x10" + "\x10\x10\x10\x10\x4E\x28\x00\x00" + "\xA2\xFC\xA1\xA3", + .alen = 20, + .input = "\x6A\x6B\x45\x0B\xA7\x06\x52\xF6" + "\x10\x60\xCF\x01\x6B\x4F\x97\x20" + "\xEA\xB3\x23\x94\xC9\x21\x1D\x33" + "\xA1\xE5\x90\x40\x05\x37\x45\x70" + "\xB5\xD6\x09\x0A\x23\x73\x33\xF9" + "\x08\xB4\x22\xE4", + .ilen = 44, + }, { + .key = "\xFE\xFF\xE9\x92\x86\x65\x73\x1C" + "\x6D\x6A\x8F\x94\x67\x30\x83\x08" + "\xFE\xFF\xE9\x92\x86\x65\x73\x1C" + "\xCA\xFE\xBA", + .klen = 27, + .iv = "\xFA\xCE\xDB\xAD\xDE\xCA\xF8\x88", + .result = "\x45\x00\x00\x28\xA4\xAD\x40\x00" + "\x40\x06\x78\x80\x0A\x01\x03\x8F" + "\x0A\x01\x06\x12\x80\x23\x06\xB8" + "\xCB\x71\x26\x02\xDD\x6B\xB0\x3E" + "\x50\x10\x16\xD0\x75\x68\x00\x01", + .rlen = 40, + .assoc = "\x00\x00\xA5\xF8\x00\x00\x00\x0A" + "\xFA\xCE\xDB\xAD\xDE\xCA\xF8\x88", + .alen = 16, + .input = "\x05\x22\x15\xD1\x52\x56\x85\x04" + "\xA8\x5C\x5D\x6D\x7E\x6E\xF5\xFA" + "\xEA\x16\x37\x50\xF3\xDF\x84\x3B" + "\x2F\x32\x18\x57\x34\x2A\x8C\x23" + "\x67\xDF\x6D\x35\x7B\x54\x0D\xFB" + "\x34\xA5\x9F\x6C\x48\x30\x1E\x22" + "\xFE\xB1\x22\x17\x17\x8A\xB9\x5B", + .ilen = 56, + }, { + .key = "\xAB\xBC\xCD\xDE\xF0\x01\x12\x23" + "\x34\x45\x56\x67\x78\x89\x9A\xAB" + "\xDE\xCA\xF8", + .klen = 19, + .iv = "\xCA\xFE\xDE\xBA\xCE\xFA\xCE\x74", + .result = "\x45\x00\x00\x49\x33\xBA\x00\x00" + "\x7F\x11\x91\x06\xC3\xFB\x1D\x10" + "\xC2\xB1\xD3\x26\xC0\x28\x31\xCE" + "\x00\x35\xDD\x7B\x80\x03\x02\xD5" + "\x00\x00\x4E\x20\x00\x1E\x8C\x18" + "\xD7\x5B\x81\xDC\x91\xBA\xA0\x47" + "\x6B\x91\xB9\x24\xB2\x80\x38\x9D" + "\x92\xC9\x63\xBA\xC0\x46\xEC\x95" + "\x9B\x62\x66\xC0\x47\x22\xB1\x49" + "\x23\x01\x01\x01", + .rlen = 76, + .assoc = "\x00\x00\x01\x00\x00\x00\x00\x00" + "\x00\x00\x00\x01\xCA\xFE\xDE\xBA" + "\xCE\xFA\xCE\x74", + .alen = 20, + .input = "\x92\xD0\x53\x79\x33\x38\xD5\xF3" + "\x7D\xE4\x7A\x8E\x86\x03\xC9\x90" + "\x96\x35\xAB\x9C\xFB\xE8\xA3\x76" + "\xE9\xE9\xE2\xD1\x2E\x11\x0E\x00" + "\xFA\xCE\xB5\x9E\x02\xA7\x7B\xEA" + "\x71\x9A\x58\xFB\xA5\x8A\xE1\xB7" + "\x9C\x39\x9D\xE3\xB5\x6E\x69\xE6" + "\x63\xC9\xDB\x05\x69\x51\x12\xAD" + "\x3E\x00\x32\x73\x86\xF2\xEE\xF5" + "\x0F\xE8\x81\x7E\x84\xD3\xC0\x0D" + "\x76\xD6\x55\xC6\xB4\xC2\x34\xC7" + "\x12\x25\x0B\xF9", + .ilen = 92, + }, { + .key = "\xAB\xBC\xCD\xDE\xF0\x01\x12\x23" + "\x34\x45\x56\x67\x78\x89\x9A\xAB" + "\xAB\xBC\xCD\xDE\xF0\x01\x12\x23" + "\x34\x45\x56\x67\x78\x89\x9A\xAB" + "\x73\x61\x6C", + .klen = 35, + .iv = "\x61\x6E\x64\x01\x69\x76\x65\x63", + .result = "\x45\x08\x00\x28\x73\x2C\x00\x00" + "\x40\x06\xE9\xF9\x0A\x01\x06\x12" + "\x0A\x01\x03\x8F\x06\xB8\x80\x23" + "\xDD\x6B\xAF\xBE\xCB\x71\x26\x02" + "\x50\x10\x1F\x64\x6D\x54\x00\x01", + .rlen = 40, + .assoc = "\x17\x40\x5E\x67\x15\x6F\x31\x26" + "\xDD\x0D\xB9\x9B\x61\x6E\x64\x01" + "\x69\x76\x65\x63", + .alen = 20, + .input = "\xCC\x74\xB7\xD3\xB0\x38\x50\x42" + "\x2C\x64\x87\x46\x1E\x34\x10\x05" + "\x29\x6B\xBB\x36\xE9\x69\xAD\x92" + "\x82\xA1\x10\x6A\xEB\x0F\xDC\x7D" + "\x08\xBA\xF3\x91\xCA\xAA\x61\xDA" + "\x62\xF4\x14\x61\x5C\x9D\xB5\xA7" + "\xEE\xD7\xB9\x7E\x87\x99\x9B\x7D", + .ilen = 56, + }, { + .key = "\x3D\xE0\x98\x74\xB3\x88\xE6\x49" + "\x19\x88\xD0\xC3\x60\x7E\xAE\x1F" + "\x57\x69\x0E", + .klen = 19, + .iv = "\x4E\x28\x00\x00\xA2\xFC\xA1\xA3", + .result = "\x45\x00\x00\x49\x33\x3E\x00\x00" + "\x7F\x11\x91\x82\xC3\xFB\x1D\x10" + "\xC2\xB1\xD3\x26\xC0\x28\x31\xCE" + "\x00\x35\xCB\x45\x80\x03\x02\x5B" + "\x00\x00\x01\xE0\x00\x1E\x8C\x18" + "\xD6\x57\x59\xD5\x22\x84\xA0\x35" + "\x2C\x71\x47\x5C\x88\x80\x39\x1C" + "\x76\x4D\x6E\x5E\xE0\x49\x6B\x32" + "\x5A\xE2\x70\xC0\x38\x99\x49\x39" + "\x15\x01\x01\x01", + .rlen = 76, + .assoc = "\x42\xF6\x7E\x3F\x10\x10\x10\x10" + "\x10\x10\x10\x10\x4E\x28\x00\x00" + "\xA2\xFC\xA1\xA3", + .alen = 20, + .input = "\x6A\x6B\x45\x5E\xD6\x9A\x52\xF6" + "\xEF\x70\x1A\x9C\xE8\xD3\x19\x86" + "\xC8\x02\xF0\xB0\x03\x09\xD9\x02" + "\xA0\xD2\x59\x04\xD1\x85\x2A\x24" + "\x1C\x67\x3E\xD8\x68\x72\x06\x94" + "\x97\xBA\x4F\x76\x8D\xB0\x44\x5B" + "\x69\xBF\xD5\xE2\x3D\xF1\x0B\x0C" + "\xC0\xBF\xB1\x8F\x70\x09\x9E\xCE" + "\xA5\xF2\x55\x58\x84\xFA\xF9\xB5" + "\x23\xF4\x84\x40\x74\x14\x8A\x6B" + "\xDB\xD7\x67\xED\xA4\x93\xF3\x47" + "\xCC\xF7\x46\x6F", + .ilen = 92, + }, { + .key = "\xAB\xBC\xCD\xDE\xF0\x01\x12\x23" + "\x34\x45\x56\x67\x78\x89\x9A\xAB" + "\xAB\xBC\xCD\xDE\xF0\x01\x12\x23" + "\x34\x45\x56\x67\x78\x89\x9A\xAB" + "\x73\x61\x6C", + .klen = 35, + .iv = "\x61\x6E\x64\x01\x69\x76\x65\x63", + .result = "\x63\x69\x73\x63\x6F\x01\x72\x75" + "\x6C\x65\x73\x01\x74\x68\x65\x01" + "\x6E\x65\x74\x77\x65\x01\x64\x65" + "\x66\x69\x6E\x65\x01\x74\x68\x65" + "\x74\x65\x63\x68\x6E\x6F\x6C\x6F" + "\x67\x69\x65\x73\x01\x74\x68\x61" + "\x74\x77\x69\x6C\x6C\x01\x64\x65" + "\x66\x69\x6E\x65\x74\x6F\x6D\x6F" + "\x72\x72\x6F\x77\x01\x02\x02\x01", + .rlen = 72, + .assoc = "\x17\x40\x5E\x67\x15\x6F\x31\x26" + "\xDD\x0D\xB9\x9B\x61\x6E\x64\x01" + "\x69\x76\x65\x63", + .alen = 20, + .input = "\xEA\x15\xC4\x98\xAC\x15\x22\x37" + "\x00\x07\x1D\xBE\x60\x5D\x73\x16" + "\x4D\x0F\xCC\xCE\x8A\xD0\x49\xD4" + "\x39\xA3\xD1\xB1\x21\x0A\x92\x1A" + "\x2C\xCF\x8F\x9D\xC9\x91\x0D\xB4" + "\x15\xFC\xBC\xA5\xC5\xBF\x54\xE5" + "\x1C\xC7\x32\x41\x07\x7B\x2C\xB6" + "\x5C\x23\x7C\x93\xEA\xEF\x23\x1C" + "\x73\xF4\xE7\x12\x84\x4C\x37\x0A" + "\x4A\x8F\x06\x37\x48\xF9\xF9\x05" + "\x55\x13\x40\xC3\xD5\x55\x3A\x3D", + .ilen = 88, + }, { + .key = "\x7D\x77\x3D\x00\xC1\x44\xC5\x25" + "\xAC\x61\x9D\x18\xC8\x4A\x3F\x47" + "\xD9\x66\x42", + .klen = 19, + .iv = "\x43\x45\x7E\x91\x82\x44\x3B\xC6", + .result = "\x01\x02\x02\x01", + .rlen = 4, + .assoc = "\x33\x54\x67\xAE\xFF\xFF\xFF\xFF" + "\x43\x45\x7E\x91\x82\x44\x3B\xC6", + .alen = 16, + .input = "\x4C\x72\x63\x30\x2F\xE6\x56\xDD" + "\xD0\xD8\x60\x9D\x8B\xEF\x85\x90" + "\xF7\x61\x24\x62", + .ilen = 20, + }, { + .key = "\xAB\xBC\xCD\xDE\xF0\x01\x12\x23" + "\x34\x45\x56\x67\x78\x89\x9A\xAB" + "\xDE\xCA\xF8", + .klen = 19, + .iv = "\xCA\xFE\xDE\xBA\xCE\xFA\xCE\x74", + .result = "\x74\x6F\x01\x62\x65\x01\x6F\x72" + "\x01\x6E\x6F\x74\x01\x74\x6F\x01" + "\x62\x65\x00\x01", + .rlen = 20, + .assoc = "\x00\x00\x01\x00\x00\x00\x00\x00" + "\x00\x00\x00\x01\xCA\xFE\xDE\xBA" + "\xCE\xFA\xCE\x74", + .alen = 20, + .input = "\xA3\xBF\x52\x52\x65\x83\xBA\x81" + "\x03\x9B\x84\xFC\x44\x8C\xBB\x81" + "\x36\xE1\x78\xBB\xA5\x49\x3A\xD0" + "\xF0\x6B\x21\xAF\x98\xC0\x34\xDC" + "\x17\x17\x65\xAD", + .ilen = 36, + }, { + .key = "\x6C\x65\x67\x61\x6C\x69\x7A\x65" + "\x6D\x61\x72\x69\x6A\x75\x61\x6E" + "\x61\x61\x6E\x64\x64\x6F\x69\x74" + "\x62\x65\x66\x6F\x72\x65\x69\x61" + "\x74\x75\x72", + .klen = 35, + .iv = "\x33\x30\x21\x69\x67\x65\x74\x6D", + .result = "\x45\x00\x00\x30\xDA\x3A\x00\x00" + "\x80\x01\xDF\x3B\xC0\xA8\x00\x05" + "\xC0\xA8\x00\x01\x08\x00\xC6\xCD" + "\x02\x00\x07\x00\x61\x62\x63\x64" + "\x65\x66\x67\x68\x69\x6A\x6B\x6C" + "\x6D\x6E\x6F\x70\x71\x72\x73\x74" + "\x01\x02\x02\x01", + .rlen = 52, + .assoc = "\x79\x6B\x69\x63\xFF\xFF\xFF\xFF" + "\xFF\xFF\xFF\xFF\x33\x30\x21\x69" + "\x67\x65\x74\x6D", + .alen = 20, + .input = "\x96\xFD\x86\xF8\xD1\x98\xFF\x10" + "\xAB\x8C\xDA\x8A\x5A\x08\x38\x1A" + "\x48\x59\x80\x18\x1A\x18\x1A\x04" + "\xC9\x0D\xE3\xE7\x0E\xA4\x0B\x75" + "\x92\x9C\x52\x5C\x0B\xFB\xF8\xAF" + "\x16\xC3\x35\xA8\xE7\xCE\x84\x04" + "\xEB\x40\x6B\x7A\x8E\x75\xBB\x42" + "\xE0\x63\x4B\x21\x44\xA2\x2B\x2B" + "\x39\xDB\xC8\xDC", + .ilen = 68, + }, { + .key = "\x3D\xE0\x98\x74\xB3\x88\xE6\x49" + "\x19\x88\xD0\xC3\x60\x7E\xAE\x1F" + "\x57\x69\x0E", + .klen = 19, + .iv = "\x4E\x28\x00\x00\xA2\xFC\xA1\xA3", + .result = "\x45\x00\x00\x30\xDA\x3A\x00\x00" + "\x80\x01\xDF\x3B\xC0\xA8\x00\x05" + "\xC0\xA8\x00\x01\x08\x00\xC6\xCD" + "\x02\x00\x07\x00\x61\x62\x63\x64" + "\x65\x66\x67\x68\x69\x6A\x6B\x6C" + "\x6D\x6E\x6F\x70\x71\x72\x73\x74" + "\x01\x02\x02\x01", + .rlen = 52, + .assoc = "\x3F\x7E\xF6\x42\x10\x10\x10\x10" + "\x10\x10\x10\x10\x4E\x28\x00\x00" + "\xA2\xFC\xA1\xA3", + .alen = 20, + .input = "\x6A\x6B\x45\x27\x3F\x9E\x52\xF6" + "\x10\x60\x54\x25\xEB\x80\x04\x93" + "\xCA\x1B\x23\x97\xCB\x21\x2E\x01" + "\xA2\xE7\x95\x41\x30\xE4\x4B\x1B" + "\x79\x01\x58\x50\x01\x06\xE1\xE0" + "\x2C\x83\x79\xD3\xDE\x46\x97\x1A" + "\x44\xCC\x90\xBF\x00\x94\x94\x92" + "\x20\x17\x0C\x1B\x55\xDE\x7E\x68" + "\xF4\x95\x5D\x4F", + .ilen = 68, + }, { + .key = "\x4C\x80\xCD\xEF\xBB\x5D\x10\xDA" + "\x90\x6A\xC7\x3C\x36\x13\xA6\x34" + "\x22\x43\x3C", + .klen = 19, + .iv = "\x48\x55\xEC\x7D\x3A\x23\x4B\xFD", + .result = "\x08\x00\xC6\xCD\x02\x00\x07\x00" + "\x61\x62\x63\x64\x65\x66\x67\x68" + "\x69\x6A\x6B\x6C\x6D\x6E\x6F\x70" + "\x71\x72\x73\x74\x01\x02\x02\x01", + .rlen = 32, + .assoc = "\x00\x00\x43\x21\x87\x65\x43\x21" + "\x00\x00\x00\x07\x48\x55\xEC\x7D" + "\x3A\x23\x4B\xFD", + .alen = 20, + .input = "\x67\xE9\x28\xB3\x1C\xA4\x6D\x02" + "\xF0\xB5\x37\xB6\x6B\x2F\xF5\x4F" + "\xF8\xA3\x4C\x53\xB8\x12\x09\xBF" + "\x58\x7D\xCF\x29\xA3\x41\x68\x6B" + "\xCE\xE8\x79\x85\x3C\xB0\x3A\x8F" + "\x16\xB0\xA1\x26\xC9\xBC\xBC\xA6", + .ilen = 48, + } +}; + +/* * ChaCha20-Poly1305 AEAD test vectors from RFC7539 2.8.2./A.5. */ #define RFC7539_ENC_TEST_VECTORS 2 @@ -22343,8 +23647,9 @@ static struct aead_testvec rfc7539esp_enc_tv_template[] = { .klen = 36, .iv = "\x01\x02\x03\x04\x05\x06\x07\x08", .assoc = "\xf3\x33\x88\x86\x00\x00\x00\x00" - "\x00\x00\x4e\x91", - .alen = 12, + "\x00\x00\x4e\x91\x01\x02\x03\x04" + "\x05\x06\x07\x08", + .alen = 20, .input = "\x49\x6e\x74\x65\x72\x6e\x65\x74" "\x2d\x44\x72\x61\x66\x74\x73\x20" "\x61\x72\x65\x20\x64\x72\x61\x66" @@ -22430,8 +23735,9 @@ static struct aead_testvec rfc7539esp_dec_tv_template[] = { .klen = 36, .iv = "\x01\x02\x03\x04\x05\x06\x07\x08", .assoc = "\xf3\x33\x88\x86\x00\x00\x00\x00" - "\x00\x00\x4e\x91", - .alen = 12, + "\x00\x00\x4e\x91\x01\x02\x03\x04" + "\x05\x06\x07\x08", + .alen = 20, .input = "\x64\xa0\x86\x15\x75\x86\x1a\xf4" "\x60\xf0\x62\xc7\x9b\xe6\x43\xbd" "\x5e\x80\x5c\xfd\x34\x5c\xf3\x89" @@ -30174,7 +31480,7 @@ static struct cipher_testvec salsa20_stream_enc_tv_template[] = { }, }; -#define CHACHA20_ENC_TEST_VECTORS 3 +#define CHACHA20_ENC_TEST_VECTORS 4 static struct cipher_testvec chacha20_enc_tv_template[] = { { /* RFC7539 A.2. Test Vector #1 */ .key = "\x00\x00\x00\x00\x00\x00\x00\x00" @@ -30348,6 +31654,338 @@ static struct cipher_testvec chacha20_enc_tv_template[] = { "\x87\xb5\x8d\xfd\x72\x8a\xfa\x36" "\x75\x7a\x79\x7a\xc1\x88\xd1", .rlen = 127, + }, { /* Self-made test vector for long data */ + .key = "\x1c\x92\x40\xa5\xeb\x55\xd3\x8a" + "\xf3\x33\x88\x86\x04\xf6\xb5\xf0" + "\x47\x39\x17\xc1\x40\x2b\x80\x09" + "\x9d\xca\x5c\xbc\x20\x70\x75\xc0", + .klen = 32, + .iv = "\x1c\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x01", + .input = "\x49\xee\xe0\xdc\x24\x90\x40\xcd" + "\xc5\x40\x8f\x47\x05\xbc\xdd\x81" + "\x47\xc6\x8d\xe6\xb1\x8f\xd7\xcb" + "\x09\x0e\x6e\x22\x48\x1f\xbf\xb8" + "\x5c\xf7\x1e\x8a\xc1\x23\xf2\xd4" + "\x19\x4b\x01\x0f\x4e\xa4\x43\xce" + "\x01\xc6\x67\xda\x03\x91\x18\x90" + "\xa5\xa4\x8e\x45\x03\xb3\x2d\xac" + "\x74\x92\xd3\x53\x47\xc8\xdd\x25" + "\x53\x6c\x02\x03\x87\x0d\x11\x0c" + "\x58\xe3\x12\x18\xfd\x2a\x5b\x40" + "\x0c\x30\xf0\xb8\x3f\x43\xce\xae" + "\x65\x3a\x7d\x7c\xf4\x54\xaa\xcc" + "\x33\x97\xc3\x77\xba\xc5\x70\xde" + "\xd7\xd5\x13\xa5\x65\xc4\x5f\x0f" + "\x46\x1a\x0d\x97\xb5\xf3\xbb\x3c" + "\x84\x0f\x2b\xc5\xaa\xea\xf2\x6c" + "\xc9\xb5\x0c\xee\x15\xf3\x7d\xbe" + "\x9f\x7b\x5a\xa6\xae\x4f\x83\xb6" + "\x79\x49\x41\xf4\x58\x18\xcb\x86" + "\x7f\x30\x0e\xf8\x7d\x44\x36\xea" + "\x75\xeb\x88\x84\x40\x3c\xad\x4f" + "\x6f\x31\x6b\xaa\x5d\xe5\xa5\xc5" + "\x21\x66\xe9\xa7\xe3\xb2\x15\x88" + "\x78\xf6\x79\xa1\x59\x47\x12\x4e" + "\x9f\x9f\x64\x1a\xa0\x22\x5b\x08" + "\xbe\x7c\x36\xc2\x2b\x66\x33\x1b" + "\xdd\x60\x71\xf7\x47\x8c\x61\xc3" + "\xda\x8a\x78\x1e\x16\xfa\x1e\x86" + "\x81\xa6\x17\x2a\xa7\xb5\xc2\xe7" + "\xa4\xc7\x42\xf1\xcf\x6a\xca\xb4" + "\x45\xcf\xf3\x93\xf0\xe7\xea\xf6" + "\xf4\xe6\x33\x43\x84\x93\xa5\x67" + "\x9b\x16\x58\x58\x80\x0f\x2b\x5c" + "\x24\x74\x75\x7f\x95\x81\xb7\x30" + "\x7a\x33\xa7\xf7\x94\x87\x32\x27" + "\x10\x5d\x14\x4c\x43\x29\xdd\x26" + "\xbd\x3e\x3c\x0e\xfe\x0e\xa5\x10" + "\xea\x6b\x64\xfd\x73\xc6\xed\xec" + "\xa8\xc9\xbf\xb3\xba\x0b\x4d\x07" + "\x70\xfc\x16\xfd\x79\x1e\xd7\xc5" + "\x49\x4e\x1c\x8b\x8d\x79\x1b\xb1" + "\xec\xca\x60\x09\x4c\x6a\xd5\x09" + "\x49\x46\x00\x88\x22\x8d\xce\xea" + "\xb1\x17\x11\xde\x42\xd2\x23\xc1" + "\x72\x11\xf5\x50\x73\x04\x40\x47" + "\xf9\x5d\xe7\xa7\x26\xb1\x7e\xb0" + "\x3f\x58\xc1\x52\xab\x12\x67\x9d" + "\x3f\x43\x4b\x68\xd4\x9c\x68\x38" + "\x07\x8a\x2d\x3e\xf3\xaf\x6a\x4b" + "\xf9\xe5\x31\x69\x22\xf9\xa6\x69" + "\xc6\x9c\x96\x9a\x12\x35\x95\x1d" + "\x95\xd5\xdd\xbe\xbf\x93\x53\x24" + "\xfd\xeb\xc2\x0a\x64\xb0\x77\x00" + "\x6f\x88\xc4\x37\x18\x69\x7c\xd7" + "\x41\x92\x55\x4c\x03\xa1\x9a\x4b" + "\x15\xe5\xdf\x7f\x37\x33\x72\xc1" + "\x8b\x10\x67\xa3\x01\x57\x94\x25" + "\x7b\x38\x71\x7e\xdd\x1e\xcc\x73" + "\x55\xd2\x8e\xeb\x07\xdd\xf1\xda" + "\x58\xb1\x47\x90\xfe\x42\x21\x72" + "\xa3\x54\x7a\xa0\x40\xec\x9f\xdd" + "\xc6\x84\x6e\xca\xae\xe3\x68\xb4" + "\x9d\xe4\x78\xff\x57\xf2\xf8\x1b" + "\x03\xa1\x31\xd9\xde\x8d\xf5\x22" + "\x9c\xdd\x20\xa4\x1e\x27\xb1\x76" + "\x4f\x44\x55\xe2\x9b\xa1\x9c\xfe" + "\x54\xf7\x27\x1b\xf4\xde\x02\xf5" + "\x1b\x55\x48\x5c\xdc\x21\x4b\x9e" + "\x4b\x6e\xed\x46\x23\xdc\x65\xb2" + "\xcf\x79\x5f\x28\xe0\x9e\x8b\xe7" + "\x4c\x9d\x8a\xff\xc1\xa6\x28\xb8" + "\x65\x69\x8a\x45\x29\xef\x74\x85" + "\xde\x79\xc7\x08\xae\x30\xb0\xf4" + "\xa3\x1d\x51\x41\xab\xce\xcb\xf6" + "\xb5\xd8\x6d\xe0\x85\xe1\x98\xb3" + "\x43\xbb\x86\x83\x0a\xa0\xf5\xb7" + "\x04\x0b\xfa\x71\x1f\xb0\xf6\xd9" + "\x13\x00\x15\xf0\xc7\xeb\x0d\x5a" + "\x9f\xd7\xb9\x6c\x65\x14\x22\x45" + "\x6e\x45\x32\x3e\x7e\x60\x1a\x12" + "\x97\x82\x14\xfb\xaa\x04\x22\xfa" + "\xa0\xe5\x7e\x8c\x78\x02\x48\x5d" + "\x78\x33\x5a\x7c\xad\xdb\x29\xce" + "\xbb\x8b\x61\xa4\xb7\x42\xe2\xac" + "\x8b\x1a\xd9\x2f\x0b\x8b\x62\x21" + "\x83\x35\x7e\xad\x73\xc2\xb5\x6c" + "\x10\x26\x38\x07\xe5\xc7\x36\x80" + "\xe2\x23\x12\x61\xf5\x48\x4b\x2b" + "\xc5\xdf\x15\xd9\x87\x01\xaa\xac" + "\x1e\x7c\xad\x73\x78\x18\x63\xe0" + "\x8b\x9f\x81\xd8\x12\x6a\x28\x10" + "\xbe\x04\x68\x8a\x09\x7c\x1b\x1c" + "\x83\x66\x80\x47\x80\xe8\xfd\x35" + "\x1c\x97\x6f\xae\x49\x10\x66\xcc" + "\xc6\xd8\xcc\x3a\x84\x91\x20\x77" + "\x72\xe4\x24\xd2\x37\x9f\xc5\xc9" + "\x25\x94\x10\x5f\x40\x00\x64\x99" + "\xdc\xae\xd7\x21\x09\x78\x50\x15" + "\xac\x5f\xc6\x2c\xa2\x0b\xa9\x39" + "\x87\x6e\x6d\xab\xde\x08\x51\x16" + "\xc7\x13\xe9\xea\xed\x06\x8e\x2c" + "\xf8\x37\x8c\xf0\xa6\x96\x8d\x43" + "\xb6\x98\x37\xb2\x43\xed\xde\xdf" + "\x89\x1a\xe7\xeb\x9d\xa1\x7b\x0b" + "\x77\xb0\xe2\x75\xc0\xf1\x98\xd9" + "\x80\x55\xc9\x34\x91\xd1\x59\xe8" + "\x4b\x0f\xc1\xa9\x4b\x7a\x84\x06" + "\x20\xa8\x5d\xfa\xd1\xde\x70\x56" + "\x2f\x9e\x91\x9c\x20\xb3\x24\xd8" + "\x84\x3d\xe1\x8c\x7e\x62\x52\xe5" + "\x44\x4b\x9f\xc2\x93\x03\xea\x2b" + "\x59\xc5\xfa\x3f\x91\x2b\xbb\x23" + "\xf5\xb2\x7b\xf5\x38\xaf\xb3\xee" + "\x63\xdc\x7b\xd1\xff\xaa\x8b\xab" + "\x82\x6b\x37\x04\xeb\x74\xbe\x79" + "\xb9\x83\x90\xef\x20\x59\x46\xff" + "\xe9\x97\x3e\x2f\xee\xb6\x64\x18" + "\x38\x4c\x7a\x4a\xf9\x61\xe8\x9a" + "\xa1\xb5\x01\xa6\x47\xd3\x11\xd4" + "\xce\xd3\x91\x49\x88\xc7\xb8\x4d" + "\xb1\xb9\x07\x6d\x16\x72\xae\x46" + "\x5e\x03\xa1\x4b\xb6\x02\x30\xa8" + "\x3d\xa9\x07\x2a\x7c\x19\xe7\x62" + "\x87\xe3\x82\x2f\x6f\xe1\x09\xd9" + "\x94\x97\xea\xdd\x58\x9e\xae\x76" + "\x7e\x35\xe5\xb4\xda\x7e\xf4\xde" + "\xf7\x32\x87\xcd\x93\xbf\x11\x56" + "\x11\xbe\x08\x74\xe1\x69\xad\xe2" + "\xd7\xf8\x86\x75\x8a\x3c\xa4\xbe" + "\x70\xa7\x1b\xfc\x0b\x44\x2a\x76" + "\x35\xea\x5d\x85\x81\xaf\x85\xeb" + "\xa0\x1c\x61\xc2\xf7\x4f\xa5\xdc" + "\x02\x7f\xf6\x95\x40\x6e\x8a\x9a" + "\xf3\x5d\x25\x6e\x14\x3a\x22\xc9" + "\x37\x1c\xeb\x46\x54\x3f\xa5\x91" + "\xc2\xb5\x8c\xfe\x53\x08\x97\x32" + "\x1b\xb2\x30\x27\xfe\x25\x5d\xdc" + "\x08\x87\xd0\xe5\x94\x1a\xd4\xf1" + "\xfe\xd6\xb4\xa3\xe6\x74\x81\x3c" + "\x1b\xb7\x31\xa7\x22\xfd\xd4\xdd" + "\x20\x4e\x7c\x51\xb0\x60\x73\xb8" + "\x9c\xac\x91\x90\x7e\x01\xb0\xe1" + "\x8a\x2f\x75\x1c\x53\x2a\x98\x2a" + "\x06\x52\x95\x52\xb2\xe9\x25\x2e" + "\x4c\xe2\x5a\x00\xb2\x13\x81\x03" + "\x77\x66\x0d\xa5\x99\xda\x4e\x8c" + "\xac\xf3\x13\x53\x27\x45\xaf\x64" + "\x46\xdc\xea\x23\xda\x97\xd1\xab" + "\x7d\x6c\x30\x96\x1f\xbc\x06\x34" + "\x18\x0b\x5e\x21\x35\x11\x8d\x4c" + "\xe0\x2d\xe9\x50\x16\x74\x81\xa8" + "\xb4\x34\xb9\x72\x42\xa6\xcc\xbc" + "\xca\x34\x83\x27\x10\x5b\x68\x45" + "\x8f\x52\x22\x0c\x55\x3d\x29\x7c" + "\xe3\xc0\x66\x05\x42\x91\x5f\x58" + "\xfe\x4a\x62\xd9\x8c\xa9\x04\x19" + "\x04\xa9\x08\x4b\x57\xfc\x67\x53" + "\x08\x7c\xbc\x66\x8a\xb0\xb6\x9f" + "\x92\xd6\x41\x7c\x5b\x2a\x00\x79" + "\x72", + .ilen = 1281, + .result = "\x45\xe8\xe0\xb6\x9c\xca\xfd\x87" + "\xe8\x1d\x37\x96\x8a\xe3\x40\x35" + "\xcf\x5e\x3a\x46\x3d\xfb\xd0\x69" + "\xde\xaf\x7a\xd5\x0d\xe9\x52\xec" + "\xc2\x82\xe5\x3e\x7d\xb2\x4a\xd9" + "\xbb\xc3\x9f\xc0\x5d\xac\x93\x8d" + "\x0e\x6f\xd3\xd7\xfb\x6a\x0d\xce" + "\x92\x2c\xf7\xbb\x93\x57\xcc\xee" + "\x42\x72\x6f\xc8\x4b\xd2\x76\xbf" + "\xa0\xe3\x7a\x39\xf9\x5c\x8e\xfd" + "\xa1\x1d\x41\xe5\x08\xc1\x1c\x11" + "\x92\xfd\x39\x5c\x51\xd0\x2f\x66" + "\x33\x4a\x71\x15\xfe\xee\x12\x54" + "\x8c\x8f\x34\xd8\x50\x3c\x18\xa6" + "\xc5\xe1\x46\x8a\xfb\x5f\x7e\x25" + "\x9b\xe2\xc3\x66\x41\x2b\xb3\xa5" + "\x57\x0e\x94\x17\x26\x39\xbb\x54" + "\xae\x2e\x6f\x42\xfb\x4d\x89\x6f" + "\x9d\xf1\x16\x2e\xe3\xe7\xfc\xe3" + "\xb2\x4b\x2b\xa6\x7c\x04\x69\x3a" + "\x70\x5a\xa7\xf1\x31\x64\x19\xca" + "\x45\x79\xd8\x58\x23\x61\xaf\xc2" + "\x52\x05\xc3\x0b\xc1\x64\x7c\x81" + "\xd9\x11\xcf\xff\x02\x3d\x51\x84" + "\x01\xac\xc6\x2e\x34\x2b\x09\x3a" + "\xa8\x5d\x98\x0e\x89\xd9\xef\x8f" + "\xd9\xd7\x7d\xdd\x63\x47\x46\x7d" + "\xa1\xda\x0b\x53\x7d\x79\xcd\xc9" + "\x86\xdd\x6b\x13\xa1\x9a\x70\xdd" + "\x5c\xa1\x69\x3c\xe4\x5d\xe3\x8c" + "\xe5\xf4\x87\x9c\x10\xcf\x0f\x0b" + "\xc8\x43\xdc\xf8\x1d\x62\x5e\x5b" + "\xe2\x03\x06\xc5\x71\xb6\x48\xa5" + "\xf0\x0f\x2d\xd5\xa2\x73\x55\x8f" + "\x01\xa7\x59\x80\x5f\x11\x6c\x40" + "\xff\xb1\xf2\xc6\x7e\x01\xbb\x1c" + "\x69\x9c\xc9\x3f\x71\x5f\x07\x7e" + "\xdf\x6f\x99\xca\x9c\xfd\xf9\xb9" + "\x49\xe7\xcc\x91\xd5\x9b\x8f\x03" + "\xae\xe7\x61\x32\xef\x41\x6c\x75" + "\x84\x9b\x8c\xce\x1d\x6b\x93\x21" + "\x41\xec\xc6\xad\x8e\x0c\x48\xa8" + "\xe2\xf5\x57\xde\xf7\x38\xfd\x4a" + "\x6f\xa7\x4a\xf9\xac\x7d\xb1\x85" + "\x7d\x6c\x95\x0a\x5a\xcf\x68\xd2" + "\xe0\x7a\x26\xd9\xc1\x6d\x3e\xc6" + "\x37\xbd\xbe\x24\x36\x77\x9f\x1b" + "\xc1\x22\xf3\x79\xae\x95\x78\x66" + "\x97\x11\xc0\x1a\xf1\xe8\x0d\x38" + "\x09\xc2\xee\xb7\xd3\x46\x7b\x59" + "\x77\x23\xe8\xb4\x92\x3d\x78\xbe" + "\xe2\x25\x63\xa5\x2a\x06\x70\x92" + "\x32\x63\xf9\x19\x21\x68\xe1\x0b" + "\x9a\xd0\xee\x21\xdb\x1f\xe0\xde" + "\x3e\x64\x02\x4d\x0e\xe0\x0a\xa9" + "\xed\x19\x8c\xa8\xbf\xe3\x2e\x75" + "\x24\x2b\xb0\xe5\x82\x6a\x1e\x6f" + "\x71\x2a\x3a\x60\xed\x06\x0d\x17" + "\xa2\xdb\x29\x1d\xae\xb2\xc4\xfb" + "\x94\x04\xd8\x58\xfc\xc4\x04\x4e" + "\xee\xc7\xc1\x0f\xe9\x9b\x63\x2d" + "\x02\x3e\x02\x67\xe5\xd8\xbb\x79" + "\xdf\xd2\xeb\x50\xe9\x0a\x02\x46" + "\xdf\x68\xcf\xe7\x2b\x0a\x56\xd6" + "\xf7\xbc\x44\xad\xb8\xb5\x5f\xeb" + "\xbc\x74\x6b\xe8\x7e\xb0\x60\xc6" + "\x0d\x96\x09\xbb\x19\xba\xe0\x3c" + "\xc4\x6c\xbf\x0f\x58\xc0\x55\x62" + "\x23\xa0\xff\xb5\x1c\xfd\x18\xe1" + "\xcf\x6d\xd3\x52\xb4\xce\xa6\xfa" + "\xaa\xfb\x1b\x0b\x42\x6d\x79\x42" + "\x48\x70\x5b\x0e\xdd\x3a\xc9\x69" + "\x8b\x73\x67\xf6\x95\xdb\x8c\xfb" + "\xfd\xb5\x08\x47\x42\x84\x9a\xfa" + "\xcc\x67\xb2\x3c\xb6\xfd\xd8\x32" + "\xd6\x04\xb6\x4a\xea\x53\x4b\xf5" + "\x94\x16\xad\xf0\x10\x2e\x2d\xb4" + "\x8b\xab\xe5\x89\xc7\x39\x12\xf3" + "\x8d\xb5\x96\x0b\x87\x5d\xa7\x7c" + "\xb0\xc2\xf6\x2e\x57\x97\x2c\xdc" + "\x54\x1c\x34\x72\xde\x0c\x68\x39" + "\x9d\x32\xa5\x75\x92\x13\x32\xea" + "\x90\x27\xbd\x5b\x1d\xb9\x21\x02" + "\x1c\xcc\xba\x97\x5e\x49\x58\xe8" + "\xac\x8b\xf3\xce\x3c\xf0\x00\xe9" + "\x6c\xae\xe9\x77\xdf\xf4\x02\xcd" + "\x55\x25\x89\x9e\x90\xf3\x6b\x8f" + "\xb7\xd6\x47\x98\x26\x2f\x31\x2f" + "\x8d\xbf\x54\xcd\x99\xeb\x80\xd7" + "\xac\xc3\x08\xc2\xa6\x32\xf1\x24" + "\x76\x7c\x4f\x78\x53\x55\xfb\x00" + "\x8a\xd6\x52\x53\x25\x45\xfb\x0a" + "\x6b\xb9\xbe\x3c\x5e\x11\xcc\x6a" + "\xdd\xfc\xa7\xc4\x79\x4d\xbd\xfb" + "\xce\x3a\xf1\x7a\xda\xeb\xfe\x64" + "\x28\x3d\x0f\xee\x80\xba\x0c\xf8" + "\xe9\x5b\x3a\xd4\xae\xc9\xf3\x0e" + "\xe8\x5d\xc5\x5c\x0b\x20\x20\xee" + "\x40\x0d\xde\x07\xa7\x14\xb4\x90" + "\xb6\xbd\x3b\xae\x7d\x2b\xa7\xc7" + "\xdc\x0b\x4c\x5d\x65\xb0\xd2\xc5" + "\x79\x61\x23\xe0\xa2\x99\x73\x55" + "\xad\xc6\xfb\xc7\x54\xb5\x98\x1f" + "\x8c\x86\xc2\x3f\xbe\x5e\xea\x64" + "\xa3\x60\x18\x9f\x80\xaf\x52\x74" + "\x1a\xfe\x22\xc2\x92\x67\x40\x02" + "\x08\xee\x67\x5b\x67\xe0\x3d\xde" + "\x7a\xaf\x8e\x28\xf3\x5e\x0e\xf4" + "\x48\x56\xaa\x85\x22\xd8\x36\xed" + "\x3b\x3d\x68\x69\x30\xbc\x71\x23" + "\xb1\x6e\x61\x03\x89\x44\x03\xf4" + "\x32\xaa\x4c\x40\x9f\x69\xfb\x70" + "\x91\xcc\x1f\x11\xbd\x76\x67\xe6" + "\x10\x8b\x29\x39\x68\xea\x4e\x6d" + "\xae\xfb\x40\xcf\xe2\xd0\x0d\x8d" + "\x6f\xed\x9b\x8d\x64\x7a\x94\x8e" + "\x32\x38\x78\xeb\x7d\x5f\xf9\x4d" + "\x13\xbe\x21\xea\x16\xe7\x5c\xee" + "\xcd\xf6\x5f\xc6\x45\xb2\x8f\x2b" + "\xb5\x93\x3e\x45\xdb\xfd\xa2\x6a" + "\xec\x83\x92\x99\x87\x47\xe0\x7c" + "\xa2\x7b\xc4\x2a\xcd\xc0\x81\x03" + "\x98\xb0\x87\xb6\x86\x13\x64\x33" + "\x4c\xd7\x99\xbf\xdb\x7b\x6e\xaa" + "\x76\xcc\xa0\x74\x1b\xa3\x6e\x83" + "\xd4\xba\x7a\x84\x9d\x91\x71\xcd" + "\x60\x2d\x56\xfd\x26\x35\xcb\xeb" + "\xac\xe9\xee\xa4\xfc\x18\x5b\x91" + "\xd5\xfe\x84\x45\xe0\xc7\xfd\x11" + "\xe9\x00\xb6\x54\xdf\xe1\x94\xde" + "\x2b\x70\x9f\x94\x7f\x15\x0e\x83" + "\x63\x10\xb3\xf5\xea\xd3\xe8\xd1" + "\xa5\xfc\x17\x19\x68\x9a\xbc\x17" + "\x30\x43\x0a\x1a\x33\x92\xd4\x2a" + "\x2e\x68\x99\xbc\x49\xf0\x68\xe3" + "\xf0\x1f\xcb\xcc\xfa\xbb\x05\x56" + "\x46\x84\x8b\x69\x83\x64\xc5\xe0" + "\xc5\x52\x99\x07\x3c\xa6\x5c\xaf" + "\xa3\xde\xd7\xdb\x43\xe6\xb7\x76" + "\x4e\x4d\xd6\x71\x60\x63\x4a\x0c" + "\x5f\xae\x25\x84\x22\x90\x5f\x26" + "\x61\x4d\x8f\xaf\xc9\x22\xf2\x05" + "\xcf\xc1\xdc\x68\xe5\x57\x8e\x24" + "\x1b\x30\x59\xca\xd7\x0d\xc3\xd3" + "\x52\x9e\x09\x3e\x0e\xaf\xdb\x5f" + "\xc7\x2b\xde\x3a\xfd\xad\x93\x04" + "\x74\x06\x89\x0e\x90\xeb\x85\xff" + "\xe6\x3c\x12\x42\xf4\xfa\x80\x75" + "\x5e\x4e\xd7\x2f\x93\x0b\x34\x41" + "\x02\x85\x68\xd0\x03\x12\xde\x92" + "\x54\x7a\x7e\xfb\x55\xe7\x88\xfb" + "\xa4\xa9\xf2\xd1\xc6\x70\x06\x37" + "\x25\xee\xa7\x6e\xd9\x89\x86\x50" + "\x2e\x07\xdb\xfb\x2a\x86\x45\x0e" + "\x91\xf4\x7c\xbb\x12\x60\xe8\x3f" + "\x71\xbe\x8f\x9d\x26\xef\xd9\x89" + "\xc4\x8f\xd8\xc5\x73\xd8\x84\xaa" + "\x2f\xad\x22\x1e\x7e\xcf\xa2\x08" + "\x23\x45\x89\x42\xa0\x30\xeb\xbf" + "\xa1\xed\xad\xd5\x76\xfa\x24\x8f" + "\x98", + .rlen = 1281, }, }; diff --git a/drivers/clk/imx/clk-imx6q.c b/drivers/clk/imx/clk-imx6q.c index c507bcad2c37..b2c1c047dc94 100644 --- a/drivers/clk/imx/clk-imx6q.c +++ b/drivers/clk/imx/clk-imx6q.c @@ -381,6 +381,9 @@ static void __init imx6q_clocks_init(struct device_node *ccm_node) clk[IMX6QDL_CLK_ASRC] = imx_clk_gate2_shared("asrc", "asrc_podf", base + 0x68, 6, &share_count_asrc); clk[IMX6QDL_CLK_ASRC_IPG] = imx_clk_gate2_shared("asrc_ipg", "ahb", base + 0x68, 6, &share_count_asrc); clk[IMX6QDL_CLK_ASRC_MEM] = imx_clk_gate2_shared("asrc_mem", "ahb", base + 0x68, 6, &share_count_asrc); + clk[IMX6QDL_CLK_CAAM_MEM] = imx_clk_gate2("caam_mem", "ahb", base + 0x68, 8); + clk[IMX6QDL_CLK_CAAM_ACLK] = imx_clk_gate2("caam_aclk", "ahb", base + 0x68, 10); + clk[IMX6QDL_CLK_CAAM_IPG] = imx_clk_gate2("caam_ipg", "ipg", base + 0x68, 12); clk[IMX6QDL_CLK_CAN1_IPG] = imx_clk_gate2("can1_ipg", "ipg", base + 0x68, 14); clk[IMX6QDL_CLK_CAN1_SERIAL] = imx_clk_gate2("can1_serial", "can_root", base + 0x68, 16); clk[IMX6QDL_CLK_CAN2_IPG] = imx_clk_gate2("can2_ipg", "ipg", base + 0x68, 18); diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig index 4044125fb5d5..07bc7aa6b224 100644 --- a/drivers/crypto/Kconfig +++ b/drivers/crypto/Kconfig @@ -480,4 +480,21 @@ config CRYPTO_DEV_IMGTEC_HASH hardware hash accelerator. Supporting MD5/SHA1/SHA224/SHA256 hashing algorithms. +config CRYPTO_DEV_SUN4I_SS + tristate "Support for Allwinner Security System cryptographic accelerator" + depends on ARCH_SUNXI + select CRYPTO_MD5 + select CRYPTO_SHA1 + select CRYPTO_AES + select CRYPTO_DES + select CRYPTO_BLKCIPHER + help + Some Allwinner SoC have a crypto accelerator named + Security System. Select this if you want to use it. + The Security System handle AES/DES/3DES ciphers in CBC mode + and SHA1 and MD5 hash algorithms. + + To compile this driver as a module, choose M here: the module + will be called sun4i-ss. + endif # CRYPTO_HW diff --git a/drivers/crypto/Makefile b/drivers/crypto/Makefile index e35c07a8da85..c3ced6fbd1b8 100644 --- a/drivers/crypto/Makefile +++ b/drivers/crypto/Makefile @@ -28,3 +28,4 @@ obj-$(CONFIG_CRYPTO_DEV_UX500) += ux500/ obj-$(CONFIG_CRYPTO_DEV_QAT) += qat/ obj-$(CONFIG_CRYPTO_DEV_QCE) += qce/ obj-$(CONFIG_CRYPTO_DEV_VMX) += vmx/ +obj-$(CONFIG_CRYPTO_DEV_SUN4I_SS) += sunxi-ss/ diff --git a/drivers/crypto/amcc/crypto4xx_core.c b/drivers/crypto/amcc/crypto4xx_core.c index 3b28e8c3de28..192a8fa325c1 100644 --- a/drivers/crypto/amcc/crypto4xx_core.c +++ b/drivers/crypto/amcc/crypto4xx_core.c @@ -1113,7 +1113,7 @@ static irqreturn_t crypto4xx_ce_interrupt_handler(int irq, void *data) struct device *dev = (struct device *)data; struct crypto4xx_core_device *core_dev = dev_get_drvdata(dev); - if (core_dev->dev->ce_base == 0) + if (!core_dev->dev->ce_base) return 0; writel(PPC4XX_INTERRUPT_CLR, diff --git a/drivers/crypto/caam/Kconfig b/drivers/crypto/caam/Kconfig index e286e285aa8a..5652a53415dc 100644 --- a/drivers/crypto/caam/Kconfig +++ b/drivers/crypto/caam/Kconfig @@ -1,6 +1,6 @@ config CRYPTO_DEV_FSL_CAAM tristate "Freescale CAAM-Multicore driver backend" - depends on FSL_SOC + depends on FSL_SOC || ARCH_MXC help Enables the driver module for Freescale's Cryptographic Accelerator and Assurance Module (CAAM), also known as the SEC version 4 (SEC4). @@ -112,6 +112,14 @@ config CRYPTO_DEV_FSL_CAAM_RNG_API To compile this as a module, choose M here: the module will be called caamrng. +config CRYPTO_DEV_FSL_CAAM_IMX + def_bool SOC_IMX6 || SOC_IMX7D + depends on CRYPTO_DEV_FSL_CAAM + +config CRYPTO_DEV_FSL_CAAM_LE + def_bool CRYPTO_DEV_FSL_CAAM_IMX || SOC_LS1021A + depends on CRYPTO_DEV_FSL_CAAM + config CRYPTO_DEV_FSL_CAAM_DEBUG bool "Enable debug output in CAAM driver" depends on CRYPTO_DEV_FSL_CAAM diff --git a/drivers/crypto/caam/caamalg.c b/drivers/crypto/caam/caamalg.c index daca933a82ec..ba79d638f782 100644 --- a/drivers/crypto/caam/caamalg.c +++ b/drivers/crypto/caam/caamalg.c @@ -68,27 +68,29 @@ #define AEAD_DESC_JOB_IO_LEN (DESC_JOB_IO_LEN + CAAM_CMD_SZ * 2) #define GCM_DESC_JOB_IO_LEN (AEAD_DESC_JOB_IO_LEN + \ CAAM_CMD_SZ * 4) +#define AUTHENC_DESC_JOB_IO_LEN (AEAD_DESC_JOB_IO_LEN + \ + CAAM_CMD_SZ * 5) /* length of descriptors text */ #define DESC_AEAD_BASE (4 * CAAM_CMD_SZ) -#define DESC_AEAD_ENC_LEN (DESC_AEAD_BASE + 15 * CAAM_CMD_SZ) -#define DESC_AEAD_DEC_LEN (DESC_AEAD_BASE + 18 * CAAM_CMD_SZ) -#define DESC_AEAD_GIVENC_LEN (DESC_AEAD_ENC_LEN + 7 * CAAM_CMD_SZ) +#define DESC_AEAD_ENC_LEN (DESC_AEAD_BASE + 11 * CAAM_CMD_SZ) +#define DESC_AEAD_DEC_LEN (DESC_AEAD_BASE + 15 * CAAM_CMD_SZ) +#define DESC_AEAD_GIVENC_LEN (DESC_AEAD_ENC_LEN + 9 * CAAM_CMD_SZ) /* Note: Nonce is counted in enckeylen */ -#define DESC_AEAD_CTR_RFC3686_LEN (6 * CAAM_CMD_SZ) +#define DESC_AEAD_CTR_RFC3686_LEN (4 * CAAM_CMD_SZ) #define DESC_AEAD_NULL_BASE (3 * CAAM_CMD_SZ) -#define DESC_AEAD_NULL_ENC_LEN (DESC_AEAD_NULL_BASE + 14 * CAAM_CMD_SZ) -#define DESC_AEAD_NULL_DEC_LEN (DESC_AEAD_NULL_BASE + 17 * CAAM_CMD_SZ) +#define DESC_AEAD_NULL_ENC_LEN (DESC_AEAD_NULL_BASE + 11 * CAAM_CMD_SZ) +#define DESC_AEAD_NULL_DEC_LEN (DESC_AEAD_NULL_BASE + 13 * CAAM_CMD_SZ) #define DESC_GCM_BASE (3 * CAAM_CMD_SZ) #define DESC_GCM_ENC_LEN (DESC_GCM_BASE + 16 * CAAM_CMD_SZ) #define DESC_GCM_DEC_LEN (DESC_GCM_BASE + 12 * CAAM_CMD_SZ) #define DESC_RFC4106_BASE (3 * CAAM_CMD_SZ) -#define DESC_RFC4106_ENC_LEN (DESC_RFC4106_BASE + 10 * CAAM_CMD_SZ) -#define DESC_RFC4106_DEC_LEN (DESC_RFC4106_BASE + 10 * CAAM_CMD_SZ) +#define DESC_RFC4106_ENC_LEN (DESC_RFC4106_BASE + 13 * CAAM_CMD_SZ) +#define DESC_RFC4106_DEC_LEN (DESC_RFC4106_BASE + 13 * CAAM_CMD_SZ) #define DESC_RFC4543_BASE (3 * CAAM_CMD_SZ) #define DESC_RFC4543_ENC_LEN (DESC_RFC4543_BASE + 11 * CAAM_CMD_SZ) @@ -111,6 +113,20 @@ #endif static struct list_head alg_list; +struct caam_alg_entry { + int class1_alg_type; + int class2_alg_type; + int alg_op; + bool rfc3686; + bool geniv; +}; + +struct caam_aead_alg { + struct aead_alg aead; + struct caam_alg_entry caam; + bool registered; +}; + /* Set DK bit in class 1 operation if shared */ static inline void append_dec_op1(u32 *desc, u32 type) { @@ -145,18 +161,6 @@ static inline void aead_append_src_dst(u32 *desc, u32 msg_type) } /* - * For aead encrypt and decrypt, read iv for both classes - */ -static inline void aead_append_ld_iv(u32 *desc, int ivsize, int ivoffset) -{ - append_seq_load(desc, ivsize, LDST_CLASS_1_CCB | - LDST_SRCDST_BYTE_CONTEXT | - (ivoffset << LDST_OFFSET_SHIFT)); - append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_CLASS2INFIFO | - (ivoffset << MOVE_OFFSET_SHIFT) | ivsize); -} - -/* * For ablkcipher encrypt and decrypt, read from req->src and * write to req->dst */ @@ -170,13 +174,6 @@ static inline void ablkcipher_append_src_dst(u32 *desc) } /* - * If all data, including src (with assoc and iv) or dst (with iv only) are - * contiguous - */ -#define GIV_SRC_CONTIG 1 -#define GIV_DST_CONTIG (1 << 1) - -/* * per-session context */ struct caam_ctx { @@ -259,7 +256,6 @@ static void init_sh_desc_key_aead(u32 *desc, struct caam_ctx *ctx, static int aead_null_set_sh_desc(struct crypto_aead *aead) { - unsigned int ivsize = crypto_aead_ivsize(aead); struct caam_ctx *ctx = crypto_aead_ctx(aead); struct device *jrdev = ctx->jrdev; bool keys_fit_inline = false; @@ -270,11 +266,11 @@ static int aead_null_set_sh_desc(struct crypto_aead *aead) * Job Descriptor and Shared Descriptors * must all fit into the 64-word Descriptor h/w Buffer */ - if (DESC_AEAD_NULL_ENC_LEN + DESC_JOB_IO_LEN + + if (DESC_AEAD_NULL_ENC_LEN + AEAD_DESC_JOB_IO_LEN + ctx->split_key_pad_len <= CAAM_DESC_BYTES_MAX) keys_fit_inline = true; - /* old_aead_encrypt shared descriptor */ + /* aead_encrypt shared descriptor */ desc = ctx->sh_desc_enc; init_sh_desc(desc, HDR_SHARE_SERIAL); @@ -291,20 +287,10 @@ static int aead_null_set_sh_desc(struct crypto_aead *aead) KEY_DEST_MDHA_SPLIT | KEY_ENC); set_jump_tgt_here(desc, key_jump_cmd); - /* cryptlen = seqoutlen - authsize */ - append_math_sub_imm_u32(desc, REG3, SEQOUTLEN, IMM, ctx->authsize); - - /* - * NULL encryption; IV is zero - * assoclen = (assoclen + cryptlen) - cryptlen - */ - append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG3, CAAM_CMD_SZ); - - /* read assoc before reading payload */ - append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG | - KEY_VLF); + /* assoclen + cryptlen = seqinlen */ + append_math_sub(desc, REG3, SEQINLEN, REG0, CAAM_CMD_SZ); - /* Prepare to read and write cryptlen bytes */ + /* Prepare to read and write cryptlen + assoclen bytes */ append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); @@ -363,7 +349,7 @@ static int aead_null_set_sh_desc(struct crypto_aead *aead) desc = ctx->sh_desc_dec; - /* old_aead_decrypt shared descriptor */ + /* aead_decrypt shared descriptor */ init_sh_desc(desc, HDR_SHARE_SERIAL); /* Skip if already shared */ @@ -382,18 +368,10 @@ static int aead_null_set_sh_desc(struct crypto_aead *aead) append_operation(desc, ctx->class2_alg_type | OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON); - /* assoclen + cryptlen = seqinlen - ivsize - authsize */ - append_math_sub_imm_u32(desc, REG3, SEQINLEN, IMM, - ctx->authsize + ivsize); - /* assoclen = (assoclen + cryptlen) - cryptlen */ + /* assoclen + cryptlen = seqoutlen */ append_math_sub(desc, REG2, SEQOUTLEN, REG0, CAAM_CMD_SZ); - append_math_sub(desc, VARSEQINLEN, REG3, REG2, CAAM_CMD_SZ); - /* read assoc before reading payload */ - append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG | - KEY_VLF); - - /* Prepare to read and write cryptlen bytes */ + /* Prepare to read and write cryptlen + assoclen bytes */ append_math_add(desc, VARSEQINLEN, ZERO, REG2, CAAM_CMD_SZ); append_math_add(desc, VARSEQOUTLEN, ZERO, REG2, CAAM_CMD_SZ); @@ -450,10 +428,10 @@ static int aead_null_set_sh_desc(struct crypto_aead *aead) static int aead_set_sh_desc(struct crypto_aead *aead) { + struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead), + struct caam_aead_alg, aead); unsigned int ivsize = crypto_aead_ivsize(aead); struct caam_ctx *ctx = crypto_aead_ctx(aead); - struct crypto_tfm *ctfm = crypto_aead_tfm(aead); - const char *alg_name = crypto_tfm_alg_name(ctfm); struct device *jrdev = ctx->jrdev; bool keys_fit_inline; u32 geniv, moveiv; @@ -461,11 +439,7 @@ static int aead_set_sh_desc(struct crypto_aead *aead) u32 *desc; const bool ctr_mode = ((ctx->class1_alg_type & OP_ALG_AAI_MASK) == OP_ALG_AAI_CTR_MOD128); - const bool is_rfc3686 = (ctr_mode && - (strstr(alg_name, "rfc3686") != NULL)); - - if (!ctx->authsize) - return 0; + const bool is_rfc3686 = alg->caam.rfc3686; /* NULL encryption / decryption */ if (!ctx->enckeylen) @@ -486,18 +460,21 @@ static int aead_set_sh_desc(struct crypto_aead *aead) if (is_rfc3686) ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE; + if (alg->caam.geniv) + goto skip_enc; + /* * Job Descriptor and Shared Descriptors * must all fit into the 64-word Descriptor h/w Buffer */ keys_fit_inline = false; - if (DESC_AEAD_ENC_LEN + DESC_JOB_IO_LEN + + if (DESC_AEAD_ENC_LEN + AUTHENC_DESC_JOB_IO_LEN + ctx->split_key_pad_len + ctx->enckeylen + (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0) <= CAAM_DESC_BYTES_MAX) keys_fit_inline = true; - /* old_aead_encrypt shared descriptor */ + /* aead_encrypt shared descriptor */ desc = ctx->sh_desc_enc; /* Note: Context registers are saved. */ @@ -507,19 +484,16 @@ static int aead_set_sh_desc(struct crypto_aead *aead) append_operation(desc, ctx->class2_alg_type | OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); - /* cryptlen = seqoutlen - authsize */ - append_math_sub_imm_u32(desc, REG3, SEQOUTLEN, IMM, ctx->authsize); - - /* assoclen + cryptlen = seqinlen - ivsize */ - append_math_sub_imm_u32(desc, REG2, SEQINLEN, IMM, ivsize); + /* Read and write assoclen bytes */ + append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); + append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); - /* assoclen = (assoclen + cryptlen) - cryptlen */ - append_math_sub(desc, VARSEQINLEN, REG2, REG3, CAAM_CMD_SZ); + /* Skip assoc data */ + append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); /* read assoc before reading payload */ append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG | - KEY_VLF); - aead_append_ld_iv(desc, ivsize, ctx1_iv_off); + FIFOLDST_VLF); /* Load Counter into CONTEXT1 reg */ if (is_rfc3686) @@ -534,8 +508,8 @@ static int aead_set_sh_desc(struct crypto_aead *aead) OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); /* Read and write cryptlen bytes */ - append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); - append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); + append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); + append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ); aead_append_src_dst(desc, FIFOLD_TYPE_MSG1OUT2); /* Write ICV */ @@ -555,18 +529,19 @@ static int aead_set_sh_desc(struct crypto_aead *aead) desc_bytes(desc), 1); #endif +skip_enc: /* * Job Descriptor and Shared Descriptors * must all fit into the 64-word Descriptor h/w Buffer */ keys_fit_inline = false; - if (DESC_AEAD_DEC_LEN + DESC_JOB_IO_LEN + + if (DESC_AEAD_DEC_LEN + AUTHENC_DESC_JOB_IO_LEN + ctx->split_key_pad_len + ctx->enckeylen + (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0) <= CAAM_DESC_BYTES_MAX) keys_fit_inline = true; - /* old_aead_decrypt shared descriptor */ + /* aead_decrypt shared descriptor */ desc = ctx->sh_desc_dec; /* Note: Context registers are saved. */ @@ -576,19 +551,17 @@ static int aead_set_sh_desc(struct crypto_aead *aead) append_operation(desc, ctx->class2_alg_type | OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON); - /* assoclen + cryptlen = seqinlen - ivsize - authsize */ - append_math_sub_imm_u32(desc, REG3, SEQINLEN, IMM, - ctx->authsize + ivsize); - /* assoclen = (assoclen + cryptlen) - cryptlen */ - append_math_sub(desc, REG2, SEQOUTLEN, REG0, CAAM_CMD_SZ); - append_math_sub(desc, VARSEQINLEN, REG3, REG2, CAAM_CMD_SZ); + /* Read and write assoclen bytes */ + append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); + append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); + + /* Skip assoc data */ + append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); /* read assoc before reading payload */ append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG | KEY_VLF); - aead_append_ld_iv(desc, ivsize, ctx1_iv_off); - /* Load Counter into CONTEXT1 reg */ if (is_rfc3686) append_load_imm_u32(desc, be32_to_cpu(1), LDST_IMM | @@ -605,8 +578,8 @@ static int aead_set_sh_desc(struct crypto_aead *aead) append_dec_op1(desc, ctx->class1_alg_type); /* Read and write cryptlen bytes */ - append_math_add(desc, VARSEQINLEN, ZERO, REG2, CAAM_CMD_SZ); - append_math_add(desc, VARSEQOUTLEN, ZERO, REG2, CAAM_CMD_SZ); + append_math_add(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); + append_math_add(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); aead_append_src_dst(desc, FIFOLD_TYPE_MSG); /* Load ICV */ @@ -626,12 +599,15 @@ static int aead_set_sh_desc(struct crypto_aead *aead) desc_bytes(desc), 1); #endif + if (!alg->caam.geniv) + goto skip_givenc; + /* * Job Descriptor and Shared Descriptors * must all fit into the 64-word Descriptor h/w Buffer */ keys_fit_inline = false; - if (DESC_AEAD_GIVENC_LEN + DESC_JOB_IO_LEN + + if (DESC_AEAD_GIVENC_LEN + AUTHENC_DESC_JOB_IO_LEN + ctx->split_key_pad_len + ctx->enckeylen + (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0) <= CAAM_DESC_BYTES_MAX) @@ -643,6 +619,9 @@ static int aead_set_sh_desc(struct crypto_aead *aead) /* Note: Context registers are saved. */ init_sh_desc_key_aead(desc, ctx, keys_fit_inline, is_rfc3686); + if (is_rfc3686) + goto copy_iv; + /* Generate IV */ geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO | NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 | @@ -656,6 +635,7 @@ static int aead_set_sh_desc(struct crypto_aead *aead) (ivsize << MOVE_LEN_SHIFT)); append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO); +copy_iv: /* Copy IV to class 1 context */ append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_OUTFIFO | (ctx1_iv_off << MOVE_OFFSET_SHIFT) | @@ -668,8 +648,12 @@ static int aead_set_sh_desc(struct crypto_aead *aead) /* ivsize + cryptlen = seqoutlen - authsize */ append_math_sub_imm_u32(desc, REG3, SEQOUTLEN, IMM, ctx->authsize); - /* assoclen = seqinlen - (ivsize + cryptlen) */ - append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG3, CAAM_CMD_SZ); + /* Read and write assoclen bytes */ + append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); + append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); + + /* Skip assoc data */ + append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); /* read assoc before reading payload */ append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG | @@ -710,9 +694,9 @@ static int aead_set_sh_desc(struct crypto_aead *aead) append_seq_store(desc, ctx->authsize, LDST_CLASS_2_CCB | LDST_SRCDST_BYTE_CONTEXT); - ctx->sh_desc_givenc_dma = dma_map_single(jrdev, desc, - desc_bytes(desc), - DMA_TO_DEVICE); + ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc, + desc_bytes(desc), + DMA_TO_DEVICE); if (dma_mapping_error(jrdev, ctx->sh_desc_givenc_dma)) { dev_err(jrdev, "unable to map shared descriptor\n"); return -ENOMEM; @@ -723,6 +707,7 @@ static int aead_set_sh_desc(struct crypto_aead *aead) desc_bytes(desc), 1); #endif +skip_givenc: return 0; } @@ -976,22 +961,28 @@ static int rfc4106_set_sh_desc(struct crypto_aead *aead) append_operation(desc, ctx->class1_alg_type | OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); - append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); + append_math_sub_imm_u32(desc, VARSEQINLEN, REG3, IMM, 8); append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); - /* Skip assoc data */ - append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); - /* Read assoc data */ append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1); - /* cryptlen = seqoutlen - assoclen */ - append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ); + /* Skip IV */ + append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP); /* Will read cryptlen bytes */ append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); + /* Workaround for erratum A-005473 (simultaneous SEQ FIFO skips) */ + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_MSG); + + /* Skip assoc data */ + append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); + + /* cryptlen = seqoutlen - assoclen */ + append_math_sub(desc, VARSEQOUTLEN, VARSEQINLEN, REG0, CAAM_CMD_SZ); + /* Write encrypted data */ append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); @@ -1044,21 +1035,27 @@ static int rfc4106_set_sh_desc(struct crypto_aead *aead) append_operation(desc, ctx->class1_alg_type | OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON); - append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); + append_math_sub_imm_u32(desc, VARSEQINLEN, REG3, IMM, 8); append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); - /* Skip assoc data */ - append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); - /* Read assoc data */ append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1); - /* Will write cryptlen bytes */ - append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); + /* Skip IV */ + append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP); /* Will read cryptlen bytes */ - append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); + append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG3, CAAM_CMD_SZ); + + /* Workaround for erratum A-005473 (simultaneous SEQ FIFO skips) */ + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_MSG); + + /* Skip assoc data */ + append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); + + /* Will write cryptlen bytes */ + append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); /* Store payload data */ append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); @@ -1793,22 +1790,6 @@ static void aead_unmap(struct device *dev, edesc->sec4_sg_dma, edesc->sec4_sg_bytes); } -static void old_aead_unmap(struct device *dev, - struct aead_edesc *edesc, - struct aead_request *req) -{ - struct crypto_aead *aead = crypto_aead_reqtfm(req); - int ivsize = crypto_aead_ivsize(aead); - - dma_unmap_sg_chained(dev, req->assoc, edesc->assoc_nents, - DMA_TO_DEVICE, edesc->assoc_chained); - - caam_unmap(dev, req->src, req->dst, - edesc->src_nents, edesc->src_chained, edesc->dst_nents, - edesc->dst_chained, edesc->iv_dma, ivsize, - edesc->sec4_sg_dma, edesc->sec4_sg_bytes); -} - static void ablkcipher_unmap(struct device *dev, struct ablkcipher_edesc *edesc, struct ablkcipher_request *req) @@ -1844,45 +1825,6 @@ static void aead_encrypt_done(struct device *jrdev, u32 *desc, u32 err, aead_request_complete(req, err); } -static void old_aead_encrypt_done(struct device *jrdev, u32 *desc, u32 err, - void *context) -{ - struct aead_request *req = context; - struct aead_edesc *edesc; -#ifdef DEBUG - struct crypto_aead *aead = crypto_aead_reqtfm(req); - struct caam_ctx *ctx = crypto_aead_ctx(aead); - int ivsize = crypto_aead_ivsize(aead); - - dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); -#endif - - edesc = (struct aead_edesc *)((char *)desc - - offsetof(struct aead_edesc, hw_desc)); - - if (err) - caam_jr_strstatus(jrdev, err); - - old_aead_unmap(jrdev, edesc, req); - -#ifdef DEBUG - print_hex_dump(KERN_ERR, "assoc @"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->assoc), - req->assoclen , 1); - print_hex_dump(KERN_ERR, "dstiv @"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src) - ivsize, - edesc->src_nents ? 100 : ivsize, 1); - print_hex_dump(KERN_ERR, "dst @"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src), - edesc->src_nents ? 100 : req->cryptlen + - ctx->authsize + 4, 1); -#endif - - kfree(edesc); - - aead_request_complete(req, err); -} - static void aead_decrypt_done(struct device *jrdev, u32 *desc, u32 err, void *context) { @@ -1911,62 +1853,6 @@ static void aead_decrypt_done(struct device *jrdev, u32 *desc, u32 err, aead_request_complete(req, err); } -static void old_aead_decrypt_done(struct device *jrdev, u32 *desc, u32 err, - void *context) -{ - struct aead_request *req = context; - struct aead_edesc *edesc; -#ifdef DEBUG - struct crypto_aead *aead = crypto_aead_reqtfm(req); - struct caam_ctx *ctx = crypto_aead_ctx(aead); - int ivsize = crypto_aead_ivsize(aead); - - dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); -#endif - - edesc = (struct aead_edesc *)((char *)desc - - offsetof(struct aead_edesc, hw_desc)); - -#ifdef DEBUG - print_hex_dump(KERN_ERR, "dstiv @"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, req->iv, - ivsize, 1); - print_hex_dump(KERN_ERR, "dst @"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->dst), - req->cryptlen - ctx->authsize, 1); -#endif - - if (err) - caam_jr_strstatus(jrdev, err); - - old_aead_unmap(jrdev, edesc, req); - - /* - * verify hw auth check passed else return -EBADMSG - */ - if ((err & JRSTA_CCBERR_ERRID_MASK) == JRSTA_CCBERR_ERRID_ICVCHK) - err = -EBADMSG; - -#ifdef DEBUG - print_hex_dump(KERN_ERR, "iphdrout@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, - ((char *)sg_virt(req->assoc) - sizeof(struct iphdr)), - sizeof(struct iphdr) + req->assoclen + - ((req->cryptlen > 1500) ? 1500 : req->cryptlen) + - ctx->authsize + 36, 1); - if (!err && edesc->sec4_sg_bytes) { - struct scatterlist *sg = sg_last(req->src, edesc->src_nents); - print_hex_dump(KERN_ERR, "sglastout@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(sg), - sg->length + ctx->authsize + 16, 1); - } -#endif - - kfree(edesc); - - aead_request_complete(req, err); -} - static void ablkcipher_encrypt_done(struct device *jrdev, u32 *desc, u32 err, void *context) { @@ -2035,91 +1921,6 @@ static void ablkcipher_decrypt_done(struct device *jrdev, u32 *desc, u32 err, /* * Fill in aead job descriptor */ -static void old_init_aead_job(u32 *sh_desc, dma_addr_t ptr, - struct aead_edesc *edesc, - struct aead_request *req, - bool all_contig, bool encrypt) -{ - struct crypto_aead *aead = crypto_aead_reqtfm(req); - struct caam_ctx *ctx = crypto_aead_ctx(aead); - int ivsize = crypto_aead_ivsize(aead); - int authsize = ctx->authsize; - u32 *desc = edesc->hw_desc; - u32 out_options = 0, in_options; - dma_addr_t dst_dma, src_dma; - int len, sec4_sg_index = 0; - bool is_gcm = false; - -#ifdef DEBUG - debug("assoclen %d cryptlen %d authsize %d\n", - req->assoclen, req->cryptlen, authsize); - print_hex_dump(KERN_ERR, "assoc @"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->assoc), - req->assoclen , 1); - print_hex_dump(KERN_ERR, "presciv@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, req->iv, - edesc->src_nents ? 100 : ivsize, 1); - print_hex_dump(KERN_ERR, "src @"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src), - edesc->src_nents ? 100 : req->cryptlen, 1); - print_hex_dump(KERN_ERR, "shrdesc@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, sh_desc, - desc_bytes(sh_desc), 1); -#endif - - if (((ctx->class1_alg_type & OP_ALG_ALGSEL_MASK) == - OP_ALG_ALGSEL_AES) && - ((ctx->class1_alg_type & OP_ALG_AAI_MASK) == OP_ALG_AAI_GCM)) - is_gcm = true; - - len = desc_len(sh_desc); - init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE); - - if (all_contig) { - if (is_gcm) - src_dma = edesc->iv_dma; - else - src_dma = sg_dma_address(req->assoc); - in_options = 0; - } else { - src_dma = edesc->sec4_sg_dma; - sec4_sg_index += (edesc->assoc_nents ? : 1) + 1 + - (edesc->src_nents ? : 1); - in_options = LDST_SGF; - } - - append_seq_in_ptr(desc, src_dma, req->assoclen + ivsize + req->cryptlen, - in_options); - - if (likely(req->src == req->dst)) { - if (all_contig) { - dst_dma = sg_dma_address(req->src); - } else { - dst_dma = src_dma + sizeof(struct sec4_sg_entry) * - ((edesc->assoc_nents ? : 1) + 1); - out_options = LDST_SGF; - } - } else { - if (!edesc->dst_nents) { - dst_dma = sg_dma_address(req->dst); - } else { - dst_dma = edesc->sec4_sg_dma + - sec4_sg_index * - sizeof(struct sec4_sg_entry); - out_options = LDST_SGF; - } - } - if (encrypt) - append_seq_out_ptr(desc, dst_dma, req->cryptlen + authsize, - out_options); - else - append_seq_out_ptr(desc, dst_dma, req->cryptlen - authsize, - out_options); -} - -/* - * Fill in aead job descriptor - */ static void init_aead_job(struct aead_request *req, struct aead_edesc *edesc, bool all_contig, bool encrypt) @@ -2208,80 +2009,43 @@ static void init_gcm_job(struct aead_request *req, /* End of blank commands */ } -/* - * Fill in aead givencrypt job descriptor - */ -static void init_aead_giv_job(u32 *sh_desc, dma_addr_t ptr, - struct aead_edesc *edesc, - struct aead_request *req, - int contig) +static void init_authenc_job(struct aead_request *req, + struct aead_edesc *edesc, + bool all_contig, bool encrypt) { struct crypto_aead *aead = crypto_aead_reqtfm(req); + struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead), + struct caam_aead_alg, aead); + unsigned int ivsize = crypto_aead_ivsize(aead); struct caam_ctx *ctx = crypto_aead_ctx(aead); - int ivsize = crypto_aead_ivsize(aead); - int authsize = ctx->authsize; + const bool ctr_mode = ((ctx->class1_alg_type & OP_ALG_AAI_MASK) == + OP_ALG_AAI_CTR_MOD128); + const bool is_rfc3686 = alg->caam.rfc3686; u32 *desc = edesc->hw_desc; - u32 out_options = 0, in_options; - dma_addr_t dst_dma, src_dma; - int len, sec4_sg_index = 0; - bool is_gcm = false; - -#ifdef DEBUG - debug("assoclen %d cryptlen %d authsize %d\n", - req->assoclen, req->cryptlen, authsize); - print_hex_dump(KERN_ERR, "assoc @"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->assoc), - req->assoclen , 1); - print_hex_dump(KERN_ERR, "presciv@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, req->iv, ivsize, 1); - print_hex_dump(KERN_ERR, "src @"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src), - edesc->src_nents > 1 ? 100 : req->cryptlen, 1); - print_hex_dump(KERN_ERR, "shrdesc@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, sh_desc, - desc_bytes(sh_desc), 1); -#endif + u32 ivoffset = 0; - if (((ctx->class1_alg_type & OP_ALG_ALGSEL_MASK) == - OP_ALG_ALGSEL_AES) && - ((ctx->class1_alg_type & OP_ALG_AAI_MASK) == OP_ALG_AAI_GCM)) - is_gcm = true; - - len = desc_len(sh_desc); - init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE); + /* + * AES-CTR needs to load IV in CONTEXT1 reg + * at an offset of 128bits (16bytes) + * CONTEXT1[255:128] = IV + */ + if (ctr_mode) + ivoffset = 16; - if (contig & GIV_SRC_CONTIG) { - if (is_gcm) - src_dma = edesc->iv_dma; - else - src_dma = sg_dma_address(req->assoc); - in_options = 0; - } else { - src_dma = edesc->sec4_sg_dma; - sec4_sg_index += edesc->assoc_nents + 1 + edesc->src_nents; - in_options = LDST_SGF; - } - append_seq_in_ptr(desc, src_dma, req->assoclen + ivsize + req->cryptlen, - in_options); + /* + * RFC3686 specific: + * CONTEXT1[255:128] = {NONCE, IV, COUNTER} + */ + if (is_rfc3686) + ivoffset = 16 + CTR_RFC3686_NONCE_SIZE; - if (contig & GIV_DST_CONTIG) { - dst_dma = edesc->iv_dma; - } else { - if (likely(req->src == req->dst)) { - dst_dma = src_dma + sizeof(struct sec4_sg_entry) * - (edesc->assoc_nents + - (is_gcm ? 1 + edesc->src_nents : 0)); - out_options = LDST_SGF; - } else { - dst_dma = edesc->sec4_sg_dma + - sec4_sg_index * - sizeof(struct sec4_sg_entry); - out_options = LDST_SGF; - } - } + init_aead_job(req, edesc, all_contig, encrypt); - append_seq_out_ptr(desc, dst_dma, ivsize + req->cryptlen + authsize, - out_options); + if (ivsize && (is_rfc3686 || !(alg->caam.geniv && encrypt))) + append_load_as_imm(desc, req->iv, ivsize, + LDST_CLASS_1_CCB | + LDST_SRCDST_BYTE_CONTEXT | + (ivoffset << LDST_OFFSET_SHIFT)); } /* @@ -2392,150 +2156,6 @@ static void init_ablkcipher_giv_job(u32 *sh_desc, dma_addr_t ptr, /* * allocate and map the aead extended descriptor */ -static struct aead_edesc *old_aead_edesc_alloc(struct aead_request *req, - int desc_bytes, - bool *all_contig_ptr, - bool encrypt) -{ - struct crypto_aead *aead = crypto_aead_reqtfm(req); - struct caam_ctx *ctx = crypto_aead_ctx(aead); - struct device *jrdev = ctx->jrdev; - gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | - CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC; - int assoc_nents, src_nents, dst_nents = 0; - struct aead_edesc *edesc; - dma_addr_t iv_dma = 0; - int sgc; - bool all_contig = true; - bool assoc_chained = false, src_chained = false, dst_chained = false; - int ivsize = crypto_aead_ivsize(aead); - int sec4_sg_index, sec4_sg_len = 0, sec4_sg_bytes; - unsigned int authsize = ctx->authsize; - bool is_gcm = false; - - assoc_nents = sg_count(req->assoc, req->assoclen, &assoc_chained); - - if (unlikely(req->dst != req->src)) { - src_nents = sg_count(req->src, req->cryptlen, &src_chained); - dst_nents = sg_count(req->dst, - req->cryptlen + - (encrypt ? authsize : (-authsize)), - &dst_chained); - } else { - src_nents = sg_count(req->src, - req->cryptlen + - (encrypt ? authsize : 0), - &src_chained); - } - - sgc = dma_map_sg_chained(jrdev, req->assoc, assoc_nents ? : 1, - DMA_TO_DEVICE, assoc_chained); - if (likely(req->src == req->dst)) { - sgc = dma_map_sg_chained(jrdev, req->src, src_nents ? : 1, - DMA_BIDIRECTIONAL, src_chained); - } else { - sgc = dma_map_sg_chained(jrdev, req->src, src_nents ? : 1, - DMA_TO_DEVICE, src_chained); - sgc = dma_map_sg_chained(jrdev, req->dst, dst_nents ? : 1, - DMA_FROM_DEVICE, dst_chained); - } - - iv_dma = dma_map_single(jrdev, req->iv, ivsize, DMA_TO_DEVICE); - if (dma_mapping_error(jrdev, iv_dma)) { - dev_err(jrdev, "unable to map IV\n"); - return ERR_PTR(-ENOMEM); - } - - if (((ctx->class1_alg_type & OP_ALG_ALGSEL_MASK) == - OP_ALG_ALGSEL_AES) && - ((ctx->class1_alg_type & OP_ALG_AAI_MASK) == OP_ALG_AAI_GCM)) - is_gcm = true; - - /* - * Check if data are contiguous. - * GCM expected input sequence: IV, AAD, text - * All other - expected input sequence: AAD, IV, text - */ - if (is_gcm) - all_contig = (!assoc_nents && - iv_dma + ivsize == sg_dma_address(req->assoc) && - !src_nents && sg_dma_address(req->assoc) + - req->assoclen == sg_dma_address(req->src)); - else - all_contig = (!assoc_nents && sg_dma_address(req->assoc) + - req->assoclen == iv_dma && !src_nents && - iv_dma + ivsize == sg_dma_address(req->src)); - if (!all_contig) { - assoc_nents = assoc_nents ? : 1; - src_nents = src_nents ? : 1; - sec4_sg_len = assoc_nents + 1 + src_nents; - } - - sec4_sg_len += dst_nents; - - sec4_sg_bytes = sec4_sg_len * sizeof(struct sec4_sg_entry); - - /* allocate space for base edesc and hw desc commands, link tables */ - edesc = kmalloc(sizeof(struct aead_edesc) + desc_bytes + - sec4_sg_bytes, GFP_DMA | flags); - if (!edesc) { - dev_err(jrdev, "could not allocate extended descriptor\n"); - return ERR_PTR(-ENOMEM); - } - - edesc->assoc_nents = assoc_nents; - edesc->assoc_chained = assoc_chained; - edesc->src_nents = src_nents; - edesc->src_chained = src_chained; - edesc->dst_nents = dst_nents; - edesc->dst_chained = dst_chained; - edesc->iv_dma = iv_dma; - edesc->sec4_sg_bytes = sec4_sg_bytes; - edesc->sec4_sg = (void *)edesc + sizeof(struct aead_edesc) + - desc_bytes; - *all_contig_ptr = all_contig; - - sec4_sg_index = 0; - if (!all_contig) { - if (!is_gcm) { - sg_to_sec4_sg_len(req->assoc, req->assoclen, - edesc->sec4_sg + sec4_sg_index); - sec4_sg_index += assoc_nents; - } - - dma_to_sec4_sg_one(edesc->sec4_sg + sec4_sg_index, - iv_dma, ivsize, 0); - sec4_sg_index += 1; - - if (is_gcm) { - sg_to_sec4_sg_len(req->assoc, req->assoclen, - edesc->sec4_sg + sec4_sg_index); - sec4_sg_index += assoc_nents; - } - - sg_to_sec4_sg_last(req->src, - src_nents, - edesc->sec4_sg + - sec4_sg_index, 0); - sec4_sg_index += src_nents; - } - if (dst_nents) { - sg_to_sec4_sg_last(req->dst, dst_nents, - edesc->sec4_sg + sec4_sg_index, 0); - } - edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, - sec4_sg_bytes, DMA_TO_DEVICE); - if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) { - dev_err(jrdev, "unable to map S/G table\n"); - return ERR_PTR(-ENOMEM); - } - - return edesc; -} - -/* - * allocate and map the aead extended descriptor - */ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, int desc_bytes, bool *all_contig_ptr, bool encrypt) @@ -2579,8 +2199,8 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, sec4_sg_bytes = sec4_sg_len * sizeof(struct sec4_sg_entry); /* allocate space for base edesc and hw desc commands, link tables */ - edesc = kzalloc(sizeof(struct aead_edesc) + desc_bytes + - sec4_sg_bytes, GFP_DMA | flags); + edesc = kzalloc(sizeof(*edesc) + desc_bytes + sec4_sg_bytes, + GFP_DMA | flags); if (!edesc) { dev_err(jrdev, "could not allocate extended descriptor\n"); return ERR_PTR(-ENOMEM); @@ -2685,7 +2305,15 @@ static int gcm_encrypt(struct aead_request *req) return ret; } -static int old_aead_encrypt(struct aead_request *req) +static int ipsec_gcm_encrypt(struct aead_request *req) +{ + if (req->assoclen < 8) + return -EINVAL; + + return gcm_encrypt(req); +} + +static int aead_encrypt(struct aead_request *req) { struct aead_edesc *edesc; struct crypto_aead *aead = crypto_aead_reqtfm(req); @@ -2696,14 +2324,13 @@ static int old_aead_encrypt(struct aead_request *req) int ret = 0; /* allocate extended descriptor */ - edesc = old_aead_edesc_alloc(req, DESC_JOB_IO_LEN * - CAAM_CMD_SZ, &all_contig, true); + edesc = aead_edesc_alloc(req, AUTHENC_DESC_JOB_IO_LEN, + &all_contig, true); if (IS_ERR(edesc)) return PTR_ERR(edesc); /* Create and submit job descriptor */ - old_init_aead_job(ctx->sh_desc_enc, ctx->sh_desc_enc_dma, edesc, req, - all_contig, true); + init_authenc_job(req, edesc, all_contig, true); #ifdef DEBUG print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, @@ -2711,11 +2338,11 @@ static int old_aead_encrypt(struct aead_request *req) #endif desc = edesc->hw_desc; - ret = caam_jr_enqueue(jrdev, desc, old_aead_encrypt_done, req); + ret = caam_jr_enqueue(jrdev, desc, aead_encrypt_done, req); if (!ret) { ret = -EINPROGRESS; } else { - old_aead_unmap(jrdev, edesc, req); + aead_unmap(jrdev, edesc, req); kfree(edesc); } @@ -2757,7 +2384,15 @@ static int gcm_decrypt(struct aead_request *req) return ret; } -static int old_aead_decrypt(struct aead_request *req) +static int ipsec_gcm_decrypt(struct aead_request *req) +{ + if (req->assoclen < 8) + return -EINVAL; + + return gcm_decrypt(req); +} + +static int aead_decrypt(struct aead_request *req) { struct aead_edesc *edesc; struct crypto_aead *aead = crypto_aead_reqtfm(req); @@ -2768,20 +2403,19 @@ static int old_aead_decrypt(struct aead_request *req) int ret = 0; /* allocate extended descriptor */ - edesc = old_aead_edesc_alloc(req, DESC_JOB_IO_LEN * - CAAM_CMD_SZ, &all_contig, false); + edesc = aead_edesc_alloc(req, AUTHENC_DESC_JOB_IO_LEN, + &all_contig, false); if (IS_ERR(edesc)) return PTR_ERR(edesc); #ifdef DEBUG print_hex_dump(KERN_ERR, "dec src@"__stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src), - req->cryptlen, 1); + req->assoclen + req->cryptlen, 1); #endif /* Create and submit job descriptor*/ - old_init_aead_job(ctx->sh_desc_dec, - ctx->sh_desc_dec_dma, edesc, req, all_contig, false); + init_authenc_job(req, edesc, all_contig, false); #ifdef DEBUG print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, @@ -2789,232 +2423,29 @@ static int old_aead_decrypt(struct aead_request *req) #endif desc = edesc->hw_desc; - ret = caam_jr_enqueue(jrdev, desc, old_aead_decrypt_done, req); + ret = caam_jr_enqueue(jrdev, desc, aead_decrypt_done, req); if (!ret) { ret = -EINPROGRESS; } else { - old_aead_unmap(jrdev, edesc, req); + aead_unmap(jrdev, edesc, req); kfree(edesc); } return ret; } -/* - * allocate and map the aead extended descriptor for aead givencrypt - */ -static struct aead_edesc *aead_giv_edesc_alloc(struct aead_givcrypt_request - *greq, int desc_bytes, - u32 *contig_ptr) -{ - struct aead_request *req = &greq->areq; - struct crypto_aead *aead = crypto_aead_reqtfm(req); - struct caam_ctx *ctx = crypto_aead_ctx(aead); - struct device *jrdev = ctx->jrdev; - gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | - CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC; - int assoc_nents, src_nents, dst_nents = 0; - struct aead_edesc *edesc; - dma_addr_t iv_dma = 0; - int sgc; - u32 contig = GIV_SRC_CONTIG | GIV_DST_CONTIG; - int ivsize = crypto_aead_ivsize(aead); - bool assoc_chained = false, src_chained = false, dst_chained = false; - int sec4_sg_index, sec4_sg_len = 0, sec4_sg_bytes; - bool is_gcm = false; - - assoc_nents = sg_count(req->assoc, req->assoclen, &assoc_chained); - src_nents = sg_count(req->src, req->cryptlen, &src_chained); - - if (unlikely(req->dst != req->src)) - dst_nents = sg_count(req->dst, req->cryptlen + ctx->authsize, - &dst_chained); - - sgc = dma_map_sg_chained(jrdev, req->assoc, assoc_nents ? : 1, - DMA_TO_DEVICE, assoc_chained); - if (likely(req->src == req->dst)) { - sgc = dma_map_sg_chained(jrdev, req->src, src_nents ? : 1, - DMA_BIDIRECTIONAL, src_chained); - } else { - sgc = dma_map_sg_chained(jrdev, req->src, src_nents ? : 1, - DMA_TO_DEVICE, src_chained); - sgc = dma_map_sg_chained(jrdev, req->dst, dst_nents ? : 1, - DMA_FROM_DEVICE, dst_chained); - } - - iv_dma = dma_map_single(jrdev, greq->giv, ivsize, DMA_TO_DEVICE); - if (dma_mapping_error(jrdev, iv_dma)) { - dev_err(jrdev, "unable to map IV\n"); - return ERR_PTR(-ENOMEM); - } - - if (((ctx->class1_alg_type & OP_ALG_ALGSEL_MASK) == - OP_ALG_ALGSEL_AES) && - ((ctx->class1_alg_type & OP_ALG_AAI_MASK) == OP_ALG_AAI_GCM)) - is_gcm = true; - - /* - * Check if data are contiguous. - * GCM expected input sequence: IV, AAD, text - * All other - expected input sequence: AAD, IV, text - */ - - if (is_gcm) { - if (assoc_nents || iv_dma + ivsize != - sg_dma_address(req->assoc) || src_nents || - sg_dma_address(req->assoc) + req->assoclen != - sg_dma_address(req->src)) - contig &= ~GIV_SRC_CONTIG; - } else { - if (assoc_nents || - sg_dma_address(req->assoc) + req->assoclen != iv_dma || - src_nents || iv_dma + ivsize != sg_dma_address(req->src)) - contig &= ~GIV_SRC_CONTIG; - } - - if (dst_nents || iv_dma + ivsize != sg_dma_address(req->dst)) - contig &= ~GIV_DST_CONTIG; - - if (!(contig & GIV_SRC_CONTIG)) { - assoc_nents = assoc_nents ? : 1; - src_nents = src_nents ? : 1; - sec4_sg_len += assoc_nents + 1 + src_nents; - if (req->src == req->dst && - (src_nents || iv_dma + ivsize != sg_dma_address(req->src))) - contig &= ~GIV_DST_CONTIG; - } - - /* - * Add new sg entries for GCM output sequence. - * Expected output sequence: IV, encrypted text. - */ - if (is_gcm && req->src == req->dst && !(contig & GIV_DST_CONTIG)) - sec4_sg_len += 1 + src_nents; - - if (unlikely(req->src != req->dst)) { - dst_nents = dst_nents ? : 1; - sec4_sg_len += 1 + dst_nents; - } - - sec4_sg_bytes = sec4_sg_len * sizeof(struct sec4_sg_entry); - - /* allocate space for base edesc and hw desc commands, link tables */ - edesc = kmalloc(sizeof(struct aead_edesc) + desc_bytes + - sec4_sg_bytes, GFP_DMA | flags); - if (!edesc) { - dev_err(jrdev, "could not allocate extended descriptor\n"); - return ERR_PTR(-ENOMEM); - } - - edesc->assoc_nents = assoc_nents; - edesc->assoc_chained = assoc_chained; - edesc->src_nents = src_nents; - edesc->src_chained = src_chained; - edesc->dst_nents = dst_nents; - edesc->dst_chained = dst_chained; - edesc->iv_dma = iv_dma; - edesc->sec4_sg_bytes = sec4_sg_bytes; - edesc->sec4_sg = (void *)edesc + sizeof(struct aead_edesc) + - desc_bytes; - *contig_ptr = contig; - - sec4_sg_index = 0; - if (!(contig & GIV_SRC_CONTIG)) { - if (!is_gcm) { - sg_to_sec4_sg_len(req->assoc, req->assoclen, - edesc->sec4_sg + sec4_sg_index); - sec4_sg_index += assoc_nents; - } - - dma_to_sec4_sg_one(edesc->sec4_sg + sec4_sg_index, - iv_dma, ivsize, 0); - sec4_sg_index += 1; - - if (is_gcm) { - sg_to_sec4_sg_len(req->assoc, req->assoclen, - edesc->sec4_sg + sec4_sg_index); - sec4_sg_index += assoc_nents; - } - - sg_to_sec4_sg_last(req->src, src_nents, - edesc->sec4_sg + - sec4_sg_index, 0); - sec4_sg_index += src_nents; - } - - if (is_gcm && req->src == req->dst && !(contig & GIV_DST_CONTIG)) { - dma_to_sec4_sg_one(edesc->sec4_sg + sec4_sg_index, - iv_dma, ivsize, 0); - sec4_sg_index += 1; - sg_to_sec4_sg_last(req->src, src_nents, - edesc->sec4_sg + sec4_sg_index, 0); - } - - if (unlikely(req->src != req->dst && !(contig & GIV_DST_CONTIG))) { - dma_to_sec4_sg_one(edesc->sec4_sg + sec4_sg_index, - iv_dma, ivsize, 0); - sec4_sg_index += 1; - sg_to_sec4_sg_last(req->dst, dst_nents, - edesc->sec4_sg + sec4_sg_index, 0); - } - edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, - sec4_sg_bytes, DMA_TO_DEVICE); - if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) { - dev_err(jrdev, "unable to map S/G table\n"); - return ERR_PTR(-ENOMEM); - } - - return edesc; -} - -static int old_aead_givencrypt(struct aead_givcrypt_request *areq) +static int aead_givdecrypt(struct aead_request *req) { - struct aead_request *req = &areq->areq; - struct aead_edesc *edesc; struct crypto_aead *aead = crypto_aead_reqtfm(req); - struct caam_ctx *ctx = crypto_aead_ctx(aead); - struct device *jrdev = ctx->jrdev; - u32 contig; - u32 *desc; - int ret = 0; - - /* allocate extended descriptor */ - edesc = aead_giv_edesc_alloc(areq, DESC_JOB_IO_LEN * - CAAM_CMD_SZ, &contig); - - if (IS_ERR(edesc)) - return PTR_ERR(edesc); - -#ifdef DEBUG - print_hex_dump(KERN_ERR, "giv src@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src), - req->cryptlen, 1); -#endif + unsigned int ivsize = crypto_aead_ivsize(aead); - /* Create and submit job descriptor*/ - init_aead_giv_job(ctx->sh_desc_givenc, - ctx->sh_desc_givenc_dma, edesc, req, contig); -#ifdef DEBUG - print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, - desc_bytes(edesc->hw_desc), 1); -#endif + if (req->cryptlen < ivsize) + return -EINVAL; - desc = edesc->hw_desc; - ret = caam_jr_enqueue(jrdev, desc, old_aead_encrypt_done, req); - if (!ret) { - ret = -EINPROGRESS; - } else { - old_aead_unmap(jrdev, edesc, req); - kfree(edesc); - } + req->cryptlen -= ivsize; + req->assoclen += ivsize; - return ret; -} - -static int aead_null_givencrypt(struct aead_givcrypt_request *areq) -{ - return old_aead_encrypt(&areq->areq); + return aead_decrypt(req); } /* @@ -3072,8 +2503,8 @@ static struct ablkcipher_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request sizeof(struct sec4_sg_entry); /* allocate space for base edesc and hw desc commands, link tables */ - edesc = kmalloc(sizeof(struct ablkcipher_edesc) + desc_bytes + - sec4_sg_bytes, GFP_DMA | flags); + edesc = kzalloc(sizeof(*edesc) + desc_bytes + sec4_sg_bytes, + GFP_DMA | flags); if (!edesc) { dev_err(jrdev, "could not allocate extended descriptor\n"); return ERR_PTR(-ENOMEM); @@ -3251,8 +2682,8 @@ static struct ablkcipher_edesc *ablkcipher_giv_edesc_alloc( sizeof(struct sec4_sg_entry); /* allocate space for base edesc and hw desc commands, link tables */ - edesc = kmalloc(sizeof(*edesc) + desc_bytes + - sec4_sg_bytes, GFP_DMA | flags); + edesc = kzalloc(sizeof(*edesc) + desc_bytes + sec4_sg_bytes, + GFP_DMA | flags); if (!edesc) { dev_err(jrdev, "could not allocate extended descriptor\n"); return ERR_PTR(-ENOMEM); @@ -3347,7 +2778,6 @@ struct caam_alg_template { u32 type; union { struct ablkcipher_alg ablkcipher; - struct old_aead_alg aead; } template_u; u32 class1_alg_type; u32 class2_alg_type; @@ -3355,753 +2785,1426 @@ struct caam_alg_template { }; static struct caam_alg_template driver_algs[] = { + /* ablkcipher descriptor */ + { + .name = "cbc(aes)", + .driver_name = "cbc-aes-caam", + .blocksize = AES_BLOCK_SIZE, + .type = CRYPTO_ALG_TYPE_GIVCIPHER, + .template_ablkcipher = { + .setkey = ablkcipher_setkey, + .encrypt = ablkcipher_encrypt, + .decrypt = ablkcipher_decrypt, + .givencrypt = ablkcipher_givencrypt, + .geniv = "<built-in>", + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + }, + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + }, + { + .name = "cbc(des3_ede)", + .driver_name = "cbc-3des-caam", + .blocksize = DES3_EDE_BLOCK_SIZE, + .type = CRYPTO_ALG_TYPE_GIVCIPHER, + .template_ablkcipher = { + .setkey = ablkcipher_setkey, + .encrypt = ablkcipher_encrypt, + .decrypt = ablkcipher_decrypt, + .givencrypt = ablkcipher_givencrypt, + .geniv = "<built-in>", + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .ivsize = DES3_EDE_BLOCK_SIZE, + }, + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + }, + { + .name = "cbc(des)", + .driver_name = "cbc-des-caam", + .blocksize = DES_BLOCK_SIZE, + .type = CRYPTO_ALG_TYPE_GIVCIPHER, + .template_ablkcipher = { + .setkey = ablkcipher_setkey, + .encrypt = ablkcipher_encrypt, + .decrypt = ablkcipher_decrypt, + .givencrypt = ablkcipher_givencrypt, + .geniv = "<built-in>", + .min_keysize = DES_KEY_SIZE, + .max_keysize = DES_KEY_SIZE, + .ivsize = DES_BLOCK_SIZE, + }, + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + }, + { + .name = "ctr(aes)", + .driver_name = "ctr-aes-caam", + .blocksize = 1, + .type = CRYPTO_ALG_TYPE_ABLKCIPHER, + .template_ablkcipher = { + .setkey = ablkcipher_setkey, + .encrypt = ablkcipher_encrypt, + .decrypt = ablkcipher_decrypt, + .geniv = "chainiv", + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + }, + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128, + }, + { + .name = "rfc3686(ctr(aes))", + .driver_name = "rfc3686-ctr-aes-caam", + .blocksize = 1, + .type = CRYPTO_ALG_TYPE_GIVCIPHER, + .template_ablkcipher = { + .setkey = ablkcipher_setkey, + .encrypt = ablkcipher_encrypt, + .decrypt = ablkcipher_decrypt, + .givencrypt = ablkcipher_givencrypt, + .geniv = "<built-in>", + .min_keysize = AES_MIN_KEY_SIZE + + CTR_RFC3686_NONCE_SIZE, + .max_keysize = AES_MAX_KEY_SIZE + + CTR_RFC3686_NONCE_SIZE, + .ivsize = CTR_RFC3686_IV_SIZE, + }, + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128, + } +}; + +static struct caam_aead_alg driver_aeads[] = { + { + .aead = { + .base = { + .cra_name = "rfc4106(gcm(aes))", + .cra_driver_name = "rfc4106-gcm-aes-caam", + .cra_blocksize = 1, + }, + .setkey = rfc4106_setkey, + .setauthsize = rfc4106_setauthsize, + .encrypt = ipsec_gcm_encrypt, + .decrypt = ipsec_gcm_decrypt, + .ivsize = 8, + .maxauthsize = AES_BLOCK_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, + }, + }, + { + .aead = { + .base = { + .cra_name = "rfc4543(gcm(aes))", + .cra_driver_name = "rfc4543-gcm-aes-caam", + .cra_blocksize = 1, + }, + .setkey = rfc4543_setkey, + .setauthsize = rfc4543_setauthsize, + .encrypt = ipsec_gcm_encrypt, + .decrypt = ipsec_gcm_decrypt, + .ivsize = 8, + .maxauthsize = AES_BLOCK_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, + }, + }, + /* Galois Counter Mode */ + { + .aead = { + .base = { + .cra_name = "gcm(aes)", + .cra_driver_name = "gcm-aes-caam", + .cra_blocksize = 1, + }, + .setkey = gcm_setkey, + .setauthsize = gcm_setauthsize, + .encrypt = gcm_encrypt, + .decrypt = gcm_decrypt, + .ivsize = 12, + .maxauthsize = AES_BLOCK_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, + }, + }, /* single-pass ipsec_esp descriptor */ { - .name = "authenc(hmac(md5),ecb(cipher_null))", - .driver_name = "authenc-hmac-md5-ecb-cipher_null-caam", - .blocksize = NULL_BLOCK_SIZE, - .type = CRYPTO_ALG_TYPE_AEAD, - .template_aead = { + .aead = { + .base = { + .cra_name = "authenc(hmac(md5)," + "ecb(cipher_null))", + .cra_driver_name = "authenc-hmac-md5-" + "ecb-cipher_null-caam", + .cra_blocksize = NULL_BLOCK_SIZE, + }, .setkey = aead_setkey, .setauthsize = aead_setauthsize, - .encrypt = old_aead_encrypt, - .decrypt = old_aead_decrypt, - .givencrypt = aead_null_givencrypt, - .geniv = "<built-in>", + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, .ivsize = NULL_IV_SIZE, .maxauthsize = MD5_DIGEST_SIZE, - }, - .class1_alg_type = 0, - .class2_alg_type = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, + }, + .caam = { + .class2_alg_type = OP_ALG_ALGSEL_MD5 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, + }, }, { - .name = "authenc(hmac(sha1),ecb(cipher_null))", - .driver_name = "authenc-hmac-sha1-ecb-cipher_null-caam", - .blocksize = NULL_BLOCK_SIZE, - .type = CRYPTO_ALG_TYPE_AEAD, - .template_aead = { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha1)," + "ecb(cipher_null))", + .cra_driver_name = "authenc-hmac-sha1-" + "ecb-cipher_null-caam", + .cra_blocksize = NULL_BLOCK_SIZE, + }, .setkey = aead_setkey, .setauthsize = aead_setauthsize, - .encrypt = old_aead_encrypt, - .decrypt = old_aead_decrypt, - .givencrypt = aead_null_givencrypt, - .geniv = "<built-in>", + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, .ivsize = NULL_IV_SIZE, .maxauthsize = SHA1_DIGEST_SIZE, - }, - .class1_alg_type = 0, - .class2_alg_type = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, + }, + .caam = { + .class2_alg_type = OP_ALG_ALGSEL_SHA1 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, + }, }, { - .name = "authenc(hmac(sha224),ecb(cipher_null))", - .driver_name = "authenc-hmac-sha224-ecb-cipher_null-caam", - .blocksize = NULL_BLOCK_SIZE, - .type = CRYPTO_ALG_TYPE_AEAD, - .template_aead = { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha224)," + "ecb(cipher_null))", + .cra_driver_name = "authenc-hmac-sha224-" + "ecb-cipher_null-caam", + .cra_blocksize = NULL_BLOCK_SIZE, + }, .setkey = aead_setkey, .setauthsize = aead_setauthsize, - .encrypt = old_aead_encrypt, - .decrypt = old_aead_decrypt, - .givencrypt = aead_null_givencrypt, - .geniv = "<built-in>", + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, .ivsize = NULL_IV_SIZE, .maxauthsize = SHA224_DIGEST_SIZE, - }, - .class1_alg_type = 0, - .class2_alg_type = OP_ALG_ALGSEL_SHA224 | - OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, + }, + .caam = { + .class2_alg_type = OP_ALG_ALGSEL_SHA224 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, + }, }, { - .name = "authenc(hmac(sha256),ecb(cipher_null))", - .driver_name = "authenc-hmac-sha256-ecb-cipher_null-caam", - .blocksize = NULL_BLOCK_SIZE, - .type = CRYPTO_ALG_TYPE_AEAD, - .template_aead = { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha256)," + "ecb(cipher_null))", + .cra_driver_name = "authenc-hmac-sha256-" + "ecb-cipher_null-caam", + .cra_blocksize = NULL_BLOCK_SIZE, + }, .setkey = aead_setkey, .setauthsize = aead_setauthsize, - .encrypt = old_aead_encrypt, - .decrypt = old_aead_decrypt, - .givencrypt = aead_null_givencrypt, - .geniv = "<built-in>", + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, .ivsize = NULL_IV_SIZE, .maxauthsize = SHA256_DIGEST_SIZE, - }, - .class1_alg_type = 0, - .class2_alg_type = OP_ALG_ALGSEL_SHA256 | - OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, + }, + .caam = { + .class2_alg_type = OP_ALG_ALGSEL_SHA256 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, + }, }, { - .name = "authenc(hmac(sha384),ecb(cipher_null))", - .driver_name = "authenc-hmac-sha384-ecb-cipher_null-caam", - .blocksize = NULL_BLOCK_SIZE, - .type = CRYPTO_ALG_TYPE_AEAD, - .template_aead = { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha384)," + "ecb(cipher_null))", + .cra_driver_name = "authenc-hmac-sha384-" + "ecb-cipher_null-caam", + .cra_blocksize = NULL_BLOCK_SIZE, + }, .setkey = aead_setkey, .setauthsize = aead_setauthsize, - .encrypt = old_aead_encrypt, - .decrypt = old_aead_decrypt, - .givencrypt = aead_null_givencrypt, - .geniv = "<built-in>", + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, .ivsize = NULL_IV_SIZE, .maxauthsize = SHA384_DIGEST_SIZE, - }, - .class1_alg_type = 0, - .class2_alg_type = OP_ALG_ALGSEL_SHA384 | - OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, + }, + .caam = { + .class2_alg_type = OP_ALG_ALGSEL_SHA384 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, + }, }, { - .name = "authenc(hmac(sha512),ecb(cipher_null))", - .driver_name = "authenc-hmac-sha512-ecb-cipher_null-caam", - .blocksize = NULL_BLOCK_SIZE, - .type = CRYPTO_ALG_TYPE_AEAD, - .template_aead = { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha512)," + "ecb(cipher_null))", + .cra_driver_name = "authenc-hmac-sha512-" + "ecb-cipher_null-caam", + .cra_blocksize = NULL_BLOCK_SIZE, + }, .setkey = aead_setkey, .setauthsize = aead_setauthsize, - .encrypt = old_aead_encrypt, - .decrypt = old_aead_decrypt, - .givencrypt = aead_null_givencrypt, - .geniv = "<built-in>", + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, .ivsize = NULL_IV_SIZE, .maxauthsize = SHA512_DIGEST_SIZE, - }, - .class1_alg_type = 0, - .class2_alg_type = OP_ALG_ALGSEL_SHA512 | - OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, + }, + .caam = { + .class2_alg_type = OP_ALG_ALGSEL_SHA512 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, + }, }, { - .name = "authenc(hmac(md5),cbc(aes))", - .driver_name = "authenc-hmac-md5-cbc-aes-caam", - .blocksize = AES_BLOCK_SIZE, - .type = CRYPTO_ALG_TYPE_AEAD, - .template_aead = { + .aead = { + .base = { + .cra_name = "authenc(hmac(md5),cbc(aes))", + .cra_driver_name = "authenc-hmac-md5-" + "cbc-aes-caam", + .cra_blocksize = AES_BLOCK_SIZE, + }, .setkey = aead_setkey, .setauthsize = aead_setauthsize, - .encrypt = old_aead_encrypt, - .decrypt = old_aead_decrypt, - .givencrypt = old_aead_givencrypt, - .geniv = "<built-in>", + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, .ivsize = AES_BLOCK_SIZE, .maxauthsize = MD5_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_MD5 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, + }, + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(md5)," + "cbc(aes)))", + .cra_driver_name = "echainiv-authenc-hmac-md5-" + "cbc-aes-caam", + .cra_blocksize = AES_BLOCK_SIZE, }, - .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, - .class2_alg_type = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_givdecrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = MD5_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_MD5 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, + .geniv = true, + }, }, { - .name = "authenc(hmac(sha1),cbc(aes))", - .driver_name = "authenc-hmac-sha1-cbc-aes-caam", - .blocksize = AES_BLOCK_SIZE, - .type = CRYPTO_ALG_TYPE_AEAD, - .template_aead = { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha1),cbc(aes))", + .cra_driver_name = "authenc-hmac-sha1-" + "cbc-aes-caam", + .cra_blocksize = AES_BLOCK_SIZE, + }, .setkey = aead_setkey, .setauthsize = aead_setauthsize, - .encrypt = old_aead_encrypt, - .decrypt = old_aead_decrypt, - .givencrypt = old_aead_givencrypt, - .geniv = "<built-in>", + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, .ivsize = AES_BLOCK_SIZE, .maxauthsize = SHA1_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA1 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, + }, + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha1)," + "cbc(aes)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha1-cbc-aes-caam", + .cra_blocksize = AES_BLOCK_SIZE, }, - .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, - .class2_alg_type = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_givdecrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA1 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, + .geniv = true, + }, }, { - .name = "authenc(hmac(sha224),cbc(aes))", - .driver_name = "authenc-hmac-sha224-cbc-aes-caam", - .blocksize = AES_BLOCK_SIZE, - .type = CRYPTO_ALG_TYPE_AEAD, - .template_aead = { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha224),cbc(aes))", + .cra_driver_name = "authenc-hmac-sha224-" + "cbc-aes-caam", + .cra_blocksize = AES_BLOCK_SIZE, + }, .setkey = aead_setkey, .setauthsize = aead_setauthsize, - .encrypt = old_aead_encrypt, - .decrypt = old_aead_decrypt, - .givencrypt = old_aead_givencrypt, - .geniv = "<built-in>", + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, .ivsize = AES_BLOCK_SIZE, .maxauthsize = SHA224_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA224 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, + }, + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha224)," + "cbc(aes)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha224-cbc-aes-caam", + .cra_blocksize = AES_BLOCK_SIZE, }, - .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, - .class2_alg_type = OP_ALG_ALGSEL_SHA224 | - OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_givdecrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA224_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA224 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, + .geniv = true, + }, }, { - .name = "authenc(hmac(sha256),cbc(aes))", - .driver_name = "authenc-hmac-sha256-cbc-aes-caam", - .blocksize = AES_BLOCK_SIZE, - .type = CRYPTO_ALG_TYPE_AEAD, - .template_aead = { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha256),cbc(aes))", + .cra_driver_name = "authenc-hmac-sha256-" + "cbc-aes-caam", + .cra_blocksize = AES_BLOCK_SIZE, + }, .setkey = aead_setkey, .setauthsize = aead_setauthsize, - .encrypt = old_aead_encrypt, - .decrypt = old_aead_decrypt, - .givencrypt = old_aead_givencrypt, - .geniv = "<built-in>", + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, .ivsize = AES_BLOCK_SIZE, .maxauthsize = SHA256_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA256 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, + }, + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha256)," + "cbc(aes)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha256-cbc-aes-caam", + .cra_blocksize = AES_BLOCK_SIZE, }, - .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, - .class2_alg_type = OP_ALG_ALGSEL_SHA256 | - OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_givdecrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA256 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, + .geniv = true, + }, }, { - .name = "authenc(hmac(sha384),cbc(aes))", - .driver_name = "authenc-hmac-sha384-cbc-aes-caam", - .blocksize = AES_BLOCK_SIZE, - .type = CRYPTO_ALG_TYPE_AEAD, - .template_aead = { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha384),cbc(aes))", + .cra_driver_name = "authenc-hmac-sha384-" + "cbc-aes-caam", + .cra_blocksize = AES_BLOCK_SIZE, + }, .setkey = aead_setkey, .setauthsize = aead_setauthsize, - .encrypt = old_aead_encrypt, - .decrypt = old_aead_decrypt, - .givencrypt = old_aead_givencrypt, - .geniv = "<built-in>", + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, .ivsize = AES_BLOCK_SIZE, .maxauthsize = SHA384_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA384 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, + }, + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha384)," + "cbc(aes)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha384-cbc-aes-caam", + .cra_blocksize = AES_BLOCK_SIZE, }, - .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, - .class2_alg_type = OP_ALG_ALGSEL_SHA384 | - OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_givdecrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA384_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA384 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, + .geniv = true, + }, }, - { - .name = "authenc(hmac(sha512),cbc(aes))", - .driver_name = "authenc-hmac-sha512-cbc-aes-caam", - .blocksize = AES_BLOCK_SIZE, - .type = CRYPTO_ALG_TYPE_AEAD, - .template_aead = { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha512),cbc(aes))", + .cra_driver_name = "authenc-hmac-sha512-" + "cbc-aes-caam", + .cra_blocksize = AES_BLOCK_SIZE, + }, .setkey = aead_setkey, .setauthsize = aead_setauthsize, - .encrypt = old_aead_encrypt, - .decrypt = old_aead_decrypt, - .givencrypt = old_aead_givencrypt, - .geniv = "<built-in>", + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, .ivsize = AES_BLOCK_SIZE, .maxauthsize = SHA512_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA512 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, + }, + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha512)," + "cbc(aes)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha512-cbc-aes-caam", + .cra_blocksize = AES_BLOCK_SIZE, }, - .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, - .class2_alg_type = OP_ALG_ALGSEL_SHA512 | - OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_givdecrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA512_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA512 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, + .geniv = true, + }, }, { - .name = "authenc(hmac(md5),cbc(des3_ede))", - .driver_name = "authenc-hmac-md5-cbc-des3_ede-caam", - .blocksize = DES3_EDE_BLOCK_SIZE, - .type = CRYPTO_ALG_TYPE_AEAD, - .template_aead = { + .aead = { + .base = { + .cra_name = "authenc(hmac(md5),cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-md5-" + "cbc-des3_ede-caam", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, .setkey = aead_setkey, .setauthsize = aead_setauthsize, - .encrypt = old_aead_encrypt, - .decrypt = old_aead_decrypt, - .givencrypt = old_aead_givencrypt, - .geniv = "<built-in>", + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, .ivsize = DES3_EDE_BLOCK_SIZE, .maxauthsize = MD5_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_MD5 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, + } + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(md5)," + "cbc(des3_ede)))", + .cra_driver_name = "echainiv-authenc-hmac-md5-" + "cbc-des3_ede-caam", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, }, - .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, - .class2_alg_type = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_givdecrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = MD5_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_MD5 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, + .geniv = true, + } }, { - .name = "authenc(hmac(sha1),cbc(des3_ede))", - .driver_name = "authenc-hmac-sha1-cbc-des3_ede-caam", - .blocksize = DES3_EDE_BLOCK_SIZE, - .type = CRYPTO_ALG_TYPE_AEAD, - .template_aead = { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha1)," + "cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-sha1-" + "cbc-des3_ede-caam", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, .setkey = aead_setkey, .setauthsize = aead_setauthsize, - .encrypt = old_aead_encrypt, - .decrypt = old_aead_decrypt, - .givencrypt = old_aead_givencrypt, - .geniv = "<built-in>", + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, .ivsize = DES3_EDE_BLOCK_SIZE, .maxauthsize = SHA1_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA1 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, + }, + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha1)," + "cbc(des3_ede)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha1-" + "cbc-des3_ede-caam", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, }, - .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, - .class2_alg_type = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_givdecrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA1 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, + .geniv = true, + }, }, { - .name = "authenc(hmac(sha224),cbc(des3_ede))", - .driver_name = "authenc-hmac-sha224-cbc-des3_ede-caam", - .blocksize = DES3_EDE_BLOCK_SIZE, - .type = CRYPTO_ALG_TYPE_AEAD, - .template_aead = { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha224)," + "cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-sha224-" + "cbc-des3_ede-caam", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, .setkey = aead_setkey, .setauthsize = aead_setauthsize, - .encrypt = old_aead_encrypt, - .decrypt = old_aead_decrypt, - .givencrypt = old_aead_givencrypt, - .geniv = "<built-in>", + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, .ivsize = DES3_EDE_BLOCK_SIZE, .maxauthsize = SHA224_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA224 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, + }, + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha224)," + "cbc(des3_ede)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha224-" + "cbc-des3_ede-caam", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, }, - .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, - .class2_alg_type = OP_ALG_ALGSEL_SHA224 | - OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_givdecrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA224_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA224 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, + .geniv = true, + }, }, { - .name = "authenc(hmac(sha256),cbc(des3_ede))", - .driver_name = "authenc-hmac-sha256-cbc-des3_ede-caam", - .blocksize = DES3_EDE_BLOCK_SIZE, - .type = CRYPTO_ALG_TYPE_AEAD, - .template_aead = { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha256)," + "cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-sha256-" + "cbc-des3_ede-caam", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, .setkey = aead_setkey, .setauthsize = aead_setauthsize, - .encrypt = old_aead_encrypt, - .decrypt = old_aead_decrypt, - .givencrypt = old_aead_givencrypt, - .geniv = "<built-in>", + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, .ivsize = DES3_EDE_BLOCK_SIZE, .maxauthsize = SHA256_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA256 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, + }, + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha256)," + "cbc(des3_ede)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha256-" + "cbc-des3_ede-caam", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, }, - .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, - .class2_alg_type = OP_ALG_ALGSEL_SHA256 | - OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_givdecrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA256 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, + .geniv = true, + }, }, { - .name = "authenc(hmac(sha384),cbc(des3_ede))", - .driver_name = "authenc-hmac-sha384-cbc-des3_ede-caam", - .blocksize = DES3_EDE_BLOCK_SIZE, - .type = CRYPTO_ALG_TYPE_AEAD, - .template_aead = { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha384)," + "cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-sha384-" + "cbc-des3_ede-caam", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, .setkey = aead_setkey, .setauthsize = aead_setauthsize, - .encrypt = old_aead_encrypt, - .decrypt = old_aead_decrypt, - .givencrypt = old_aead_givencrypt, - .geniv = "<built-in>", + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, .ivsize = DES3_EDE_BLOCK_SIZE, .maxauthsize = SHA384_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA384 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, + }, + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha384)," + "cbc(des3_ede)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha384-" + "cbc-des3_ede-caam", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, }, - .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, - .class2_alg_type = OP_ALG_ALGSEL_SHA384 | - OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_givdecrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA384_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA384 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, + .geniv = true, + }, }, { - .name = "authenc(hmac(sha512),cbc(des3_ede))", - .driver_name = "authenc-hmac-sha512-cbc-des3_ede-caam", - .blocksize = DES3_EDE_BLOCK_SIZE, - .type = CRYPTO_ALG_TYPE_AEAD, - .template_aead = { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha512)," + "cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-sha512-" + "cbc-des3_ede-caam", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, .setkey = aead_setkey, .setauthsize = aead_setauthsize, - .encrypt = old_aead_encrypt, - .decrypt = old_aead_decrypt, - .givencrypt = old_aead_givencrypt, - .geniv = "<built-in>", + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, .ivsize = DES3_EDE_BLOCK_SIZE, .maxauthsize = SHA512_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA512 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, + }, + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha512)," + "cbc(des3_ede)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha512-" + "cbc-des3_ede-caam", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, }, - .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, - .class2_alg_type = OP_ALG_ALGSEL_SHA512 | - OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_givdecrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA512_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA512 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, + .geniv = true, + }, }, { - .name = "authenc(hmac(md5),cbc(des))", - .driver_name = "authenc-hmac-md5-cbc-des-caam", - .blocksize = DES_BLOCK_SIZE, - .type = CRYPTO_ALG_TYPE_AEAD, - .template_aead = { + .aead = { + .base = { + .cra_name = "authenc(hmac(md5),cbc(des))", + .cra_driver_name = "authenc-hmac-md5-" + "cbc-des-caam", + .cra_blocksize = DES_BLOCK_SIZE, + }, .setkey = aead_setkey, .setauthsize = aead_setauthsize, - .encrypt = old_aead_encrypt, - .decrypt = old_aead_decrypt, - .givencrypt = old_aead_givencrypt, - .geniv = "<built-in>", + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, .ivsize = DES_BLOCK_SIZE, .maxauthsize = MD5_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_MD5 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, + }, + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(md5)," + "cbc(des)))", + .cra_driver_name = "echainiv-authenc-hmac-md5-" + "cbc-des-caam", + .cra_blocksize = DES_BLOCK_SIZE, }, - .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, - .class2_alg_type = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_givdecrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = MD5_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_MD5 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, + .geniv = true, + }, }, { - .name = "authenc(hmac(sha1),cbc(des))", - .driver_name = "authenc-hmac-sha1-cbc-des-caam", - .blocksize = DES_BLOCK_SIZE, - .type = CRYPTO_ALG_TYPE_AEAD, - .template_aead = { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha1),cbc(des))", + .cra_driver_name = "authenc-hmac-sha1-" + "cbc-des-caam", + .cra_blocksize = DES_BLOCK_SIZE, + }, .setkey = aead_setkey, .setauthsize = aead_setauthsize, - .encrypt = old_aead_encrypt, - .decrypt = old_aead_decrypt, - .givencrypt = old_aead_givencrypt, - .geniv = "<built-in>", + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, .ivsize = DES_BLOCK_SIZE, .maxauthsize = SHA1_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA1 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, + }, + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha1)," + "cbc(des)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha1-cbc-des-caam", + .cra_blocksize = DES_BLOCK_SIZE, }, - .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, - .class2_alg_type = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_givdecrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA1 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, + .geniv = true, + }, }, { - .name = "authenc(hmac(sha224),cbc(des))", - .driver_name = "authenc-hmac-sha224-cbc-des-caam", - .blocksize = DES_BLOCK_SIZE, - .type = CRYPTO_ALG_TYPE_AEAD, - .template_aead = { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha224),cbc(des))", + .cra_driver_name = "authenc-hmac-sha224-" + "cbc-des-caam", + .cra_blocksize = DES_BLOCK_SIZE, + }, .setkey = aead_setkey, .setauthsize = aead_setauthsize, - .encrypt = old_aead_encrypt, - .decrypt = old_aead_decrypt, - .givencrypt = old_aead_givencrypt, - .geniv = "<built-in>", + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, .ivsize = DES_BLOCK_SIZE, .maxauthsize = SHA224_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA224 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, + }, + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha224)," + "cbc(des)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha224-cbc-des-caam", + .cra_blocksize = DES_BLOCK_SIZE, }, - .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, - .class2_alg_type = OP_ALG_ALGSEL_SHA224 | - OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_givdecrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA224_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA224 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, + .geniv = true, + }, }, { - .name = "authenc(hmac(sha256),cbc(des))", - .driver_name = "authenc-hmac-sha256-cbc-des-caam", - .blocksize = DES_BLOCK_SIZE, - .type = CRYPTO_ALG_TYPE_AEAD, - .template_aead = { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha256),cbc(des))", + .cra_driver_name = "authenc-hmac-sha256-" + "cbc-des-caam", + .cra_blocksize = DES_BLOCK_SIZE, + }, .setkey = aead_setkey, .setauthsize = aead_setauthsize, - .encrypt = old_aead_encrypt, - .decrypt = old_aead_decrypt, - .givencrypt = old_aead_givencrypt, - .geniv = "<built-in>", + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, .ivsize = DES_BLOCK_SIZE, .maxauthsize = SHA256_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA256 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, + }, + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha256)," + "cbc(des)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha256-cbc-des-caam", + .cra_blocksize = DES_BLOCK_SIZE, }, - .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, - .class2_alg_type = OP_ALG_ALGSEL_SHA256 | - OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_givdecrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA256 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, + .geniv = true, + }, }, { - .name = "authenc(hmac(sha384),cbc(des))", - .driver_name = "authenc-hmac-sha384-cbc-des-caam", - .blocksize = DES_BLOCK_SIZE, - .type = CRYPTO_ALG_TYPE_AEAD, - .template_aead = { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha384),cbc(des))", + .cra_driver_name = "authenc-hmac-sha384-" + "cbc-des-caam", + .cra_blocksize = DES_BLOCK_SIZE, + }, .setkey = aead_setkey, .setauthsize = aead_setauthsize, - .encrypt = old_aead_encrypt, - .decrypt = old_aead_decrypt, - .givencrypt = old_aead_givencrypt, - .geniv = "<built-in>", + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, .ivsize = DES_BLOCK_SIZE, .maxauthsize = SHA384_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA384 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, + }, + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha384)," + "cbc(des)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha384-cbc-des-caam", + .cra_blocksize = DES_BLOCK_SIZE, }, - .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, - .class2_alg_type = OP_ALG_ALGSEL_SHA384 | - OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_givdecrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA384_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA384 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, + .geniv = true, + }, }, { - .name = "authenc(hmac(sha512),cbc(des))", - .driver_name = "authenc-hmac-sha512-cbc-des-caam", - .blocksize = DES_BLOCK_SIZE, - .type = CRYPTO_ALG_TYPE_AEAD, - .template_aead = { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha512),cbc(des))", + .cra_driver_name = "authenc-hmac-sha512-" + "cbc-des-caam", + .cra_blocksize = DES_BLOCK_SIZE, + }, .setkey = aead_setkey, .setauthsize = aead_setauthsize, - .encrypt = old_aead_encrypt, - .decrypt = old_aead_decrypt, - .givencrypt = old_aead_givencrypt, - .geniv = "<built-in>", + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, .ivsize = DES_BLOCK_SIZE, .maxauthsize = SHA512_DIGEST_SIZE, - }, - .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, - .class2_alg_type = OP_ALG_ALGSEL_SHA512 | - OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA512 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, + }, }, { - .name = "authenc(hmac(md5),rfc3686(ctr(aes)))", - .driver_name = "authenc-hmac-md5-rfc3686-ctr-aes-caam", - .blocksize = 1, - .type = CRYPTO_ALG_TYPE_AEAD, - .template_aead = { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha512)," + "cbc(des)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha512-cbc-des-caam", + .cra_blocksize = DES_BLOCK_SIZE, + }, .setkey = aead_setkey, .setauthsize = aead_setauthsize, - .encrypt = old_aead_encrypt, - .decrypt = old_aead_decrypt, - .givencrypt = old_aead_givencrypt, - .geniv = "<built-in>", - .ivsize = CTR_RFC3686_IV_SIZE, - .maxauthsize = MD5_DIGEST_SIZE, - }, - .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128, - .class2_alg_type = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, + .encrypt = aead_encrypt, + .decrypt = aead_givdecrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA512_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA512 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, + .geniv = true, + }, }, { - .name = "authenc(hmac(sha1),rfc3686(ctr(aes)))", - .driver_name = "authenc-hmac-sha1-rfc3686-ctr-aes-caam", - .blocksize = 1, - .type = CRYPTO_ALG_TYPE_AEAD, - .template_aead = { + .aead = { + .base = { + .cra_name = "authenc(hmac(md5)," + "rfc3686(ctr(aes)))", + .cra_driver_name = "authenc-hmac-md5-" + "rfc3686-ctr-aes-caam", + .cra_blocksize = 1, + }, .setkey = aead_setkey, .setauthsize = aead_setauthsize, - .encrypt = old_aead_encrypt, - .decrypt = old_aead_decrypt, - .givencrypt = old_aead_givencrypt, - .geniv = "<built-in>", + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, .ivsize = CTR_RFC3686_IV_SIZE, - .maxauthsize = SHA1_DIGEST_SIZE, - }, - .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128, - .class2_alg_type = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, + .maxauthsize = MD5_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | + OP_ALG_AAI_CTR_MOD128, + .class2_alg_type = OP_ALG_ALGSEL_MD5 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, + .rfc3686 = true, + }, }, { - .name = "authenc(hmac(sha224),rfc3686(ctr(aes)))", - .driver_name = "authenc-hmac-sha224-rfc3686-ctr-aes-caam", - .blocksize = 1, - .type = CRYPTO_ALG_TYPE_AEAD, - .template_aead = { + .aead = { + .base = { + .cra_name = "seqiv(authenc(" + "hmac(md5),rfc3686(ctr(aes))))", + .cra_driver_name = "seqiv-authenc-hmac-md5-" + "rfc3686-ctr-aes-caam", + .cra_blocksize = 1, + }, .setkey = aead_setkey, .setauthsize = aead_setauthsize, - .encrypt = old_aead_encrypt, - .decrypt = old_aead_decrypt, - .givencrypt = old_aead_givencrypt, - .geniv = "<built-in>", + .encrypt = aead_encrypt, + .decrypt = aead_givdecrypt, .ivsize = CTR_RFC3686_IV_SIZE, - .maxauthsize = SHA224_DIGEST_SIZE, - }, - .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128, - .class2_alg_type = OP_ALG_ALGSEL_SHA224 | - OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, + .maxauthsize = MD5_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | + OP_ALG_AAI_CTR_MOD128, + .class2_alg_type = OP_ALG_ALGSEL_MD5 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, + .rfc3686 = true, + .geniv = true, + }, }, { - .name = "authenc(hmac(sha256),rfc3686(ctr(aes)))", - .driver_name = "authenc-hmac-sha256-rfc3686-ctr-aes-caam", - .blocksize = 1, - .type = CRYPTO_ALG_TYPE_AEAD, - .template_aead = { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha1)," + "rfc3686(ctr(aes)))", + .cra_driver_name = "authenc-hmac-sha1-" + "rfc3686-ctr-aes-caam", + .cra_blocksize = 1, + }, .setkey = aead_setkey, .setauthsize = aead_setauthsize, - .encrypt = old_aead_encrypt, - .decrypt = old_aead_decrypt, - .givencrypt = old_aead_givencrypt, - .geniv = "<built-in>", + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, .ivsize = CTR_RFC3686_IV_SIZE, - .maxauthsize = SHA256_DIGEST_SIZE, - }, - .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128, - .class2_alg_type = OP_ALG_ALGSEL_SHA256 | - OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, + .maxauthsize = SHA1_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | + OP_ALG_AAI_CTR_MOD128, + .class2_alg_type = OP_ALG_ALGSEL_SHA1 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, + .rfc3686 = true, + }, }, { - .name = "authenc(hmac(sha384),rfc3686(ctr(aes)))", - .driver_name = "authenc-hmac-sha384-rfc3686-ctr-aes-caam", - .blocksize = 1, - .type = CRYPTO_ALG_TYPE_AEAD, - .template_aead = { + .aead = { + .base = { + .cra_name = "seqiv(authenc(" + "hmac(sha1),rfc3686(ctr(aes))))", + .cra_driver_name = "seqiv-authenc-hmac-sha1-" + "rfc3686-ctr-aes-caam", + .cra_blocksize = 1, + }, .setkey = aead_setkey, .setauthsize = aead_setauthsize, - .encrypt = old_aead_encrypt, - .decrypt = old_aead_decrypt, - .givencrypt = old_aead_givencrypt, - .geniv = "<built-in>", + .encrypt = aead_encrypt, + .decrypt = aead_givdecrypt, .ivsize = CTR_RFC3686_IV_SIZE, - .maxauthsize = SHA384_DIGEST_SIZE, - }, - .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128, - .class2_alg_type = OP_ALG_ALGSEL_SHA384 | - OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, + .maxauthsize = SHA1_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | + OP_ALG_AAI_CTR_MOD128, + .class2_alg_type = OP_ALG_ALGSEL_SHA1 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, + .rfc3686 = true, + .geniv = true, + }, }, { - .name = "authenc(hmac(sha512),rfc3686(ctr(aes)))", - .driver_name = "authenc-hmac-sha512-rfc3686-ctr-aes-caam", - .blocksize = 1, - .type = CRYPTO_ALG_TYPE_AEAD, - .template_aead = { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha224)," + "rfc3686(ctr(aes)))", + .cra_driver_name = "authenc-hmac-sha224-" + "rfc3686-ctr-aes-caam", + .cra_blocksize = 1, + }, .setkey = aead_setkey, .setauthsize = aead_setauthsize, - .encrypt = old_aead_encrypt, - .decrypt = old_aead_decrypt, - .givencrypt = old_aead_givencrypt, - .geniv = "<built-in>", + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, .ivsize = CTR_RFC3686_IV_SIZE, - .maxauthsize = SHA512_DIGEST_SIZE, - }, - .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128, - .class2_alg_type = OP_ALG_ALGSEL_SHA512 | - OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, + .maxauthsize = SHA224_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | + OP_ALG_AAI_CTR_MOD128, + .class2_alg_type = OP_ALG_ALGSEL_SHA224 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, + .rfc3686 = true, + }, }, - /* ablkcipher descriptor */ { - .name = "cbc(aes)", - .driver_name = "cbc-aes-caam", - .blocksize = AES_BLOCK_SIZE, - .type = CRYPTO_ALG_TYPE_GIVCIPHER, - .template_ablkcipher = { - .setkey = ablkcipher_setkey, - .encrypt = ablkcipher_encrypt, - .decrypt = ablkcipher_decrypt, - .givencrypt = ablkcipher_givencrypt, - .geniv = "<built-in>", - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, - .ivsize = AES_BLOCK_SIZE, + .aead = { + .base = { + .cra_name = "seqiv(authenc(" + "hmac(sha224),rfc3686(ctr(aes))))", + .cra_driver_name = "seqiv-authenc-hmac-sha224-" + "rfc3686-ctr-aes-caam", + .cra_blocksize = 1, }, - .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_givdecrypt, + .ivsize = CTR_RFC3686_IV_SIZE, + .maxauthsize = SHA224_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | + OP_ALG_AAI_CTR_MOD128, + .class2_alg_type = OP_ALG_ALGSEL_SHA224 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, + .rfc3686 = true, + .geniv = true, + }, }, { - .name = "cbc(des3_ede)", - .driver_name = "cbc-3des-caam", - .blocksize = DES3_EDE_BLOCK_SIZE, - .type = CRYPTO_ALG_TYPE_GIVCIPHER, - .template_ablkcipher = { - .setkey = ablkcipher_setkey, - .encrypt = ablkcipher_encrypt, - .decrypt = ablkcipher_decrypt, - .givencrypt = ablkcipher_givencrypt, - .geniv = "<built-in>", - .min_keysize = DES3_EDE_KEY_SIZE, - .max_keysize = DES3_EDE_KEY_SIZE, - .ivsize = DES3_EDE_BLOCK_SIZE, + .aead = { + .base = { + .cra_name = "authenc(hmac(sha256)," + "rfc3686(ctr(aes)))", + .cra_driver_name = "authenc-hmac-sha256-" + "rfc3686-ctr-aes-caam", + .cra_blocksize = 1, }, - .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = CTR_RFC3686_IV_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | + OP_ALG_AAI_CTR_MOD128, + .class2_alg_type = OP_ALG_ALGSEL_SHA256 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, + .rfc3686 = true, + }, }, { - .name = "cbc(des)", - .driver_name = "cbc-des-caam", - .blocksize = DES_BLOCK_SIZE, - .type = CRYPTO_ALG_TYPE_GIVCIPHER, - .template_ablkcipher = { - .setkey = ablkcipher_setkey, - .encrypt = ablkcipher_encrypt, - .decrypt = ablkcipher_decrypt, - .givencrypt = ablkcipher_givencrypt, - .geniv = "<built-in>", - .min_keysize = DES_KEY_SIZE, - .max_keysize = DES_KEY_SIZE, - .ivsize = DES_BLOCK_SIZE, + .aead = { + .base = { + .cra_name = "seqiv(authenc(hmac(sha256)," + "rfc3686(ctr(aes))))", + .cra_driver_name = "seqiv-authenc-hmac-sha256-" + "rfc3686-ctr-aes-caam", + .cra_blocksize = 1, }, - .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_givdecrypt, + .ivsize = CTR_RFC3686_IV_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | + OP_ALG_AAI_CTR_MOD128, + .class2_alg_type = OP_ALG_ALGSEL_SHA256 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, + .rfc3686 = true, + .geniv = true, + }, }, { - .name = "ctr(aes)", - .driver_name = "ctr-aes-caam", - .blocksize = 1, - .type = CRYPTO_ALG_TYPE_ABLKCIPHER, - .template_ablkcipher = { - .setkey = ablkcipher_setkey, - .encrypt = ablkcipher_encrypt, - .decrypt = ablkcipher_decrypt, - .geniv = "chainiv", - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, - .ivsize = AES_BLOCK_SIZE, + .aead = { + .base = { + .cra_name = "authenc(hmac(sha384)," + "rfc3686(ctr(aes)))", + .cra_driver_name = "authenc-hmac-sha384-" + "rfc3686-ctr-aes-caam", + .cra_blocksize = 1, }, - .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128, - }, - { - .name = "rfc3686(ctr(aes))", - .driver_name = "rfc3686-ctr-aes-caam", - .blocksize = 1, - .type = CRYPTO_ALG_TYPE_GIVCIPHER, - .template_ablkcipher = { - .setkey = ablkcipher_setkey, - .encrypt = ablkcipher_encrypt, - .decrypt = ablkcipher_decrypt, - .givencrypt = ablkcipher_givencrypt, - .geniv = "<built-in>", - .min_keysize = AES_MIN_KEY_SIZE + - CTR_RFC3686_NONCE_SIZE, - .max_keysize = AES_MAX_KEY_SIZE + - CTR_RFC3686_NONCE_SIZE, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, .ivsize = CTR_RFC3686_IV_SIZE, - }, - .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128, - } -}; - -struct caam_alg_entry { - int class1_alg_type; - int class2_alg_type; - int alg_op; -}; - -struct caam_aead_alg { - struct aead_alg aead; - struct caam_alg_entry caam; - bool registered; -}; - -static struct caam_aead_alg driver_aeads[] = { + .maxauthsize = SHA384_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | + OP_ALG_AAI_CTR_MOD128, + .class2_alg_type = OP_ALG_ALGSEL_SHA384 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, + .rfc3686 = true, + }, + }, { .aead = { .base = { - .cra_name = "rfc4106(gcm(aes))", - .cra_driver_name = "rfc4106-gcm-aes-caam", + .cra_name = "seqiv(authenc(hmac(sha384)," + "rfc3686(ctr(aes))))", + .cra_driver_name = "seqiv-authenc-hmac-sha384-" + "rfc3686-ctr-aes-caam", .cra_blocksize = 1, }, - .setkey = rfc4106_setkey, - .setauthsize = rfc4106_setauthsize, - .encrypt = gcm_encrypt, - .decrypt = gcm_decrypt, - .ivsize = 8, - .maxauthsize = AES_BLOCK_SIZE, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_givdecrypt, + .ivsize = CTR_RFC3686_IV_SIZE, + .maxauthsize = SHA384_DIGEST_SIZE, }, .caam = { - .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, + .class1_alg_type = OP_ALG_ALGSEL_AES | + OP_ALG_AAI_CTR_MOD128, + .class2_alg_type = OP_ALG_ALGSEL_SHA384 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, + .rfc3686 = true, + .geniv = true, }, }, { .aead = { .base = { - .cra_name = "rfc4543(gcm(aes))", - .cra_driver_name = "rfc4543-gcm-aes-caam", + .cra_name = "authenc(hmac(sha512)," + "rfc3686(ctr(aes)))", + .cra_driver_name = "authenc-hmac-sha512-" + "rfc3686-ctr-aes-caam", .cra_blocksize = 1, }, - .setkey = rfc4543_setkey, - .setauthsize = rfc4543_setauthsize, - .encrypt = gcm_encrypt, - .decrypt = gcm_decrypt, - .ivsize = 8, - .maxauthsize = AES_BLOCK_SIZE, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = CTR_RFC3686_IV_SIZE, + .maxauthsize = SHA512_DIGEST_SIZE, }, .caam = { - .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, + .class1_alg_type = OP_ALG_ALGSEL_AES | + OP_ALG_AAI_CTR_MOD128, + .class2_alg_type = OP_ALG_ALGSEL_SHA512 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, + .rfc3686 = true, }, }, - /* Galois Counter Mode */ { .aead = { .base = { - .cra_name = "gcm(aes)", - .cra_driver_name = "gcm-aes-caam", + .cra_name = "seqiv(authenc(hmac(sha512)," + "rfc3686(ctr(aes))))", + .cra_driver_name = "seqiv-authenc-hmac-sha512-" + "rfc3686-ctr-aes-caam", .cra_blocksize = 1, }, - .setkey = gcm_setkey, - .setauthsize = gcm_setauthsize, - .encrypt = gcm_encrypt, - .decrypt = gcm_decrypt, - .ivsize = 12, - .maxauthsize = AES_BLOCK_SIZE, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_givdecrypt, + .ivsize = CTR_RFC3686_IV_SIZE, + .maxauthsize = SHA512_DIGEST_SIZE, }, .caam = { - .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, + .class1_alg_type = OP_ALG_ALGSEL_AES | + OP_ALG_AAI_CTR_MOD128, + .class2_alg_type = OP_ALG_ALGSEL_SHA512 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, + .rfc3686 = true, + .geniv = true, }, }, }; @@ -4211,7 +4314,7 @@ static struct caam_crypto_alg *caam_alg_alloc(struct caam_alg_template struct caam_crypto_alg *t_alg; struct crypto_alg *alg; - t_alg = kzalloc(sizeof(struct caam_crypto_alg), GFP_KERNEL); + t_alg = kzalloc(sizeof(*t_alg), GFP_KERNEL); if (!t_alg) { pr_err("failed to allocate t_alg\n"); return ERR_PTR(-ENOMEM); @@ -4240,10 +4343,6 @@ static struct caam_crypto_alg *caam_alg_alloc(struct caam_alg_template alg->cra_type = &crypto_ablkcipher_type; alg->cra_ablkcipher = template->template_ablkcipher; break; - case CRYPTO_ALG_TYPE_AEAD: - alg->cra_type = &crypto_aead_type; - alg->cra_aead = template->template_aead; - break; } t_alg->caam.class1_alg_type = template->class1_alg_type; @@ -4271,8 +4370,10 @@ static int __init caam_algapi_init(void) struct device_node *dev_node; struct platform_device *pdev; struct device *ctrldev; - void *priv; + struct caam_drv_private *priv; int i = 0, err = 0; + u32 cha_vid, cha_inst, des_inst, aes_inst, md_inst; + unsigned int md_limit = SHA512_DIGEST_SIZE; bool registered = false; dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0"); @@ -4302,16 +4403,39 @@ static int __init caam_algapi_init(void) INIT_LIST_HEAD(&alg_list); - /* register crypto algorithms the device supports */ + /* + * Register crypto algorithms the device supports. + * First, detect presence and attributes of DES, AES, and MD blocks. + */ + cha_vid = rd_reg32(&priv->ctrl->perfmon.cha_id_ls); + cha_inst = rd_reg32(&priv->ctrl->perfmon.cha_num_ls); + des_inst = (cha_inst & CHA_ID_LS_DES_MASK) >> CHA_ID_LS_DES_SHIFT; + aes_inst = (cha_inst & CHA_ID_LS_AES_MASK) >> CHA_ID_LS_AES_SHIFT; + md_inst = (cha_inst & CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT; + + /* If MD is present, limit digest size based on LP256 */ + if (md_inst && ((cha_vid & CHA_ID_LS_MD_MASK) == CHA_ID_LS_MD_LP256)) + md_limit = SHA256_DIGEST_SIZE; + for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { - /* TODO: check if h/w supports alg */ struct caam_crypto_alg *t_alg; + struct caam_alg_template *alg = driver_algs + i; + u32 alg_sel = alg->class1_alg_type & OP_ALG_ALGSEL_MASK; + + /* Skip DES algorithms if not supported by device */ + if (!des_inst && + ((alg_sel == OP_ALG_ALGSEL_3DES) || + (alg_sel == OP_ALG_ALGSEL_DES))) + continue; + + /* Skip AES algorithms if not supported by device */ + if (!aes_inst && (alg_sel == OP_ALG_ALGSEL_AES)) + continue; - t_alg = caam_alg_alloc(&driver_algs[i]); + t_alg = caam_alg_alloc(alg); if (IS_ERR(t_alg)) { err = PTR_ERR(t_alg); - pr_warn("%s alg allocation failed\n", - driver_algs[i].driver_name); + pr_warn("%s alg allocation failed\n", alg->driver_name); continue; } @@ -4329,6 +4453,37 @@ static int __init caam_algapi_init(void) for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) { struct caam_aead_alg *t_alg = driver_aeads + i; + u32 c1_alg_sel = t_alg->caam.class1_alg_type & + OP_ALG_ALGSEL_MASK; + u32 c2_alg_sel = t_alg->caam.class2_alg_type & + OP_ALG_ALGSEL_MASK; + u32 alg_aai = t_alg->caam.class1_alg_type & OP_ALG_AAI_MASK; + + /* Skip DES algorithms if not supported by device */ + if (!des_inst && + ((c1_alg_sel == OP_ALG_ALGSEL_3DES) || + (c1_alg_sel == OP_ALG_ALGSEL_DES))) + continue; + + /* Skip AES algorithms if not supported by device */ + if (!aes_inst && (c1_alg_sel == OP_ALG_ALGSEL_AES)) + continue; + + /* + * Check support for AES algorithms not available + * on LP devices. + */ + if ((cha_vid & CHA_ID_LS_AES_MASK) == CHA_ID_LS_AES_LP) + if (alg_aai == OP_ALG_AAI_GCM) + continue; + + /* + * Skip algorithms requiring message digests + * if MD or MD size is not supported by device. + */ + if (c2_alg_sel && + (!md_inst || (t_alg->aead.maxauthsize > md_limit))) + continue; caam_aead_alg_init(t_alg); diff --git a/drivers/crypto/caam/caamhash.c b/drivers/crypto/caam/caamhash.c index f9c78751989e..94433b9fc200 100644 --- a/drivers/crypto/caam/caamhash.c +++ b/drivers/crypto/caam/caamhash.c @@ -127,7 +127,7 @@ struct caam_hash_state { int buflen_0; u8 buf_1[CAAM_MAX_HASH_BLOCK_SIZE] ____cacheline_aligned; int buflen_1; - u8 caam_ctx[MAX_CTX_LEN]; + u8 caam_ctx[MAX_CTX_LEN] ____cacheline_aligned; int (*update)(struct ahash_request *req); int (*final)(struct ahash_request *req); int (*finup)(struct ahash_request *req); @@ -807,7 +807,7 @@ static int ahash_update_ctx(struct ahash_request *req) * allocate space for base edesc and hw desc commands, * link tables */ - edesc = kmalloc(sizeof(struct ahash_edesc) + DESC_JOB_IO_LEN + + edesc = kzalloc(sizeof(*edesc) + DESC_JOB_IO_LEN + sec4_sg_bytes, GFP_DMA | flags); if (!edesc) { dev_err(jrdev, @@ -829,7 +829,7 @@ static int ahash_update_ctx(struct ahash_request *req) state->buf_dma = try_buf_map_to_sec4_sg(jrdev, edesc->sec4_sg + 1, buf, state->buf_dma, - *buflen, last_buflen); + *next_buflen, *buflen); if (src_nents) { src_map_to_sec4_sg(jrdev, req->src, src_nents, @@ -919,8 +919,8 @@ static int ahash_final_ctx(struct ahash_request *req) sec4_sg_bytes = sec4_sg_src_index * sizeof(struct sec4_sg_entry); /* allocate space for base edesc and hw desc commands, link tables */ - edesc = kmalloc(sizeof(struct ahash_edesc) + DESC_JOB_IO_LEN + - sec4_sg_bytes, GFP_DMA | flags); + edesc = kzalloc(sizeof(*edesc) + DESC_JOB_IO_LEN + sec4_sg_bytes, + GFP_DMA | flags); if (!edesc) { dev_err(jrdev, "could not allocate extended descriptor\n"); return -ENOMEM; @@ -1006,8 +1006,8 @@ static int ahash_finup_ctx(struct ahash_request *req) sizeof(struct sec4_sg_entry); /* allocate space for base edesc and hw desc commands, link tables */ - edesc = kmalloc(sizeof(struct ahash_edesc) + DESC_JOB_IO_LEN + - sec4_sg_bytes, GFP_DMA | flags); + edesc = kzalloc(sizeof(*edesc) + DESC_JOB_IO_LEN + sec4_sg_bytes, + GFP_DMA | flags); if (!edesc) { dev_err(jrdev, "could not allocate extended descriptor\n"); return -ENOMEM; @@ -1092,8 +1092,8 @@ static int ahash_digest(struct ahash_request *req) sec4_sg_bytes = src_nents * sizeof(struct sec4_sg_entry); /* allocate space for base edesc and hw desc commands, link tables */ - edesc = kmalloc(sizeof(struct ahash_edesc) + sec4_sg_bytes + - DESC_JOB_IO_LEN, GFP_DMA | flags); + edesc = kzalloc(sizeof(*edesc) + sec4_sg_bytes + DESC_JOB_IO_LEN, + GFP_DMA | flags); if (!edesc) { dev_err(jrdev, "could not allocate extended descriptor\n"); return -ENOMEM; @@ -1166,8 +1166,7 @@ static int ahash_final_no_ctx(struct ahash_request *req) int sh_len; /* allocate space for base edesc and hw desc commands, link tables */ - edesc = kmalloc(sizeof(struct ahash_edesc) + DESC_JOB_IO_LEN, - GFP_DMA | flags); + edesc = kzalloc(sizeof(*edesc) + DESC_JOB_IO_LEN, GFP_DMA | flags); if (!edesc) { dev_err(jrdev, "could not allocate extended descriptor\n"); return -ENOMEM; @@ -1246,7 +1245,7 @@ static int ahash_update_no_ctx(struct ahash_request *req) * allocate space for base edesc and hw desc commands, * link tables */ - edesc = kmalloc(sizeof(struct ahash_edesc) + DESC_JOB_IO_LEN + + edesc = kzalloc(sizeof(*edesc) + DESC_JOB_IO_LEN + sec4_sg_bytes, GFP_DMA | flags); if (!edesc) { dev_err(jrdev, @@ -1354,8 +1353,8 @@ static int ahash_finup_no_ctx(struct ahash_request *req) sizeof(struct sec4_sg_entry); /* allocate space for base edesc and hw desc commands, link tables */ - edesc = kmalloc(sizeof(struct ahash_edesc) + DESC_JOB_IO_LEN + - sec4_sg_bytes, GFP_DMA | flags); + edesc = kzalloc(sizeof(*edesc) + DESC_JOB_IO_LEN + sec4_sg_bytes, + GFP_DMA | flags); if (!edesc) { dev_err(jrdev, "could not allocate extended descriptor\n"); return -ENOMEM; @@ -1449,7 +1448,7 @@ static int ahash_update_first(struct ahash_request *req) * allocate space for base edesc and hw desc commands, * link tables */ - edesc = kmalloc(sizeof(struct ahash_edesc) + DESC_JOB_IO_LEN + + edesc = kzalloc(sizeof(*edesc) + DESC_JOB_IO_LEN + sec4_sg_bytes, GFP_DMA | flags); if (!edesc) { dev_err(jrdev, @@ -1843,7 +1842,7 @@ caam_hash_alloc(struct caam_hash_template *template, struct ahash_alg *halg; struct crypto_alg *alg; - t_alg = kzalloc(sizeof(struct caam_hash_alg), GFP_KERNEL); + t_alg = kzalloc(sizeof(*t_alg), GFP_KERNEL); if (!t_alg) { pr_err("failed to allocate t_alg\n"); return ERR_PTR(-ENOMEM); @@ -1885,8 +1884,10 @@ static int __init caam_algapi_hash_init(void) struct device_node *dev_node; struct platform_device *pdev; struct device *ctrldev; - void *priv; int i = 0, err = 0; + struct caam_drv_private *priv; + unsigned int md_limit = SHA512_DIGEST_SIZE; + u32 cha_inst, cha_vid; dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0"); if (!dev_node) { @@ -1912,19 +1913,40 @@ static int __init caam_algapi_hash_init(void) if (!priv) return -ENODEV; + /* + * Register crypto algorithms the device supports. First, identify + * presence and attributes of MD block. + */ + cha_vid = rd_reg32(&priv->ctrl->perfmon.cha_id_ls); + cha_inst = rd_reg32(&priv->ctrl->perfmon.cha_num_ls); + + /* + * Skip registration of any hashing algorithms if MD block + * is not present. + */ + if (!((cha_inst & CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT)) + return -ENODEV; + + /* Limit digest size based on LP256 */ + if ((cha_vid & CHA_ID_LS_MD_MASK) == CHA_ID_LS_MD_LP256) + md_limit = SHA256_DIGEST_SIZE; + INIT_LIST_HEAD(&hash_list); /* register crypto algorithms the device supports */ for (i = 0; i < ARRAY_SIZE(driver_hash); i++) { - /* TODO: check if h/w supports alg */ struct caam_hash_alg *t_alg; + struct caam_hash_template *alg = driver_hash + i; + + /* If MD size is not supported by device, skip registration */ + if (alg->template_ahash.halg.digestsize > md_limit) + continue; /* register hmac version */ - t_alg = caam_hash_alloc(&driver_hash[i], true); + t_alg = caam_hash_alloc(alg, true); if (IS_ERR(t_alg)) { err = PTR_ERR(t_alg); - pr_warn("%s alg allocation failed\n", - driver_hash[i].driver_name); + pr_warn("%s alg allocation failed\n", alg->driver_name); continue; } @@ -1937,11 +1959,10 @@ static int __init caam_algapi_hash_init(void) list_add_tail(&t_alg->entry, &hash_list); /* register unkeyed version */ - t_alg = caam_hash_alloc(&driver_hash[i], false); + t_alg = caam_hash_alloc(alg, false); if (IS_ERR(t_alg)) { err = PTR_ERR(t_alg); - pr_warn("%s alg allocation failed\n", - driver_hash[i].driver_name); + pr_warn("%s alg allocation failed\n", alg->driver_name); continue; } diff --git a/drivers/crypto/caam/caamrng.c b/drivers/crypto/caam/caamrng.c index 5095337205b8..9b92af2c7241 100644 --- a/drivers/crypto/caam/caamrng.c +++ b/drivers/crypto/caam/caamrng.c @@ -108,6 +108,10 @@ static void rng_done(struct device *jrdev, u32 *desc, u32 err, void *context) atomic_set(&bd->empty, BUF_NOT_EMPTY); complete(&bd->filled); + + /* Buffer refilled, invalidate cache */ + dma_sync_single_for_cpu(jrdev, bd->addr, RN_BUF_SIZE, DMA_FROM_DEVICE); + #ifdef DEBUG print_hex_dump(KERN_ERR, "rng refreshed buf@: ", DUMP_PREFIX_ADDRESS, 16, 4, bd->buf, RN_BUF_SIZE, 1); @@ -311,7 +315,7 @@ static int __init caam_rng_init(void) struct device_node *dev_node; struct platform_device *pdev; struct device *ctrldev; - void *priv; + struct caam_drv_private *priv; int err; dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0"); @@ -338,20 +342,32 @@ static int __init caam_rng_init(void) if (!priv) return -ENODEV; + /* Check for an instantiated RNG before registration */ + if (!(rd_reg32(&priv->ctrl->perfmon.cha_num_ls) & CHA_ID_LS_RNG_MASK)) + return -ENODEV; + dev = caam_jr_alloc(); if (IS_ERR(dev)) { pr_err("Job Ring Device allocation for transform failed\n"); return PTR_ERR(dev); } - rng_ctx = kmalloc(sizeof(struct caam_rng_ctx), GFP_DMA); - if (!rng_ctx) - return -ENOMEM; + rng_ctx = kmalloc(sizeof(*rng_ctx), GFP_DMA); + if (!rng_ctx) { + err = -ENOMEM; + goto free_caam_alloc; + } err = caam_init_rng(rng_ctx, dev); if (err) - return err; + goto free_rng_ctx; dev_info(dev, "registering rng-caam\n"); return hwrng_register(&caam_rng); + +free_rng_ctx: + kfree(rng_ctx); +free_caam_alloc: + caam_jr_free(dev); + return err; } module_init(caam_rng_init); diff --git a/drivers/crypto/caam/compat.h b/drivers/crypto/caam/compat.h index f57f395db33f..b6955ecdfb3f 100644 --- a/drivers/crypto/caam/compat.h +++ b/drivers/crypto/caam/compat.h @@ -23,6 +23,7 @@ #include <linux/types.h> #include <linux/debugfs.h> #include <linux/circ_buf.h> +#include <linux/clk.h> #include <net/xfrm.h> #include <crypto/algapi.h> diff --git a/drivers/crypto/caam/ctrl.c b/drivers/crypto/caam/ctrl.c index efacab7539ef..8abb4bc548cc 100644 --- a/drivers/crypto/caam/ctrl.c +++ b/drivers/crypto/caam/ctrl.c @@ -16,6 +16,24 @@ #include "error.h" /* + * i.MX targets tend to have clock control subsystems that can + * enable/disable clocking to our device. + */ +#ifdef CONFIG_CRYPTO_DEV_FSL_CAAM_IMX +static inline struct clk *caam_drv_identify_clk(struct device *dev, + char *clk_name) +{ + return devm_clk_get(dev, clk_name); +} +#else +static inline struct clk *caam_drv_identify_clk(struct device *dev, + char *clk_name) +{ + return NULL; +} +#endif + +/* * Descriptor to instantiate RNG State Handle 0 in normal mode and * load the JDKEK, TDKEK and TDSK registers */ @@ -121,7 +139,7 @@ static inline int run_descriptor_deco0(struct device *ctrldev, u32 *desc, flags |= DECO_JQCR_FOUR; /* Instruct the DECO to execute it */ - wr_reg32(&deco->jr_ctl_hi, flags); + setbits32(&deco->jr_ctl_hi, flags); timeout = 10000000; do { @@ -175,7 +193,7 @@ static int instantiate_rng(struct device *ctrldev, int state_handle_mask, { struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctrldev); struct caam_ctrl __iomem *ctrl; - u32 *desc, status, rdsta_val; + u32 *desc, status = 0, rdsta_val; int ret = 0, sh_idx; ctrl = (struct caam_ctrl __iomem *)ctrlpriv->ctrl; @@ -207,7 +225,8 @@ static int instantiate_rng(struct device *ctrldev, int state_handle_mask, * CAAM eras), then try again. */ rdsta_val = rd_reg32(&ctrl->r4tst[0].rdsta) & RDSTA_IFMASK; - if (status || !(rdsta_val & (1 << sh_idx))) + if ((status && status != JRSTA_SSRC_JUMP_HALT_CC) || + !(rdsta_val & (1 << sh_idx))) ret = -EAGAIN; if (ret) break; @@ -279,7 +298,7 @@ static int caam_remove(struct platform_device *pdev) struct device *ctrldev; struct caam_drv_private *ctrlpriv; struct caam_ctrl __iomem *ctrl; - int ring, ret = 0; + int ring; ctrldev = &pdev->dev; ctrlpriv = dev_get_drvdata(ctrldev); @@ -303,7 +322,13 @@ static int caam_remove(struct platform_device *pdev) /* Unmap controller region */ iounmap(ctrl); - return ret; + /* shut clocks off before finalizing shutdown */ + clk_disable_unprepare(ctrlpriv->caam_ipg); + clk_disable_unprepare(ctrlpriv->caam_mem); + clk_disable_unprepare(ctrlpriv->caam_aclk); + clk_disable_unprepare(ctrlpriv->caam_emi_slow); + + return 0; } /* @@ -370,14 +395,14 @@ static void kick_trng(struct platform_device *pdev, int ent_delay) int caam_get_era(void) { struct device_node *caam_node; - for_each_compatible_node(caam_node, NULL, "fsl,sec-v4.0") { - const uint32_t *prop = (uint32_t *)of_get_property(caam_node, - "fsl,sec-era", - NULL); - return prop ? *prop : -ENOTSUPP; - } + int ret; + u32 prop; - return -ENOTSUPP; + caam_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0"); + ret = of_property_read_u32(caam_node, "fsl,sec-era", &prop); + of_node_put(caam_node); + + return IS_ERR_VALUE(ret) ? -ENOTSUPP : prop; } EXPORT_SYMBOL(caam_get_era); @@ -390,6 +415,7 @@ static int caam_probe(struct platform_device *pdev) struct device_node *nprop, *np; struct caam_ctrl __iomem *ctrl; struct caam_drv_private *ctrlpriv; + struct clk *clk; #ifdef CONFIG_DEBUG_FS struct caam_perfmon *perfmon; #endif @@ -398,8 +424,7 @@ static int caam_probe(struct platform_device *pdev) int pg_size; int BLOCK_OFFSET = 0; - ctrlpriv = devm_kzalloc(&pdev->dev, sizeof(struct caam_drv_private), - GFP_KERNEL); + ctrlpriv = devm_kzalloc(&pdev->dev, sizeof(*ctrlpriv), GFP_KERNEL); if (!ctrlpriv) return -ENOMEM; @@ -408,12 +433,76 @@ static int caam_probe(struct platform_device *pdev) ctrlpriv->pdev = pdev; nprop = pdev->dev.of_node; + /* Enable clocking */ + clk = caam_drv_identify_clk(&pdev->dev, "ipg"); + if (IS_ERR(clk)) { + ret = PTR_ERR(clk); + dev_err(&pdev->dev, + "can't identify CAAM ipg clk: %d\n", ret); + return ret; + } + ctrlpriv->caam_ipg = clk; + + clk = caam_drv_identify_clk(&pdev->dev, "mem"); + if (IS_ERR(clk)) { + ret = PTR_ERR(clk); + dev_err(&pdev->dev, + "can't identify CAAM mem clk: %d\n", ret); + return ret; + } + ctrlpriv->caam_mem = clk; + + clk = caam_drv_identify_clk(&pdev->dev, "aclk"); + if (IS_ERR(clk)) { + ret = PTR_ERR(clk); + dev_err(&pdev->dev, + "can't identify CAAM aclk clk: %d\n", ret); + return ret; + } + ctrlpriv->caam_aclk = clk; + + clk = caam_drv_identify_clk(&pdev->dev, "emi_slow"); + if (IS_ERR(clk)) { + ret = PTR_ERR(clk); + dev_err(&pdev->dev, + "can't identify CAAM emi_slow clk: %d\n", ret); + return ret; + } + ctrlpriv->caam_emi_slow = clk; + + ret = clk_prepare_enable(ctrlpriv->caam_ipg); + if (ret < 0) { + dev_err(&pdev->dev, "can't enable CAAM ipg clock: %d\n", ret); + return ret; + } + + ret = clk_prepare_enable(ctrlpriv->caam_mem); + if (ret < 0) { + dev_err(&pdev->dev, "can't enable CAAM secure mem clock: %d\n", + ret); + goto disable_caam_ipg; + } + + ret = clk_prepare_enable(ctrlpriv->caam_aclk); + if (ret < 0) { + dev_err(&pdev->dev, "can't enable CAAM aclk clock: %d\n", ret); + goto disable_caam_mem; + } + + ret = clk_prepare_enable(ctrlpriv->caam_emi_slow); + if (ret < 0) { + dev_err(&pdev->dev, "can't enable CAAM emi slow clock: %d\n", + ret); + goto disable_caam_aclk; + } + /* Get configuration properties from device tree */ /* First, get register page */ ctrl = of_iomap(nprop, 0); if (ctrl == NULL) { dev_err(dev, "caam: of_iomap() failed\n"); - return -ENOMEM; + ret = -ENOMEM; + goto disable_caam_emi_slow; } /* Finding the page size for using the CTPR_MS register */ comp_params = rd_reg32(&ctrl->perfmon.comp_parms_ms); @@ -444,8 +533,9 @@ static int caam_probe(struct platform_device *pdev) * Enable DECO watchdogs and, if this is a PHYS_ADDR_T_64BIT kernel, * long pointers in master configuration register */ - setbits32(&ctrl->mcr, MCFGR_WDENABLE | - (sizeof(dma_addr_t) == sizeof(u64) ? MCFGR_LONG_PTR : 0)); + clrsetbits_32(&ctrl->mcr, MCFGR_AWCACHE_MASK, MCFGR_AWCACHE_CACH | + MCFGR_WDENABLE | (sizeof(dma_addr_t) == sizeof(u64) ? + MCFGR_LONG_PTR : 0)); /* * Read the Compile Time paramters and SCFGR to determine @@ -492,12 +582,11 @@ static int caam_probe(struct platform_device *pdev) of_device_is_compatible(np, "fsl,sec4.0-job-ring")) rspec++; - ctrlpriv->jrpdev = devm_kzalloc(&pdev->dev, - sizeof(struct platform_device *) * rspec, - GFP_KERNEL); + ctrlpriv->jrpdev = devm_kcalloc(&pdev->dev, rspec, + sizeof(*ctrlpriv->jrpdev), GFP_KERNEL); if (ctrlpriv->jrpdev == NULL) { - iounmap(ctrl); - return -ENOMEM; + ret = -ENOMEM; + goto iounmap_ctrl; } ring = 0; @@ -537,8 +626,8 @@ static int caam_probe(struct platform_device *pdev) /* If no QI and no rings specified, quit and go home */ if ((!ctrlpriv->qi_present) && (!ctrlpriv->total_jobrs)) { dev_err(dev, "no queues configured, terminating\n"); - caam_remove(pdev); - return -ENOMEM; + ret = -ENOMEM; + goto caam_remove; } cha_vid_ls = rd_reg32(&ctrl->perfmon.cha_id_ls); @@ -595,8 +684,7 @@ static int caam_probe(struct platform_device *pdev) } while ((ret == -EAGAIN) && (ent_delay < RTSDCTL_ENT_DLY_MAX)); if (ret) { dev_err(dev, "failed to instantiate RNG"); - caam_remove(pdev); - return ret; + goto caam_remove; } /* * Set handles init'ed by this module as the complement of the @@ -700,6 +788,20 @@ static int caam_probe(struct platform_device *pdev) &ctrlpriv->ctl_tdsk_wrap); #endif return 0; + +caam_remove: + caam_remove(pdev); +iounmap_ctrl: + iounmap(ctrl); +disable_caam_emi_slow: + clk_disable_unprepare(ctrlpriv->caam_emi_slow); +disable_caam_aclk: + clk_disable_unprepare(ctrlpriv->caam_aclk); +disable_caam_mem: + clk_disable_unprepare(ctrlpriv->caam_mem); +disable_caam_ipg: + clk_disable_unprepare(ctrlpriv->caam_ipg); + return ret; } static struct of_device_id caam_match[] = { diff --git a/drivers/crypto/caam/desc.h b/drivers/crypto/caam/desc.h index d397ff9d56fd..983d663ef671 100644 --- a/drivers/crypto/caam/desc.h +++ b/drivers/crypto/caam/desc.h @@ -8,12 +8,29 @@ #ifndef DESC_H #define DESC_H +/* + * 16-byte hardware scatter/gather table + * An 8-byte table exists in the hardware spec, but has never been + * implemented to date. The 8/16 option is selected at RTL-compile-time. + * and this selection is visible in the Compile Time Parameters Register + */ + +#define SEC4_SG_LEN_EXT 0x80000000 /* Entry points to table */ +#define SEC4_SG_LEN_FIN 0x40000000 /* Last ent in table */ +#define SEC4_SG_BPID_MASK 0x000000ff +#define SEC4_SG_BPID_SHIFT 16 +#define SEC4_SG_LEN_MASK 0x3fffffff /* Excludes EXT and FINAL */ +#define SEC4_SG_OFFS_MASK 0x00001fff + struct sec4_sg_entry { +#ifdef CONFIG_CRYPTO_DEV_FSL_CAAM_IMX + u32 rsvd1; + dma_addr_t ptr; +#else u64 ptr; -#define SEC4_SG_LEN_FIN 0x40000000 -#define SEC4_SG_LEN_EXT 0x80000000 +#endif /* CONFIG_CRYPTO_DEV_FSL_CAAM_IMX */ u32 len; - u8 reserved; + u8 rsvd2; u8 buf_pool_id; u16 offset; }; diff --git a/drivers/crypto/caam/desc_constr.h b/drivers/crypto/caam/desc_constr.h index 9f79fd7bd4d7..98d07de24fc4 100644 --- a/drivers/crypto/caam/desc_constr.h +++ b/drivers/crypto/caam/desc_constr.h @@ -367,7 +367,7 @@ do { \ if (upper) \ append_u64(desc, data); \ else \ - append_u32(desc, data); \ + append_u32(desc, lower_32_bits(data)); \ } while (0) #define append_math_add_imm_u64(desc, dest, src0, src1, data) \ diff --git a/drivers/crypto/caam/intern.h b/drivers/crypto/caam/intern.h index 89b94cc9e7a2..e2bcacc1a921 100644 --- a/drivers/crypto/caam/intern.h +++ b/drivers/crypto/caam/intern.h @@ -91,6 +91,11 @@ struct caam_drv_private { Handles of the RNG4 block are initialized by this driver */ + struct clk *caam_ipg; + struct clk *caam_mem; + struct clk *caam_aclk; + struct clk *caam_emi_slow; + /* * debugfs entries for developer view into driver/device * variables at runtime. diff --git a/drivers/crypto/caam/jr.c b/drivers/crypto/caam/jr.c index b8b5d47acd7a..f7e0d8d4c3da 100644 --- a/drivers/crypto/caam/jr.c +++ b/drivers/crypto/caam/jr.c @@ -202,6 +202,13 @@ static void caam_jr_dequeue(unsigned long devarg) userdesc = jrp->entinfo[sw_idx].desc_addr_virt; userstatus = jrp->outring[hw_idx].jrstatus; + /* + * Make sure all information from the job has been obtained + * before telling CAAM that the job has been removed from the + * output ring. + */ + mb(); + /* set done */ wr_reg32(&jrp->rregs->outring_rmvd, 1); @@ -351,12 +358,23 @@ int caam_jr_enqueue(struct device *dev, u32 *desc, jrp->inpring[jrp->inp_ring_write_index] = desc_dma; + /* + * Guarantee that the descriptor's DMA address has been written to + * the next slot in the ring before the write index is updated, since + * other cores may update this index independently. + */ smp_wmb(); jrp->inp_ring_write_index = (jrp->inp_ring_write_index + 1) & (JOBR_DEPTH - 1); jrp->head = (head + 1) & (JOBR_DEPTH - 1); + /* + * Ensure that all job information has been written before + * notifying CAAM that a new job was added to the input ring. + */ + wmb(); + wr_reg32(&jrp->rregs->inpring_jobadd, 1); spin_unlock_bh(&jrp->inplock); @@ -392,18 +410,17 @@ static int caam_jr_init(struct device *dev) goto out_free_irq; error = -ENOMEM; - jrp->inpring = dma_alloc_coherent(dev, sizeof(dma_addr_t) * JOBR_DEPTH, - &inpbusaddr, GFP_KERNEL); + jrp->inpring = dma_alloc_coherent(dev, sizeof(*jrp->inpring) * + JOBR_DEPTH, &inpbusaddr, GFP_KERNEL); if (!jrp->inpring) goto out_free_irq; - jrp->outring = dma_alloc_coherent(dev, sizeof(struct jr_outentry) * + jrp->outring = dma_alloc_coherent(dev, sizeof(*jrp->outring) * JOBR_DEPTH, &outbusaddr, GFP_KERNEL); if (!jrp->outring) goto out_free_inpring; - jrp->entinfo = kzalloc(sizeof(struct caam_jrentry_info) * JOBR_DEPTH, - GFP_KERNEL); + jrp->entinfo = kcalloc(JOBR_DEPTH, sizeof(*jrp->entinfo), GFP_KERNEL); if (!jrp->entinfo) goto out_free_outring; @@ -461,8 +478,7 @@ static int caam_jr_probe(struct platform_device *pdev) int error; jrdev = &pdev->dev; - jrpriv = devm_kmalloc(jrdev, sizeof(struct caam_drv_private_jr), - GFP_KERNEL); + jrpriv = devm_kmalloc(jrdev, sizeof(*jrpriv), GFP_KERNEL); if (!jrpriv) return -ENOMEM; diff --git a/drivers/crypto/caam/regs.h b/drivers/crypto/caam/regs.h index 672c97489505..a8a79975682f 100644 --- a/drivers/crypto/caam/regs.h +++ b/drivers/crypto/caam/regs.h @@ -65,9 +65,31 @@ * */ +#ifdef CONFIG_ARM +/* These are common macros for Power, put here for ARM */ +#define setbits32(_addr, _v) writel((readl(_addr) | (_v)), (_addr)) +#define clrbits32(_addr, _v) writel((readl(_addr) & ~(_v)), (_addr)) + +#define out_arch(type, endian, a, v) __raw_write##type(cpu_to_##endian(v), a) +#define in_arch(type, endian, a) endian##_to_cpu(__raw_read##type(a)) + +#define out_le32(a, v) out_arch(l, le32, a, v) +#define in_le32(a) in_arch(l, le32, a) + +#define out_be32(a, v) out_arch(l, be32, a, v) +#define in_be32(a) in_arch(l, be32, a) + +#define clrsetbits(type, addr, clear, set) \ + out_##type((addr), (in_##type(addr) & ~(clear)) | (set)) + +#define clrsetbits_be32(addr, clear, set) clrsetbits(be32, addr, clear, set) +#define clrsetbits_le32(addr, clear, set) clrsetbits(le32, addr, clear, set) +#endif + #ifdef __BIG_ENDIAN #define wr_reg32(reg, data) out_be32(reg, data) #define rd_reg32(reg) in_be32(reg) +#define clrsetbits_32(addr, clear, set) clrsetbits_be32(addr, clear, set) #ifdef CONFIG_64BIT #define wr_reg64(reg, data) out_be64(reg, data) #define rd_reg64(reg) in_be64(reg) @@ -76,6 +98,7 @@ #ifdef __LITTLE_ENDIAN #define wr_reg32(reg, data) __raw_writel(data, reg) #define rd_reg32(reg) __raw_readl(reg) +#define clrsetbits_32(addr, clear, set) clrsetbits_le32(addr, clear, set) #ifdef CONFIG_64BIT #define wr_reg64(reg, data) __raw_writeq(data, reg) #define rd_reg64(reg) __raw_readq(reg) @@ -85,20 +108,31 @@ /* * The only users of these wr/rd_reg64 functions is the Job Ring (JR). - * The DMA address registers in the JR are a pair of 32-bit registers. - * The layout is: + * The DMA address registers in the JR are handled differently depending on + * platform: + * + * 1. All BE CAAM platforms and i.MX platforms (LE CAAM): * * base + 0x0000 : most-significant 32 bits * base + 0x0004 : least-significant 32 bits * * The 32-bit version of this core therefore has to write to base + 0x0004 - * to set the 32-bit wide DMA address. This seems to be independent of the - * endianness of the written/read data. + * to set the 32-bit wide DMA address. + * + * 2. All other LE CAAM platforms (LS1021A etc.) + * base + 0x0000 : least-significant 32 bits + * base + 0x0004 : most-significant 32 bits */ #ifndef CONFIG_64BIT +#if !defined(CONFIG_CRYPTO_DEV_FSL_CAAM_LE) || \ + defined(CONFIG_CRYPTO_DEV_FSL_CAAM_IMX) #define REG64_MS32(reg) ((u32 __iomem *)(reg)) #define REG64_LS32(reg) ((u32 __iomem *)(reg) + 1) +#else +#define REG64_MS32(reg) ((u32 __iomem *)(reg) + 1) +#define REG64_LS32(reg) ((u32 __iomem *)(reg)) +#endif static inline void wr_reg64(u64 __iomem *reg, u64 data) { @@ -133,18 +167,28 @@ struct jr_outentry { #define CHA_NUM_MS_DECONUM_SHIFT 24 #define CHA_NUM_MS_DECONUM_MASK (0xfull << CHA_NUM_MS_DECONUM_SHIFT) -/* CHA Version IDs */ +/* + * CHA version IDs / instantiation bitfields + * Defined for use with the cha_id fields in perfmon, but the same shift/mask + * selectors can be used to pull out the number of instantiated blocks within + * cha_num fields in perfmon because the locations are the same. + */ #define CHA_ID_LS_AES_SHIFT 0 -#define CHA_ID_LS_AES_MASK (0xfull << CHA_ID_LS_AES_SHIFT) +#define CHA_ID_LS_AES_MASK (0xfull << CHA_ID_LS_AES_SHIFT) +#define CHA_ID_LS_AES_LP (0x3ull << CHA_ID_LS_AES_SHIFT) +#define CHA_ID_LS_AES_HP (0x4ull << CHA_ID_LS_AES_SHIFT) #define CHA_ID_LS_DES_SHIFT 4 -#define CHA_ID_LS_DES_MASK (0xfull << CHA_ID_LS_DES_SHIFT) +#define CHA_ID_LS_DES_MASK (0xfull << CHA_ID_LS_DES_SHIFT) #define CHA_ID_LS_ARC4_SHIFT 8 #define CHA_ID_LS_ARC4_MASK (0xfull << CHA_ID_LS_ARC4_SHIFT) #define CHA_ID_LS_MD_SHIFT 12 #define CHA_ID_LS_MD_MASK (0xfull << CHA_ID_LS_MD_SHIFT) +#define CHA_ID_LS_MD_LP256 (0x0ull << CHA_ID_LS_MD_SHIFT) +#define CHA_ID_LS_MD_LP512 (0x1ull << CHA_ID_LS_MD_SHIFT) +#define CHA_ID_LS_MD_HP (0x2ull << CHA_ID_LS_MD_SHIFT) #define CHA_ID_LS_RNG_SHIFT 16 #define CHA_ID_LS_RNG_MASK (0xfull << CHA_ID_LS_RNG_SHIFT) @@ -395,10 +439,16 @@ struct caam_ctrl { /* AXI read cache control */ #define MCFGR_ARCACHE_SHIFT 12 #define MCFGR_ARCACHE_MASK (0xf << MCFGR_ARCACHE_SHIFT) +#define MCFGR_ARCACHE_BUFF (0x1 << MCFGR_ARCACHE_SHIFT) +#define MCFGR_ARCACHE_CACH (0x2 << MCFGR_ARCACHE_SHIFT) +#define MCFGR_ARCACHE_RALL (0x4 << MCFGR_ARCACHE_SHIFT) /* AXI write cache control */ #define MCFGR_AWCACHE_SHIFT 8 #define MCFGR_AWCACHE_MASK (0xf << MCFGR_AWCACHE_SHIFT) +#define MCFGR_AWCACHE_BUFF (0x1 << MCFGR_AWCACHE_SHIFT) +#define MCFGR_AWCACHE_CACH (0x2 << MCFGR_AWCACHE_SHIFT) +#define MCFGR_AWCACHE_WALL (0x8 << MCFGR_AWCACHE_SHIFT) /* AXI pipeline depth */ #define MCFGR_AXIPIPE_SHIFT 4 diff --git a/drivers/crypto/caam/sg_sw_sec4.h b/drivers/crypto/caam/sg_sw_sec4.h index b68b74cc7b77..18cd6d1f5870 100644 --- a/drivers/crypto/caam/sg_sw_sec4.h +++ b/drivers/crypto/caam/sg_sw_sec4.h @@ -15,7 +15,6 @@ static inline void dma_to_sec4_sg_one(struct sec4_sg_entry *sec4_sg_ptr, { sec4_sg_ptr->ptr = dma; sec4_sg_ptr->len = len; - sec4_sg_ptr->reserved = 0; sec4_sg_ptr->buf_pool_id = 0; sec4_sg_ptr->offset = offset; #ifdef DEBUG @@ -106,9 +105,15 @@ static inline void dma_unmap_sg_chained( { if (unlikely(chained)) { int i; + struct scatterlist *tsg = sg; + + /* + * Use a local copy of the sg pointer to avoid moving the + * head of the list pointed to by sg as we walk the list. + */ for (i = 0; i < nents; i++) { - dma_unmap_sg(dev, sg, 1, dir); - sg = sg_next(sg); + dma_unmap_sg(dev, tsg, 1, dir); + tsg = sg_next(tsg); } } else if (nents) { dma_unmap_sg(dev, sg, nents, dir); @@ -119,19 +124,23 @@ static inline int dma_map_sg_chained( struct device *dev, struct scatterlist *sg, unsigned int nents, enum dma_data_direction dir, bool chained) { - struct scatterlist *first = sg; - if (unlikely(chained)) { int i; + struct scatterlist *tsg = sg; + + /* + * Use a local copy of the sg pointer to avoid moving the + * head of the list pointed to by sg as we walk the list. + */ for (i = 0; i < nents; i++) { - if (!dma_map_sg(dev, sg, 1, dir)) { - dma_unmap_sg_chained(dev, first, i, dir, + if (!dma_map_sg(dev, tsg, 1, dir)) { + dma_unmap_sg_chained(dev, sg, i, dir, chained); nents = 0; break; } - sg = sg_next(sg); + tsg = sg_next(tsg); } } else nents = dma_map_sg(dev, sg, nents, dir); diff --git a/drivers/crypto/ccp/ccp-platform.c b/drivers/crypto/ccp/ccp-platform.c index f2e6de361fd1..bb241c3ab6b9 100644 --- a/drivers/crypto/ccp/ccp-platform.c +++ b/drivers/crypto/ccp/ccp-platform.c @@ -216,6 +216,7 @@ static const struct acpi_device_id ccp_acpi_match[] = { { "AMDI0C00", 0 }, { }, }; +MODULE_DEVICE_TABLE(acpi, ccp_acpi_match); #endif #ifdef CONFIG_OF @@ -223,6 +224,7 @@ static const struct of_device_id ccp_of_match[] = { { .compatible = "amd,ccp-seattle-v1a" }, { }, }; +MODULE_DEVICE_TABLE(of, ccp_of_match); #endif static struct platform_driver ccp_platform_driver = { diff --git a/drivers/crypto/img-hash.c b/drivers/crypto/img-hash.c index ad47d0d61098..68e8aa90fe01 100644 --- a/drivers/crypto/img-hash.c +++ b/drivers/crypto/img-hash.c @@ -334,7 +334,7 @@ static int img_hash_dma_init(struct img_hash_dev *hdev) hdev->dma_lch = dma_request_slave_channel(hdev->dev, "tx"); if (!hdev->dma_lch) { - dev_err(hdev->dev, "Couldn't aquire a slave DMA channel.\n"); + dev_err(hdev->dev, "Couldn't acquire a slave DMA channel.\n"); return -EBUSY; } dma_conf.direction = DMA_MEM_TO_DEV; diff --git a/drivers/crypto/ixp4xx_crypto.c b/drivers/crypto/ixp4xx_crypto.c index 402631a19a11..8f2790353281 100644 --- a/drivers/crypto/ixp4xx_crypto.c +++ b/drivers/crypto/ixp4xx_crypto.c @@ -156,7 +156,8 @@ struct ablk_ctx { }; struct aead_ctx { - struct buffer_desc *buffer; + struct buffer_desc *src; + struct buffer_desc *dst; struct scatterlist ivlist; /* used when the hmac is not on one sg entry */ u8 *hmac_virt; @@ -198,6 +199,15 @@ struct ixp_alg { int registered; }; +struct ixp_aead_alg { + struct aead_alg crypto; + const struct ix_hash_algo *hash; + u32 cfg_enc; + u32 cfg_dec; + + int registered; +}; + static const struct ix_hash_algo hash_alg_md5 = { .cfgword = 0xAA010004, .icv = "\x01\x23\x45\x67\x89\xAB\xCD\xEF" @@ -339,11 +349,11 @@ static void finish_scattered_hmac(struct crypt_ctl *crypt) struct aead_ctx *req_ctx = aead_request_ctx(req); struct crypto_aead *tfm = crypto_aead_reqtfm(req); int authsize = crypto_aead_authsize(tfm); - int decryptlen = req->cryptlen - authsize; + int decryptlen = req->assoclen + req->cryptlen - authsize; if (req_ctx->encrypt) { scatterwalk_map_and_copy(req_ctx->hmac_virt, - req->src, decryptlen, authsize, 1); + req->dst, decryptlen, authsize, 1); } dma_pool_free(buffer_pool, req_ctx->hmac_virt, crypt->icv_rev_aes); } @@ -364,7 +374,8 @@ static void one_packet(dma_addr_t phys) struct aead_request *req = crypt->data.aead_req; struct aead_ctx *req_ctx = aead_request_ctx(req); - free_buf_chain(dev, req_ctx->buffer, crypt->src_buf); + free_buf_chain(dev, req_ctx->src, crypt->src_buf); + free_buf_chain(dev, req_ctx->dst, crypt->dst_buf); if (req_ctx->hmac_virt) { finish_scattered_hmac(crypt); } @@ -573,11 +584,10 @@ static int init_tfm_ablk(struct crypto_tfm *tfm) return init_tfm(tfm); } -static int init_tfm_aead(struct crypto_tfm *tfm) +static int init_tfm_aead(struct crypto_aead *tfm) { - crypto_aead_set_reqsize(__crypto_aead_cast(tfm), - sizeof(struct aead_ctx)); - return init_tfm(tfm); + crypto_aead_set_reqsize(tfm, sizeof(struct aead_ctx)); + return init_tfm(crypto_aead_tfm(tfm)); } static void exit_tfm(struct crypto_tfm *tfm) @@ -587,6 +597,11 @@ static void exit_tfm(struct crypto_tfm *tfm) free_sa_dir(&ctx->decrypt); } +static void exit_tfm_aead(struct crypto_aead *tfm) +{ + exit_tfm(crypto_aead_tfm(tfm)); +} + static int register_chain_var(struct crypto_tfm *tfm, u8 xpad, u32 target, int init_len, u32 ctx_addr, const u8 *key, int key_len) { @@ -969,24 +984,6 @@ static int ablk_rfc3686_crypt(struct ablkcipher_request *req) return ret; } -static int hmac_inconsistent(struct scatterlist *sg, unsigned start, - unsigned int nbytes) -{ - int offset = 0; - - if (!nbytes) - return 0; - - for (;;) { - if (start < offset + sg->length) - break; - - offset += sg->length; - sg = sg_next(sg); - } - return (start + nbytes > offset + sg->length); -} - static int aead_perform(struct aead_request *req, int encrypt, int cryptoffset, int eff_cryptlen, u8 *iv) { @@ -1002,6 +999,8 @@ static int aead_perform(struct aead_request *req, int encrypt, struct device *dev = &pdev->dev; gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : GFP_ATOMIC; + enum dma_data_direction src_direction = DMA_BIDIRECTIONAL; + unsigned int lastlen; if (qmgr_stat_full(SEND_QID)) return -EAGAIN; @@ -1030,35 +1029,55 @@ static int aead_perform(struct aead_request *req, int encrypt, crypt->crypt_len = eff_cryptlen; crypt->auth_offs = 0; - crypt->auth_len = req->assoclen + ivsize + cryptlen; + crypt->auth_len = req->assoclen + cryptlen; BUG_ON(ivsize && !req->iv); memcpy(crypt->iv, req->iv, ivsize); + req_ctx->dst = NULL; + if (req->src != req->dst) { - BUG(); /* -ENOTSUP because of my laziness */ + struct buffer_desc dst_hook; + + crypt->mode |= NPE_OP_NOT_IN_PLACE; + src_direction = DMA_TO_DEVICE; + + buf = chainup_buffers(dev, req->dst, crypt->auth_len, + &dst_hook, flags, DMA_FROM_DEVICE); + req_ctx->dst = dst_hook.next; + crypt->dst_buf = dst_hook.phys_next; + + if (!buf) + goto free_buf_dst; + + if (encrypt) { + lastlen = buf->buf_len; + if (lastlen >= authsize) + crypt->icv_rev_aes = buf->phys_addr + + buf->buf_len - authsize; + } } - /* ASSOC data */ - buf = chainup_buffers(dev, req->assoc, req->assoclen, &src_hook, - flags, DMA_TO_DEVICE); - req_ctx->buffer = src_hook.next; + buf = chainup_buffers(dev, req->src, crypt->auth_len, + &src_hook, flags, src_direction); + req_ctx->src = src_hook.next; crypt->src_buf = src_hook.phys_next; if (!buf) - goto out; - /* IV */ - sg_init_table(&req_ctx->ivlist, 1); - sg_set_buf(&req_ctx->ivlist, iv, ivsize); - buf = chainup_buffers(dev, &req_ctx->ivlist, ivsize, buf, flags, - DMA_BIDIRECTIONAL); - if (!buf) - goto free_chain; - if (unlikely(hmac_inconsistent(req->src, cryptlen, authsize))) { + goto free_buf_src; + + if (!encrypt || !req_ctx->dst) { + lastlen = buf->buf_len; + if (lastlen >= authsize) + crypt->icv_rev_aes = buf->phys_addr + + buf->buf_len - authsize; + } + + if (unlikely(lastlen < authsize)) { /* The 12 hmac bytes are scattered, * we need to copy them into a safe buffer */ req_ctx->hmac_virt = dma_pool_alloc(buffer_pool, flags, &crypt->icv_rev_aes); if (unlikely(!req_ctx->hmac_virt)) - goto free_chain; + goto free_buf_src; if (!encrypt) { scatterwalk_map_and_copy(req_ctx->hmac_virt, req->src, cryptlen, authsize, 0); @@ -1067,27 +1086,16 @@ static int aead_perform(struct aead_request *req, int encrypt, } else { req_ctx->hmac_virt = NULL; } - /* Crypt */ - buf = chainup_buffers(dev, req->src, cryptlen + authsize, buf, flags, - DMA_BIDIRECTIONAL); - if (!buf) - goto free_hmac_virt; - if (!req_ctx->hmac_virt) { - crypt->icv_rev_aes = buf->phys_addr + buf->buf_len - authsize; - } crypt->ctl_flags |= CTL_FLAG_PERFORM_AEAD; qmgr_put_entry(SEND_QID, crypt_virt2phys(crypt)); BUG_ON(qmgr_stat_overflow(SEND_QID)); return -EINPROGRESS; -free_hmac_virt: - if (req_ctx->hmac_virt) { - dma_pool_free(buffer_pool, req_ctx->hmac_virt, - crypt->icv_rev_aes); - } -free_chain: - free_buf_chain(dev, req_ctx->buffer, crypt->src_buf); -out: + +free_buf_src: + free_buf_chain(dev, req_ctx->src, crypt->src_buf); +free_buf_dst: + free_buf_chain(dev, req_ctx->dst, crypt->dst_buf); crypt->ctl_flags = CTL_FLAG_UNUSED; return -ENOMEM; } @@ -1173,40 +1181,12 @@ badkey: static int aead_encrypt(struct aead_request *req) { - unsigned ivsize = crypto_aead_ivsize(crypto_aead_reqtfm(req)); - return aead_perform(req, 1, req->assoclen + ivsize, - req->cryptlen, req->iv); + return aead_perform(req, 1, req->assoclen, req->cryptlen, req->iv); } static int aead_decrypt(struct aead_request *req) { - unsigned ivsize = crypto_aead_ivsize(crypto_aead_reqtfm(req)); - return aead_perform(req, 0, req->assoclen + ivsize, - req->cryptlen, req->iv); -} - -static int aead_givencrypt(struct aead_givcrypt_request *req) -{ - struct crypto_aead *tfm = aead_givcrypt_reqtfm(req); - struct ixp_ctx *ctx = crypto_aead_ctx(tfm); - unsigned len, ivsize = crypto_aead_ivsize(tfm); - __be64 seq; - - /* copied from eseqiv.c */ - if (!ctx->salted) { - get_random_bytes(ctx->salt, ivsize); - ctx->salted = 1; - } - memcpy(req->areq.iv, ctx->salt, ivsize); - len = ivsize; - if (ivsize > sizeof(u64)) { - memset(req->giv, 0, ivsize - sizeof(u64)); - len = sizeof(u64); - } - seq = cpu_to_be64(req->seq); - memcpy(req->giv + ivsize - len, &seq, len); - return aead_perform(&req->areq, 1, req->areq.assoclen, - req->areq.cryptlen +ivsize, req->giv); + return aead_perform(req, 0, req->assoclen, req->cryptlen, req->iv); } static struct ixp_alg ixp4xx_algos[] = { @@ -1319,80 +1299,77 @@ static struct ixp_alg ixp4xx_algos[] = { }, .cfg_enc = CIPH_ENCR | MOD_AES | MOD_CTR, .cfg_dec = CIPH_ENCR | MOD_AES | MOD_CTR, -}, { +} }; + +static struct ixp_aead_alg ixp4xx_aeads[] = { +{ .crypto = { - .cra_name = "authenc(hmac(md5),cbc(des))", - .cra_blocksize = DES_BLOCK_SIZE, - .cra_u = { .aead = { - .ivsize = DES_BLOCK_SIZE, - .maxauthsize = MD5_DIGEST_SIZE, - } - } + .base = { + .cra_name = "authenc(hmac(md5),cbc(des))", + .cra_blocksize = DES_BLOCK_SIZE, + }, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = MD5_DIGEST_SIZE, }, .hash = &hash_alg_md5, .cfg_enc = CIPH_ENCR | MOD_DES | MOD_CBC_ENC | KEYLEN_192, .cfg_dec = CIPH_DECR | MOD_DES | MOD_CBC_DEC | KEYLEN_192, }, { .crypto = { - .cra_name = "authenc(hmac(md5),cbc(des3_ede))", - .cra_blocksize = DES3_EDE_BLOCK_SIZE, - .cra_u = { .aead = { - .ivsize = DES3_EDE_BLOCK_SIZE, - .maxauthsize = MD5_DIGEST_SIZE, - } - } + .base = { + .cra_name = "authenc(hmac(md5),cbc(des3_ede))", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = MD5_DIGEST_SIZE, }, .hash = &hash_alg_md5, .cfg_enc = CIPH_ENCR | MOD_3DES | MOD_CBC_ENC | KEYLEN_192, .cfg_dec = CIPH_DECR | MOD_3DES | MOD_CBC_DEC | KEYLEN_192, }, { .crypto = { - .cra_name = "authenc(hmac(sha1),cbc(des))", - .cra_blocksize = DES_BLOCK_SIZE, - .cra_u = { .aead = { + .base = { + .cra_name = "authenc(hmac(sha1),cbc(des))", + .cra_blocksize = DES_BLOCK_SIZE, + }, .ivsize = DES_BLOCK_SIZE, .maxauthsize = SHA1_DIGEST_SIZE, - } - } }, .hash = &hash_alg_sha1, .cfg_enc = CIPH_ENCR | MOD_DES | MOD_CBC_ENC | KEYLEN_192, .cfg_dec = CIPH_DECR | MOD_DES | MOD_CBC_DEC | KEYLEN_192, }, { .crypto = { - .cra_name = "authenc(hmac(sha1),cbc(des3_ede))", - .cra_blocksize = DES3_EDE_BLOCK_SIZE, - .cra_u = { .aead = { - .ivsize = DES3_EDE_BLOCK_SIZE, - .maxauthsize = SHA1_DIGEST_SIZE, - } - } + .base = { + .cra_name = "authenc(hmac(sha1),cbc(des3_ede))", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, }, .hash = &hash_alg_sha1, .cfg_enc = CIPH_ENCR | MOD_3DES | MOD_CBC_ENC | KEYLEN_192, .cfg_dec = CIPH_DECR | MOD_3DES | MOD_CBC_DEC | KEYLEN_192, }, { .crypto = { - .cra_name = "authenc(hmac(md5),cbc(aes))", - .cra_blocksize = AES_BLOCK_SIZE, - .cra_u = { .aead = { - .ivsize = AES_BLOCK_SIZE, - .maxauthsize = MD5_DIGEST_SIZE, - } - } + .base = { + .cra_name = "authenc(hmac(md5),cbc(aes))", + .cra_blocksize = AES_BLOCK_SIZE, + }, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = MD5_DIGEST_SIZE, }, .hash = &hash_alg_md5, .cfg_enc = CIPH_ENCR | MOD_AES | MOD_CBC_ENC, .cfg_dec = CIPH_DECR | MOD_AES | MOD_CBC_DEC, }, { .crypto = { - .cra_name = "authenc(hmac(sha1),cbc(aes))", - .cra_blocksize = AES_BLOCK_SIZE, - .cra_u = { .aead = { - .ivsize = AES_BLOCK_SIZE, - .maxauthsize = SHA1_DIGEST_SIZE, - } - } + .base = { + .cra_name = "authenc(hmac(sha1),cbc(aes))", + .cra_blocksize = AES_BLOCK_SIZE, + }, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, }, .hash = &hash_alg_sha1, .cfg_enc = CIPH_ENCR | MOD_AES | MOD_CBC_ENC, @@ -1436,32 +1413,20 @@ static int __init ixp_module_init(void) if (!support_aes && (ixp4xx_algos[i].cfg_enc & MOD_AES)) { continue; } - if (!ixp4xx_algos[i].hash) { - /* block ciphers */ - cra->cra_type = &crypto_ablkcipher_type; - cra->cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | - CRYPTO_ALG_KERN_DRIVER_ONLY | - CRYPTO_ALG_ASYNC; - if (!cra->cra_ablkcipher.setkey) - cra->cra_ablkcipher.setkey = ablk_setkey; - if (!cra->cra_ablkcipher.encrypt) - cra->cra_ablkcipher.encrypt = ablk_encrypt; - if (!cra->cra_ablkcipher.decrypt) - cra->cra_ablkcipher.decrypt = ablk_decrypt; - cra->cra_init = init_tfm_ablk; - } else { - /* authenc */ - cra->cra_type = &crypto_aead_type; - cra->cra_flags = CRYPTO_ALG_TYPE_AEAD | - CRYPTO_ALG_KERN_DRIVER_ONLY | - CRYPTO_ALG_ASYNC; - cra->cra_aead.setkey = aead_setkey; - cra->cra_aead.setauthsize = aead_setauthsize; - cra->cra_aead.encrypt = aead_encrypt; - cra->cra_aead.decrypt = aead_decrypt; - cra->cra_aead.givencrypt = aead_givencrypt; - cra->cra_init = init_tfm_aead; - } + + /* block ciphers */ + cra->cra_type = &crypto_ablkcipher_type; + cra->cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC; + if (!cra->cra_ablkcipher.setkey) + cra->cra_ablkcipher.setkey = ablk_setkey; + if (!cra->cra_ablkcipher.encrypt) + cra->cra_ablkcipher.encrypt = ablk_encrypt; + if (!cra->cra_ablkcipher.decrypt) + cra->cra_ablkcipher.decrypt = ablk_decrypt; + cra->cra_init = init_tfm_ablk; + cra->cra_ctxsize = sizeof(struct ixp_ctx); cra->cra_module = THIS_MODULE; cra->cra_alignmask = 3; @@ -1473,6 +1438,38 @@ static int __init ixp_module_init(void) else ixp4xx_algos[i].registered = 1; } + + for (i = 0; i < ARRAY_SIZE(ixp4xx_aeads); i++) { + struct aead_alg *cra = &ixp4xx_aeads[i].crypto; + + if (snprintf(cra->base.cra_driver_name, CRYPTO_MAX_ALG_NAME, + "%s"IXP_POSTFIX, cra->base.cra_name) >= + CRYPTO_MAX_ALG_NAME) + continue; + if (!support_aes && (ixp4xx_algos[i].cfg_enc & MOD_AES)) + continue; + + /* authenc */ + cra->base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC; + cra->setkey = aead_setkey; + cra->setauthsize = aead_setauthsize; + cra->encrypt = aead_encrypt; + cra->decrypt = aead_decrypt; + cra->init = init_tfm_aead; + cra->exit = exit_tfm_aead; + + cra->base.cra_ctxsize = sizeof(struct ixp_ctx); + cra->base.cra_module = THIS_MODULE; + cra->base.cra_alignmask = 3; + cra->base.cra_priority = 300; + + if (crypto_register_aead(cra)) + printk(KERN_ERR "Failed to register '%s'\n", + cra->base.cra_driver_name); + else + ixp4xx_aeads[i].registered = 1; + } return 0; } @@ -1481,6 +1478,11 @@ static void __exit ixp_module_exit(void) int num = ARRAY_SIZE(ixp4xx_algos); int i; + for (i = 0; i < ARRAY_SIZE(ixp4xx_aeads); i++) { + if (ixp4xx_aeads[i].registered) + crypto_unregister_aead(&ixp4xx_aeads[i].crypto); + } + for (i=0; i< num; i++) { if (ixp4xx_algos[i].registered) crypto_unregister_alg(&ixp4xx_algos[i].crypto); diff --git a/drivers/crypto/marvell/cesa.c b/drivers/crypto/marvell/cesa.c index 1c6f98dd88f4..0643e3366e33 100644 --- a/drivers/crypto/marvell/cesa.c +++ b/drivers/crypto/marvell/cesa.c @@ -533,7 +533,6 @@ static struct platform_driver marvell_cesa = { .probe = mv_cesa_probe, .remove = mv_cesa_remove, .driver = { - .owner = THIS_MODULE, .name = "marvell-cesa", .of_match_table = mv_cesa_of_match_table, }, diff --git a/drivers/crypto/nx/Kconfig b/drivers/crypto/nx/Kconfig index e421c96c763a..ad7552a6998c 100644 --- a/drivers/crypto/nx/Kconfig +++ b/drivers/crypto/nx/Kconfig @@ -14,11 +14,14 @@ config CRYPTO_DEV_NX_ENCRYPT config CRYPTO_DEV_NX_COMPRESS tristate "Compression acceleration support" default y + select CRYPTO_ALGAPI + select 842_DECOMPRESS help Support for PowerPC Nest (NX) compression acceleration. This module supports acceleration for compressing memory with the 842 - algorithm. One of the platform drivers must be selected also. - If you choose 'M' here, this module will be called nx_compress. + algorithm using the cryptographic API. One of the platform + drivers must be selected also. If you choose 'M' here, this + module will be called nx_compress. if CRYPTO_DEV_NX_COMPRESS @@ -42,14 +45,4 @@ config CRYPTO_DEV_NX_COMPRESS_POWERNV algorithm. This supports NX hardware on the PowerNV platform. If you choose 'M' here, this module will be called nx_compress_powernv. -config CRYPTO_DEV_NX_COMPRESS_CRYPTO - tristate "Compression acceleration cryptographic interface" - select CRYPTO_ALGAPI - select 842_DECOMPRESS - default y - help - Support for PowerPC Nest (NX) accelerators using the cryptographic - API. If you choose 'M' here, this module will be called - nx_compress_crypto. - endif diff --git a/drivers/crypto/nx/Makefile b/drivers/crypto/nx/Makefile index e1684f5adb11..b727821c8ed4 100644 --- a/drivers/crypto/nx/Makefile +++ b/drivers/crypto/nx/Makefile @@ -10,12 +10,8 @@ nx-crypto-objs := nx.o \ nx-sha256.o \ nx-sha512.o -obj-$(CONFIG_CRYPTO_DEV_NX_COMPRESS) += nx-compress.o nx-compress-platform.o -obj-$(CONFIG_CRYPTO_DEV_NX_COMPRESS_PSERIES) += nx-compress-pseries.o -obj-$(CONFIG_CRYPTO_DEV_NX_COMPRESS_POWERNV) += nx-compress-powernv.o -obj-$(CONFIG_CRYPTO_DEV_NX_COMPRESS_CRYPTO) += nx-compress-crypto.o +obj-$(CONFIG_CRYPTO_DEV_NX_COMPRESS_PSERIES) += nx-compress-pseries.o nx-compress.o +obj-$(CONFIG_CRYPTO_DEV_NX_COMPRESS_POWERNV) += nx-compress-powernv.o nx-compress.o nx-compress-objs := nx-842.o -nx-compress-platform-objs := nx-842-platform.o nx-compress-pseries-objs := nx-842-pseries.o nx-compress-powernv-objs := nx-842-powernv.o -nx-compress-crypto-objs := nx-842-crypto.o diff --git a/drivers/crypto/nx/nx-842-crypto.c b/drivers/crypto/nx/nx-842-crypto.c deleted file mode 100644 index d53a1dcd7b4e..000000000000 --- a/drivers/crypto/nx/nx-842-crypto.c +++ /dev/null @@ -1,580 +0,0 @@ -/* - * Cryptographic API for the NX-842 hardware compression. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * Copyright (C) IBM Corporation, 2011-2015 - * - * Original Authors: Robert Jennings <rcj@linux.vnet.ibm.com> - * Seth Jennings <sjenning@linux.vnet.ibm.com> - * - * Rewrite: Dan Streetman <ddstreet@ieee.org> - * - * This is an interface to the NX-842 compression hardware in PowerPC - * processors. Most of the complexity of this drvier is due to the fact that - * the NX-842 compression hardware requires the input and output data buffers - * to be specifically aligned, to be a specific multiple in length, and within - * specific minimum and maximum lengths. Those restrictions, provided by the - * nx-842 driver via nx842_constraints, mean this driver must use bounce - * buffers and headers to correct misaligned in or out buffers, and to split - * input buffers that are too large. - * - * This driver will fall back to software decompression if the hardware - * decompression fails, so this driver's decompression should never fail as - * long as the provided compressed buffer is valid. Any compressed buffer - * created by this driver will have a header (except ones where the input - * perfectly matches the constraints); so users of this driver cannot simply - * pass a compressed buffer created by this driver over to the 842 software - * decompression library. Instead, users must use this driver to decompress; - * if the hardware fails or is unavailable, the compressed buffer will be - * parsed and the header removed, and the raw 842 buffer(s) passed to the 842 - * software decompression library. - * - * This does not fall back to software compression, however, since the caller - * of this function is specifically requesting hardware compression; if the - * hardware compression fails, the caller can fall back to software - * compression, and the raw 842 compressed buffer that the software compressor - * creates can be passed to this driver for hardware decompression; any - * buffer without our specific header magic is assumed to be a raw 842 buffer - * and passed directly to the hardware. Note that the software compression - * library will produce a compressed buffer that is incompatible with the - * hardware decompressor if the original input buffer length is not a multiple - * of 8; if such a compressed buffer is passed to this driver for - * decompression, the hardware will reject it and this driver will then pass - * it over to the software library for decompression. - */ - -#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt - -#include <linux/init.h> -#include <linux/module.h> -#include <linux/crypto.h> -#include <linux/vmalloc.h> -#include <linux/sw842.h> -#include <linux/ratelimit.h> - -#include "nx-842.h" - -/* The first 5 bits of this magic are 0x1f, which is an invalid 842 5-bit - * template (see lib/842/842.h), so this magic number will never appear at - * the start of a raw 842 compressed buffer. That is important, as any buffer - * passed to us without this magic is assumed to be a raw 842 compressed - * buffer, and passed directly to the hardware to decompress. - */ -#define NX842_CRYPTO_MAGIC (0xf842) -#define NX842_CRYPTO_GROUP_MAX (0x20) -#define NX842_CRYPTO_HEADER_SIZE(g) \ - (sizeof(struct nx842_crypto_header) + \ - sizeof(struct nx842_crypto_header_group) * (g)) -#define NX842_CRYPTO_HEADER_MAX_SIZE \ - NX842_CRYPTO_HEADER_SIZE(NX842_CRYPTO_GROUP_MAX) - -/* bounce buffer size */ -#define BOUNCE_BUFFER_ORDER (2) -#define BOUNCE_BUFFER_SIZE \ - ((unsigned int)(PAGE_SIZE << BOUNCE_BUFFER_ORDER)) - -/* try longer on comp because we can fallback to sw decomp if hw is busy */ -#define COMP_BUSY_TIMEOUT (250) /* ms */ -#define DECOMP_BUSY_TIMEOUT (50) /* ms */ - -struct nx842_crypto_header_group { - __be16 padding; /* unused bytes at start of group */ - __be32 compressed_length; /* compressed bytes in group */ - __be32 uncompressed_length; /* bytes after decompression */ -} __packed; - -struct nx842_crypto_header { - __be16 magic; /* NX842_CRYPTO_MAGIC */ - __be16 ignore; /* decompressed end bytes to ignore */ - u8 groups; /* total groups in this header */ - struct nx842_crypto_header_group group[]; -} __packed; - -struct nx842_crypto_param { - u8 *in; - unsigned int iremain; - u8 *out; - unsigned int oremain; - unsigned int ototal; -}; - -static int update_param(struct nx842_crypto_param *p, - unsigned int slen, unsigned int dlen) -{ - if (p->iremain < slen) - return -EOVERFLOW; - if (p->oremain < dlen) - return -ENOSPC; - - p->in += slen; - p->iremain -= slen; - p->out += dlen; - p->oremain -= dlen; - p->ototal += dlen; - - return 0; -} - -struct nx842_crypto_ctx { - u8 *wmem; - u8 *sbounce, *dbounce; - - struct nx842_crypto_header header; - struct nx842_crypto_header_group group[NX842_CRYPTO_GROUP_MAX]; -}; - -static int nx842_crypto_init(struct crypto_tfm *tfm) -{ - struct nx842_crypto_ctx *ctx = crypto_tfm_ctx(tfm); - - ctx->wmem = kmalloc(nx842_workmem_size(), GFP_KERNEL); - ctx->sbounce = (u8 *)__get_free_pages(GFP_KERNEL, BOUNCE_BUFFER_ORDER); - ctx->dbounce = (u8 *)__get_free_pages(GFP_KERNEL, BOUNCE_BUFFER_ORDER); - if (!ctx->wmem || !ctx->sbounce || !ctx->dbounce) { - kfree(ctx->wmem); - free_page((unsigned long)ctx->sbounce); - free_page((unsigned long)ctx->dbounce); - return -ENOMEM; - } - - return 0; -} - -static void nx842_crypto_exit(struct crypto_tfm *tfm) -{ - struct nx842_crypto_ctx *ctx = crypto_tfm_ctx(tfm); - - kfree(ctx->wmem); - free_page((unsigned long)ctx->sbounce); - free_page((unsigned long)ctx->dbounce); -} - -static int read_constraints(struct nx842_constraints *c) -{ - int ret; - - ret = nx842_constraints(c); - if (ret) { - pr_err_ratelimited("could not get nx842 constraints : %d\n", - ret); - return ret; - } - - /* limit maximum, to always have enough bounce buffer to decompress */ - if (c->maximum > BOUNCE_BUFFER_SIZE) { - c->maximum = BOUNCE_BUFFER_SIZE; - pr_info_once("limiting nx842 maximum to %x\n", c->maximum); - } - - return 0; -} - -static int nx842_crypto_add_header(struct nx842_crypto_header *hdr, u8 *buf) -{ - int s = NX842_CRYPTO_HEADER_SIZE(hdr->groups); - - /* compress should have added space for header */ - if (s > be16_to_cpu(hdr->group[0].padding)) { - pr_err("Internal error: no space for header\n"); - return -EINVAL; - } - - memcpy(buf, hdr, s); - - print_hex_dump_debug("header ", DUMP_PREFIX_OFFSET, 16, 1, buf, s, 0); - - return 0; -} - -static int compress(struct nx842_crypto_ctx *ctx, - struct nx842_crypto_param *p, - struct nx842_crypto_header_group *g, - struct nx842_constraints *c, - u16 *ignore, - unsigned int hdrsize) -{ - unsigned int slen = p->iremain, dlen = p->oremain, tmplen; - unsigned int adj_slen = slen; - u8 *src = p->in, *dst = p->out; - int ret, dskip = 0; - ktime_t timeout; - - if (p->iremain == 0) - return -EOVERFLOW; - - if (p->oremain == 0 || hdrsize + c->minimum > dlen) - return -ENOSPC; - - if (slen % c->multiple) - adj_slen = round_up(slen, c->multiple); - if (slen < c->minimum) - adj_slen = c->minimum; - if (slen > c->maximum) - adj_slen = slen = c->maximum; - if (adj_slen > slen || (u64)src % c->alignment) { - adj_slen = min(adj_slen, BOUNCE_BUFFER_SIZE); - slen = min(slen, BOUNCE_BUFFER_SIZE); - if (adj_slen > slen) - memset(ctx->sbounce + slen, 0, adj_slen - slen); - memcpy(ctx->sbounce, src, slen); - src = ctx->sbounce; - slen = adj_slen; - pr_debug("using comp sbounce buffer, len %x\n", slen); - } - - dst += hdrsize; - dlen -= hdrsize; - - if ((u64)dst % c->alignment) { - dskip = (int)(PTR_ALIGN(dst, c->alignment) - dst); - dst += dskip; - dlen -= dskip; - } - if (dlen % c->multiple) - dlen = round_down(dlen, c->multiple); - if (dlen < c->minimum) { -nospc: - dst = ctx->dbounce; - dlen = min(p->oremain, BOUNCE_BUFFER_SIZE); - dlen = round_down(dlen, c->multiple); - dskip = 0; - pr_debug("using comp dbounce buffer, len %x\n", dlen); - } - if (dlen > c->maximum) - dlen = c->maximum; - - tmplen = dlen; - timeout = ktime_add_ms(ktime_get(), COMP_BUSY_TIMEOUT); - do { - dlen = tmplen; /* reset dlen, if we're retrying */ - ret = nx842_compress(src, slen, dst, &dlen, ctx->wmem); - /* possibly we should reduce the slen here, instead of - * retrying with the dbounce buffer? - */ - if (ret == -ENOSPC && dst != ctx->dbounce) - goto nospc; - } while (ret == -EBUSY && ktime_before(ktime_get(), timeout)); - if (ret) - return ret; - - dskip += hdrsize; - - if (dst == ctx->dbounce) - memcpy(p->out + dskip, dst, dlen); - - g->padding = cpu_to_be16(dskip); - g->compressed_length = cpu_to_be32(dlen); - g->uncompressed_length = cpu_to_be32(slen); - - if (p->iremain < slen) { - *ignore = slen - p->iremain; - slen = p->iremain; - } - - pr_debug("compress slen %x ignore %x dlen %x padding %x\n", - slen, *ignore, dlen, dskip); - - return update_param(p, slen, dskip + dlen); -} - -static int nx842_crypto_compress(struct crypto_tfm *tfm, - const u8 *src, unsigned int slen, - u8 *dst, unsigned int *dlen) -{ - struct nx842_crypto_ctx *ctx = crypto_tfm_ctx(tfm); - struct nx842_crypto_header *hdr = &ctx->header; - struct nx842_crypto_param p; - struct nx842_constraints c; - unsigned int groups, hdrsize, h; - int ret, n; - bool add_header; - u16 ignore = 0; - - p.in = (u8 *)src; - p.iremain = slen; - p.out = dst; - p.oremain = *dlen; - p.ototal = 0; - - *dlen = 0; - - ret = read_constraints(&c); - if (ret) - return ret; - - groups = min_t(unsigned int, NX842_CRYPTO_GROUP_MAX, - DIV_ROUND_UP(p.iremain, c.maximum)); - hdrsize = NX842_CRYPTO_HEADER_SIZE(groups); - - /* skip adding header if the buffers meet all constraints */ - add_header = (p.iremain % c.multiple || - p.iremain < c.minimum || - p.iremain > c.maximum || - (u64)p.in % c.alignment || - p.oremain % c.multiple || - p.oremain < c.minimum || - p.oremain > c.maximum || - (u64)p.out % c.alignment); - - hdr->magic = cpu_to_be16(NX842_CRYPTO_MAGIC); - hdr->groups = 0; - hdr->ignore = 0; - - while (p.iremain > 0) { - n = hdr->groups++; - if (hdr->groups > NX842_CRYPTO_GROUP_MAX) - return -ENOSPC; - - /* header goes before first group */ - h = !n && add_header ? hdrsize : 0; - - if (ignore) - pr_warn("interal error, ignore is set %x\n", ignore); - - ret = compress(ctx, &p, &hdr->group[n], &c, &ignore, h); - if (ret) - return ret; - } - - if (!add_header && hdr->groups > 1) { - pr_err("Internal error: No header but multiple groups\n"); - return -EINVAL; - } - - /* ignore indicates the input stream needed to be padded */ - hdr->ignore = cpu_to_be16(ignore); - if (ignore) - pr_debug("marked %d bytes as ignore\n", ignore); - - if (add_header) - ret = nx842_crypto_add_header(hdr, dst); - if (ret) - return ret; - - *dlen = p.ototal; - - pr_debug("compress total slen %x dlen %x\n", slen, *dlen); - - return 0; -} - -static int decompress(struct nx842_crypto_ctx *ctx, - struct nx842_crypto_param *p, - struct nx842_crypto_header_group *g, - struct nx842_constraints *c, - u16 ignore, - bool usehw) -{ - unsigned int slen = be32_to_cpu(g->compressed_length); - unsigned int required_len = be32_to_cpu(g->uncompressed_length); - unsigned int dlen = p->oremain, tmplen; - unsigned int adj_slen = slen; - u8 *src = p->in, *dst = p->out; - u16 padding = be16_to_cpu(g->padding); - int ret, spadding = 0, dpadding = 0; - ktime_t timeout; - - if (!slen || !required_len) - return -EINVAL; - - if (p->iremain <= 0 || padding + slen > p->iremain) - return -EOVERFLOW; - - if (p->oremain <= 0 || required_len - ignore > p->oremain) - return -ENOSPC; - - src += padding; - - if (!usehw) - goto usesw; - - if (slen % c->multiple) - adj_slen = round_up(slen, c->multiple); - if (slen < c->minimum) - adj_slen = c->minimum; - if (slen > c->maximum) - goto usesw; - if (slen < adj_slen || (u64)src % c->alignment) { - /* we can append padding bytes because the 842 format defines - * an "end" template (see lib/842/842_decompress.c) and will - * ignore any bytes following it. - */ - if (slen < adj_slen) - memset(ctx->sbounce + slen, 0, adj_slen - slen); - memcpy(ctx->sbounce, src, slen); - src = ctx->sbounce; - spadding = adj_slen - slen; - slen = adj_slen; - pr_debug("using decomp sbounce buffer, len %x\n", slen); - } - - if (dlen % c->multiple) - dlen = round_down(dlen, c->multiple); - if (dlen < required_len || (u64)dst % c->alignment) { - dst = ctx->dbounce; - dlen = min(required_len, BOUNCE_BUFFER_SIZE); - pr_debug("using decomp dbounce buffer, len %x\n", dlen); - } - if (dlen < c->minimum) - goto usesw; - if (dlen > c->maximum) - dlen = c->maximum; - - tmplen = dlen; - timeout = ktime_add_ms(ktime_get(), DECOMP_BUSY_TIMEOUT); - do { - dlen = tmplen; /* reset dlen, if we're retrying */ - ret = nx842_decompress(src, slen, dst, &dlen, ctx->wmem); - } while (ret == -EBUSY && ktime_before(ktime_get(), timeout)); - if (ret) { -usesw: - /* reset everything, sw doesn't have constraints */ - src = p->in + padding; - slen = be32_to_cpu(g->compressed_length); - spadding = 0; - dst = p->out; - dlen = p->oremain; - dpadding = 0; - if (dlen < required_len) { /* have ignore bytes */ - dst = ctx->dbounce; - dlen = BOUNCE_BUFFER_SIZE; - } - pr_info_ratelimited("using software 842 decompression\n"); - ret = sw842_decompress(src, slen, dst, &dlen); - } - if (ret) - return ret; - - slen -= spadding; - - dlen -= ignore; - if (ignore) - pr_debug("ignoring last %x bytes\n", ignore); - - if (dst == ctx->dbounce) - memcpy(p->out, dst, dlen); - - pr_debug("decompress slen %x padding %x dlen %x ignore %x\n", - slen, padding, dlen, ignore); - - return update_param(p, slen + padding, dlen); -} - -static int nx842_crypto_decompress(struct crypto_tfm *tfm, - const u8 *src, unsigned int slen, - u8 *dst, unsigned int *dlen) -{ - struct nx842_crypto_ctx *ctx = crypto_tfm_ctx(tfm); - struct nx842_crypto_header *hdr; - struct nx842_crypto_param p; - struct nx842_constraints c; - int n, ret, hdr_len; - u16 ignore = 0; - bool usehw = true; - - p.in = (u8 *)src; - p.iremain = slen; - p.out = dst; - p.oremain = *dlen; - p.ototal = 0; - - *dlen = 0; - - if (read_constraints(&c)) - usehw = false; - - hdr = (struct nx842_crypto_header *)src; - - /* If it doesn't start with our header magic number, assume it's a raw - * 842 compressed buffer and pass it directly to the hardware driver - */ - if (be16_to_cpu(hdr->magic) != NX842_CRYPTO_MAGIC) { - struct nx842_crypto_header_group g = { - .padding = 0, - .compressed_length = cpu_to_be32(p.iremain), - .uncompressed_length = cpu_to_be32(p.oremain), - }; - - ret = decompress(ctx, &p, &g, &c, 0, usehw); - if (ret) - return ret; - - *dlen = p.ototal; - - return 0; - } - - if (!hdr->groups) { - pr_err("header has no groups\n"); - return -EINVAL; - } - if (hdr->groups > NX842_CRYPTO_GROUP_MAX) { - pr_err("header has too many groups %x, max %x\n", - hdr->groups, NX842_CRYPTO_GROUP_MAX); - return -EINVAL; - } - - hdr_len = NX842_CRYPTO_HEADER_SIZE(hdr->groups); - if (hdr_len > slen) - return -EOVERFLOW; - - memcpy(&ctx->header, src, hdr_len); - hdr = &ctx->header; - - for (n = 0; n < hdr->groups; n++) { - /* ignore applies to last group */ - if (n + 1 == hdr->groups) - ignore = be16_to_cpu(hdr->ignore); - - ret = decompress(ctx, &p, &hdr->group[n], &c, ignore, usehw); - if (ret) - return ret; - } - - *dlen = p.ototal; - - pr_debug("decompress total slen %x dlen %x\n", slen, *dlen); - - return 0; -} - -static struct crypto_alg alg = { - .cra_name = "842", - .cra_driver_name = "842-nx", - .cra_priority = 300, - .cra_flags = CRYPTO_ALG_TYPE_COMPRESS, - .cra_ctxsize = sizeof(struct nx842_crypto_ctx), - .cra_module = THIS_MODULE, - .cra_init = nx842_crypto_init, - .cra_exit = nx842_crypto_exit, - .cra_u = { .compress = { - .coa_compress = nx842_crypto_compress, - .coa_decompress = nx842_crypto_decompress } } -}; - -static int __init nx842_crypto_mod_init(void) -{ - return crypto_register_alg(&alg); -} -module_init(nx842_crypto_mod_init); - -static void __exit nx842_crypto_mod_exit(void) -{ - crypto_unregister_alg(&alg); -} -module_exit(nx842_crypto_mod_exit); - -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("IBM PowerPC Nest (NX) 842 Hardware Compression Interface"); -MODULE_ALIAS_CRYPTO("842"); -MODULE_ALIAS_CRYPTO("842-nx"); -MODULE_AUTHOR("Dan Streetman <ddstreet@ieee.org>"); diff --git a/drivers/crypto/nx/nx-842-platform.c b/drivers/crypto/nx/nx-842-platform.c deleted file mode 100644 index 664f13dd06ed..000000000000 --- a/drivers/crypto/nx/nx-842-platform.c +++ /dev/null @@ -1,84 +0,0 @@ - -#include "nx-842.h" - -/* this is needed, separate from the main nx-842.c driver, because that main - * driver loads the platform drivers during its init(), and it expects one - * (or none) of the platform drivers to set this pointer to its driver. - * That means this pointer can't be in the main nx-842 driver, because it - * wouldn't be accessible until after the main driver loaded, which wouldn't - * be possible as it's waiting for the platform driver to load. So place it - * here. - */ -static struct nx842_driver *driver; -static DEFINE_SPINLOCK(driver_lock); - -struct nx842_driver *nx842_platform_driver(void) -{ - return driver; -} -EXPORT_SYMBOL_GPL(nx842_platform_driver); - -bool nx842_platform_driver_set(struct nx842_driver *_driver) -{ - bool ret = false; - - spin_lock(&driver_lock); - - if (!driver) { - driver = _driver; - ret = true; - } else - WARN(1, "can't set platform driver, already set to %s\n", - driver->name); - - spin_unlock(&driver_lock); - return ret; -} -EXPORT_SYMBOL_GPL(nx842_platform_driver_set); - -/* only call this from the platform driver exit function */ -void nx842_platform_driver_unset(struct nx842_driver *_driver) -{ - spin_lock(&driver_lock); - - if (driver == _driver) - driver = NULL; - else if (driver) - WARN(1, "can't unset platform driver %s, currently set to %s\n", - _driver->name, driver->name); - else - WARN(1, "can't unset platform driver, already unset\n"); - - spin_unlock(&driver_lock); -} -EXPORT_SYMBOL_GPL(nx842_platform_driver_unset); - -bool nx842_platform_driver_get(void) -{ - bool ret = false; - - spin_lock(&driver_lock); - - if (driver) - ret = try_module_get(driver->owner); - - spin_unlock(&driver_lock); - - return ret; -} -EXPORT_SYMBOL_GPL(nx842_platform_driver_get); - -void nx842_platform_driver_put(void) -{ - spin_lock(&driver_lock); - - if (driver) - module_put(driver->owner); - - spin_unlock(&driver_lock); -} -EXPORT_SYMBOL_GPL(nx842_platform_driver_put); - -MODULE_LICENSE("GPL"); -MODULE_AUTHOR("Dan Streetman <ddstreet@ieee.org>"); -MODULE_DESCRIPTION("842 H/W Compression platform driver"); diff --git a/drivers/crypto/nx/nx-842-powernv.c b/drivers/crypto/nx/nx-842-powernv.c index 33b3b0abf4ae..3750e13d8721 100644 --- a/drivers/crypto/nx/nx-842-powernv.c +++ b/drivers/crypto/nx/nx-842-powernv.c @@ -26,6 +26,8 @@ MODULE_LICENSE("GPL"); MODULE_AUTHOR("Dan Streetman <ddstreet@ieee.org>"); MODULE_DESCRIPTION("842 H/W Compression driver for IBM PowerNV processors"); +MODULE_ALIAS_CRYPTO("842"); +MODULE_ALIAS_CRYPTO("842-nx"); #define WORKMEM_ALIGN (CRB_ALIGN) #define CSB_WAIT_MAX (5000) /* ms */ @@ -344,7 +346,8 @@ static int wait_for_csb(struct nx842_workmem *wmem, } /* successful completion */ - pr_debug_ratelimited("Processed %u bytes in %lu us\n", csb->count, + pr_debug_ratelimited("Processed %u bytes in %lu us\n", + be32_to_cpu(csb->count), (unsigned long)ktime_us_delta(now, start)); return 0; @@ -581,9 +584,29 @@ static struct nx842_driver nx842_powernv_driver = { .decompress = nx842_powernv_decompress, }; +static int nx842_powernv_crypto_init(struct crypto_tfm *tfm) +{ + return nx842_crypto_init(tfm, &nx842_powernv_driver); +} + +static struct crypto_alg nx842_powernv_alg = { + .cra_name = "842", + .cra_driver_name = "842-nx", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_COMPRESS, + .cra_ctxsize = sizeof(struct nx842_crypto_ctx), + .cra_module = THIS_MODULE, + .cra_init = nx842_powernv_crypto_init, + .cra_exit = nx842_crypto_exit, + .cra_u = { .compress = { + .coa_compress = nx842_crypto_compress, + .coa_decompress = nx842_crypto_decompress } } +}; + static __init int nx842_powernv_init(void) { struct device_node *dn; + int ret; /* verify workmem size/align restrictions */ BUILD_BUG_ON(WORKMEM_ALIGN % CRB_ALIGN); @@ -594,17 +617,14 @@ static __init int nx842_powernv_init(void) BUILD_BUG_ON(DDE_BUFFER_ALIGN % DDE_BUFFER_SIZE_MULT); BUILD_BUG_ON(DDE_BUFFER_SIZE_MULT % DDE_BUFFER_LAST_MULT); - pr_info("loading\n"); - for_each_compatible_node(dn, NULL, "ibm,power-nx") nx842_powernv_probe(dn); - if (!nx842_ct) { - pr_err("no coprocessors found\n"); + if (!nx842_ct) return -ENODEV; - } - if (!nx842_platform_driver_set(&nx842_powernv_driver)) { + ret = crypto_register_alg(&nx842_powernv_alg); + if (ret) { struct nx842_coproc *coproc, *n; list_for_each_entry_safe(coproc, n, &nx842_coprocs, list) { @@ -612,11 +632,9 @@ static __init int nx842_powernv_init(void) kfree(coproc); } - return -EEXIST; + return ret; } - pr_info("loaded\n"); - return 0; } module_init(nx842_powernv_init); @@ -625,13 +643,11 @@ static void __exit nx842_powernv_exit(void) { struct nx842_coproc *coproc, *n; - nx842_platform_driver_unset(&nx842_powernv_driver); + crypto_unregister_alg(&nx842_powernv_alg); list_for_each_entry_safe(coproc, n, &nx842_coprocs, list) { list_del(&coproc->list); kfree(coproc); } - - pr_info("unloaded\n"); } module_exit(nx842_powernv_exit); diff --git a/drivers/crypto/nx/nx-842-pseries.c b/drivers/crypto/nx/nx-842-pseries.c index 3040a6091bf2..f4cbde03c6ad 100644 --- a/drivers/crypto/nx/nx-842-pseries.c +++ b/drivers/crypto/nx/nx-842-pseries.c @@ -29,6 +29,8 @@ MODULE_LICENSE("GPL"); MODULE_AUTHOR("Robert Jennings <rcj@linux.vnet.ibm.com>"); MODULE_DESCRIPTION("842 H/W Compression driver for IBM Power processors"); +MODULE_ALIAS_CRYPTO("842"); +MODULE_ALIAS_CRYPTO("842-nx"); static struct nx842_constraints nx842_pseries_constraints = { .alignment = DDE_BUFFER_ALIGN, @@ -99,11 +101,6 @@ struct nx842_workmem { #define NX842_HW_PAGE_SIZE (4096) #define NX842_HW_PAGE_MASK (~(NX842_HW_PAGE_SIZE-1)) -enum nx842_status { - UNAVAILABLE, - AVAILABLE -}; - struct ibm_nx842_counters { atomic64_t comp_complete; atomic64_t comp_failed; @@ -121,7 +118,6 @@ static struct nx842_devdata { unsigned int max_sg_len; unsigned int max_sync_size; unsigned int max_sync_sg; - enum nx842_status status; } __rcu *devdata; static DEFINE_SPINLOCK(devdata_mutex); @@ -230,9 +226,12 @@ static int nx842_validate_result(struct device *dev, switch (csb->completion_code) { case 0: /* Completed without error */ break; - case 64: /* Target bytes > Source bytes during compression */ + case 64: /* Compression ok, but output larger than input */ + dev_dbg(dev, "%s: output size larger than input size\n", + __func__); + break; case 13: /* Output buffer too small */ - dev_dbg(dev, "%s: Compression output larger than input\n", + dev_dbg(dev, "%s: Out of space in output buffer\n", __func__); return -ENOSPC; case 66: /* Input data contains an illegal template field */ @@ -537,41 +536,36 @@ static int nx842_OF_set_defaults(struct nx842_devdata *devdata) devdata->max_sync_size = 0; devdata->max_sync_sg = 0; devdata->max_sg_len = 0; - devdata->status = UNAVAILABLE; return 0; } else return -ENOENT; } /** - * nx842_OF_upd_status -- Update the device info from OF status prop + * nx842_OF_upd_status -- Check the device info from OF status prop * * The status property indicates if the accelerator is enabled. If the * device is in the OF tree it indicates that the hardware is present. * The status field indicates if the device is enabled when the status * is 'okay'. Otherwise the device driver will be disabled. * - * @devdata - struct nx842_devdata to update * @prop - struct property point containing the maxsyncop for the update * * Returns: * 0 - Device is available - * -EINVAL - Device is not available + * -ENODEV - Device is not available */ -static int nx842_OF_upd_status(struct nx842_devdata *devdata, - struct property *prop) { - int ret = 0; +static int nx842_OF_upd_status(struct property *prop) +{ const char *status = (const char *)prop->value; - if (!strncmp(status, "okay", (size_t)prop->length)) { - devdata->status = AVAILABLE; - } else { - dev_info(devdata->dev, "%s: status '%s' is not 'okay'\n", - __func__, status); - devdata->status = UNAVAILABLE; - } + if (!strncmp(status, "okay", (size_t)prop->length)) + return 0; + if (!strncmp(status, "disabled", (size_t)prop->length)) + return -ENODEV; + dev_info(devdata->dev, "%s: unknown status '%s'\n", __func__, status); - return ret; + return -EINVAL; } /** @@ -735,6 +729,10 @@ static int nx842_OF_upd(struct property *new_prop) int ret = 0; unsigned long flags; + new_devdata = kzalloc(sizeof(*new_devdata), GFP_NOFS); + if (!new_devdata) + return -ENOMEM; + spin_lock_irqsave(&devdata_mutex, flags); old_devdata = rcu_dereference_check(devdata, lockdep_is_held(&devdata_mutex)); @@ -744,16 +742,10 @@ static int nx842_OF_upd(struct property *new_prop) if (!old_devdata || !of_node) { pr_err("%s: device is not available\n", __func__); spin_unlock_irqrestore(&devdata_mutex, flags); + kfree(new_devdata); return -ENODEV; } - new_devdata = kzalloc(sizeof(*new_devdata), GFP_NOFS); - if (!new_devdata) { - dev_err(old_devdata->dev, "%s: Could not allocate memory for device data\n", __func__); - ret = -ENOMEM; - goto error_out; - } - memcpy(new_devdata, old_devdata, sizeof(*old_devdata)); new_devdata->counters = old_devdata->counters; @@ -777,7 +769,7 @@ static int nx842_OF_upd(struct property *new_prop) goto out; /* Perform property updates */ - ret = nx842_OF_upd_status(new_devdata, status); + ret = nx842_OF_upd_status(status); if (ret) goto error_out; @@ -970,13 +962,43 @@ static struct nx842_driver nx842_pseries_driver = { .decompress = nx842_pseries_decompress, }; -static int __init nx842_probe(struct vio_dev *viodev, - const struct vio_device_id *id) +static int nx842_pseries_crypto_init(struct crypto_tfm *tfm) +{ + return nx842_crypto_init(tfm, &nx842_pseries_driver); +} + +static struct crypto_alg nx842_pseries_alg = { + .cra_name = "842", + .cra_driver_name = "842-nx", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_COMPRESS, + .cra_ctxsize = sizeof(struct nx842_crypto_ctx), + .cra_module = THIS_MODULE, + .cra_init = nx842_pseries_crypto_init, + .cra_exit = nx842_crypto_exit, + .cra_u = { .compress = { + .coa_compress = nx842_crypto_compress, + .coa_decompress = nx842_crypto_decompress } } +}; + +static int nx842_probe(struct vio_dev *viodev, + const struct vio_device_id *id) { struct nx842_devdata *old_devdata, *new_devdata = NULL; unsigned long flags; int ret = 0; + new_devdata = kzalloc(sizeof(*new_devdata), GFP_NOFS); + if (!new_devdata) + return -ENOMEM; + + new_devdata->counters = kzalloc(sizeof(*new_devdata->counters), + GFP_NOFS); + if (!new_devdata->counters) { + kfree(new_devdata); + return -ENOMEM; + } + spin_lock_irqsave(&devdata_mutex, flags); old_devdata = rcu_dereference_check(devdata, lockdep_is_held(&devdata_mutex)); @@ -989,21 +1011,6 @@ static int __init nx842_probe(struct vio_dev *viodev, dev_set_drvdata(&viodev->dev, NULL); - new_devdata = kzalloc(sizeof(*new_devdata), GFP_NOFS); - if (!new_devdata) { - dev_err(&viodev->dev, "%s: Could not allocate memory for device data\n", __func__); - ret = -ENOMEM; - goto error_unlock; - } - - new_devdata->counters = kzalloc(sizeof(*new_devdata->counters), - GFP_NOFS); - if (!new_devdata->counters) { - dev_err(&viodev->dev, "%s: Could not allocate memory for performance counters\n", __func__); - ret = -ENOMEM; - goto error_unlock; - } - new_devdata->vdev = viodev; new_devdata->dev = &viodev->dev; nx842_OF_set_defaults(new_devdata); @@ -1016,9 +1023,12 @@ static int __init nx842_probe(struct vio_dev *viodev, of_reconfig_notifier_register(&nx842_of_nb); ret = nx842_OF_upd(NULL); - if (ret && ret != -ENODEV) { - dev_err(&viodev->dev, "could not parse device tree. %d\n", ret); - ret = -1; + if (ret) + goto error; + + ret = crypto_register_alg(&nx842_pseries_alg); + if (ret) { + dev_err(&viodev->dev, "could not register comp alg: %d\n", ret); goto error; } @@ -1043,7 +1053,7 @@ error: return ret; } -static int __exit nx842_remove(struct vio_dev *viodev) +static int nx842_remove(struct vio_dev *viodev) { struct nx842_devdata *old_devdata; unsigned long flags; @@ -1051,6 +1061,8 @@ static int __exit nx842_remove(struct vio_dev *viodev) pr_info("Removing IBM Power 842 compression device\n"); sysfs_remove_group(&viodev->dev.kobj, &nx842_attribute_group); + crypto_unregister_alg(&nx842_pseries_alg); + spin_lock_irqsave(&devdata_mutex, flags); old_devdata = rcu_dereference_check(devdata, lockdep_is_held(&devdata_mutex)); @@ -1074,18 +1086,16 @@ static struct vio_device_id nx842_vio_driver_ids[] = { static struct vio_driver nx842_vio_driver = { .name = KBUILD_MODNAME, .probe = nx842_probe, - .remove = __exit_p(nx842_remove), + .remove = nx842_remove, .get_desired_dma = nx842_get_desired_dma, .id_table = nx842_vio_driver_ids, }; -static int __init nx842_init(void) +static int __init nx842_pseries_init(void) { struct nx842_devdata *new_devdata; int ret; - pr_info("Registering IBM Power 842 compression driver\n"); - if (!of_find_compatible_node(NULL, NULL, "ibm,compression")) return -ENODEV; @@ -1095,7 +1105,6 @@ static int __init nx842_init(void) pr_err("Could not allocate memory for device data\n"); return -ENOMEM; } - new_devdata->status = UNAVAILABLE; RCU_INIT_POINTER(devdata, new_devdata); ret = vio_register_driver(&nx842_vio_driver); @@ -1106,24 +1115,18 @@ static int __init nx842_init(void) return ret; } - if (!nx842_platform_driver_set(&nx842_pseries_driver)) { - vio_unregister_driver(&nx842_vio_driver); - kfree(new_devdata); - return -EEXIST; - } - return 0; } -module_init(nx842_init); +module_init(nx842_pseries_init); -static void __exit nx842_exit(void) +static void __exit nx842_pseries_exit(void) { struct nx842_devdata *old_devdata; unsigned long flags; - pr_info("Exiting IBM Power 842 compression driver\n"); - nx842_platform_driver_unset(&nx842_pseries_driver); + crypto_unregister_alg(&nx842_pseries_alg); + spin_lock_irqsave(&devdata_mutex, flags); old_devdata = rcu_dereference_check(devdata, lockdep_is_held(&devdata_mutex)); @@ -1136,5 +1139,5 @@ static void __exit nx842_exit(void) vio_unregister_driver(&nx842_vio_driver); } -module_exit(nx842_exit); +module_exit(nx842_pseries_exit); diff --git a/drivers/crypto/nx/nx-842.c b/drivers/crypto/nx/nx-842.c index 6e5e0d60d0c8..046c1c45411b 100644 --- a/drivers/crypto/nx/nx-842.c +++ b/drivers/crypto/nx/nx-842.c @@ -1,10 +1,5 @@ /* - * Driver frontend for IBM Power 842 compression accelerator - * - * Copyright (C) 2015 Dan Streetman, IBM Corp - * - * Designer of the Power data compression engine: - * Bulent Abali <abali@us.ibm.com> + * Cryptographic API for the NX-842 hardware compression. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -15,89 +10,522 @@ * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. + * + * Copyright (C) IBM Corporation, 2011-2015 + * + * Designer of the Power data compression engine: + * Bulent Abali <abali@us.ibm.com> + * + * Original Authors: Robert Jennings <rcj@linux.vnet.ibm.com> + * Seth Jennings <sjenning@linux.vnet.ibm.com> + * + * Rewrite: Dan Streetman <ddstreet@ieee.org> + * + * This is an interface to the NX-842 compression hardware in PowerPC + * processors. Most of the complexity of this drvier is due to the fact that + * the NX-842 compression hardware requires the input and output data buffers + * to be specifically aligned, to be a specific multiple in length, and within + * specific minimum and maximum lengths. Those restrictions, provided by the + * nx-842 driver via nx842_constraints, mean this driver must use bounce + * buffers and headers to correct misaligned in or out buffers, and to split + * input buffers that are too large. + * + * This driver will fall back to software decompression if the hardware + * decompression fails, so this driver's decompression should never fail as + * long as the provided compressed buffer is valid. Any compressed buffer + * created by this driver will have a header (except ones where the input + * perfectly matches the constraints); so users of this driver cannot simply + * pass a compressed buffer created by this driver over to the 842 software + * decompression library. Instead, users must use this driver to decompress; + * if the hardware fails or is unavailable, the compressed buffer will be + * parsed and the header removed, and the raw 842 buffer(s) passed to the 842 + * software decompression library. + * + * This does not fall back to software compression, however, since the caller + * of this function is specifically requesting hardware compression; if the + * hardware compression fails, the caller can fall back to software + * compression, and the raw 842 compressed buffer that the software compressor + * creates can be passed to this driver for hardware decompression; any + * buffer without our specific header magic is assumed to be a raw 842 buffer + * and passed directly to the hardware. Note that the software compression + * library will produce a compressed buffer that is incompatible with the + * hardware decompressor if the original input buffer length is not a multiple + * of 8; if such a compressed buffer is passed to this driver for + * decompression, the hardware will reject it and this driver will then pass + * it over to the software library for decompression. */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -#include "nx-842.h" +#include <linux/vmalloc.h> +#include <linux/sw842.h> +#include <linux/spinlock.h> -MODULE_LICENSE("GPL"); -MODULE_AUTHOR("Dan Streetman <ddstreet@ieee.org>"); -MODULE_DESCRIPTION("842 H/W Compression driver for IBM Power processors"); +#include "nx-842.h" -/** - * nx842_constraints - * - * This provides the driver's constraints. Different nx842 implementations - * may have varying requirements. The constraints are: - * @alignment: All buffers should be aligned to this - * @multiple: All buffer lengths should be a multiple of this - * @minimum: Buffer lengths must not be less than this amount - * @maximum: Buffer lengths must not be more than this amount - * - * The constraints apply to all buffers and lengths, both input and output, - * for both compression and decompression, except for the minimum which - * only applies to compression input and decompression output; the - * compressed data can be less than the minimum constraint. It can be - * assumed that compressed data will always adhere to the multiple - * constraint. - * - * The driver may succeed even if these constraints are violated; - * however the driver can return failure or suffer reduced performance - * if any constraint is not met. +/* The first 5 bits of this magic are 0x1f, which is an invalid 842 5-bit + * template (see lib/842/842.h), so this magic number will never appear at + * the start of a raw 842 compressed buffer. That is important, as any buffer + * passed to us without this magic is assumed to be a raw 842 compressed + * buffer, and passed directly to the hardware to decompress. */ -int nx842_constraints(struct nx842_constraints *c) +#define NX842_CRYPTO_MAGIC (0xf842) +#define NX842_CRYPTO_HEADER_SIZE(g) \ + (sizeof(struct nx842_crypto_header) + \ + sizeof(struct nx842_crypto_header_group) * (g)) +#define NX842_CRYPTO_HEADER_MAX_SIZE \ + NX842_CRYPTO_HEADER_SIZE(NX842_CRYPTO_GROUP_MAX) + +/* bounce buffer size */ +#define BOUNCE_BUFFER_ORDER (2) +#define BOUNCE_BUFFER_SIZE \ + ((unsigned int)(PAGE_SIZE << BOUNCE_BUFFER_ORDER)) + +/* try longer on comp because we can fallback to sw decomp if hw is busy */ +#define COMP_BUSY_TIMEOUT (250) /* ms */ +#define DECOMP_BUSY_TIMEOUT (50) /* ms */ + +struct nx842_crypto_param { + u8 *in; + unsigned int iremain; + u8 *out; + unsigned int oremain; + unsigned int ototal; +}; + +static int update_param(struct nx842_crypto_param *p, + unsigned int slen, unsigned int dlen) { - memcpy(c, nx842_platform_driver()->constraints, sizeof(*c)); + if (p->iremain < slen) + return -EOVERFLOW; + if (p->oremain < dlen) + return -ENOSPC; + + p->in += slen; + p->iremain -= slen; + p->out += dlen; + p->oremain -= dlen; + p->ototal += dlen; + return 0; } -EXPORT_SYMBOL_GPL(nx842_constraints); -/** - * nx842_workmem_size - * - * Get the amount of working memory the driver requires. - */ -size_t nx842_workmem_size(void) +int nx842_crypto_init(struct crypto_tfm *tfm, struct nx842_driver *driver) { - return nx842_platform_driver()->workmem_size; + struct nx842_crypto_ctx *ctx = crypto_tfm_ctx(tfm); + + spin_lock_init(&ctx->lock); + ctx->driver = driver; + ctx->wmem = kmalloc(driver->workmem_size, GFP_KERNEL); + ctx->sbounce = (u8 *)__get_free_pages(GFP_KERNEL, BOUNCE_BUFFER_ORDER); + ctx->dbounce = (u8 *)__get_free_pages(GFP_KERNEL, BOUNCE_BUFFER_ORDER); + if (!ctx->wmem || !ctx->sbounce || !ctx->dbounce) { + kfree(ctx->wmem); + free_page((unsigned long)ctx->sbounce); + free_page((unsigned long)ctx->dbounce); + return -ENOMEM; + } + + return 0; } -EXPORT_SYMBOL_GPL(nx842_workmem_size); +EXPORT_SYMBOL_GPL(nx842_crypto_init); -int nx842_compress(const unsigned char *in, unsigned int ilen, - unsigned char *out, unsigned int *olen, void *wmem) +void nx842_crypto_exit(struct crypto_tfm *tfm) { - return nx842_platform_driver()->compress(in, ilen, out, olen, wmem); + struct nx842_crypto_ctx *ctx = crypto_tfm_ctx(tfm); + + kfree(ctx->wmem); + free_page((unsigned long)ctx->sbounce); + free_page((unsigned long)ctx->dbounce); } -EXPORT_SYMBOL_GPL(nx842_compress); +EXPORT_SYMBOL_GPL(nx842_crypto_exit); -int nx842_decompress(const unsigned char *in, unsigned int ilen, - unsigned char *out, unsigned int *olen, void *wmem) +static void check_constraints(struct nx842_constraints *c) { - return nx842_platform_driver()->decompress(in, ilen, out, olen, wmem); + /* limit maximum, to always have enough bounce buffer to decompress */ + if (c->maximum > BOUNCE_BUFFER_SIZE) + c->maximum = BOUNCE_BUFFER_SIZE; } -EXPORT_SYMBOL_GPL(nx842_decompress); -static __init int nx842_init(void) +static int nx842_crypto_add_header(struct nx842_crypto_header *hdr, u8 *buf) { - request_module("nx-compress-powernv"); - request_module("nx-compress-pseries"); + int s = NX842_CRYPTO_HEADER_SIZE(hdr->groups); - /* we prevent loading if there's no platform driver, and we get the - * module that set it so it won't unload, so we don't need to check - * if it's set in any of the above functions - */ - if (!nx842_platform_driver_get()) { - pr_err("no nx842 driver found.\n"); - return -ENODEV; + /* compress should have added space for header */ + if (s > be16_to_cpu(hdr->group[0].padding)) { + pr_err("Internal error: no space for header\n"); + return -EINVAL; } + memcpy(buf, hdr, s); + + print_hex_dump_debug("header ", DUMP_PREFIX_OFFSET, 16, 1, buf, s, 0); + return 0; } -module_init(nx842_init); -static void __exit nx842_exit(void) +static int compress(struct nx842_crypto_ctx *ctx, + struct nx842_crypto_param *p, + struct nx842_crypto_header_group *g, + struct nx842_constraints *c, + u16 *ignore, + unsigned int hdrsize) +{ + unsigned int slen = p->iremain, dlen = p->oremain, tmplen; + unsigned int adj_slen = slen; + u8 *src = p->in, *dst = p->out; + int ret, dskip = 0; + ktime_t timeout; + + if (p->iremain == 0) + return -EOVERFLOW; + + if (p->oremain == 0 || hdrsize + c->minimum > dlen) + return -ENOSPC; + + if (slen % c->multiple) + adj_slen = round_up(slen, c->multiple); + if (slen < c->minimum) + adj_slen = c->minimum; + if (slen > c->maximum) + adj_slen = slen = c->maximum; + if (adj_slen > slen || (u64)src % c->alignment) { + adj_slen = min(adj_slen, BOUNCE_BUFFER_SIZE); + slen = min(slen, BOUNCE_BUFFER_SIZE); + if (adj_slen > slen) + memset(ctx->sbounce + slen, 0, adj_slen - slen); + memcpy(ctx->sbounce, src, slen); + src = ctx->sbounce; + slen = adj_slen; + pr_debug("using comp sbounce buffer, len %x\n", slen); + } + + dst += hdrsize; + dlen -= hdrsize; + + if ((u64)dst % c->alignment) { + dskip = (int)(PTR_ALIGN(dst, c->alignment) - dst); + dst += dskip; + dlen -= dskip; + } + if (dlen % c->multiple) + dlen = round_down(dlen, c->multiple); + if (dlen < c->minimum) { +nospc: + dst = ctx->dbounce; + dlen = min(p->oremain, BOUNCE_BUFFER_SIZE); + dlen = round_down(dlen, c->multiple); + dskip = 0; + pr_debug("using comp dbounce buffer, len %x\n", dlen); + } + if (dlen > c->maximum) + dlen = c->maximum; + + tmplen = dlen; + timeout = ktime_add_ms(ktime_get(), COMP_BUSY_TIMEOUT); + do { + dlen = tmplen; /* reset dlen, if we're retrying */ + ret = ctx->driver->compress(src, slen, dst, &dlen, ctx->wmem); + /* possibly we should reduce the slen here, instead of + * retrying with the dbounce buffer? + */ + if (ret == -ENOSPC && dst != ctx->dbounce) + goto nospc; + } while (ret == -EBUSY && ktime_before(ktime_get(), timeout)); + if (ret) + return ret; + + dskip += hdrsize; + + if (dst == ctx->dbounce) + memcpy(p->out + dskip, dst, dlen); + + g->padding = cpu_to_be16(dskip); + g->compressed_length = cpu_to_be32(dlen); + g->uncompressed_length = cpu_to_be32(slen); + + if (p->iremain < slen) { + *ignore = slen - p->iremain; + slen = p->iremain; + } + + pr_debug("compress slen %x ignore %x dlen %x padding %x\n", + slen, *ignore, dlen, dskip); + + return update_param(p, slen, dskip + dlen); +} + +int nx842_crypto_compress(struct crypto_tfm *tfm, + const u8 *src, unsigned int slen, + u8 *dst, unsigned int *dlen) +{ + struct nx842_crypto_ctx *ctx = crypto_tfm_ctx(tfm); + struct nx842_crypto_header *hdr = &ctx->header; + struct nx842_crypto_param p; + struct nx842_constraints c = *ctx->driver->constraints; + unsigned int groups, hdrsize, h; + int ret, n; + bool add_header; + u16 ignore = 0; + + check_constraints(&c); + + p.in = (u8 *)src; + p.iremain = slen; + p.out = dst; + p.oremain = *dlen; + p.ototal = 0; + + *dlen = 0; + + groups = min_t(unsigned int, NX842_CRYPTO_GROUP_MAX, + DIV_ROUND_UP(p.iremain, c.maximum)); + hdrsize = NX842_CRYPTO_HEADER_SIZE(groups); + + spin_lock_bh(&ctx->lock); + + /* skip adding header if the buffers meet all constraints */ + add_header = (p.iremain % c.multiple || + p.iremain < c.minimum || + p.iremain > c.maximum || + (u64)p.in % c.alignment || + p.oremain % c.multiple || + p.oremain < c.minimum || + p.oremain > c.maximum || + (u64)p.out % c.alignment); + + hdr->magic = cpu_to_be16(NX842_CRYPTO_MAGIC); + hdr->groups = 0; + hdr->ignore = 0; + + while (p.iremain > 0) { + n = hdr->groups++; + ret = -ENOSPC; + if (hdr->groups > NX842_CRYPTO_GROUP_MAX) + goto unlock; + + /* header goes before first group */ + h = !n && add_header ? hdrsize : 0; + + if (ignore) + pr_warn("interal error, ignore is set %x\n", ignore); + + ret = compress(ctx, &p, &hdr->group[n], &c, &ignore, h); + if (ret) + goto unlock; + } + + if (!add_header && hdr->groups > 1) { + pr_err("Internal error: No header but multiple groups\n"); + ret = -EINVAL; + goto unlock; + } + + /* ignore indicates the input stream needed to be padded */ + hdr->ignore = cpu_to_be16(ignore); + if (ignore) + pr_debug("marked %d bytes as ignore\n", ignore); + + if (add_header) + ret = nx842_crypto_add_header(hdr, dst); + if (ret) + goto unlock; + + *dlen = p.ototal; + + pr_debug("compress total slen %x dlen %x\n", slen, *dlen); + +unlock: + spin_unlock_bh(&ctx->lock); + return ret; +} +EXPORT_SYMBOL_GPL(nx842_crypto_compress); + +static int decompress(struct nx842_crypto_ctx *ctx, + struct nx842_crypto_param *p, + struct nx842_crypto_header_group *g, + struct nx842_constraints *c, + u16 ignore) { - nx842_platform_driver_put(); + unsigned int slen = be32_to_cpu(g->compressed_length); + unsigned int required_len = be32_to_cpu(g->uncompressed_length); + unsigned int dlen = p->oremain, tmplen; + unsigned int adj_slen = slen; + u8 *src = p->in, *dst = p->out; + u16 padding = be16_to_cpu(g->padding); + int ret, spadding = 0, dpadding = 0; + ktime_t timeout; + + if (!slen || !required_len) + return -EINVAL; + + if (p->iremain <= 0 || padding + slen > p->iremain) + return -EOVERFLOW; + + if (p->oremain <= 0 || required_len - ignore > p->oremain) + return -ENOSPC; + + src += padding; + + if (slen % c->multiple) + adj_slen = round_up(slen, c->multiple); + if (slen < c->minimum) + adj_slen = c->minimum; + if (slen > c->maximum) + goto usesw; + if (slen < adj_slen || (u64)src % c->alignment) { + /* we can append padding bytes because the 842 format defines + * an "end" template (see lib/842/842_decompress.c) and will + * ignore any bytes following it. + */ + if (slen < adj_slen) + memset(ctx->sbounce + slen, 0, adj_slen - slen); + memcpy(ctx->sbounce, src, slen); + src = ctx->sbounce; + spadding = adj_slen - slen; + slen = adj_slen; + pr_debug("using decomp sbounce buffer, len %x\n", slen); + } + + if (dlen % c->multiple) + dlen = round_down(dlen, c->multiple); + if (dlen < required_len || (u64)dst % c->alignment) { + dst = ctx->dbounce; + dlen = min(required_len, BOUNCE_BUFFER_SIZE); + pr_debug("using decomp dbounce buffer, len %x\n", dlen); + } + if (dlen < c->minimum) + goto usesw; + if (dlen > c->maximum) + dlen = c->maximum; + + tmplen = dlen; + timeout = ktime_add_ms(ktime_get(), DECOMP_BUSY_TIMEOUT); + do { + dlen = tmplen; /* reset dlen, if we're retrying */ + ret = ctx->driver->decompress(src, slen, dst, &dlen, ctx->wmem); + } while (ret == -EBUSY && ktime_before(ktime_get(), timeout)); + if (ret) { +usesw: + /* reset everything, sw doesn't have constraints */ + src = p->in + padding; + slen = be32_to_cpu(g->compressed_length); + spadding = 0; + dst = p->out; + dlen = p->oremain; + dpadding = 0; + if (dlen < required_len) { /* have ignore bytes */ + dst = ctx->dbounce; + dlen = BOUNCE_BUFFER_SIZE; + } + pr_info_ratelimited("using software 842 decompression\n"); + ret = sw842_decompress(src, slen, dst, &dlen); + } + if (ret) + return ret; + + slen -= spadding; + + dlen -= ignore; + if (ignore) + pr_debug("ignoring last %x bytes\n", ignore); + + if (dst == ctx->dbounce) + memcpy(p->out, dst, dlen); + + pr_debug("decompress slen %x padding %x dlen %x ignore %x\n", + slen, padding, dlen, ignore); + + return update_param(p, slen + padding, dlen); } -module_exit(nx842_exit); + +int nx842_crypto_decompress(struct crypto_tfm *tfm, + const u8 *src, unsigned int slen, + u8 *dst, unsigned int *dlen) +{ + struct nx842_crypto_ctx *ctx = crypto_tfm_ctx(tfm); + struct nx842_crypto_header *hdr; + struct nx842_crypto_param p; + struct nx842_constraints c = *ctx->driver->constraints; + int n, ret, hdr_len; + u16 ignore = 0; + + check_constraints(&c); + + p.in = (u8 *)src; + p.iremain = slen; + p.out = dst; + p.oremain = *dlen; + p.ototal = 0; + + *dlen = 0; + + hdr = (struct nx842_crypto_header *)src; + + spin_lock_bh(&ctx->lock); + + /* If it doesn't start with our header magic number, assume it's a raw + * 842 compressed buffer and pass it directly to the hardware driver + */ + if (be16_to_cpu(hdr->magic) != NX842_CRYPTO_MAGIC) { + struct nx842_crypto_header_group g = { + .padding = 0, + .compressed_length = cpu_to_be32(p.iremain), + .uncompressed_length = cpu_to_be32(p.oremain), + }; + + ret = decompress(ctx, &p, &g, &c, 0); + if (ret) + goto unlock; + + goto success; + } + + if (!hdr->groups) { + pr_err("header has no groups\n"); + ret = -EINVAL; + goto unlock; + } + if (hdr->groups > NX842_CRYPTO_GROUP_MAX) { + pr_err("header has too many groups %x, max %x\n", + hdr->groups, NX842_CRYPTO_GROUP_MAX); + ret = -EINVAL; + goto unlock; + } + + hdr_len = NX842_CRYPTO_HEADER_SIZE(hdr->groups); + if (hdr_len > slen) { + ret = -EOVERFLOW; + goto unlock; + } + + memcpy(&ctx->header, src, hdr_len); + hdr = &ctx->header; + + for (n = 0; n < hdr->groups; n++) { + /* ignore applies to last group */ + if (n + 1 == hdr->groups) + ignore = be16_to_cpu(hdr->ignore); + + ret = decompress(ctx, &p, &hdr->group[n], &c, ignore); + if (ret) + goto unlock; + } + +success: + *dlen = p.ototal; + + pr_debug("decompress total slen %x dlen %x\n", slen, *dlen); + + ret = 0; + +unlock: + spin_unlock_bh(&ctx->lock); + + return ret; +} +EXPORT_SYMBOL_GPL(nx842_crypto_decompress); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("IBM PowerPC Nest (NX) 842 Hardware Compression Driver"); +MODULE_AUTHOR("Dan Streetman <ddstreet@ieee.org>"); diff --git a/drivers/crypto/nx/nx-842.h b/drivers/crypto/nx/nx-842.h index ac0ea79d0f8b..a4eee3bba937 100644 --- a/drivers/crypto/nx/nx-842.h +++ b/drivers/crypto/nx/nx-842.h @@ -3,8 +3,9 @@ #define __NX_842_H__ #include <linux/kernel.h> +#include <linux/init.h> #include <linux/module.h> -#include <linux/sw842.h> +#include <linux/crypto.h> #include <linux/of.h> #include <linux/slab.h> #include <linux/io.h> @@ -104,6 +105,25 @@ static inline unsigned long nx842_get_pa(void *addr) #define GET_FIELD(v, m) (((v) & (m)) >> MASK_LSH(m)) #define SET_FIELD(v, m, val) (((v) & ~(m)) | (((val) << MASK_LSH(m)) & (m))) +/** + * This provides the driver's constraints. Different nx842 implementations + * may have varying requirements. The constraints are: + * @alignment: All buffers should be aligned to this + * @multiple: All buffer lengths should be a multiple of this + * @minimum: Buffer lengths must not be less than this amount + * @maximum: Buffer lengths must not be more than this amount + * + * The constraints apply to all buffers and lengths, both input and output, + * for both compression and decompression, except for the minimum which + * only applies to compression input and decompression output; the + * compressed data can be less than the minimum constraint. It can be + * assumed that compressed data will always adhere to the multiple + * constraint. + * + * The driver may succeed even if these constraints are violated; + * however the driver can return failure or suffer reduced performance + * if any constraint is not met. + */ struct nx842_constraints { int alignment; int multiple; @@ -126,19 +146,40 @@ struct nx842_driver { void *wrkmem); }; -struct nx842_driver *nx842_platform_driver(void); -bool nx842_platform_driver_set(struct nx842_driver *driver); -void nx842_platform_driver_unset(struct nx842_driver *driver); -bool nx842_platform_driver_get(void); -void nx842_platform_driver_put(void); +struct nx842_crypto_header_group { + __be16 padding; /* unused bytes at start of group */ + __be32 compressed_length; /* compressed bytes in group */ + __be32 uncompressed_length; /* bytes after decompression */ +} __packed; + +struct nx842_crypto_header { + __be16 magic; /* NX842_CRYPTO_MAGIC */ + __be16 ignore; /* decompressed end bytes to ignore */ + u8 groups; /* total groups in this header */ + struct nx842_crypto_header_group group[]; +} __packed; -size_t nx842_workmem_size(void); +#define NX842_CRYPTO_GROUP_MAX (0x20) -int nx842_constraints(struct nx842_constraints *constraints); +struct nx842_crypto_ctx { + spinlock_t lock; + + u8 *wmem; + u8 *sbounce, *dbounce; + + struct nx842_crypto_header header; + struct nx842_crypto_header_group group[NX842_CRYPTO_GROUP_MAX]; + + struct nx842_driver *driver; +}; -int nx842_compress(const unsigned char *in, unsigned int in_len, - unsigned char *out, unsigned int *out_len, void *wrkmem); -int nx842_decompress(const unsigned char *in, unsigned int in_len, - unsigned char *out, unsigned int *out_len, void *wrkmem); +int nx842_crypto_init(struct crypto_tfm *tfm, struct nx842_driver *driver); +void nx842_crypto_exit(struct crypto_tfm *tfm); +int nx842_crypto_compress(struct crypto_tfm *tfm, + const u8 *src, unsigned int slen, + u8 *dst, unsigned int *dlen); +int nx842_crypto_decompress(struct crypto_tfm *tfm, + const u8 *src, unsigned int slen, + u8 *dst, unsigned int *dlen); #endif /* __NX_842_H__ */ diff --git a/drivers/crypto/nx/nx-aes-ccm.c b/drivers/crypto/nx/nx-aes-ccm.c index e4311ce0cd78..73ef49922788 100644 --- a/drivers/crypto/nx/nx-aes-ccm.c +++ b/drivers/crypto/nx/nx-aes-ccm.c @@ -94,8 +94,6 @@ static int ccm_aes_nx_setauthsize(struct crypto_aead *tfm, return -EINVAL; } - crypto_aead_crt(tfm)->authsize = authsize; - return 0; } @@ -111,8 +109,6 @@ static int ccm4309_aes_nx_setauthsize(struct crypto_aead *tfm, return -EINVAL; } - crypto_aead_crt(tfm)->authsize = authsize; - return 0; } @@ -174,6 +170,7 @@ static int generate_pat(u8 *iv, struct nx_crypto_ctx *nx_ctx, unsigned int authsize, unsigned int nbytes, + unsigned int assoclen, u8 *out) { struct nx_sg *nx_insg = nx_ctx->in_sg; @@ -200,16 +197,16 @@ static int generate_pat(u8 *iv, * greater than 2^32. */ - if (!req->assoclen) { + if (!assoclen) { b0 = nx_ctx->csbcpb->cpb.aes_ccm.in_pat_or_b0; - } else if (req->assoclen <= 14) { + } else if (assoclen <= 14) { /* if associated data is 14 bytes or less, we do 1 GCM * operation on 2 AES blocks, B0 (stored in the csbcpb) and B1, * which is fed in through the source buffers here */ b0 = nx_ctx->csbcpb->cpb.aes_ccm.in_pat_or_b0; b1 = nx_ctx->priv.ccm.iauth_tag; - iauth_len = req->assoclen; - } else if (req->assoclen <= 65280) { + iauth_len = assoclen; + } else if (assoclen <= 65280) { /* if associated data is less than (2^16 - 2^8), we construct * B1 differently and feed in the associated data to a CCA * operation */ @@ -223,7 +220,7 @@ static int generate_pat(u8 *iv, } /* generate B0 */ - rc = generate_b0(iv, req->assoclen, authsize, nbytes, b0); + rc = generate_b0(iv, assoclen, authsize, nbytes, b0); if (rc) return rc; @@ -233,22 +230,22 @@ static int generate_pat(u8 *iv, */ if (b1) { memset(b1, 0, 16); - if (req->assoclen <= 65280) { - *(u16 *)b1 = (u16)req->assoclen; - scatterwalk_map_and_copy(b1 + 2, req->assoc, 0, + if (assoclen <= 65280) { + *(u16 *)b1 = assoclen; + scatterwalk_map_and_copy(b1 + 2, req->src, 0, iauth_len, SCATTERWALK_FROM_SG); } else { *(u16 *)b1 = (u16)(0xfffe); - *(u32 *)&b1[2] = (u32)req->assoclen; - scatterwalk_map_and_copy(b1 + 6, req->assoc, 0, + *(u32 *)&b1[2] = assoclen; + scatterwalk_map_and_copy(b1 + 6, req->src, 0, iauth_len, SCATTERWALK_FROM_SG); } } /* now copy any remaining AAD to scatterlist and call nx... */ - if (!req->assoclen) { + if (!assoclen) { return rc; - } else if (req->assoclen <= 14) { + } else if (assoclen <= 14) { unsigned int len = 16; nx_insg = nx_build_sg_list(nx_insg, b1, &len, nx_ctx->ap->sglen); @@ -280,7 +277,7 @@ static int generate_pat(u8 *iv, return rc; atomic_inc(&(nx_ctx->stats->aes_ops)); - atomic64_add(req->assoclen, &(nx_ctx->stats->aes_bytes)); + atomic64_add(assoclen, &nx_ctx->stats->aes_bytes); } else { unsigned int processed = 0, to_process; @@ -294,15 +291,15 @@ static int generate_pat(u8 *iv, nx_ctx->ap->databytelen/NX_PAGE_SIZE); do { - to_process = min_t(u32, req->assoclen - processed, + to_process = min_t(u32, assoclen - processed, nx_ctx->ap->databytelen); nx_insg = nx_walk_and_build(nx_ctx->in_sg, nx_ctx->ap->sglen, - req->assoc, processed, + req->src, processed, &to_process); - if ((to_process + processed) < req->assoclen) { + if ((to_process + processed) < assoclen) { NX_CPB_FDM(nx_ctx->csbcpb_aead) |= NX_FDM_INTERMEDIATE; } else { @@ -328,11 +325,10 @@ static int generate_pat(u8 *iv, NX_CPB_FDM(nx_ctx->csbcpb_aead) |= NX_FDM_CONTINUATION; atomic_inc(&(nx_ctx->stats->aes_ops)); - atomic64_add(req->assoclen, - &(nx_ctx->stats->aes_bytes)); + atomic64_add(assoclen, &nx_ctx->stats->aes_bytes); processed += to_process; - } while (processed < req->assoclen); + } while (processed < assoclen); result = nx_ctx->csbcpb_aead->cpb.aes_cca.out_pat_or_b0; } @@ -343,7 +339,8 @@ static int generate_pat(u8 *iv, } static int ccm_nx_decrypt(struct aead_request *req, - struct blkcipher_desc *desc) + struct blkcipher_desc *desc, + unsigned int assoclen) { struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(req->base.tfm); struct nx_csbcpb *csbcpb = nx_ctx->csbcpb; @@ -360,10 +357,10 @@ static int ccm_nx_decrypt(struct aead_request *req, /* copy out the auth tag to compare with later */ scatterwalk_map_and_copy(priv->oauth_tag, - req->src, nbytes, authsize, + req->src, nbytes + req->assoclen, authsize, SCATTERWALK_FROM_SG); - rc = generate_pat(desc->info, req, nx_ctx, authsize, nbytes, + rc = generate_pat(desc->info, req, nx_ctx, authsize, nbytes, assoclen, csbcpb->cpb.aes_ccm.in_pat_or_b0); if (rc) goto out; @@ -383,8 +380,8 @@ static int ccm_nx_decrypt(struct aead_request *req, NX_CPB_FDM(nx_ctx->csbcpb) &= ~NX_FDM_ENDE_ENCRYPT; rc = nx_build_sg_lists(nx_ctx, desc, req->dst, req->src, - &to_process, processed, - csbcpb->cpb.aes_ccm.iv_or_ctr); + &to_process, processed + req->assoclen, + csbcpb->cpb.aes_ccm.iv_or_ctr); if (rc) goto out; @@ -420,7 +417,8 @@ out: } static int ccm_nx_encrypt(struct aead_request *req, - struct blkcipher_desc *desc) + struct blkcipher_desc *desc, + unsigned int assoclen) { struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(req->base.tfm); struct nx_csbcpb *csbcpb = nx_ctx->csbcpb; @@ -432,7 +430,7 @@ static int ccm_nx_encrypt(struct aead_request *req, spin_lock_irqsave(&nx_ctx->lock, irq_flags); - rc = generate_pat(desc->info, req, nx_ctx, authsize, nbytes, + rc = generate_pat(desc->info, req, nx_ctx, authsize, nbytes, assoclen, csbcpb->cpb.aes_ccm.in_pat_or_b0); if (rc) goto out; @@ -451,7 +449,7 @@ static int ccm_nx_encrypt(struct aead_request *req, NX_CPB_FDM(csbcpb) |= NX_FDM_ENDE_ENCRYPT; rc = nx_build_sg_lists(nx_ctx, desc, req->dst, req->src, - &to_process, processed, + &to_process, processed + req->assoclen, csbcpb->cpb.aes_ccm.iv_or_ctr); if (rc) goto out; @@ -483,7 +481,7 @@ static int ccm_nx_encrypt(struct aead_request *req, /* copy out the auth tag */ scatterwalk_map_and_copy(csbcpb->cpb.aes_ccm.out_pat_or_mac, - req->dst, nbytes, authsize, + req->dst, nbytes + req->assoclen, authsize, SCATTERWALK_TO_SG); out: @@ -503,9 +501,8 @@ static int ccm4309_aes_nx_encrypt(struct aead_request *req) memcpy(iv + 4, req->iv, 8); desc.info = iv; - desc.tfm = (struct crypto_blkcipher *)req->base.tfm; - return ccm_nx_encrypt(req, &desc); + return ccm_nx_encrypt(req, &desc, req->assoclen - 8); } static int ccm_aes_nx_encrypt(struct aead_request *req) @@ -514,13 +511,12 @@ static int ccm_aes_nx_encrypt(struct aead_request *req) int rc; desc.info = req->iv; - desc.tfm = (struct crypto_blkcipher *)req->base.tfm; rc = crypto_ccm_check_iv(desc.info); if (rc) return rc; - return ccm_nx_encrypt(req, &desc); + return ccm_nx_encrypt(req, &desc, req->assoclen); } static int ccm4309_aes_nx_decrypt(struct aead_request *req) @@ -535,9 +531,8 @@ static int ccm4309_aes_nx_decrypt(struct aead_request *req) memcpy(iv + 4, req->iv, 8); desc.info = iv; - desc.tfm = (struct crypto_blkcipher *)req->base.tfm; - return ccm_nx_decrypt(req, &desc); + return ccm_nx_decrypt(req, &desc, req->assoclen - 8); } static int ccm_aes_nx_decrypt(struct aead_request *req) @@ -546,13 +541,12 @@ static int ccm_aes_nx_decrypt(struct aead_request *req) int rc; desc.info = req->iv; - desc.tfm = (struct crypto_blkcipher *)req->base.tfm; rc = crypto_ccm_check_iv(desc.info); if (rc) return rc; - return ccm_nx_decrypt(req, &desc); + return ccm_nx_decrypt(req, &desc, req->assoclen); } /* tell the block cipher walk routines that this is a stream cipher by @@ -560,47 +554,42 @@ static int ccm_aes_nx_decrypt(struct aead_request *req) * during encrypt/decrypt doesn't solve this problem, because it calls * blkcipher_walk_done under the covers, which doesn't use walk->blocksize, * but instead uses this tfm->blocksize. */ -struct crypto_alg nx_ccm_aes_alg = { - .cra_name = "ccm(aes)", - .cra_driver_name = "ccm-aes-nx", - .cra_priority = 300, - .cra_flags = CRYPTO_ALG_TYPE_AEAD | - CRYPTO_ALG_NEED_FALLBACK, - .cra_blocksize = 1, - .cra_ctxsize = sizeof(struct nx_crypto_ctx), - .cra_type = &crypto_aead_type, - .cra_module = THIS_MODULE, - .cra_init = nx_crypto_ctx_aes_ccm_init, - .cra_exit = nx_crypto_ctx_exit, - .cra_aead = { - .ivsize = AES_BLOCK_SIZE, - .maxauthsize = AES_BLOCK_SIZE, - .setkey = ccm_aes_nx_set_key, - .setauthsize = ccm_aes_nx_setauthsize, - .encrypt = ccm_aes_nx_encrypt, - .decrypt = ccm_aes_nx_decrypt, - } +struct aead_alg nx_ccm_aes_alg = { + .base = { + .cra_name = "ccm(aes)", + .cra_driver_name = "ccm-aes-nx", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct nx_crypto_ctx), + .cra_module = THIS_MODULE, + }, + .init = nx_crypto_ctx_aes_ccm_init, + .exit = nx_crypto_ctx_aead_exit, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = AES_BLOCK_SIZE, + .setkey = ccm_aes_nx_set_key, + .setauthsize = ccm_aes_nx_setauthsize, + .encrypt = ccm_aes_nx_encrypt, + .decrypt = ccm_aes_nx_decrypt, }; -struct crypto_alg nx_ccm4309_aes_alg = { - .cra_name = "rfc4309(ccm(aes))", - .cra_driver_name = "rfc4309-ccm-aes-nx", - .cra_priority = 300, - .cra_flags = CRYPTO_ALG_TYPE_AEAD | - CRYPTO_ALG_NEED_FALLBACK, - .cra_blocksize = 1, - .cra_ctxsize = sizeof(struct nx_crypto_ctx), - .cra_type = &crypto_nivaead_type, - .cra_module = THIS_MODULE, - .cra_init = nx_crypto_ctx_aes_ccm_init, - .cra_exit = nx_crypto_ctx_exit, - .cra_aead = { - .ivsize = 8, - .maxauthsize = AES_BLOCK_SIZE, - .setkey = ccm4309_aes_nx_set_key, - .setauthsize = ccm4309_aes_nx_setauthsize, - .encrypt = ccm4309_aes_nx_encrypt, - .decrypt = ccm4309_aes_nx_decrypt, - .geniv = "seqiv", - } +struct aead_alg nx_ccm4309_aes_alg = { + .base = { + .cra_name = "rfc4309(ccm(aes))", + .cra_driver_name = "rfc4309-ccm-aes-nx", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct nx_crypto_ctx), + .cra_module = THIS_MODULE, + }, + .init = nx_crypto_ctx_aes_ccm_init, + .exit = nx_crypto_ctx_aead_exit, + .ivsize = 8, + .maxauthsize = AES_BLOCK_SIZE, + .setkey = ccm4309_aes_nx_set_key, + .setauthsize = ccm4309_aes_nx_setauthsize, + .encrypt = ccm4309_aes_nx_encrypt, + .decrypt = ccm4309_aes_nx_decrypt, }; diff --git a/drivers/crypto/nx/nx-aes-ctr.c b/drivers/crypto/nx/nx-aes-ctr.c index dd7e9f3f5b6b..898c0a280511 100644 --- a/drivers/crypto/nx/nx-aes-ctr.c +++ b/drivers/crypto/nx/nx-aes-ctr.c @@ -144,27 +144,6 @@ static int ctr3686_aes_nx_crypt(struct blkcipher_desc *desc, return ctr_aes_nx_crypt(desc, dst, src, nbytes); } -struct crypto_alg nx_ctr_aes_alg = { - .cra_name = "ctr(aes)", - .cra_driver_name = "ctr-aes-nx", - .cra_priority = 300, - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, - .cra_blocksize = 1, - .cra_ctxsize = sizeof(struct nx_crypto_ctx), - .cra_type = &crypto_blkcipher_type, - .cra_module = THIS_MODULE, - .cra_init = nx_crypto_ctx_aes_ctr_init, - .cra_exit = nx_crypto_ctx_exit, - .cra_blkcipher = { - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, - .ivsize = AES_BLOCK_SIZE, - .setkey = ctr_aes_nx_set_key, - .encrypt = ctr_aes_nx_crypt, - .decrypt = ctr_aes_nx_crypt, - } -}; - struct crypto_alg nx_ctr3686_aes_alg = { .cra_name = "rfc3686(ctr(aes))", .cra_driver_name = "rfc3686-ctr-aes-nx", diff --git a/drivers/crypto/nx/nx-aes-gcm.c b/drivers/crypto/nx/nx-aes-gcm.c index 92c993f08213..eee624f589b6 100644 --- a/drivers/crypto/nx/nx-aes-gcm.c +++ b/drivers/crypto/nx/nx-aes-gcm.c @@ -21,11 +21,9 @@ #include <crypto/internal/aead.h> #include <crypto/aes.h> -#include <crypto/algapi.h> #include <crypto/scatterwalk.h> #include <linux/module.h> #include <linux/types.h> -#include <linux/crypto.h> #include <asm/vio.h> #include "nx_csbcpb.h" @@ -36,7 +34,7 @@ static int gcm_aes_nx_set_key(struct crypto_aead *tfm, const u8 *in_key, unsigned int key_len) { - struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&tfm->base); + struct nx_crypto_ctx *nx_ctx = crypto_aead_ctx(tfm); struct nx_csbcpb *csbcpb = nx_ctx->csbcpb; struct nx_csbcpb *csbcpb_aead = nx_ctx->csbcpb_aead; @@ -75,7 +73,7 @@ static int gcm4106_aes_nx_set_key(struct crypto_aead *tfm, const u8 *in_key, unsigned int key_len) { - struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&tfm->base); + struct nx_crypto_ctx *nx_ctx = crypto_aead_ctx(tfm); char *nonce = nx_ctx->priv.gcm.nonce; int rc; @@ -110,13 +108,14 @@ static int gcm4106_aes_nx_setauthsize(struct crypto_aead *tfm, static int nx_gca(struct nx_crypto_ctx *nx_ctx, struct aead_request *req, - u8 *out) + u8 *out, + unsigned int assoclen) { int rc; struct nx_csbcpb *csbcpb_aead = nx_ctx->csbcpb_aead; struct scatter_walk walk; struct nx_sg *nx_sg = nx_ctx->in_sg; - unsigned int nbytes = req->assoclen; + unsigned int nbytes = assoclen; unsigned int processed = 0, to_process; unsigned int max_sg_len; @@ -167,7 +166,7 @@ static int nx_gca(struct nx_crypto_ctx *nx_ctx, NX_CPB_FDM(csbcpb_aead) |= NX_FDM_CONTINUATION; atomic_inc(&(nx_ctx->stats->aes_ops)); - atomic64_add(req->assoclen, &(nx_ctx->stats->aes_bytes)); + atomic64_add(assoclen, &(nx_ctx->stats->aes_bytes)); processed += to_process; } while (processed < nbytes); @@ -177,13 +176,15 @@ static int nx_gca(struct nx_crypto_ctx *nx_ctx, return rc; } -static int gmac(struct aead_request *req, struct blkcipher_desc *desc) +static int gmac(struct aead_request *req, struct blkcipher_desc *desc, + unsigned int assoclen) { int rc; - struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(req->base.tfm); + struct nx_crypto_ctx *nx_ctx = + crypto_aead_ctx(crypto_aead_reqtfm(req)); struct nx_csbcpb *csbcpb = nx_ctx->csbcpb; struct nx_sg *nx_sg; - unsigned int nbytes = req->assoclen; + unsigned int nbytes = assoclen; unsigned int processed = 0, to_process; unsigned int max_sg_len; @@ -238,7 +239,7 @@ static int gmac(struct aead_request *req, struct blkcipher_desc *desc) NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION; atomic_inc(&(nx_ctx->stats->aes_ops)); - atomic64_add(req->assoclen, &(nx_ctx->stats->aes_bytes)); + atomic64_add(assoclen, &(nx_ctx->stats->aes_bytes)); processed += to_process; } while (processed < nbytes); @@ -253,7 +254,8 @@ static int gcm_empty(struct aead_request *req, struct blkcipher_desc *desc, int enc) { int rc; - struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(req->base.tfm); + struct nx_crypto_ctx *nx_ctx = + crypto_aead_ctx(crypto_aead_reqtfm(req)); struct nx_csbcpb *csbcpb = nx_ctx->csbcpb; char out[AES_BLOCK_SIZE]; struct nx_sg *in_sg, *out_sg; @@ -314,9 +316,11 @@ out: return rc; } -static int gcm_aes_nx_crypt(struct aead_request *req, int enc) +static int gcm_aes_nx_crypt(struct aead_request *req, int enc, + unsigned int assoclen) { - struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(req->base.tfm); + struct nx_crypto_ctx *nx_ctx = + crypto_aead_ctx(crypto_aead_reqtfm(req)); struct nx_gcm_rctx *rctx = aead_request_ctx(req); struct nx_csbcpb *csbcpb = nx_ctx->csbcpb; struct blkcipher_desc desc; @@ -332,10 +336,10 @@ static int gcm_aes_nx_crypt(struct aead_request *req, int enc) *(u32 *)(desc.info + NX_GCM_CTR_OFFSET) = 1; if (nbytes == 0) { - if (req->assoclen == 0) + if (assoclen == 0) rc = gcm_empty(req, &desc, enc); else - rc = gmac(req, &desc); + rc = gmac(req, &desc, assoclen); if (rc) goto out; else @@ -343,9 +347,10 @@ static int gcm_aes_nx_crypt(struct aead_request *req, int enc) } /* Process associated data */ - csbcpb->cpb.aes_gcm.bit_length_aad = req->assoclen * 8; - if (req->assoclen) { - rc = nx_gca(nx_ctx, req, csbcpb->cpb.aes_gcm.in_pat_or_aad); + csbcpb->cpb.aes_gcm.bit_length_aad = assoclen * 8; + if (assoclen) { + rc = nx_gca(nx_ctx, req, csbcpb->cpb.aes_gcm.in_pat_or_aad, + assoclen); if (rc) goto out; } @@ -363,7 +368,6 @@ static int gcm_aes_nx_crypt(struct aead_request *req, int enc) to_process = nbytes - processed; csbcpb->cpb.aes_gcm.bit_length_data = nbytes * 8; - desc.tfm = (struct crypto_blkcipher *) req->base.tfm; rc = nx_build_sg_lists(nx_ctx, &desc, req->dst, req->src, &to_process, processed + req->assoclen, @@ -430,7 +434,7 @@ static int gcm_aes_nx_encrypt(struct aead_request *req) memcpy(iv, req->iv, 12); - return gcm_aes_nx_crypt(req, 1); + return gcm_aes_nx_crypt(req, 1, req->assoclen); } static int gcm_aes_nx_decrypt(struct aead_request *req) @@ -440,12 +444,13 @@ static int gcm_aes_nx_decrypt(struct aead_request *req) memcpy(iv, req->iv, 12); - return gcm_aes_nx_crypt(req, 0); + return gcm_aes_nx_crypt(req, 0, req->assoclen); } static int gcm4106_aes_nx_encrypt(struct aead_request *req) { - struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(req->base.tfm); + struct nx_crypto_ctx *nx_ctx = + crypto_aead_ctx(crypto_aead_reqtfm(req)); struct nx_gcm_rctx *rctx = aead_request_ctx(req); char *iv = rctx->iv; char *nonce = nx_ctx->priv.gcm.nonce; @@ -453,12 +458,16 @@ static int gcm4106_aes_nx_encrypt(struct aead_request *req) memcpy(iv, nonce, NX_GCM4106_NONCE_LEN); memcpy(iv + NX_GCM4106_NONCE_LEN, req->iv, 8); - return gcm_aes_nx_crypt(req, 1); + if (req->assoclen < 8) + return -EINVAL; + + return gcm_aes_nx_crypt(req, 1, req->assoclen - 8); } static int gcm4106_aes_nx_decrypt(struct aead_request *req) { - struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(req->base.tfm); + struct nx_crypto_ctx *nx_ctx = + crypto_aead_ctx(crypto_aead_reqtfm(req)); struct nx_gcm_rctx *rctx = aead_request_ctx(req); char *iv = rctx->iv; char *nonce = nx_ctx->priv.gcm.nonce; @@ -466,7 +475,10 @@ static int gcm4106_aes_nx_decrypt(struct aead_request *req) memcpy(iv, nonce, NX_GCM4106_NONCE_LEN); memcpy(iv + NX_GCM4106_NONCE_LEN, req->iv, 8); - return gcm_aes_nx_crypt(req, 0); + if (req->assoclen < 8) + return -EINVAL; + + return gcm_aes_nx_crypt(req, 0, req->assoclen - 8); } /* tell the block cipher walk routines that this is a stream cipher by diff --git a/drivers/crypto/nx/nx.c b/drivers/crypto/nx/nx.c index 436971343ff7..0794f1cc0018 100644 --- a/drivers/crypto/nx/nx.c +++ b/drivers/crypto/nx/nx.c @@ -596,13 +596,9 @@ static int nx_register_algs(void) if (rc) goto out_unreg_ecb; - rc = nx_register_alg(&nx_ctr_aes_alg, NX_FC_AES, NX_MODE_AES_CTR); - if (rc) - goto out_unreg_cbc; - rc = nx_register_alg(&nx_ctr3686_aes_alg, NX_FC_AES, NX_MODE_AES_CTR); if (rc) - goto out_unreg_ctr; + goto out_unreg_cbc; rc = nx_register_aead(&nx_gcm_aes_alg, NX_FC_AES, NX_MODE_AES_GCM); if (rc) @@ -612,11 +608,11 @@ static int nx_register_algs(void) if (rc) goto out_unreg_gcm; - rc = nx_register_alg(&nx_ccm_aes_alg, NX_FC_AES, NX_MODE_AES_CCM); + rc = nx_register_aead(&nx_ccm_aes_alg, NX_FC_AES, NX_MODE_AES_CCM); if (rc) goto out_unreg_gcm4106; - rc = nx_register_alg(&nx_ccm4309_aes_alg, NX_FC_AES, NX_MODE_AES_CCM); + rc = nx_register_aead(&nx_ccm4309_aes_alg, NX_FC_AES, NX_MODE_AES_CCM); if (rc) goto out_unreg_ccm; @@ -644,17 +640,15 @@ out_unreg_s256: nx_unregister_shash(&nx_shash_sha256_alg, NX_FC_SHA, NX_MODE_SHA, NX_PROPS_SHA256); out_unreg_ccm4309: - nx_unregister_alg(&nx_ccm4309_aes_alg, NX_FC_AES, NX_MODE_AES_CCM); + nx_unregister_aead(&nx_ccm4309_aes_alg, NX_FC_AES, NX_MODE_AES_CCM); out_unreg_ccm: - nx_unregister_alg(&nx_ccm_aes_alg, NX_FC_AES, NX_MODE_AES_CCM); + nx_unregister_aead(&nx_ccm_aes_alg, NX_FC_AES, NX_MODE_AES_CCM); out_unreg_gcm4106: nx_unregister_aead(&nx_gcm4106_aes_alg, NX_FC_AES, NX_MODE_AES_GCM); out_unreg_gcm: nx_unregister_aead(&nx_gcm_aes_alg, NX_FC_AES, NX_MODE_AES_GCM); out_unreg_ctr3686: nx_unregister_alg(&nx_ctr3686_aes_alg, NX_FC_AES, NX_MODE_AES_CTR); -out_unreg_ctr: - nx_unregister_alg(&nx_ctr_aes_alg, NX_FC_AES, NX_MODE_AES_CTR); out_unreg_cbc: nx_unregister_alg(&nx_cbc_aes_alg, NX_FC_AES, NX_MODE_AES_CBC); out_unreg_ecb: @@ -711,11 +705,10 @@ static int nx_crypto_ctx_init(struct nx_crypto_ctx *nx_ctx, u32 fc, u32 mode) } /* entry points from the crypto tfm initializers */ -int nx_crypto_ctx_aes_ccm_init(struct crypto_tfm *tfm) +int nx_crypto_ctx_aes_ccm_init(struct crypto_aead *tfm) { - crypto_aead_set_reqsize(__crypto_aead_cast(tfm), - sizeof(struct nx_ccm_rctx)); - return nx_crypto_ctx_init(crypto_tfm_ctx(tfm), NX_FC_AES, + crypto_aead_set_reqsize(tfm, sizeof(struct nx_ccm_rctx)); + return nx_crypto_ctx_init(crypto_aead_ctx(tfm), NX_FC_AES, NX_MODE_AES_CCM); } @@ -813,16 +806,15 @@ static int nx_remove(struct vio_dev *viodev) NX_FC_SHA, NX_MODE_SHA, NX_PROPS_SHA256); nx_unregister_shash(&nx_shash_sha256_alg, NX_FC_SHA, NX_MODE_SHA, NX_PROPS_SHA512); - nx_unregister_alg(&nx_ccm4309_aes_alg, - NX_FC_AES, NX_MODE_AES_CCM); - nx_unregister_alg(&nx_ccm_aes_alg, NX_FC_AES, NX_MODE_AES_CCM); + nx_unregister_aead(&nx_ccm4309_aes_alg, + NX_FC_AES, NX_MODE_AES_CCM); + nx_unregister_aead(&nx_ccm_aes_alg, NX_FC_AES, NX_MODE_AES_CCM); nx_unregister_aead(&nx_gcm4106_aes_alg, NX_FC_AES, NX_MODE_AES_GCM); nx_unregister_aead(&nx_gcm_aes_alg, NX_FC_AES, NX_MODE_AES_GCM); nx_unregister_alg(&nx_ctr3686_aes_alg, NX_FC_AES, NX_MODE_AES_CTR); - nx_unregister_alg(&nx_ctr_aes_alg, NX_FC_AES, NX_MODE_AES_CTR); nx_unregister_alg(&nx_cbc_aes_alg, NX_FC_AES, NX_MODE_AES_CBC); nx_unregister_alg(&nx_ecb_aes_alg, NX_FC_AES, NX_MODE_AES_ECB); } diff --git a/drivers/crypto/nx/nx.h b/drivers/crypto/nx/nx.h index cdff03a42ae7..9347878d4f30 100644 --- a/drivers/crypto/nx/nx.h +++ b/drivers/crypto/nx/nx.h @@ -149,8 +149,10 @@ struct nx_crypto_ctx { } priv; }; +struct crypto_aead; + /* prototypes */ -int nx_crypto_ctx_aes_ccm_init(struct crypto_tfm *tfm); +int nx_crypto_ctx_aes_ccm_init(struct crypto_aead *tfm); int nx_crypto_ctx_aes_gcm_init(struct crypto_aead *tfm); int nx_crypto_ctx_aes_xcbc_init(struct crypto_tfm *tfm); int nx_crypto_ctx_aes_ctr_init(struct crypto_tfm *tfm); @@ -187,10 +189,9 @@ extern struct crypto_alg nx_cbc_aes_alg; extern struct crypto_alg nx_ecb_aes_alg; extern struct aead_alg nx_gcm_aes_alg; extern struct aead_alg nx_gcm4106_aes_alg; -extern struct crypto_alg nx_ctr_aes_alg; extern struct crypto_alg nx_ctr3686_aes_alg; -extern struct crypto_alg nx_ccm_aes_alg; -extern struct crypto_alg nx_ccm4309_aes_alg; +extern struct aead_alg nx_ccm_aes_alg; +extern struct aead_alg nx_ccm4309_aes_alg; extern struct shash_alg nx_shash_aes_xcbc_alg; extern struct shash_alg nx_shash_sha512_alg; extern struct shash_alg nx_shash_sha256_alg; diff --git a/drivers/crypto/omap-aes.c b/drivers/crypto/omap-aes.c index 9a28b7e07c71..eba23147c0ee 100644 --- a/drivers/crypto/omap-aes.c +++ b/drivers/crypto/omap-aes.c @@ -52,29 +52,30 @@ #define AES_REG_IV(dd, x) ((dd)->pdata->iv_ofs + ((x) * 0x04)) #define AES_REG_CTRL(dd) ((dd)->pdata->ctrl_ofs) -#define AES_REG_CTRL_CTR_WIDTH_MASK (3 << 7) -#define AES_REG_CTRL_CTR_WIDTH_32 (0 << 7) -#define AES_REG_CTRL_CTR_WIDTH_64 (1 << 7) -#define AES_REG_CTRL_CTR_WIDTH_96 (2 << 7) -#define AES_REG_CTRL_CTR_WIDTH_128 (3 << 7) -#define AES_REG_CTRL_CTR (1 << 6) -#define AES_REG_CTRL_CBC (1 << 5) -#define AES_REG_CTRL_KEY_SIZE (3 << 3) -#define AES_REG_CTRL_DIRECTION (1 << 2) -#define AES_REG_CTRL_INPUT_READY (1 << 1) -#define AES_REG_CTRL_OUTPUT_READY (1 << 0) +#define AES_REG_CTRL_CTR_WIDTH_MASK GENMASK(8, 7) +#define AES_REG_CTRL_CTR_WIDTH_32 0 +#define AES_REG_CTRL_CTR_WIDTH_64 BIT(7) +#define AES_REG_CTRL_CTR_WIDTH_96 BIT(8) +#define AES_REG_CTRL_CTR_WIDTH_128 GENMASK(8, 7) +#define AES_REG_CTRL_CTR BIT(6) +#define AES_REG_CTRL_CBC BIT(5) +#define AES_REG_CTRL_KEY_SIZE GENMASK(4, 3) +#define AES_REG_CTRL_DIRECTION BIT(2) +#define AES_REG_CTRL_INPUT_READY BIT(1) +#define AES_REG_CTRL_OUTPUT_READY BIT(0) +#define AES_REG_CTRL_MASK GENMASK(24, 2) #define AES_REG_DATA_N(dd, x) ((dd)->pdata->data_ofs + ((x) * 0x04)) #define AES_REG_REV(dd) ((dd)->pdata->rev_ofs) #define AES_REG_MASK(dd) ((dd)->pdata->mask_ofs) -#define AES_REG_MASK_SIDLE (1 << 6) -#define AES_REG_MASK_START (1 << 5) -#define AES_REG_MASK_DMA_OUT_EN (1 << 3) -#define AES_REG_MASK_DMA_IN_EN (1 << 2) -#define AES_REG_MASK_SOFTRESET (1 << 1) -#define AES_REG_AUTOIDLE (1 << 0) +#define AES_REG_MASK_SIDLE BIT(6) +#define AES_REG_MASK_START BIT(5) +#define AES_REG_MASK_DMA_OUT_EN BIT(3) +#define AES_REG_MASK_DMA_IN_EN BIT(2) +#define AES_REG_MASK_SOFTRESET BIT(1) +#define AES_REG_AUTOIDLE BIT(0) #define AES_REG_LENGTH_N(x) (0x54 + ((x) * 0x04)) @@ -254,7 +255,7 @@ static int omap_aes_write_ctrl(struct omap_aes_dev *dd) { unsigned int key32; int i, err; - u32 val, mask = 0; + u32 val; err = omap_aes_hw_init(dd); if (err) @@ -274,17 +275,13 @@ static int omap_aes_write_ctrl(struct omap_aes_dev *dd) val = FLD_VAL(((dd->ctx->keylen >> 3) - 1), 4, 3); if (dd->flags & FLAGS_CBC) val |= AES_REG_CTRL_CBC; - if (dd->flags & FLAGS_CTR) { + if (dd->flags & FLAGS_CTR) val |= AES_REG_CTRL_CTR | AES_REG_CTRL_CTR_WIDTH_128; - mask = AES_REG_CTRL_CTR | AES_REG_CTRL_CTR_WIDTH_MASK; - } + if (dd->flags & FLAGS_ENCRYPT) val |= AES_REG_CTRL_DIRECTION; - mask |= AES_REG_CTRL_CBC | AES_REG_CTRL_DIRECTION | - AES_REG_CTRL_KEY_SIZE; - - omap_aes_write_mask(dd, AES_REG_CTRL(dd), val, mask); + omap_aes_write_mask(dd, AES_REG_CTRL(dd), val, AES_REG_CTRL_MASK); return 0; } @@ -558,6 +555,9 @@ static int omap_aes_check_aligned(struct scatterlist *sg, int total) { int len = 0; + if (!IS_ALIGNED(total, AES_BLOCK_SIZE)) + return -EINVAL; + while (sg) { if (!IS_ALIGNED(sg->offset, 4)) return -1; @@ -577,9 +577,10 @@ static int omap_aes_check_aligned(struct scatterlist *sg, int total) static int omap_aes_copy_sgs(struct omap_aes_dev *dd) { void *buf_in, *buf_out; - int pages; + int pages, total; - pages = get_order(dd->total); + total = ALIGN(dd->total, AES_BLOCK_SIZE); + pages = get_order(total); buf_in = (void *)__get_free_pages(GFP_ATOMIC, pages); buf_out = (void *)__get_free_pages(GFP_ATOMIC, pages); @@ -594,11 +595,11 @@ static int omap_aes_copy_sgs(struct omap_aes_dev *dd) sg_copy_buf(buf_in, dd->in_sg, 0, dd->total, 0); sg_init_table(&dd->in_sgl, 1); - sg_set_buf(&dd->in_sgl, buf_in, dd->total); + sg_set_buf(&dd->in_sgl, buf_in, total); dd->in_sg = &dd->in_sgl; sg_init_table(&dd->out_sgl, 1); - sg_set_buf(&dd->out_sgl, buf_out, dd->total); + sg_set_buf(&dd->out_sgl, buf_out, total); dd->out_sg = &dd->out_sgl; return 0; @@ -611,7 +612,7 @@ static int omap_aes_handle_queue(struct omap_aes_dev *dd, struct omap_aes_ctx *ctx; struct omap_aes_reqctx *rctx; unsigned long flags; - int err, ret = 0; + int err, ret = 0, len; spin_lock_irqsave(&dd->lock, flags); if (req) @@ -650,8 +651,9 @@ static int omap_aes_handle_queue(struct omap_aes_dev *dd, dd->sgs_copied = 0; } - dd->in_sg_len = scatterwalk_bytes_sglen(dd->in_sg, dd->total); - dd->out_sg_len = scatterwalk_bytes_sglen(dd->out_sg, dd->total); + len = ALIGN(dd->total, AES_BLOCK_SIZE); + dd->in_sg_len = scatterwalk_bytes_sglen(dd->in_sg, len); + dd->out_sg_len = scatterwalk_bytes_sglen(dd->out_sg, len); BUG_ON(dd->in_sg_len < 0 || dd->out_sg_len < 0); rctx = ablkcipher_request_ctx(req); @@ -678,7 +680,7 @@ static void omap_aes_done_task(unsigned long data) { struct omap_aes_dev *dd = (struct omap_aes_dev *)data; void *buf_in, *buf_out; - int pages; + int pages, len; pr_debug("enter done_task\n"); @@ -697,7 +699,8 @@ static void omap_aes_done_task(unsigned long data) sg_copy_buf(buf_out, dd->orig_out, 0, dd->total_save, 1); - pages = get_order(dd->total_save); + len = ALIGN(dd->total_save, AES_BLOCK_SIZE); + pages = get_order(len); free_pages((unsigned long)buf_in, pages); free_pages((unsigned long)buf_out, pages); } @@ -726,11 +729,6 @@ static int omap_aes_crypt(struct ablkcipher_request *req, unsigned long mode) !!(mode & FLAGS_ENCRYPT), !!(mode & FLAGS_CBC)); - if (!IS_ALIGNED(req->nbytes, AES_BLOCK_SIZE)) { - pr_err("request size is not exact amount of AES blocks\n"); - return -EINVAL; - } - dd = omap_aes_find_dev(ctx); if (!dd) return -ENODEV; @@ -833,7 +831,7 @@ static struct crypto_alg algs_ecb_cbc[] = { { .cra_name = "ecb(aes)", .cra_driver_name = "ecb-aes-omap", - .cra_priority = 100, + .cra_priority = 300, .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC, @@ -855,7 +853,7 @@ static struct crypto_alg algs_ecb_cbc[] = { { .cra_name = "cbc(aes)", .cra_driver_name = "cbc-aes-omap", - .cra_priority = 100, + .cra_priority = 300, .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC, @@ -881,7 +879,7 @@ static struct crypto_alg algs_ctr[] = { { .cra_name = "ctr(aes)", .cra_driver_name = "ctr-aes-omap", - .cra_priority = 100, + .cra_priority = 300, .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC, @@ -1046,9 +1044,7 @@ static irqreturn_t omap_aes_irq(int irq, void *dev_id) } } - dd->total -= AES_BLOCK_SIZE; - - BUG_ON(dd->total < 0); + dd->total -= min_t(size_t, AES_BLOCK_SIZE, dd->total); /* Clear IRQ status */ status &= ~AES_REG_IRQ_DATA_OUT; diff --git a/drivers/crypto/picoxcell_crypto.c b/drivers/crypto/picoxcell_crypto.c index 4f56f3681abd..da36de26a4dc 100644 --- a/drivers/crypto/picoxcell_crypto.c +++ b/drivers/crypto/picoxcell_crypto.c @@ -99,11 +99,16 @@ struct spacc_req { dma_addr_t src_addr, dst_addr; struct spacc_ddt *src_ddt, *dst_ddt; void (*complete)(struct spacc_req *req); +}; - /* AEAD specific bits. */ - u8 *giv; - size_t giv_len; - dma_addr_t giv_pa; +struct spacc_aead { + unsigned long ctrl_default; + unsigned long type; + struct aead_alg alg; + struct spacc_engine *engine; + struct list_head entry; + int key_offs; + int iv_offs; }; struct spacc_engine { @@ -121,6 +126,9 @@ struct spacc_engine { struct spacc_alg *algs; unsigned num_algs; struct list_head registered_algs; + struct spacc_aead *aeads; + unsigned num_aeads; + struct list_head registered_aeads; size_t cipher_pg_sz; size_t hash_pg_sz; const char *name; @@ -174,8 +182,6 @@ struct spacc_aead_ctx { u8 cipher_key_len; u8 hash_key_len; struct crypto_aead *sw_cipher; - size_t auth_size; - u8 salt[AES_BLOCK_SIZE]; }; static int spacc_ablk_submit(struct spacc_req *req); @@ -185,6 +191,11 @@ static inline struct spacc_alg *to_spacc_alg(struct crypto_alg *alg) return alg ? container_of(alg, struct spacc_alg, alg) : NULL; } +static inline struct spacc_aead *to_spacc_aead(struct aead_alg *alg) +{ + return container_of(alg, struct spacc_aead, alg); +} + static inline int spacc_fifo_cmd_full(struct spacc_engine *engine) { u32 fifo_stat = readl(engine->regs + SPA_FIFO_STAT_REG_OFFSET); @@ -310,120 +321,117 @@ out: return NULL; } -static int spacc_aead_make_ddts(struct spacc_req *req, u8 *giv) +static int spacc_aead_make_ddts(struct aead_request *areq) { - struct aead_request *areq = container_of(req->req, struct aead_request, - base); + struct crypto_aead *aead = crypto_aead_reqtfm(areq); + struct spacc_req *req = aead_request_ctx(areq); struct spacc_engine *engine = req->engine; struct spacc_ddt *src_ddt, *dst_ddt; - unsigned ivsize = crypto_aead_ivsize(crypto_aead_reqtfm(areq)); - unsigned nents = sg_count(areq->src, areq->cryptlen); unsigned total; - dma_addr_t iv_addr; + unsigned int src_nents, dst_nents; struct scatterlist *cur; - int i, dst_ents, src_ents, assoc_ents; - u8 *iv = giv ? giv : areq->iv; + int i, dst_ents, src_ents; + + total = areq->assoclen + areq->cryptlen; + if (req->is_encrypt) + total += crypto_aead_authsize(aead); + + src_nents = sg_count(areq->src, total); + if (src_nents + 1 > MAX_DDT_LEN) + return -E2BIG; + + dst_nents = 0; + if (areq->src != areq->dst) { + dst_nents = sg_count(areq->dst, total); + if (src_nents + 1 > MAX_DDT_LEN) + return -E2BIG; + } src_ddt = dma_pool_alloc(engine->req_pool, GFP_ATOMIC, &req->src_addr); if (!src_ddt) - return -ENOMEM; + goto err; dst_ddt = dma_pool_alloc(engine->req_pool, GFP_ATOMIC, &req->dst_addr); - if (!dst_ddt) { - dma_pool_free(engine->req_pool, src_ddt, req->src_addr); - return -ENOMEM; - } + if (!dst_ddt) + goto err_free_src; req->src_ddt = src_ddt; req->dst_ddt = dst_ddt; - assoc_ents = dma_map_sg(engine->dev, areq->assoc, - sg_count(areq->assoc, areq->assoclen), DMA_TO_DEVICE); - if (areq->src != areq->dst) { - src_ents = dma_map_sg(engine->dev, areq->src, nents, + if (dst_nents) { + src_ents = dma_map_sg(engine->dev, areq->src, src_nents, DMA_TO_DEVICE); - dst_ents = dma_map_sg(engine->dev, areq->dst, nents, + if (!src_ents) + goto err_free_dst; + + dst_ents = dma_map_sg(engine->dev, areq->dst, dst_nents, DMA_FROM_DEVICE); + + if (!dst_ents) { + dma_unmap_sg(engine->dev, areq->src, src_nents, + DMA_TO_DEVICE); + goto err_free_dst; + } } else { - src_ents = dma_map_sg(engine->dev, areq->src, nents, + src_ents = dma_map_sg(engine->dev, areq->src, src_nents, DMA_BIDIRECTIONAL); - dst_ents = 0; + if (!src_ents) + goto err_free_dst; + dst_ents = src_ents; } /* - * Map the IV/GIV. For the GIV it needs to be bidirectional as it is - * formed by the crypto block and sent as the ESP IV for IPSEC. + * Now map in the payload for the source and destination and terminate + * with the NULL pointers. */ - iv_addr = dma_map_single(engine->dev, iv, ivsize, - giv ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE); - req->giv_pa = iv_addr; + for_each_sg(areq->src, cur, src_ents, i) + ddt_set(src_ddt++, sg_dma_address(cur), sg_dma_len(cur)); - /* - * Map the associated data. For decryption we don't copy the - * associated data. - */ - total = areq->assoclen; - for_each_sg(areq->assoc, cur, assoc_ents, i) { + /* For decryption we need to skip the associated data. */ + total = req->is_encrypt ? 0 : areq->assoclen; + for_each_sg(areq->dst, cur, dst_ents, i) { unsigned len = sg_dma_len(cur); - if (len > total) - len = total; - - total -= len; + if (len <= total) { + total -= len; + continue; + } - ddt_set(src_ddt++, sg_dma_address(cur), len); - if (req->is_encrypt) - ddt_set(dst_ddt++, sg_dma_address(cur), len); + ddt_set(dst_ddt++, sg_dma_address(cur) + total, len - total); } - ddt_set(src_ddt++, iv_addr, ivsize); - - if (giv || req->is_encrypt) - ddt_set(dst_ddt++, iv_addr, ivsize); - - /* - * Now map in the payload for the source and destination and terminate - * with the NULL pointers. - */ - for_each_sg(areq->src, cur, src_ents, i) { - ddt_set(src_ddt++, sg_dma_address(cur), sg_dma_len(cur)); - if (areq->src == areq->dst) - ddt_set(dst_ddt++, sg_dma_address(cur), - sg_dma_len(cur)); - } - - for_each_sg(areq->dst, cur, dst_ents, i) - ddt_set(dst_ddt++, sg_dma_address(cur), - sg_dma_len(cur)); ddt_set(src_ddt, 0, 0); ddt_set(dst_ddt, 0, 0); return 0; + +err_free_dst: + dma_pool_free(engine->req_pool, dst_ddt, req->dst_addr); +err_free_src: + dma_pool_free(engine->req_pool, src_ddt, req->src_addr); +err: + return -ENOMEM; } static void spacc_aead_free_ddts(struct spacc_req *req) { struct aead_request *areq = container_of(req->req, struct aead_request, base); - struct spacc_alg *alg = to_spacc_alg(req->req->tfm->__crt_alg); - struct spacc_ablk_ctx *aead_ctx = crypto_tfm_ctx(req->req->tfm); + struct crypto_aead *aead = crypto_aead_reqtfm(areq); + unsigned total = areq->assoclen + areq->cryptlen + + (req->is_encrypt ? crypto_aead_authsize(aead) : 0); + struct spacc_aead_ctx *aead_ctx = crypto_aead_ctx(aead); struct spacc_engine *engine = aead_ctx->generic.engine; - unsigned ivsize = alg->alg.cra_aead.ivsize; - unsigned nents = sg_count(areq->src, areq->cryptlen); + unsigned nents = sg_count(areq->src, total); if (areq->src != areq->dst) { dma_unmap_sg(engine->dev, areq->src, nents, DMA_TO_DEVICE); dma_unmap_sg(engine->dev, areq->dst, - sg_count(areq->dst, areq->cryptlen), + sg_count(areq->dst, total), DMA_FROM_DEVICE); } else dma_unmap_sg(engine->dev, areq->src, nents, DMA_BIDIRECTIONAL); - dma_unmap_sg(engine->dev, areq->assoc, - sg_count(areq->assoc, areq->assoclen), DMA_TO_DEVICE); - - dma_unmap_single(engine->dev, req->giv_pa, ivsize, DMA_BIDIRECTIONAL); - dma_pool_free(engine->req_pool, req->src_ddt, req->src_addr); dma_pool_free(engine->req_pool, req->dst_ddt, req->dst_addr); } @@ -438,65 +446,22 @@ static void spacc_free_ddt(struct spacc_req *req, struct spacc_ddt *ddt, dma_pool_free(req->engine->req_pool, ddt, ddt_addr); } -/* - * Set key for a DES operation in an AEAD cipher. This also performs weak key - * checking if required. - */ -static int spacc_aead_des_setkey(struct crypto_aead *aead, const u8 *key, - unsigned int len) -{ - struct crypto_tfm *tfm = crypto_aead_tfm(aead); - struct spacc_aead_ctx *ctx = crypto_tfm_ctx(tfm); - u32 tmp[DES_EXPKEY_WORDS]; - - if (unlikely(!des_ekey(tmp, key)) && - (crypto_aead_get_flags(aead)) & CRYPTO_TFM_REQ_WEAK_KEY) { - tfm->crt_flags |= CRYPTO_TFM_RES_WEAK_KEY; - return -EINVAL; - } - - memcpy(ctx->cipher_key, key, len); - ctx->cipher_key_len = len; - - return 0; -} - -/* Set the key for the AES block cipher component of the AEAD transform. */ -static int spacc_aead_aes_setkey(struct crypto_aead *aead, const u8 *key, - unsigned int len) -{ - struct crypto_tfm *tfm = crypto_aead_tfm(aead); - struct spacc_aead_ctx *ctx = crypto_tfm_ctx(tfm); - - /* - * IPSec engine only supports 128 and 256 bit AES keys. If we get a - * request for any other size (192 bits) then we need to do a software - * fallback. - */ - if (len != AES_KEYSIZE_128 && len != AES_KEYSIZE_256) { - /* - * Set the fallback transform to use the same request flags as - * the hardware transform. - */ - ctx->sw_cipher->base.crt_flags &= ~CRYPTO_TFM_REQ_MASK; - ctx->sw_cipher->base.crt_flags |= - tfm->crt_flags & CRYPTO_TFM_REQ_MASK; - return crypto_aead_setkey(ctx->sw_cipher, key, len); - } - - memcpy(ctx->cipher_key, key, len); - ctx->cipher_key_len = len; - - return 0; -} - static int spacc_aead_setkey(struct crypto_aead *tfm, const u8 *key, unsigned int keylen) { struct spacc_aead_ctx *ctx = crypto_aead_ctx(tfm); - struct spacc_alg *alg = to_spacc_alg(tfm->base.__crt_alg); struct crypto_authenc_keys keys; - int err = -EINVAL; + int err; + + crypto_aead_clear_flags(ctx->sw_cipher, CRYPTO_TFM_REQ_MASK); + crypto_aead_set_flags(ctx->sw_cipher, crypto_aead_get_flags(tfm) & + CRYPTO_TFM_REQ_MASK); + err = crypto_aead_setkey(ctx->sw_cipher, key, keylen); + crypto_aead_clear_flags(tfm, CRYPTO_TFM_RES_MASK); + crypto_aead_set_flags(tfm, crypto_aead_get_flags(ctx->sw_cipher) & + CRYPTO_TFM_RES_MASK); + if (err) + return err; if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) goto badkey; @@ -507,14 +472,8 @@ static int spacc_aead_setkey(struct crypto_aead *tfm, const u8 *key, if (keys.authkeylen > sizeof(ctx->hash_ctx)) goto badkey; - if ((alg->ctrl_default & SPACC_CRYPTO_ALG_MASK) == - SPA_CTRL_CIPH_ALG_AES) - err = spacc_aead_aes_setkey(tfm, keys.enckey, keys.enckeylen); - else - err = spacc_aead_des_setkey(tfm, keys.enckey, keys.enckeylen); - - if (err) - goto badkey; + memcpy(ctx->cipher_key, keys.enckey, keys.enckeylen); + ctx->cipher_key_len = keys.enckeylen; memcpy(ctx->hash_ctx, keys.authkey, keys.authkeylen); ctx->hash_key_len = keys.authkeylen; @@ -531,9 +490,7 @@ static int spacc_aead_setauthsize(struct crypto_aead *tfm, { struct spacc_aead_ctx *ctx = crypto_tfm_ctx(crypto_aead_tfm(tfm)); - ctx->auth_size = authsize; - - return 0; + return crypto_aead_setauthsize(ctx->sw_cipher, authsize); } /* @@ -541,15 +498,13 @@ static int spacc_aead_setauthsize(struct crypto_aead *tfm, * be completed in hardware because the hardware may not support certain key * sizes. In these cases we need to complete the request in software. */ -static int spacc_aead_need_fallback(struct spacc_req *req) +static int spacc_aead_need_fallback(struct aead_request *aead_req) { - struct aead_request *aead_req; - struct crypto_tfm *tfm = req->req->tfm; - struct crypto_alg *alg = req->req->tfm->__crt_alg; - struct spacc_alg *spacc_alg = to_spacc_alg(alg); - struct spacc_aead_ctx *ctx = crypto_tfm_ctx(tfm); + struct crypto_aead *aead = crypto_aead_reqtfm(aead_req); + struct aead_alg *alg = crypto_aead_alg(aead); + struct spacc_aead *spacc_alg = to_spacc_aead(alg); + struct spacc_aead_ctx *ctx = crypto_aead_ctx(aead); - aead_req = container_of(req->req, struct aead_request, base); /* * If we have a non-supported key-length, then we need to do a * software fallback. @@ -568,22 +523,17 @@ static int spacc_aead_do_fallback(struct aead_request *req, unsigned alg_type, { struct crypto_tfm *old_tfm = crypto_aead_tfm(crypto_aead_reqtfm(req)); struct spacc_aead_ctx *ctx = crypto_tfm_ctx(old_tfm); - int err; + struct aead_request *subreq = aead_request_ctx(req); - if (ctx->sw_cipher) { - /* - * Change the request to use the software fallback transform, - * and once the ciphering has completed, put the old transform - * back into the request. - */ - aead_request_set_tfm(req, ctx->sw_cipher); - err = is_encrypt ? crypto_aead_encrypt(req) : - crypto_aead_decrypt(req); - aead_request_set_tfm(req, __crypto_aead_cast(old_tfm)); - } else - err = -EINVAL; + aead_request_set_tfm(subreq, ctx->sw_cipher); + aead_request_set_callback(subreq, req->base.flags, + req->base.complete, req->base.data); + aead_request_set_crypt(subreq, req->src, req->dst, req->cryptlen, + req->iv); + aead_request_set_ad(subreq, req->assoclen); - return err; + return is_encrypt ? crypto_aead_encrypt(subreq) : + crypto_aead_decrypt(subreq); } static void spacc_aead_complete(struct spacc_req *req) @@ -594,18 +544,19 @@ static void spacc_aead_complete(struct spacc_req *req) static int spacc_aead_submit(struct spacc_req *req) { - struct crypto_tfm *tfm = req->req->tfm; - struct spacc_aead_ctx *ctx = crypto_tfm_ctx(tfm); - struct crypto_alg *alg = req->req->tfm->__crt_alg; - struct spacc_alg *spacc_alg = to_spacc_alg(alg); - struct spacc_engine *engine = ctx->generic.engine; - u32 ctrl, proc_len, assoc_len; struct aead_request *aead_req = container_of(req->req, struct aead_request, base); + struct crypto_aead *aead = crypto_aead_reqtfm(aead_req); + unsigned int authsize = crypto_aead_authsize(aead); + struct spacc_aead_ctx *ctx = crypto_aead_ctx(aead); + struct aead_alg *alg = crypto_aead_alg(aead); + struct spacc_aead *spacc_alg = to_spacc_aead(alg); + struct spacc_engine *engine = ctx->generic.engine; + u32 ctrl, proc_len, assoc_len; req->result = -EINPROGRESS; req->ctx_id = spacc_load_ctx(&ctx->generic, ctx->cipher_key, - ctx->cipher_key_len, aead_req->iv, alg->cra_aead.ivsize, + ctx->cipher_key_len, aead_req->iv, crypto_aead_ivsize(aead), ctx->hash_ctx, ctx->hash_key_len); /* Set the source and destination DDT pointers. */ @@ -617,25 +568,15 @@ static int spacc_aead_submit(struct spacc_req *req) proc_len = aead_req->cryptlen + assoc_len; /* - * If we aren't generating an IV, then we need to include the IV in the - * associated data so that it is included in the hash. - */ - if (!req->giv) { - assoc_len += crypto_aead_ivsize(crypto_aead_reqtfm(aead_req)); - proc_len += crypto_aead_ivsize(crypto_aead_reqtfm(aead_req)); - } else - proc_len += req->giv_len; - - /* * If we are decrypting, we need to take the length of the ICV out of * the processing length. */ if (!req->is_encrypt) - proc_len -= ctx->auth_size; + proc_len -= authsize; writel(proc_len, engine->regs + SPA_PROC_LEN_REG_OFFSET); writel(assoc_len, engine->regs + SPA_AAD_LEN_REG_OFFSET); - writel(ctx->auth_size, engine->regs + SPA_ICV_LEN_REG_OFFSET); + writel(authsize, engine->regs + SPA_ICV_LEN_REG_OFFSET); writel(0, engine->regs + SPA_ICV_OFFSET_REG_OFFSET); writel(0, engine->regs + SPA_AUX_INFO_REG_OFFSET); @@ -674,32 +615,29 @@ static void spacc_push(struct spacc_engine *engine) /* * Setup an AEAD request for processing. This will configure the engine, load * the context and then start the packet processing. - * - * @giv Pointer to destination address for a generated IV. If the - * request does not need to generate an IV then this should be set to NULL. */ -static int spacc_aead_setup(struct aead_request *req, u8 *giv, +static int spacc_aead_setup(struct aead_request *req, unsigned alg_type, bool is_encrypt) { - struct crypto_alg *alg = req->base.tfm->__crt_alg; - struct spacc_engine *engine = to_spacc_alg(alg)->engine; + struct crypto_aead *aead = crypto_aead_reqtfm(req); + struct aead_alg *alg = crypto_aead_alg(aead); + struct spacc_engine *engine = to_spacc_aead(alg)->engine; struct spacc_req *dev_req = aead_request_ctx(req); - int err = -EINPROGRESS; + int err; unsigned long flags; - unsigned ivsize = crypto_aead_ivsize(crypto_aead_reqtfm(req)); - dev_req->giv = giv; - dev_req->giv_len = ivsize; dev_req->req = &req->base; dev_req->is_encrypt = is_encrypt; dev_req->result = -EBUSY; dev_req->engine = engine; dev_req->complete = spacc_aead_complete; - if (unlikely(spacc_aead_need_fallback(dev_req))) + if (unlikely(spacc_aead_need_fallback(req) || + ((err = spacc_aead_make_ddts(req)) == -E2BIG))) return spacc_aead_do_fallback(req, alg_type, is_encrypt); - spacc_aead_make_ddts(dev_req, dev_req->giv); + if (err) + goto out; err = -EINPROGRESS; spin_lock_irqsave(&engine->hw_lock, flags); @@ -728,70 +666,44 @@ out: static int spacc_aead_encrypt(struct aead_request *req) { struct crypto_aead *aead = crypto_aead_reqtfm(req); - struct crypto_tfm *tfm = crypto_aead_tfm(aead); - struct spacc_alg *alg = to_spacc_alg(tfm->__crt_alg); + struct spacc_aead *alg = to_spacc_aead(crypto_aead_alg(aead)); - return spacc_aead_setup(req, NULL, alg->type, 1); -} - -static int spacc_aead_givencrypt(struct aead_givcrypt_request *req) -{ - struct crypto_aead *tfm = aead_givcrypt_reqtfm(req); - struct spacc_aead_ctx *ctx = crypto_aead_ctx(tfm); - size_t ivsize = crypto_aead_ivsize(tfm); - struct spacc_alg *alg = to_spacc_alg(tfm->base.__crt_alg); - unsigned len; - __be64 seq; - - memcpy(req->areq.iv, ctx->salt, ivsize); - len = ivsize; - if (ivsize > sizeof(u64)) { - memset(req->giv, 0, ivsize - sizeof(u64)); - len = sizeof(u64); - } - seq = cpu_to_be64(req->seq); - memcpy(req->giv + ivsize - len, &seq, len); - - return spacc_aead_setup(&req->areq, req->giv, alg->type, 1); + return spacc_aead_setup(req, alg->type, 1); } static int spacc_aead_decrypt(struct aead_request *req) { struct crypto_aead *aead = crypto_aead_reqtfm(req); - struct crypto_tfm *tfm = crypto_aead_tfm(aead); - struct spacc_alg *alg = to_spacc_alg(tfm->__crt_alg); + struct spacc_aead *alg = to_spacc_aead(crypto_aead_alg(aead)); - return spacc_aead_setup(req, NULL, alg->type, 0); + return spacc_aead_setup(req, alg->type, 0); } /* * Initialise a new AEAD context. This is responsible for allocating the * fallback cipher and initialising the context. */ -static int spacc_aead_cra_init(struct crypto_tfm *tfm) +static int spacc_aead_cra_init(struct crypto_aead *tfm) { - struct spacc_aead_ctx *ctx = crypto_tfm_ctx(tfm); - struct crypto_alg *alg = tfm->__crt_alg; - struct spacc_alg *spacc_alg = to_spacc_alg(alg); + struct spacc_aead_ctx *ctx = crypto_aead_ctx(tfm); + struct aead_alg *alg = crypto_aead_alg(tfm); + struct spacc_aead *spacc_alg = to_spacc_aead(alg); struct spacc_engine *engine = spacc_alg->engine; ctx->generic.flags = spacc_alg->type; ctx->generic.engine = engine; - ctx->sw_cipher = crypto_alloc_aead(alg->cra_name, 0, - CRYPTO_ALG_ASYNC | + ctx->sw_cipher = crypto_alloc_aead(alg->base.cra_name, 0, CRYPTO_ALG_NEED_FALLBACK); - if (IS_ERR(ctx->sw_cipher)) { - dev_warn(engine->dev, "failed to allocate fallback for %s\n", - alg->cra_name); - ctx->sw_cipher = NULL; - } + if (IS_ERR(ctx->sw_cipher)) + return PTR_ERR(ctx->sw_cipher); ctx->generic.key_offs = spacc_alg->key_offs; ctx->generic.iv_offs = spacc_alg->iv_offs; - get_random_bytes(ctx->salt, sizeof(ctx->salt)); - - crypto_aead_set_reqsize(__crypto_aead_cast(tfm), - sizeof(struct spacc_req)); + crypto_aead_set_reqsize( + tfm, + max(sizeof(struct spacc_req), + sizeof(struct aead_request) + + crypto_aead_reqsize(ctx->sw_cipher))); return 0; } @@ -800,13 +712,11 @@ static int spacc_aead_cra_init(struct crypto_tfm *tfm) * Destructor for an AEAD context. This is called when the transform is freed * and must free the fallback cipher. */ -static void spacc_aead_cra_exit(struct crypto_tfm *tfm) +static void spacc_aead_cra_exit(struct crypto_aead *tfm) { - struct spacc_aead_ctx *ctx = crypto_tfm_ctx(tfm); + struct spacc_aead_ctx *ctx = crypto_aead_ctx(tfm); - if (ctx->sw_cipher) - crypto_free_aead(ctx->sw_cipher); - ctx->sw_cipher = NULL; + crypto_free_aead(ctx->sw_cipher); } /* @@ -1458,180 +1368,188 @@ static struct spacc_alg ipsec_engine_algs[] = { .cra_exit = spacc_ablk_cra_exit, }, }, +}; + +static struct spacc_aead ipsec_engine_aeads[] = { { - .ctrl_default = SPA_CTRL_CIPH_ALG_AES | SPA_CTRL_CIPH_MODE_CBC | - SPA_CTRL_HASH_ALG_SHA | SPA_CTRL_HASH_MODE_HMAC, + .ctrl_default = SPA_CTRL_CIPH_ALG_AES | + SPA_CTRL_CIPH_MODE_CBC | + SPA_CTRL_HASH_ALG_SHA | + SPA_CTRL_HASH_MODE_HMAC, .key_offs = 0, .iv_offs = AES_MAX_KEY_SIZE, .alg = { - .cra_name = "authenc(hmac(sha1),cbc(aes))", - .cra_driver_name = "authenc-hmac-sha1-cbc-aes-picoxcell", - .cra_priority = SPACC_CRYPTO_ALG_PRIORITY, - .cra_flags = CRYPTO_ALG_TYPE_AEAD | - CRYPTO_ALG_ASYNC | - CRYPTO_ALG_KERN_DRIVER_ONLY, - .cra_blocksize = AES_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct spacc_aead_ctx), - .cra_type = &crypto_aead_type, - .cra_module = THIS_MODULE, - .cra_aead = { - .setkey = spacc_aead_setkey, - .setauthsize = spacc_aead_setauthsize, - .encrypt = spacc_aead_encrypt, - .decrypt = spacc_aead_decrypt, - .givencrypt = spacc_aead_givencrypt, - .ivsize = AES_BLOCK_SIZE, - .maxauthsize = SHA1_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(sha1),cbc(aes))", + .cra_driver_name = "authenc-hmac-sha1-" + "cbc-aes-picoxcell", + .cra_priority = SPACC_CRYPTO_ALG_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct spacc_aead_ctx), + .cra_module = THIS_MODULE, }, - .cra_init = spacc_aead_cra_init, - .cra_exit = spacc_aead_cra_exit, + .setkey = spacc_aead_setkey, + .setauthsize = spacc_aead_setauthsize, + .encrypt = spacc_aead_encrypt, + .decrypt = spacc_aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + .init = spacc_aead_cra_init, + .exit = spacc_aead_cra_exit, }, }, { - .ctrl_default = SPA_CTRL_CIPH_ALG_AES | SPA_CTRL_CIPH_MODE_CBC | + .ctrl_default = SPA_CTRL_CIPH_ALG_AES | + SPA_CTRL_CIPH_MODE_CBC | SPA_CTRL_HASH_ALG_SHA256 | SPA_CTRL_HASH_MODE_HMAC, .key_offs = 0, .iv_offs = AES_MAX_KEY_SIZE, .alg = { - .cra_name = "authenc(hmac(sha256),cbc(aes))", - .cra_driver_name = "authenc-hmac-sha256-cbc-aes-picoxcell", - .cra_priority = SPACC_CRYPTO_ALG_PRIORITY, - .cra_flags = CRYPTO_ALG_TYPE_AEAD | - CRYPTO_ALG_ASYNC | - CRYPTO_ALG_KERN_DRIVER_ONLY, - .cra_blocksize = AES_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct spacc_aead_ctx), - .cra_type = &crypto_aead_type, - .cra_module = THIS_MODULE, - .cra_aead = { - .setkey = spacc_aead_setkey, - .setauthsize = spacc_aead_setauthsize, - .encrypt = spacc_aead_encrypt, - .decrypt = spacc_aead_decrypt, - .givencrypt = spacc_aead_givencrypt, - .ivsize = AES_BLOCK_SIZE, - .maxauthsize = SHA256_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(sha256),cbc(aes))", + .cra_driver_name = "authenc-hmac-sha256-" + "cbc-aes-picoxcell", + .cra_priority = SPACC_CRYPTO_ALG_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct spacc_aead_ctx), + .cra_module = THIS_MODULE, }, - .cra_init = spacc_aead_cra_init, - .cra_exit = spacc_aead_cra_exit, + .setkey = spacc_aead_setkey, + .setauthsize = spacc_aead_setauthsize, + .encrypt = spacc_aead_encrypt, + .decrypt = spacc_aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + .init = spacc_aead_cra_init, + .exit = spacc_aead_cra_exit, }, }, { .key_offs = 0, .iv_offs = AES_MAX_KEY_SIZE, - .ctrl_default = SPA_CTRL_CIPH_ALG_AES | SPA_CTRL_CIPH_MODE_CBC | - SPA_CTRL_HASH_ALG_MD5 | SPA_CTRL_HASH_MODE_HMAC, + .ctrl_default = SPA_CTRL_CIPH_ALG_AES | + SPA_CTRL_CIPH_MODE_CBC | + SPA_CTRL_HASH_ALG_MD5 | + SPA_CTRL_HASH_MODE_HMAC, .alg = { - .cra_name = "authenc(hmac(md5),cbc(aes))", - .cra_driver_name = "authenc-hmac-md5-cbc-aes-picoxcell", - .cra_priority = SPACC_CRYPTO_ALG_PRIORITY, - .cra_flags = CRYPTO_ALG_TYPE_AEAD | - CRYPTO_ALG_ASYNC | - CRYPTO_ALG_KERN_DRIVER_ONLY, - .cra_blocksize = AES_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct spacc_aead_ctx), - .cra_type = &crypto_aead_type, - .cra_module = THIS_MODULE, - .cra_aead = { - .setkey = spacc_aead_setkey, - .setauthsize = spacc_aead_setauthsize, - .encrypt = spacc_aead_encrypt, - .decrypt = spacc_aead_decrypt, - .givencrypt = spacc_aead_givencrypt, - .ivsize = AES_BLOCK_SIZE, - .maxauthsize = MD5_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(md5),cbc(aes))", + .cra_driver_name = "authenc-hmac-md5-" + "cbc-aes-picoxcell", + .cra_priority = SPACC_CRYPTO_ALG_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct spacc_aead_ctx), + .cra_module = THIS_MODULE, }, - .cra_init = spacc_aead_cra_init, - .cra_exit = spacc_aead_cra_exit, + .setkey = spacc_aead_setkey, + .setauthsize = spacc_aead_setauthsize, + .encrypt = spacc_aead_encrypt, + .decrypt = spacc_aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = MD5_DIGEST_SIZE, + .init = spacc_aead_cra_init, + .exit = spacc_aead_cra_exit, }, }, { .key_offs = DES_BLOCK_SIZE, .iv_offs = 0, - .ctrl_default = SPA_CTRL_CIPH_ALG_DES | SPA_CTRL_CIPH_MODE_CBC | - SPA_CTRL_HASH_ALG_SHA | SPA_CTRL_HASH_MODE_HMAC, + .ctrl_default = SPA_CTRL_CIPH_ALG_DES | + SPA_CTRL_CIPH_MODE_CBC | + SPA_CTRL_HASH_ALG_SHA | + SPA_CTRL_HASH_MODE_HMAC, .alg = { - .cra_name = "authenc(hmac(sha1),cbc(des3_ede))", - .cra_driver_name = "authenc-hmac-sha1-cbc-3des-picoxcell", - .cra_priority = SPACC_CRYPTO_ALG_PRIORITY, - .cra_flags = CRYPTO_ALG_TYPE_AEAD | - CRYPTO_ALG_ASYNC | - CRYPTO_ALG_KERN_DRIVER_ONLY, - .cra_blocksize = DES3_EDE_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct spacc_aead_ctx), - .cra_type = &crypto_aead_type, - .cra_module = THIS_MODULE, - .cra_aead = { - .setkey = spacc_aead_setkey, - .setauthsize = spacc_aead_setauthsize, - .encrypt = spacc_aead_encrypt, - .decrypt = spacc_aead_decrypt, - .givencrypt = spacc_aead_givencrypt, - .ivsize = DES3_EDE_BLOCK_SIZE, - .maxauthsize = SHA1_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(sha1),cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-sha1-" + "cbc-3des-picoxcell", + .cra_priority = SPACC_CRYPTO_ALG_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct spacc_aead_ctx), + .cra_module = THIS_MODULE, }, - .cra_init = spacc_aead_cra_init, - .cra_exit = spacc_aead_cra_exit, + .setkey = spacc_aead_setkey, + .setauthsize = spacc_aead_setauthsize, + .encrypt = spacc_aead_encrypt, + .decrypt = spacc_aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + .init = spacc_aead_cra_init, + .exit = spacc_aead_cra_exit, }, }, { .key_offs = DES_BLOCK_SIZE, .iv_offs = 0, - .ctrl_default = SPA_CTRL_CIPH_ALG_AES | SPA_CTRL_CIPH_MODE_CBC | + .ctrl_default = SPA_CTRL_CIPH_ALG_AES | + SPA_CTRL_CIPH_MODE_CBC | SPA_CTRL_HASH_ALG_SHA256 | SPA_CTRL_HASH_MODE_HMAC, .alg = { - .cra_name = "authenc(hmac(sha256),cbc(des3_ede))", - .cra_driver_name = "authenc-hmac-sha256-cbc-3des-picoxcell", - .cra_priority = SPACC_CRYPTO_ALG_PRIORITY, - .cra_flags = CRYPTO_ALG_TYPE_AEAD | - CRYPTO_ALG_ASYNC | - CRYPTO_ALG_KERN_DRIVER_ONLY, - .cra_blocksize = DES3_EDE_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct spacc_aead_ctx), - .cra_type = &crypto_aead_type, - .cra_module = THIS_MODULE, - .cra_aead = { - .setkey = spacc_aead_setkey, - .setauthsize = spacc_aead_setauthsize, - .encrypt = spacc_aead_encrypt, - .decrypt = spacc_aead_decrypt, - .givencrypt = spacc_aead_givencrypt, - .ivsize = DES3_EDE_BLOCK_SIZE, - .maxauthsize = SHA256_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(sha256)," + "cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-sha256-" + "cbc-3des-picoxcell", + .cra_priority = SPACC_CRYPTO_ALG_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct spacc_aead_ctx), + .cra_module = THIS_MODULE, }, - .cra_init = spacc_aead_cra_init, - .cra_exit = spacc_aead_cra_exit, + .setkey = spacc_aead_setkey, + .setauthsize = spacc_aead_setauthsize, + .encrypt = spacc_aead_encrypt, + .decrypt = spacc_aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + .init = spacc_aead_cra_init, + .exit = spacc_aead_cra_exit, }, }, { .key_offs = DES_BLOCK_SIZE, .iv_offs = 0, - .ctrl_default = SPA_CTRL_CIPH_ALG_DES | SPA_CTRL_CIPH_MODE_CBC | - SPA_CTRL_HASH_ALG_MD5 | SPA_CTRL_HASH_MODE_HMAC, + .ctrl_default = SPA_CTRL_CIPH_ALG_DES | + SPA_CTRL_CIPH_MODE_CBC | + SPA_CTRL_HASH_ALG_MD5 | + SPA_CTRL_HASH_MODE_HMAC, .alg = { - .cra_name = "authenc(hmac(md5),cbc(des3_ede))", - .cra_driver_name = "authenc-hmac-md5-cbc-3des-picoxcell", - .cra_priority = SPACC_CRYPTO_ALG_PRIORITY, - .cra_flags = CRYPTO_ALG_TYPE_AEAD | - CRYPTO_ALG_ASYNC | - CRYPTO_ALG_KERN_DRIVER_ONLY, - .cra_blocksize = DES3_EDE_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct spacc_aead_ctx), - .cra_type = &crypto_aead_type, - .cra_module = THIS_MODULE, - .cra_aead = { - .setkey = spacc_aead_setkey, - .setauthsize = spacc_aead_setauthsize, - .encrypt = spacc_aead_encrypt, - .decrypt = spacc_aead_decrypt, - .givencrypt = spacc_aead_givencrypt, - .ivsize = DES3_EDE_BLOCK_SIZE, - .maxauthsize = MD5_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(md5),cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-md5-" + "cbc-3des-picoxcell", + .cra_priority = SPACC_CRYPTO_ALG_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct spacc_aead_ctx), + .cra_module = THIS_MODULE, }, - .cra_init = spacc_aead_cra_init, - .cra_exit = spacc_aead_cra_exit, + .setkey = spacc_aead_setkey, + .setauthsize = spacc_aead_setauthsize, + .encrypt = spacc_aead_encrypt, + .decrypt = spacc_aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = MD5_DIGEST_SIZE, + .init = spacc_aead_cra_init, + .exit = spacc_aead_cra_exit, }, }, }; @@ -1707,6 +1625,8 @@ static int spacc_probe(struct platform_device *pdev) engine->fifo_sz = SPACC_CRYPTO_IPSEC_FIFO_SZ; engine->algs = ipsec_engine_algs; engine->num_algs = ARRAY_SIZE(ipsec_engine_algs); + engine->aeads = ipsec_engine_aeads; + engine->num_aeads = ARRAY_SIZE(ipsec_engine_aeads); } else if (spacc_is_compatible(pdev, "picochip,spacc-l2")) { engine->max_ctxs = SPACC_CRYPTO_L2_MAX_CTXS; engine->cipher_pg_sz = SPACC_CRYPTO_L2_CIPHER_PG_SZ; @@ -1815,17 +1735,40 @@ static int spacc_probe(struct platform_device *pdev) engine->algs[i].alg.cra_name); } + INIT_LIST_HEAD(&engine->registered_aeads); + for (i = 0; i < engine->num_aeads; ++i) { + engine->aeads[i].engine = engine; + err = crypto_register_aead(&engine->aeads[i].alg); + if (!err) { + list_add_tail(&engine->aeads[i].entry, + &engine->registered_aeads); + ret = 0; + } + if (err) + dev_err(engine->dev, "failed to register alg \"%s\"\n", + engine->aeads[i].alg.base.cra_name); + else + dev_dbg(engine->dev, "registered alg \"%s\"\n", + engine->aeads[i].alg.base.cra_name); + } + return ret; } static int spacc_remove(struct platform_device *pdev) { + struct spacc_aead *aead, *an; struct spacc_alg *alg, *next; struct spacc_engine *engine = platform_get_drvdata(pdev); del_timer_sync(&engine->packet_timeout); device_remove_file(&pdev->dev, &dev_attr_stat_irq_thresh); + list_for_each_entry_safe(aead, an, &engine->registered_aeads, entry) { + list_del(&aead->entry); + crypto_unregister_aead(&aead->alg); + } + list_for_each_entry_safe(alg, next, &engine->registered_algs, entry) { list_del(&alg->entry); crypto_unregister_alg(&alg->alg); diff --git a/drivers/crypto/qat/Kconfig b/drivers/crypto/qat/Kconfig index 6fdb9e8b22a7..eefccf7b8be7 100644 --- a/drivers/crypto/qat/Kconfig +++ b/drivers/crypto/qat/Kconfig @@ -3,11 +3,13 @@ config CRYPTO_DEV_QAT select CRYPTO_AEAD select CRYPTO_AUTHENC select CRYPTO_BLKCIPHER + select CRYPTO_AKCIPHER select CRYPTO_HMAC select CRYPTO_SHA1 select CRYPTO_SHA256 select CRYPTO_SHA512 select FW_LOADER + select ASN1 config CRYPTO_DEV_QAT_DH895xCC tristate "Support for Intel(R) DH895xCC" @@ -19,3 +21,16 @@ config CRYPTO_DEV_QAT_DH895xCC To compile this as a module, choose M here: the module will be called qat_dh895xcc. + +config CRYPTO_DEV_QAT_DH895xCCVF + tristate "Support for Intel(R) DH895xCC Virtual Function" + depends on X86 && PCI + select PCI_IOV + select CRYPTO_DEV_QAT + + help + Support for Intel(R) DH895xcc with Intel(R) QuickAssist Technology + Virtual Function for accelerating crypto and compression workloads. + + To compile this as a module, choose M here: the module + will be called qat_dh895xccvf. diff --git a/drivers/crypto/qat/Makefile b/drivers/crypto/qat/Makefile index d11481be225e..a3ce0b70e32f 100644 --- a/drivers/crypto/qat/Makefile +++ b/drivers/crypto/qat/Makefile @@ -1,2 +1,3 @@ obj-$(CONFIG_CRYPTO_DEV_QAT) += qat_common/ obj-$(CONFIG_CRYPTO_DEV_QAT_DH895xCC) += qat_dh895xcc/ +obj-$(CONFIG_CRYPTO_DEV_QAT_DH895xCCVF) += qat_dh895xccvf/ diff --git a/drivers/crypto/qat/qat_common/.gitignore b/drivers/crypto/qat/qat_common/.gitignore new file mode 100644 index 000000000000..ee328374dba8 --- /dev/null +++ b/drivers/crypto/qat/qat_common/.gitignore @@ -0,0 +1 @@ +*-asn1.[ch] diff --git a/drivers/crypto/qat/qat_common/Makefile b/drivers/crypto/qat/qat_common/Makefile index e0424dc382fe..df20a9de1c58 100644 --- a/drivers/crypto/qat/qat_common/Makefile +++ b/drivers/crypto/qat/qat_common/Makefile @@ -1,3 +1,6 @@ +$(obj)/qat_rsakey-asn1.o: $(obj)/qat_rsakey-asn1.c $(obj)/qat_rsakey-asn1.h +clean-files += qat_rsakey-asn1.c qat_rsakey-asn1.h + obj-$(CONFIG_CRYPTO_DEV_QAT) += intel_qat.o intel_qat-objs := adf_cfg.o \ adf_ctl_drv.o \ @@ -6,9 +9,14 @@ intel_qat-objs := adf_cfg.o \ adf_accel_engine.o \ adf_aer.o \ adf_transport.o \ + adf_admin.o \ + adf_hw_arbiter.o \ qat_crypto.o \ qat_algs.o \ + qat_rsakey-asn1.o \ + qat_asym_algs.o \ qat_uclo.o \ qat_hal.o intel_qat-$(CONFIG_DEBUG_FS) += adf_transport_debug.o +intel_qat-$(CONFIG_PCI_IOV) += adf_sriov.o adf_pf2vf_msg.o diff --git a/drivers/crypto/qat/qat_common/adf_accel_devices.h b/drivers/crypto/qat/qat_common/adf_accel_devices.h index 5fe902967620..ca853d50b4b7 100644 --- a/drivers/crypto/qat/qat_common/adf_accel_devices.h +++ b/drivers/crypto/qat/qat_common/adf_accel_devices.h @@ -46,13 +46,17 @@ */ #ifndef ADF_ACCEL_DEVICES_H_ #define ADF_ACCEL_DEVICES_H_ +#include <linux/interrupt.h> #include <linux/module.h> #include <linux/list.h> #include <linux/io.h> +#include <linux/ratelimit.h> #include "adf_cfg_common.h" #define ADF_DH895XCC_DEVICE_NAME "dh895xcc" +#define ADF_DH895XCCVF_DEVICE_NAME "dh895xccvf" #define ADF_DH895XCC_PCI_DEVICE_ID 0x435 +#define ADF_DH895XCCIOV_PCI_DEVICE_ID 0x443 #define ADF_PCI_MAX_BARS 3 #define ADF_DEVICE_NAME_LENGTH 32 #define ADF_ETR_MAX_RINGS_PER_BANK 16 @@ -79,6 +83,7 @@ struct adf_bar { struct adf_accel_msix { struct msix_entry *entries; char **names; + u32 num_entries; } __packed; struct adf_accel_pci { @@ -99,6 +104,7 @@ enum dev_sku_info { DEV_SKU_2, DEV_SKU_3, DEV_SKU_4, + DEV_SKU_VF, DEV_SKU_UNKNOWN, }; @@ -113,6 +119,8 @@ static inline const char *get_sku_info(enum dev_sku_info info) return "SKU3"; case DEV_SKU_4: return "SKU4"; + case DEV_SKU_VF: + return "SKUVF"; case DEV_SKU_UNKNOWN: default: break; @@ -135,23 +143,29 @@ struct adf_hw_device_data { struct adf_hw_device_class *dev_class; uint32_t (*get_accel_mask)(uint32_t fuse); uint32_t (*get_ae_mask)(uint32_t fuse); + uint32_t (*get_sram_bar_id)(struct adf_hw_device_data *self); uint32_t (*get_misc_bar_id)(struct adf_hw_device_data *self); uint32_t (*get_etr_bar_id)(struct adf_hw_device_data *self); uint32_t (*get_num_aes)(struct adf_hw_device_data *self); uint32_t (*get_num_accels)(struct adf_hw_device_data *self); + uint32_t (*get_pf2vf_offset)(uint32_t i); + uint32_t (*get_vintmsk_offset)(uint32_t i); enum dev_sku_info (*get_sku)(struct adf_hw_device_data *self); - void (*hw_arb_ring_enable)(struct adf_etr_ring_data *ring); - void (*hw_arb_ring_disable)(struct adf_etr_ring_data *ring); int (*alloc_irq)(struct adf_accel_dev *accel_dev); void (*free_irq)(struct adf_accel_dev *accel_dev); void (*enable_error_correction)(struct adf_accel_dev *accel_dev); int (*init_admin_comms)(struct adf_accel_dev *accel_dev); void (*exit_admin_comms)(struct adf_accel_dev *accel_dev); + int (*send_admin_init)(struct adf_accel_dev *accel_dev); int (*init_arb)(struct adf_accel_dev *accel_dev); void (*exit_arb)(struct adf_accel_dev *accel_dev); + void (*get_arb_mapping)(struct adf_accel_dev *accel_dev, + const uint32_t **cfg); + void (*disable_iov)(struct adf_accel_dev *accel_dev); void (*enable_ints)(struct adf_accel_dev *accel_dev); + int (*enable_vf2pf_comms)(struct adf_accel_dev *accel_dev); const char *fw_name; - uint32_t pci_dev_id; + const char *fw_mmp_name; uint32_t fuses; uint32_t accel_capabilities_mask; uint16_t accel_mask; @@ -163,6 +177,7 @@ struct adf_hw_device_data { uint8_t num_accel; uint8_t num_logical_accel; uint8_t num_engines; + uint8_t min_iov_compat_ver; } __packed; /* CSR write macro */ @@ -184,6 +199,16 @@ struct icp_qat_fw_loader_handle; struct adf_fw_loader_data { struct icp_qat_fw_loader_handle *fw_loader; const struct firmware *uof_fw; + const struct firmware *mmp_fw; +}; + +struct adf_accel_vf_info { + struct adf_accel_dev *accel_dev; + struct tasklet_struct vf2pf_bh_tasklet; + struct mutex pf2vf_lock; /* protect CSR access for PF2VF messages */ + struct ratelimit_state vf2pf_ratelimit; + u32 vf_nr; + bool init; }; struct adf_accel_dev { @@ -199,6 +224,21 @@ struct adf_accel_dev { struct list_head list; struct module *owner; struct adf_accel_pci accel_pci_dev; + union { + struct { + /* vf_info is non-zero when SR-IOV is init'ed */ + struct adf_accel_vf_info *vf_info; + } pf; + struct { + char *irq_name; + struct tasklet_struct pf2vf_bh_tasklet; + struct mutex vf2pf_lock; /* protect CSR access */ + struct completion iov_msg_completion; + uint8_t compatible; + uint8_t pf_version; + } vf; + }; + bool is_vf; uint8_t accel_id; } __packed; #endif diff --git a/drivers/crypto/qat/qat_common/adf_accel_engine.c b/drivers/crypto/qat/qat_common/adf_accel_engine.c index fdda8e7ae302..20b08bdcb146 100644 --- a/drivers/crypto/qat/qat_common/adf_accel_engine.c +++ b/drivers/crypto/qat/qat_common/adf_accel_engine.c @@ -55,24 +55,36 @@ int adf_ae_fw_load(struct adf_accel_dev *accel_dev) { struct adf_fw_loader_data *loader_data = accel_dev->fw_loader; struct adf_hw_device_data *hw_device = accel_dev->hw_device; - void *uof_addr; - uint32_t uof_size; + void *uof_addr, *mmp_addr; + u32 uof_size, mmp_size; + if (!hw_device->fw_name) + return 0; + + if (request_firmware(&loader_data->mmp_fw, hw_device->fw_mmp_name, + &accel_dev->accel_pci_dev.pci_dev->dev)) { + dev_err(&GET_DEV(accel_dev), "Failed to load MMP firmware %s\n", + hw_device->fw_mmp_name); + return -EFAULT; + } if (request_firmware(&loader_data->uof_fw, hw_device->fw_name, &accel_dev->accel_pci_dev.pci_dev->dev)) { - dev_err(&GET_DEV(accel_dev), "Failed to load firmware %s\n", + dev_err(&GET_DEV(accel_dev), "Failed to load UOF firmware %s\n", hw_device->fw_name); - return -EFAULT; + goto out_err; } uof_size = loader_data->uof_fw->size; uof_addr = (void *)loader_data->uof_fw->data; + mmp_size = loader_data->mmp_fw->size; + mmp_addr = (void *)loader_data->mmp_fw->data; + qat_uclo_wr_mimage(loader_data->fw_loader, mmp_addr, mmp_size); if (qat_uclo_map_uof_obj(loader_data->fw_loader, uof_addr, uof_size)) { dev_err(&GET_DEV(accel_dev), "Failed to map UOF\n"); goto out_err; } if (qat_uclo_wr_all_uimage(loader_data->fw_loader)) { - dev_err(&GET_DEV(accel_dev), "Failed to map UOF\n"); + dev_err(&GET_DEV(accel_dev), "Failed to load UOF\n"); goto out_err; } return 0; @@ -85,11 +97,17 @@ out_err: void adf_ae_fw_release(struct adf_accel_dev *accel_dev) { struct adf_fw_loader_data *loader_data = accel_dev->fw_loader; + struct adf_hw_device_data *hw_device = accel_dev->hw_device; + + if (!hw_device->fw_name) + return; qat_uclo_del_uof_obj(loader_data->fw_loader); qat_hal_deinit(loader_data->fw_loader); release_firmware(loader_data->uof_fw); + release_firmware(loader_data->mmp_fw); loader_data->uof_fw = NULL; + loader_data->mmp_fw = NULL; loader_data->fw_loader = NULL; } @@ -99,6 +117,9 @@ int adf_ae_start(struct adf_accel_dev *accel_dev) struct adf_hw_device_data *hw_data = accel_dev->hw_device; uint32_t ae_ctr, ae, max_aes = GET_MAX_ACCELENGINES(accel_dev); + if (!hw_data->fw_name) + return 0; + for (ae = 0, ae_ctr = 0; ae < max_aes; ae++) { if (hw_data->ae_mask & (1 << ae)) { qat_hal_start(loader_data->fw_loader, ae, 0xFF); @@ -117,6 +138,9 @@ int adf_ae_stop(struct adf_accel_dev *accel_dev) struct adf_hw_device_data *hw_data = accel_dev->hw_device; uint32_t ae_ctr, ae, max_aes = GET_MAX_ACCELENGINES(accel_dev); + if (!hw_data->fw_name) + return 0; + for (ae = 0, ae_ctr = 0; ae < max_aes; ae++) { if (hw_data->ae_mask & (1 << ae)) { qat_hal_stop(loader_data->fw_loader, ae, 0xFF); @@ -143,6 +167,10 @@ static int adf_ae_reset(struct adf_accel_dev *accel_dev, int ae) int adf_ae_init(struct adf_accel_dev *accel_dev) { struct adf_fw_loader_data *loader_data; + struct adf_hw_device_data *hw_device = accel_dev->hw_device; + + if (!hw_device->fw_name) + return 0; loader_data = kzalloc(sizeof(*loader_data), GFP_KERNEL); if (!loader_data) @@ -166,6 +194,10 @@ int adf_ae_init(struct adf_accel_dev *accel_dev) int adf_ae_shutdown(struct adf_accel_dev *accel_dev) { struct adf_fw_loader_data *loader_data = accel_dev->fw_loader; + struct adf_hw_device_data *hw_device = accel_dev->hw_device; + + if (!hw_device->fw_name) + return 0; qat_hal_deinit(loader_data->fw_loader); kfree(accel_dev->fw_loader); diff --git a/drivers/crypto/qat/qat_common/adf_admin.c b/drivers/crypto/qat/qat_common/adf_admin.c new file mode 100644 index 000000000000..147d755fed97 --- /dev/null +++ b/drivers/crypto/qat/qat_common/adf_admin.c @@ -0,0 +1,290 @@ +/* + This file is provided under a dual BSD/GPLv2 license. When using or + redistributing this file, you may do so under either license. + + GPL LICENSE SUMMARY + Copyright(c) 2014 Intel Corporation. + This program is free software; you can redistribute it and/or modify + it under the terms of version 2 of the GNU General Public License as + published by the Free Software Foundation. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + Contact Information: + qat-linux@intel.com + + BSD LICENSE + Copyright(c) 2014 Intel Corporation. + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#include <linux/types.h> +#include <linux/mutex.h> +#include <linux/slab.h> +#include <linux/delay.h> +#include <linux/pci.h> +#include <linux/dma-mapping.h> +#include "adf_accel_devices.h" +#include "icp_qat_fw_init_admin.h" + +/* Admin Messages Registers */ +#define ADF_DH895XCC_ADMINMSGUR_OFFSET (0x3A000 + 0x574) +#define ADF_DH895XCC_ADMINMSGLR_OFFSET (0x3A000 + 0x578) +#define ADF_DH895XCC_MAILBOX_BASE_OFFSET 0x20970 +#define ADF_DH895XCC_MAILBOX_STRIDE 0x1000 +#define ADF_ADMINMSG_LEN 32 + +static const u8 const_tab[1024] = { +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x11, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x11, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x21, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x03, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x01, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x13, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x13, 0x02, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x13, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x13, +0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x23, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x33, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x06, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef, 0xfe, 0xdc, 0xba, 0x98, 0x76, +0x54, 0x32, 0x10, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x67, 0x45, 0x23, 0x01, 0xef, 0xcd, 0xab, +0x89, 0x98, 0xba, 0xdc, 0xfe, 0x10, 0x32, 0x54, 0x76, 0xc3, 0xd2, 0xe1, 0xf0, +0x00, 0x00, 0x00, 0x00, 0x11, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc1, 0x05, 0x9e, +0xd8, 0x36, 0x7c, 0xd5, 0x07, 0x30, 0x70, 0xdd, 0x17, 0xf7, 0x0e, 0x59, 0x39, +0xff, 0xc0, 0x0b, 0x31, 0x68, 0x58, 0x15, 0x11, 0x64, 0xf9, 0x8f, 0xa7, 0xbe, +0xfa, 0x4f, 0xa4, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6a, 0x09, 0xe6, 0x67, 0xbb, 0x67, 0xae, +0x85, 0x3c, 0x6e, 0xf3, 0x72, 0xa5, 0x4f, 0xf5, 0x3a, 0x51, 0x0e, 0x52, 0x7f, +0x9b, 0x05, 0x68, 0x8c, 0x1f, 0x83, 0xd9, 0xab, 0x5b, 0xe0, 0xcd, 0x19, 0x05, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0xcb, 0xbb, 0x9d, 0x5d, 0xc1, 0x05, 0x9e, 0xd8, 0x62, 0x9a, 0x29, +0x2a, 0x36, 0x7c, 0xd5, 0x07, 0x91, 0x59, 0x01, 0x5a, 0x30, 0x70, 0xdd, 0x17, +0x15, 0x2f, 0xec, 0xd8, 0xf7, 0x0e, 0x59, 0x39, 0x67, 0x33, 0x26, 0x67, 0xff, +0xc0, 0x0b, 0x31, 0x8e, 0xb4, 0x4a, 0x87, 0x68, 0x58, 0x15, 0x11, 0xdb, 0x0c, +0x2e, 0x0d, 0x64, 0xf9, 0x8f, 0xa7, 0x47, 0xb5, 0x48, 0x1d, 0xbe, 0xfa, 0x4f, +0xa4, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x6a, 0x09, 0xe6, 0x67, 0xf3, 0xbc, 0xc9, 0x08, 0xbb, +0x67, 0xae, 0x85, 0x84, 0xca, 0xa7, 0x3b, 0x3c, 0x6e, 0xf3, 0x72, 0xfe, 0x94, +0xf8, 0x2b, 0xa5, 0x4f, 0xf5, 0x3a, 0x5f, 0x1d, 0x36, 0xf1, 0x51, 0x0e, 0x52, +0x7f, 0xad, 0xe6, 0x82, 0xd1, 0x9b, 0x05, 0x68, 0x8c, 0x2b, 0x3e, 0x6c, 0x1f, +0x1f, 0x83, 0xd9, 0xab, 0xfb, 0x41, 0xbd, 0x6b, 0x5b, 0xe0, 0xcd, 0x19, 0x13, +0x7e, 0x21, 0x79, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; + +struct adf_admin_comms { + dma_addr_t phy_addr; + dma_addr_t const_tbl_addr; + void *virt_addr; + void __iomem *mailbox_addr; + struct mutex lock; /* protects adf_admin_comms struct */ +}; + +static int adf_put_admin_msg_sync(struct adf_accel_dev *accel_dev, u32 ae, + void *in, void *out) +{ + struct adf_admin_comms *admin = accel_dev->admin; + int offset = ae * ADF_ADMINMSG_LEN * 2; + void __iomem *mailbox = admin->mailbox_addr; + int mb_offset = ae * ADF_DH895XCC_MAILBOX_STRIDE; + int times, received; + + mutex_lock(&admin->lock); + + if (ADF_CSR_RD(mailbox, mb_offset) == 1) { + mutex_unlock(&admin->lock); + return -EAGAIN; + } + + memcpy(admin->virt_addr + offset, in, ADF_ADMINMSG_LEN); + ADF_CSR_WR(mailbox, mb_offset, 1); + received = 0; + for (times = 0; times < 50; times++) { + msleep(20); + if (ADF_CSR_RD(mailbox, mb_offset) == 0) { + received = 1; + break; + } + } + if (received) + memcpy(out, admin->virt_addr + offset + + ADF_ADMINMSG_LEN, ADF_ADMINMSG_LEN); + else + dev_err(&GET_DEV(accel_dev), + "Failed to send admin msg to accelerator\n"); + + mutex_unlock(&admin->lock); + return received ? 0 : -EFAULT; +} + +static int adf_send_admin_cmd(struct adf_accel_dev *accel_dev, int cmd) +{ + struct adf_hw_device_data *hw_device = accel_dev->hw_device; + struct icp_qat_fw_init_admin_req req; + struct icp_qat_fw_init_admin_resp resp; + int i; + + memset(&req, 0, sizeof(struct icp_qat_fw_init_admin_req)); + req.init_admin_cmd_id = cmd; + + if (cmd == ICP_QAT_FW_CONSTANTS_CFG) { + req.init_cfg_sz = 1024; + req.init_cfg_ptr = accel_dev->admin->const_tbl_addr; + } + for (i = 0; i < hw_device->get_num_aes(hw_device); i++) { + memset(&resp, 0, sizeof(struct icp_qat_fw_init_admin_resp)); + if (adf_put_admin_msg_sync(accel_dev, i, &req, &resp) || + resp.init_resp_hdr.status) + return -EFAULT; + } + return 0; +} + +/** + * adf_send_admin_init() - Function sends init message to FW + * @accel_dev: Pointer to acceleration device. + * + * Function sends admin init message to the FW + * + * Return: 0 on success, error code otherwise. + */ +int adf_send_admin_init(struct adf_accel_dev *accel_dev) +{ + int ret = adf_send_admin_cmd(accel_dev, ICP_QAT_FW_INIT_ME); + + if (ret) + return ret; + return adf_send_admin_cmd(accel_dev, ICP_QAT_FW_CONSTANTS_CFG); +} +EXPORT_SYMBOL_GPL(adf_send_admin_init); + +int adf_init_admin_comms(struct adf_accel_dev *accel_dev) +{ + struct adf_admin_comms *admin; + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + struct adf_bar *pmisc = + &GET_BARS(accel_dev)[hw_data->get_misc_bar_id(hw_data)]; + void __iomem *csr = pmisc->virt_addr; + void __iomem *mailbox = csr + ADF_DH895XCC_MAILBOX_BASE_OFFSET; + u64 reg_val; + + admin = kzalloc_node(sizeof(*accel_dev->admin), GFP_KERNEL, + dev_to_node(&GET_DEV(accel_dev))); + if (!admin) + return -ENOMEM; + admin->virt_addr = dma_zalloc_coherent(&GET_DEV(accel_dev), PAGE_SIZE, + &admin->phy_addr, GFP_KERNEL); + if (!admin->virt_addr) { + dev_err(&GET_DEV(accel_dev), "Failed to allocate dma buff\n"); + kfree(admin); + return -ENOMEM; + } + + admin->const_tbl_addr = dma_map_single(&GET_DEV(accel_dev), + (void *) const_tab, 1024, + DMA_TO_DEVICE); + + if (unlikely(dma_mapping_error(&GET_DEV(accel_dev), + admin->const_tbl_addr))) { + dma_free_coherent(&GET_DEV(accel_dev), PAGE_SIZE, + admin->virt_addr, admin->phy_addr); + kfree(admin); + return -ENOMEM; + } + reg_val = (u64)admin->phy_addr; + ADF_CSR_WR(csr, ADF_DH895XCC_ADMINMSGUR_OFFSET, reg_val >> 32); + ADF_CSR_WR(csr, ADF_DH895XCC_ADMINMSGLR_OFFSET, reg_val); + mutex_init(&admin->lock); + admin->mailbox_addr = mailbox; + accel_dev->admin = admin; + return 0; +} +EXPORT_SYMBOL_GPL(adf_init_admin_comms); + +void adf_exit_admin_comms(struct adf_accel_dev *accel_dev) +{ + struct adf_admin_comms *admin = accel_dev->admin; + + if (!admin) + return; + + if (admin->virt_addr) + dma_free_coherent(&GET_DEV(accel_dev), PAGE_SIZE, + admin->virt_addr, admin->phy_addr); + + dma_unmap_single(&GET_DEV(accel_dev), admin->const_tbl_addr, 1024, + DMA_TO_DEVICE); + mutex_destroy(&admin->lock); + kfree(admin); + accel_dev->admin = NULL; +} +EXPORT_SYMBOL_GPL(adf_exit_admin_comms); diff --git a/drivers/crypto/qat/qat_common/adf_aer.c b/drivers/crypto/qat/qat_common/adf_aer.c index 2dbc733b8ab2..a57b4194de28 100644 --- a/drivers/crypto/qat/qat_common/adf_aer.c +++ b/drivers/crypto/qat/qat_common/adf_aer.c @@ -91,6 +91,9 @@ static void adf_dev_restore(struct adf_accel_dev *accel_dev) dev_info(&GET_DEV(accel_dev), "Resetting device qat_dev%d\n", accel_dev->accel_id); + if (!parent) + parent = pdev; + if (!pci_wait_for_pending_transaction(pdev)) dev_info(&GET_DEV(accel_dev), "Transaction still in progress. Proceeding\n"); @@ -206,7 +209,7 @@ static struct pci_error_handlers adf_err_handler = { * QAT acceleration device accel_dev. * To be used by QAT device specific drivers. * - * Return: 0 on success, error code othewise. + * Return: 0 on success, error code otherwise. */ int adf_enable_aer(struct adf_accel_dev *accel_dev, struct pci_driver *adf) { diff --git a/drivers/crypto/qat/qat_common/adf_cfg.c b/drivers/crypto/qat/qat_common/adf_cfg.c index ab65bc274561..d0879790561f 100644 --- a/drivers/crypto/qat/qat_common/adf_cfg.c +++ b/drivers/crypto/qat/qat_common/adf_cfg.c @@ -123,7 +123,7 @@ static const struct file_operations qat_dev_cfg_fops = { * The table stores device specific config values. * To be used by QAT device specific drivers. * - * Return: 0 on success, error code othewise. + * Return: 0 on success, error code otherwise. */ int adf_cfg_dev_add(struct adf_accel_dev *accel_dev) { @@ -178,6 +178,9 @@ void adf_cfg_dev_remove(struct adf_accel_dev *accel_dev) { struct adf_cfg_device_data *dev_cfg_data = accel_dev->cfg; + if (!dev_cfg_data) + return; + down_write(&dev_cfg_data->lock); adf_cfg_section_del_all(&dev_cfg_data->sec_list); up_write(&dev_cfg_data->lock); @@ -276,7 +279,7 @@ static int adf_cfg_key_val_get(struct adf_accel_dev *accel_dev, * in the given acceleration device * To be used by QAT device specific drivers. * - * Return: 0 on success, error code othewise. + * Return: 0 on success, error code otherwise. */ int adf_cfg_add_key_value_param(struct adf_accel_dev *accel_dev, const char *section_name, @@ -327,7 +330,7 @@ EXPORT_SYMBOL_GPL(adf_cfg_add_key_value_param); * will be stored. * To be used by QAT device specific drivers. * - * Return: 0 on success, error code othewise. + * Return: 0 on success, error code otherwise. */ int adf_cfg_section_add(struct adf_accel_dev *accel_dev, const char *name) { diff --git a/drivers/crypto/qat/qat_common/adf_cfg_common.h b/drivers/crypto/qat/qat_common/adf_cfg_common.h index 88b82187ac35..c697fb1cdfb5 100644 --- a/drivers/crypto/qat/qat_common/adf_cfg_common.h +++ b/drivers/crypto/qat/qat_common/adf_cfg_common.h @@ -60,7 +60,7 @@ #define ADF_CFG_NO_DEVICE 0xFF #define ADF_CFG_AFFINITY_WHATEVER 0xFF #define MAX_DEVICE_NAME_SIZE 32 -#define ADF_MAX_DEVICES 32 +#define ADF_MAX_DEVICES (32 * 32) enum adf_cfg_val_type { ADF_DEC, @@ -71,6 +71,7 @@ enum adf_cfg_val_type { enum adf_device_type { DEV_UNKNOWN = 0, DEV_DH895XCC, + DEV_DH895XCCVF, }; struct adf_dev_status_info { diff --git a/drivers/crypto/qat/qat_common/adf_common_drv.h b/drivers/crypto/qat/qat_common/adf_common_drv.h index 27e16c09230b..7836dffc3d47 100644 --- a/drivers/crypto/qat/qat_common/adf_common_drv.h +++ b/drivers/crypto/qat/qat_common/adf_common_drv.h @@ -54,8 +54,8 @@ #include "icp_qat_hal.h" #define ADF_MAJOR_VERSION 0 -#define ADF_MINOR_VERSION 1 -#define ADF_BUILD_VERSION 3 +#define ADF_MINOR_VERSION 2 +#define ADF_BUILD_VERSION 0 #define ADF_DRV_VERSION __stringify(ADF_MAJOR_VERSION) "." \ __stringify(ADF_MINOR_VERSION) "." \ __stringify(ADF_BUILD_VERSION) @@ -91,9 +91,13 @@ struct service_hndl { unsigned long start_status; char *name; struct list_head list; - int admin; }; +static inline int get_current_node(void) +{ + return topology_physical_package_id(smp_processor_id()); +} + int adf_service_register(struct service_hndl *service); int adf_service_unregister(struct service_hndl *service); @@ -102,13 +106,24 @@ int adf_dev_start(struct adf_accel_dev *accel_dev); int adf_dev_stop(struct adf_accel_dev *accel_dev); void adf_dev_shutdown(struct adf_accel_dev *accel_dev); +void adf_enable_pf2vf_interrupts(struct adf_accel_dev *accel_dev); +void adf_disable_pf2vf_interrupts(struct adf_accel_dev *accel_dev); +int adf_iov_putmsg(struct adf_accel_dev *accel_dev, u32 msg, u8 vf_nr); +void adf_pf2vf_notify_restarting(struct adf_accel_dev *accel_dev); +int adf_enable_vf2pf_comms(struct adf_accel_dev *accel_dev); +void adf_vf2pf_req_hndl(struct adf_accel_vf_info *vf_info); +void adf_devmgr_update_class_index(struct adf_hw_device_data *hw_data); +void adf_clean_vf_map(bool); + int adf_ctl_dev_register(void); void adf_ctl_dev_unregister(void); int adf_processes_dev_register(void); void adf_processes_dev_unregister(void); -int adf_devmgr_add_dev(struct adf_accel_dev *accel_dev); -void adf_devmgr_rm_dev(struct adf_accel_dev *accel_dev); +int adf_devmgr_add_dev(struct adf_accel_dev *accel_dev, + struct adf_accel_dev *pf); +void adf_devmgr_rm_dev(struct adf_accel_dev *accel_dev, + struct adf_accel_dev *pf); struct list_head *adf_devmgr_get_head(void); struct adf_accel_dev *adf_devmgr_get_dev_by_id(uint32_t id); struct adf_accel_dev *adf_devmgr_get_first(void); @@ -130,6 +145,12 @@ int adf_enable_aer(struct adf_accel_dev *accel_dev, struct pci_driver *adf); void adf_disable_aer(struct adf_accel_dev *accel_dev); int adf_init_aer(void); void adf_exit_aer(void); +int adf_init_admin_comms(struct adf_accel_dev *accel_dev); +void adf_exit_admin_comms(struct adf_accel_dev *accel_dev); +int adf_send_admin_init(struct adf_accel_dev *accel_dev); +int adf_init_arb(struct adf_accel_dev *accel_dev); +void adf_exit_arb(struct adf_accel_dev *accel_dev); +void adf_update_ring_arb(struct adf_etr_ring_data *ring); int adf_dev_get(struct adf_accel_dev *accel_dev); void adf_dev_put(struct adf_accel_dev *accel_dev); @@ -141,10 +162,13 @@ int qat_crypto_unregister(void); struct qat_crypto_instance *qat_crypto_get_instance_node(int node); void qat_crypto_put_instance(struct qat_crypto_instance *inst); void qat_alg_callback(void *resp); +void qat_alg_asym_callback(void *resp); int qat_algs_init(void); void qat_algs_exit(void); int qat_algs_register(void); int qat_algs_unregister(void); +int qat_asym_algs_register(void); +void qat_asym_algs_unregister(void); int qat_hal_init(struct adf_accel_dev *accel_dev); void qat_hal_deinit(struct icp_qat_fw_loader_handle *handle); @@ -196,4 +220,23 @@ int qat_uclo_wr_all_uimage(struct icp_qat_fw_loader_handle *handle); void qat_uclo_del_uof_obj(struct icp_qat_fw_loader_handle *handle); int qat_uclo_map_uof_obj(struct icp_qat_fw_loader_handle *handle, void *addr_ptr, int mem_size); +void qat_uclo_wr_mimage(struct icp_qat_fw_loader_handle *handle, + void *addr_ptr, int mem_size); +#if defined(CONFIG_PCI_IOV) +int adf_sriov_configure(struct pci_dev *pdev, int numvfs); +void adf_disable_sriov(struct adf_accel_dev *accel_dev); +void adf_disable_vf2pf_interrupts(struct adf_accel_dev *accel_dev, + uint32_t vf_mask); +void adf_enable_vf2pf_interrupts(struct adf_accel_dev *accel_dev, + uint32_t vf_mask); +#else +static inline int adf_sriov_configure(struct pci_dev *pdev, int numvfs) +{ + return 0; +} + +static inline void adf_disable_sriov(struct adf_accel_dev *accel_dev) +{ +} +#endif #endif diff --git a/drivers/crypto/qat/qat_common/adf_ctl_drv.c b/drivers/crypto/qat/qat_common/adf_ctl_drv.c index e056b9e9bf8a..cd8a12af8ec5 100644 --- a/drivers/crypto/qat/qat_common/adf_ctl_drv.c +++ b/drivers/crypto/qat/qat_common/adf_ctl_drv.c @@ -398,10 +398,9 @@ static int adf_ctl_ioctl_get_status(struct file *fp, unsigned int cmd, } accel_dev = adf_devmgr_get_dev_by_id(dev_info.accel_id); - if (!accel_dev) { - pr_err("QAT: Device %d not found\n", dev_info.accel_id); + if (!accel_dev) return -ENODEV; - } + hw_data = accel_dev->hw_device; dev_info.state = adf_dev_started(accel_dev) ? DEV_UP : DEV_DOWN; dev_info.num_ae = hw_data->get_num_aes(hw_data); @@ -495,6 +494,7 @@ static void __exit adf_unregister_ctl_device_driver(void) adf_exit_aer(); qat_crypto_unregister(); qat_algs_exit(); + adf_clean_vf_map(false); mutex_destroy(&adf_ctl_lock); } diff --git a/drivers/crypto/qat/qat_common/adf_dev_mgr.c b/drivers/crypto/qat/qat_common/adf_dev_mgr.c index 3f0ff9e7d840..8dfdb8f90797 100644 --- a/drivers/crypto/qat/qat_common/adf_dev_mgr.c +++ b/drivers/crypto/qat/qat_common/adf_dev_mgr.c @@ -50,21 +50,125 @@ #include "adf_common_drv.h" static LIST_HEAD(accel_table); +static LIST_HEAD(vfs_table); static DEFINE_MUTEX(table_lock); static uint32_t num_devices; +struct vf_id_map { + u32 bdf; + u32 id; + u32 fake_id; + bool attached; + struct list_head list; +}; + +static int adf_get_vf_id(struct adf_accel_dev *vf) +{ + return ((7 * (PCI_SLOT(accel_to_pci_dev(vf)->devfn) - 1)) + + PCI_FUNC(accel_to_pci_dev(vf)->devfn) + + (PCI_SLOT(accel_to_pci_dev(vf)->devfn) - 1)); +} + +static int adf_get_vf_num(struct adf_accel_dev *vf) +{ + return (accel_to_pci_dev(vf)->bus->number << 8) | adf_get_vf_id(vf); +} + +static struct vf_id_map *adf_find_vf(u32 bdf) +{ + struct list_head *itr; + + list_for_each(itr, &vfs_table) { + struct vf_id_map *ptr = + list_entry(itr, struct vf_id_map, list); + + if (ptr->bdf == bdf) + return ptr; + } + return NULL; +} + +static int adf_get_vf_real_id(u32 fake) +{ + struct list_head *itr; + + list_for_each(itr, &vfs_table) { + struct vf_id_map *ptr = + list_entry(itr, struct vf_id_map, list); + if (ptr->fake_id == fake) + return ptr->id; + } + return -1; +} + +/** + * adf_clean_vf_map() - Cleans VF id mapings + * + * Function cleans internal ids for virtual functions. + * @vf: flag indicating whether mappings is cleaned + * for vfs only or for vfs and pfs + */ +void adf_clean_vf_map(bool vf) +{ + struct vf_id_map *map; + struct list_head *ptr, *tmp; + + mutex_lock(&table_lock); + list_for_each_safe(ptr, tmp, &vfs_table) { + map = list_entry(ptr, struct vf_id_map, list); + if (map->bdf != -1) + num_devices--; + + if (vf && map->bdf == -1) + continue; + + list_del(ptr); + kfree(map); + } + mutex_unlock(&table_lock); +} +EXPORT_SYMBOL_GPL(adf_clean_vf_map); + +/** + * adf_devmgr_update_class_index() - Update internal index + * @hw_data: Pointer to internal device data. + * + * Function updates internal dev index for VFs + */ +void adf_devmgr_update_class_index(struct adf_hw_device_data *hw_data) +{ + struct adf_hw_device_class *class = hw_data->dev_class; + struct list_head *itr; + int i = 0; + + list_for_each(itr, &accel_table) { + struct adf_accel_dev *ptr = + list_entry(itr, struct adf_accel_dev, list); + + if (ptr->hw_device->dev_class == class) + ptr->hw_device->instance_id = i++; + + if (i == class->instances) + break; + } +} +EXPORT_SYMBOL_GPL(adf_devmgr_update_class_index); + /** * adf_devmgr_add_dev() - Add accel_dev to the acceleration framework * @accel_dev: Pointer to acceleration device. + * @pf: Corresponding PF if the accel_dev is a VF * * Function adds acceleration device to the acceleration framework. * To be used by QAT device specific drivers. * - * Return: 0 on success, error code othewise. + * Return: 0 on success, error code otherwise. */ -int adf_devmgr_add_dev(struct adf_accel_dev *accel_dev) +int adf_devmgr_add_dev(struct adf_accel_dev *accel_dev, + struct adf_accel_dev *pf) { struct list_head *itr; + int ret = 0; if (num_devices == ADF_MAX_DEVICES) { dev_err(&GET_DEV(accel_dev), "Only support up to %d devices\n", @@ -73,20 +177,77 @@ int adf_devmgr_add_dev(struct adf_accel_dev *accel_dev) } mutex_lock(&table_lock); - list_for_each(itr, &accel_table) { - struct adf_accel_dev *ptr = + atomic_set(&accel_dev->ref_count, 0); + + /* PF on host or VF on guest */ + if (!accel_dev->is_vf || (accel_dev->is_vf && !pf)) { + struct vf_id_map *map; + + list_for_each(itr, &accel_table) { + struct adf_accel_dev *ptr = list_entry(itr, struct adf_accel_dev, list); - if (ptr == accel_dev) { - mutex_unlock(&table_lock); - return -EEXIST; + if (ptr == accel_dev) { + ret = -EEXIST; + goto unlock; + } } + + list_add_tail(&accel_dev->list, &accel_table); + accel_dev->accel_id = num_devices++; + + map = kzalloc(sizeof(*map), GFP_KERNEL); + if (!map) { + ret = -ENOMEM; + goto unlock; + } + map->bdf = ~0; + map->id = accel_dev->accel_id; + map->fake_id = map->id; + map->attached = true; + list_add_tail(&map->list, &vfs_table); + } else if (accel_dev->is_vf && pf) { + /* VF on host */ + struct adf_accel_vf_info *vf_info; + struct vf_id_map *map; + + vf_info = pf->pf.vf_info + adf_get_vf_id(accel_dev); + + map = adf_find_vf(adf_get_vf_num(accel_dev)); + if (map) { + struct vf_id_map *next; + + accel_dev->accel_id = map->id; + list_add_tail(&accel_dev->list, &accel_table); + map->fake_id++; + map->attached = true; + next = list_next_entry(map, list); + while (next && &next->list != &vfs_table) { + next->fake_id++; + next = list_next_entry(next, list); + } + + ret = 0; + goto unlock; + } + + map = kzalloc(sizeof(*map), GFP_KERNEL); + if (!map) { + ret = -ENOMEM; + goto unlock; + } + + accel_dev->accel_id = num_devices++; + list_add_tail(&accel_dev->list, &accel_table); + map->bdf = adf_get_vf_num(accel_dev); + map->id = accel_dev->accel_id; + map->fake_id = map->id; + map->attached = true; + list_add_tail(&map->list, &vfs_table); } - atomic_set(&accel_dev->ref_count, 0); - list_add_tail(&accel_dev->list, &accel_table); - accel_dev->accel_id = num_devices++; +unlock: mutex_unlock(&table_lock); - return 0; + return ret; } EXPORT_SYMBOL_GPL(adf_devmgr_add_dev); @@ -98,17 +259,37 @@ struct list_head *adf_devmgr_get_head(void) /** * adf_devmgr_rm_dev() - Remove accel_dev from the acceleration framework. * @accel_dev: Pointer to acceleration device. + * @pf: Corresponding PF if the accel_dev is a VF * * Function removes acceleration device from the acceleration framework. * To be used by QAT device specific drivers. * * Return: void */ -void adf_devmgr_rm_dev(struct adf_accel_dev *accel_dev) +void adf_devmgr_rm_dev(struct adf_accel_dev *accel_dev, + struct adf_accel_dev *pf) { mutex_lock(&table_lock); + if (!accel_dev->is_vf || (accel_dev->is_vf && !pf)) { + num_devices--; + } else if (accel_dev->is_vf && pf) { + struct vf_id_map *map, *next; + + map = adf_find_vf(adf_get_vf_num(accel_dev)); + if (!map) { + dev_err(&GET_DEV(accel_dev), "Failed to find VF map\n"); + goto unlock; + } + map->fake_id--; + map->attached = false; + next = list_next_entry(map, list); + while (next && &next->list != &vfs_table) { + next->fake_id--; + next = list_next_entry(next, list); + } + } +unlock: list_del(&accel_dev->list); - num_devices--; mutex_unlock(&table_lock); } EXPORT_SYMBOL_GPL(adf_devmgr_rm_dev); @@ -154,17 +335,24 @@ EXPORT_SYMBOL_GPL(adf_devmgr_pci_to_accel_dev); struct adf_accel_dev *adf_devmgr_get_dev_by_id(uint32_t id) { struct list_head *itr; + int real_id; mutex_lock(&table_lock); + real_id = adf_get_vf_real_id(id); + if (real_id < 0) + goto unlock; + + id = real_id; + list_for_each(itr, &accel_table) { struct adf_accel_dev *ptr = list_entry(itr, struct adf_accel_dev, list); - if (ptr->accel_id == id) { mutex_unlock(&table_lock); return ptr; } } +unlock: mutex_unlock(&table_lock); return NULL; } @@ -180,21 +368,52 @@ int adf_devmgr_verify_id(uint32_t id) return -ENODEV; } -void adf_devmgr_get_num_dev(uint32_t *num) +static int adf_get_num_dettached_vfs(void) { struct list_head *itr; + int vfs = 0; - *num = 0; - list_for_each(itr, &accel_table) { - (*num)++; + mutex_lock(&table_lock); + list_for_each(itr, &vfs_table) { + struct vf_id_map *ptr = + list_entry(itr, struct vf_id_map, list); + if (ptr->bdf != ~0 && !ptr->attached) + vfs++; } + mutex_unlock(&table_lock); + return vfs; +} + +void adf_devmgr_get_num_dev(uint32_t *num) +{ + *num = num_devices - adf_get_num_dettached_vfs(); } +/** + * adf_dev_in_use() - Check whether accel_dev is currently in use + * @accel_dev: Pointer to acceleration device. + * + * To be used by QAT device specific drivers. + * + * Return: 1 when device is in use, 0 otherwise. + */ int adf_dev_in_use(struct adf_accel_dev *accel_dev) { return atomic_read(&accel_dev->ref_count) != 0; } +EXPORT_SYMBOL_GPL(adf_dev_in_use); +/** + * adf_dev_get() - Increment accel_dev reference count + * @accel_dev: Pointer to acceleration device. + * + * Increment the accel_dev refcount and if this is the first time + * incrementing it during this period the accel_dev is in use, + * increment the module refcount too. + * To be used by QAT device specific drivers. + * + * Return: 0 when successful, EFAULT when fail to bump module refcount + */ int adf_dev_get(struct adf_accel_dev *accel_dev) { if (atomic_add_return(1, &accel_dev->ref_count) == 1) @@ -202,19 +421,50 @@ int adf_dev_get(struct adf_accel_dev *accel_dev) return -EFAULT; return 0; } +EXPORT_SYMBOL_GPL(adf_dev_get); +/** + * adf_dev_put() - Decrement accel_dev reference count + * @accel_dev: Pointer to acceleration device. + * + * Decrement the accel_dev refcount and if this is the last time + * decrementing it during this period the accel_dev is in use, + * decrement the module refcount too. + * To be used by QAT device specific drivers. + * + * Return: void + */ void adf_dev_put(struct adf_accel_dev *accel_dev) { if (atomic_sub_return(1, &accel_dev->ref_count) == 0) module_put(accel_dev->owner); } +EXPORT_SYMBOL_GPL(adf_dev_put); +/** + * adf_devmgr_in_reset() - Check whether device is in reset + * @accel_dev: Pointer to acceleration device. + * + * To be used by QAT device specific drivers. + * + * Return: 1 when the device is being reset, 0 otherwise. + */ int adf_devmgr_in_reset(struct adf_accel_dev *accel_dev) { return test_bit(ADF_STATUS_RESTARTING, &accel_dev->status); } +EXPORT_SYMBOL_GPL(adf_devmgr_in_reset); +/** + * adf_dev_started() - Check whether device has started + * @accel_dev: Pointer to acceleration device. + * + * To be used by QAT device specific drivers. + * + * Return: 1 when the device has started, 0 otherwise + */ int adf_dev_started(struct adf_accel_dev *accel_dev) { return test_bit(ADF_STATUS_STARTED, &accel_dev->status); } +EXPORT_SYMBOL_GPL(adf_dev_started); diff --git a/drivers/crypto/qat/qat_dh895xcc/adf_hw_arbiter.c b/drivers/crypto/qat/qat_common/adf_hw_arbiter.c index 1864bdb36f8f..6849422e04bb 100644 --- a/drivers/crypto/qat/qat_dh895xcc/adf_hw_arbiter.c +++ b/drivers/crypto/qat/qat_common/adf_hw_arbiter.c @@ -44,9 +44,8 @@ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -#include <adf_accel_devices.h> -#include <adf_transport_internal.h> -#include "adf_drv.h" +#include "adf_accel_devices.h" +#include "adf_transport_internal.h" #define ADF_ARB_NUM 4 #define ADF_ARB_REQ_RING_NUM 8 @@ -58,7 +57,6 @@ #define ADF_ARB_RO_EN_OFFSET 0x090 #define ADF_ARB_WQCFG_OFFSET 0x100 #define ADF_ARB_WRK_2_SER_MAP_OFFSET 0x180 -#define ADF_ARB_WRK_2_SER_MAP 10 #define ADF_ARB_RINGSRVARBEN_OFFSET 0x19C #define WRITE_CSR_ARB_RINGSRVARBEN(csr_addr, index, value) \ @@ -89,10 +87,11 @@ int adf_init_arb(struct adf_accel_dev *accel_dev) { + struct adf_hw_device_data *hw_data = accel_dev->hw_device; void __iomem *csr = accel_dev->transport->banks[0].csr_addr; - uint32_t arb_cfg = 0x1 << 31 | 0x4 << 4 | 0x1; - uint32_t arb, i; - const uint32_t *thd_2_arb_cfg; + u32 arb_cfg = 0x1 << 31 | 0x4 << 4 | 0x1; + u32 arb, i; + const u32 *thd_2_arb_cfg; /* Service arb configured for 32 bytes responses and * ring flow control check enabled. */ @@ -109,30 +108,39 @@ int adf_init_arb(struct adf_accel_dev *accel_dev) WRITE_CSR_ARB_RESPORDERING(csr, i, 0xFFFFFFFF); /* Setup worker queue registers */ - for (i = 0; i < ADF_ARB_WRK_2_SER_MAP; i++) + for (i = 0; i < hw_data->num_engines; i++) WRITE_CSR_ARB_WQCFG(csr, i, i); /* Map worker threads to service arbiters */ - adf_get_arbiter_mapping(accel_dev, &thd_2_arb_cfg); + hw_data->get_arb_mapping(accel_dev, &thd_2_arb_cfg); if (!thd_2_arb_cfg) return -EFAULT; - for (i = 0; i < ADF_ARB_WRK_2_SER_MAP; i++) + for (i = 0; i < hw_data->num_engines; i++) WRITE_CSR_ARB_WRK_2_SER_MAP(csr, i, *(thd_2_arb_cfg + i)); return 0; } - -void adf_update_ring_arb_enable(struct adf_etr_ring_data *ring) +EXPORT_SYMBOL_GPL(adf_init_arb); + +/** + * adf_update_ring_arb() - update ring arbitration rgister + * @accel_dev: Pointer to ring data. + * + * Function enables or disables rings for/from arbitration. + */ +void adf_update_ring_arb(struct adf_etr_ring_data *ring) { WRITE_CSR_ARB_RINGSRVARBEN(ring->bank->csr_addr, ring->bank->bank_number, ring->bank->ring_mask & 0xFF); } +EXPORT_SYMBOL_GPL(adf_update_ring_arb); void adf_exit_arb(struct adf_accel_dev *accel_dev) { + struct adf_hw_device_data *hw_data = accel_dev->hw_device; void __iomem *csr; unsigned int i; @@ -146,14 +154,15 @@ void adf_exit_arb(struct adf_accel_dev *accel_dev) WRITE_CSR_ARB_SARCONFIG(csr, i, 0); /* Shutdown work queue */ - for (i = 0; i < ADF_ARB_WRK_2_SER_MAP; i++) + for (i = 0; i < hw_data->num_engines; i++) WRITE_CSR_ARB_WQCFG(csr, i, 0); /* Unmap worker threads to service arbiters */ - for (i = 0; i < ADF_ARB_WRK_2_SER_MAP; i++) + for (i = 0; i < hw_data->num_engines; i++) WRITE_CSR_ARB_WRK_2_SER_MAP(csr, i, 0); /* Disable arbitration on all rings */ for (i = 0; i < GET_MAX_BANKS(accel_dev); i++) WRITE_CSR_ARB_RINGSRVARBEN(csr, i, 0); } +EXPORT_SYMBOL_GPL(adf_exit_arb); diff --git a/drivers/crypto/qat/qat_common/adf_init.c b/drivers/crypto/qat/qat_common/adf_init.c index 245f43237a2d..ac37a89965ac 100644 --- a/drivers/crypto/qat/qat_common/adf_init.c +++ b/drivers/crypto/qat/qat_common/adf_init.c @@ -69,7 +69,7 @@ static void adf_service_add(struct service_hndl *service) * Function adds the acceleration service to the acceleration framework. * To be used by QAT device specific drivers. * - * Return: 0 on success, error code othewise. + * Return: 0 on success, error code otherwise. */ int adf_service_register(struct service_hndl *service) { @@ -94,7 +94,7 @@ static void adf_service_remove(struct service_hndl *service) * Function remove the acceleration service from the acceleration framework. * To be used by QAT device specific drivers. * - * Return: 0 on success, error code othewise. + * Return: 0 on success, error code otherwise. */ int adf_service_unregister(struct service_hndl *service) { @@ -114,7 +114,7 @@ EXPORT_SYMBOL_GPL(adf_service_unregister); * Initialize the ring data structures and the admin comms and arbitration * services. * - * Return: 0 on success, error code othewise. + * Return: 0 on success, error code otherwise. */ int adf_dev_init(struct adf_accel_dev *accel_dev) { @@ -177,20 +177,6 @@ int adf_dev_init(struct adf_accel_dev *accel_dev) */ list_for_each(list_itr, &service_table) { service = list_entry(list_itr, struct service_hndl, list); - if (!service->admin) - continue; - if (service->event_hld(accel_dev, ADF_EVENT_INIT)) { - dev_err(&GET_DEV(accel_dev), - "Failed to initialise service %s\n", - service->name); - return -EFAULT; - } - set_bit(accel_dev->accel_id, &service->init_status); - } - list_for_each(list_itr, &service_table) { - service = list_entry(list_itr, struct service_hndl, list); - if (service->admin) - continue; if (service->event_hld(accel_dev, ADF_EVENT_INIT)) { dev_err(&GET_DEV(accel_dev), "Failed to initialise service %s\n", @@ -201,6 +187,7 @@ int adf_dev_init(struct adf_accel_dev *accel_dev) } hw_data->enable_error_correction(accel_dev); + hw_data->enable_vf2pf_comms(accel_dev); return 0; } @@ -214,10 +201,11 @@ EXPORT_SYMBOL_GPL(adf_dev_init); * is ready to be used. * To be used by QAT device specific drivers. * - * Return: 0 on success, error code othewise. + * Return: 0 on success, error code otherwise. */ int adf_dev_start(struct adf_accel_dev *accel_dev) { + struct adf_hw_device_data *hw_data = accel_dev->hw_device; struct service_hndl *service; struct list_head *list_itr; @@ -229,22 +217,13 @@ int adf_dev_start(struct adf_accel_dev *accel_dev) } set_bit(ADF_STATUS_AE_STARTED, &accel_dev->status); - list_for_each(list_itr, &service_table) { - service = list_entry(list_itr, struct service_hndl, list); - if (!service->admin) - continue; - if (service->event_hld(accel_dev, ADF_EVENT_START)) { - dev_err(&GET_DEV(accel_dev), - "Failed to start service %s\n", - service->name); - return -EFAULT; - } - set_bit(accel_dev->accel_id, &service->start_status); + if (hw_data->send_admin_init(accel_dev)) { + dev_err(&GET_DEV(accel_dev), "Failed to send init message\n"); + return -EFAULT; } + list_for_each(list_itr, &service_table) { service = list_entry(list_itr, struct service_hndl, list); - if (service->admin) - continue; if (service->event_hld(accel_dev, ADF_EVENT_START)) { dev_err(&GET_DEV(accel_dev), "Failed to start service %s\n", @@ -257,7 +236,8 @@ int adf_dev_start(struct adf_accel_dev *accel_dev) clear_bit(ADF_STATUS_STARTING, &accel_dev->status); set_bit(ADF_STATUS_STARTED, &accel_dev->status); - if (qat_algs_register()) { + if (!list_empty(&accel_dev->crypto_list) && + (qat_algs_register() || qat_asym_algs_register())) { dev_err(&GET_DEV(accel_dev), "Failed to register crypto algs\n"); set_bit(ADF_STATUS_STARTING, &accel_dev->status); @@ -276,7 +256,7 @@ EXPORT_SYMBOL_GPL(adf_dev_start); * is shuting down. * To be used by QAT device specific drivers. * - * Return: 0 on success, error code othewise. + * Return: 0 on success, error code otherwise. */ int adf_dev_stop(struct adf_accel_dev *accel_dev) { @@ -292,14 +272,15 @@ int adf_dev_stop(struct adf_accel_dev *accel_dev) clear_bit(ADF_STATUS_STARTING, &accel_dev->status); clear_bit(ADF_STATUS_STARTED, &accel_dev->status); - if (qat_algs_unregister()) + if (!list_empty(&accel_dev->crypto_list) && qat_algs_unregister()) dev_err(&GET_DEV(accel_dev), "Failed to unregister crypto algs\n"); + if (!list_empty(&accel_dev->crypto_list)) + qat_asym_algs_unregister(); + list_for_each(list_itr, &service_table) { service = list_entry(list_itr, struct service_hndl, list); - if (service->admin) - continue; if (!test_bit(accel_dev->accel_id, &service->start_status)) continue; ret = service->event_hld(accel_dev, ADF_EVENT_STOP); @@ -310,19 +291,6 @@ int adf_dev_stop(struct adf_accel_dev *accel_dev) clear_bit(accel_dev->accel_id, &service->start_status); } } - list_for_each(list_itr, &service_table) { - service = list_entry(list_itr, struct service_hndl, list); - if (!service->admin) - continue; - if (!test_bit(accel_dev->accel_id, &service->start_status)) - continue; - if (service->event_hld(accel_dev, ADF_EVENT_STOP)) - dev_err(&GET_DEV(accel_dev), - "Failed to shutdown service %s\n", - service->name); - else - clear_bit(accel_dev->accel_id, &service->start_status); - } if (wait) msleep(100); @@ -373,21 +341,6 @@ void adf_dev_shutdown(struct adf_accel_dev *accel_dev) list_for_each(list_itr, &service_table) { service = list_entry(list_itr, struct service_hndl, list); - if (service->admin) - continue; - if (!test_bit(accel_dev->accel_id, &service->init_status)) - continue; - if (service->event_hld(accel_dev, ADF_EVENT_SHUTDOWN)) - dev_err(&GET_DEV(accel_dev), - "Failed to shutdown service %s\n", - service->name); - else - clear_bit(accel_dev->accel_id, &service->init_status); - } - list_for_each(list_itr, &service_table) { - service = list_entry(list_itr, struct service_hndl, list); - if (!service->admin) - continue; if (!test_bit(accel_dev->accel_id, &service->init_status)) continue; if (service->event_hld(accel_dev, ADF_EVENT_SHUTDOWN)) @@ -413,6 +366,7 @@ void adf_dev_shutdown(struct adf_accel_dev *accel_dev) if (hw_data->exit_admin_comms) hw_data->exit_admin_comms(accel_dev); + hw_data->disable_iov(accel_dev); adf_cleanup_etr_data(accel_dev); } EXPORT_SYMBOL_GPL(adf_dev_shutdown); @@ -424,17 +378,6 @@ int adf_dev_restarting_notify(struct adf_accel_dev *accel_dev) list_for_each(list_itr, &service_table) { service = list_entry(list_itr, struct service_hndl, list); - if (service->admin) - continue; - if (service->event_hld(accel_dev, ADF_EVENT_RESTARTING)) - dev_err(&GET_DEV(accel_dev), - "Failed to restart service %s.\n", - service->name); - } - list_for_each(list_itr, &service_table) { - service = list_entry(list_itr, struct service_hndl, list); - if (!service->admin) - continue; if (service->event_hld(accel_dev, ADF_EVENT_RESTARTING)) dev_err(&GET_DEV(accel_dev), "Failed to restart service %s.\n", @@ -450,17 +393,6 @@ int adf_dev_restarted_notify(struct adf_accel_dev *accel_dev) list_for_each(list_itr, &service_table) { service = list_entry(list_itr, struct service_hndl, list); - if (service->admin) - continue; - if (service->event_hld(accel_dev, ADF_EVENT_RESTARTED)) - dev_err(&GET_DEV(accel_dev), - "Failed to restart service %s.\n", - service->name); - } - list_for_each(list_itr, &service_table) { - service = list_entry(list_itr, struct service_hndl, list); - if (!service->admin) - continue; if (service->event_hld(accel_dev, ADF_EVENT_RESTARTED)) dev_err(&GET_DEV(accel_dev), "Failed to restart service %s.\n", diff --git a/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c b/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c new file mode 100644 index 000000000000..5fdbad809343 --- /dev/null +++ b/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c @@ -0,0 +1,438 @@ +/* + This file is provided under a dual BSD/GPLv2 license. When using or + redistributing this file, you may do so under either license. + + GPL LICENSE SUMMARY + Copyright(c) 2015 Intel Corporation. + This program is free software; you can redistribute it and/or modify + it under the terms of version 2 of the GNU General Public License as + published by the Free Software Foundation. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + Contact Information: + qat-linux@intel.com + + BSD LICENSE + Copyright(c) 2015 Intel Corporation. + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#include <linux/pci.h> +#include <linux/mutex.h> +#include <linux/delay.h> +#include "adf_accel_devices.h" +#include "adf_common_drv.h" +#include "adf_pf2vf_msg.h" + +#define ADF_DH895XCC_EP_OFFSET 0x3A000 +#define ADF_DH895XCC_ERRMSK3 (ADF_DH895XCC_EP_OFFSET + 0x1C) +#define ADF_DH895XCC_ERRMSK3_VF2PF_L_MASK(vf_mask) ((vf_mask & 0xFFFF) << 9) +#define ADF_DH895XCC_ERRMSK5 (ADF_DH895XCC_EP_OFFSET + 0xDC) +#define ADF_DH895XCC_ERRMSK5_VF2PF_U_MASK(vf_mask) (vf_mask >> 16) + +/** + * adf_enable_pf2vf_interrupts() - Enable PF to VF interrupts + * @accel_dev: Pointer to acceleration device. + * + * Function enables PF to VF interrupts + */ +void adf_enable_pf2vf_interrupts(struct adf_accel_dev *accel_dev) +{ + struct adf_accel_pci *pci_info = &accel_dev->accel_pci_dev; + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + void __iomem *pmisc_bar_addr = + pci_info->pci_bars[hw_data->get_misc_bar_id(hw_data)].virt_addr; + + ADF_CSR_WR(pmisc_bar_addr, hw_data->get_vintmsk_offset(0), 0x0); +} +EXPORT_SYMBOL_GPL(adf_enable_pf2vf_interrupts); + +/** + * adf_disable_pf2vf_interrupts() - Disable PF to VF interrupts + * @accel_dev: Pointer to acceleration device. + * + * Function disables PF to VF interrupts + */ +void adf_disable_pf2vf_interrupts(struct adf_accel_dev *accel_dev) +{ + struct adf_accel_pci *pci_info = &accel_dev->accel_pci_dev; + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + void __iomem *pmisc_bar_addr = + pci_info->pci_bars[hw_data->get_misc_bar_id(hw_data)].virt_addr; + + ADF_CSR_WR(pmisc_bar_addr, hw_data->get_vintmsk_offset(0), 0x2); +} +EXPORT_SYMBOL_GPL(adf_disable_pf2vf_interrupts); + +void adf_enable_vf2pf_interrupts(struct adf_accel_dev *accel_dev, + u32 vf_mask) +{ + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + struct adf_bar *pmisc = + &GET_BARS(accel_dev)[hw_data->get_misc_bar_id(hw_data)]; + void __iomem *pmisc_addr = pmisc->virt_addr; + u32 reg; + + /* Enable VF2PF Messaging Ints - VFs 1 through 16 per vf_mask[15:0] */ + if (vf_mask & 0xFFFF) { + reg = ADF_CSR_RD(pmisc_addr, ADF_DH895XCC_ERRMSK3); + reg &= ~ADF_DH895XCC_ERRMSK3_VF2PF_L_MASK(vf_mask); + ADF_CSR_WR(pmisc_addr, ADF_DH895XCC_ERRMSK3, reg); + } + + /* Enable VF2PF Messaging Ints - VFs 17 through 32 per vf_mask[31:16] */ + if (vf_mask >> 16) { + reg = ADF_CSR_RD(pmisc_addr, ADF_DH895XCC_ERRMSK5); + reg &= ~ADF_DH895XCC_ERRMSK5_VF2PF_U_MASK(vf_mask); + ADF_CSR_WR(pmisc_addr, ADF_DH895XCC_ERRMSK5, reg); + } +} + +/** + * adf_disable_pf2vf_interrupts() - Disable VF to PF interrupts + * @accel_dev: Pointer to acceleration device. + * + * Function disables VF to PF interrupts + */ +void adf_disable_vf2pf_interrupts(struct adf_accel_dev *accel_dev, u32 vf_mask) +{ + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + struct adf_bar *pmisc = + &GET_BARS(accel_dev)[hw_data->get_misc_bar_id(hw_data)]; + void __iomem *pmisc_addr = pmisc->virt_addr; + u32 reg; + + /* Disable VF2PF interrupts for VFs 1 through 16 per vf_mask[15:0] */ + if (vf_mask & 0xFFFF) { + reg = ADF_CSR_RD(pmisc_addr, ADF_DH895XCC_ERRMSK3) | + ADF_DH895XCC_ERRMSK3_VF2PF_L_MASK(vf_mask); + ADF_CSR_WR(pmisc_addr, ADF_DH895XCC_ERRMSK3, reg); + } + + /* Disable VF2PF interrupts for VFs 17 through 32 per vf_mask[31:16] */ + if (vf_mask >> 16) { + reg = ADF_CSR_RD(pmisc_addr, ADF_DH895XCC_ERRMSK5) | + ADF_DH895XCC_ERRMSK5_VF2PF_U_MASK(vf_mask); + ADF_CSR_WR(pmisc_addr, ADF_DH895XCC_ERRMSK5, reg); + } +} +EXPORT_SYMBOL_GPL(adf_disable_vf2pf_interrupts); + +static int __adf_iov_putmsg(struct adf_accel_dev *accel_dev, u32 msg, u8 vf_nr) +{ + struct adf_accel_pci *pci_info = &accel_dev->accel_pci_dev; + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + void __iomem *pmisc_bar_addr = + pci_info->pci_bars[hw_data->get_misc_bar_id(hw_data)].virt_addr; + u32 val, pf2vf_offset, count = 0; + u32 local_in_use_mask, local_in_use_pattern; + u32 remote_in_use_mask, remote_in_use_pattern; + struct mutex *lock; /* lock preventing concurrent acces of CSR */ + u32 int_bit; + int ret = 0; + + if (accel_dev->is_vf) { + pf2vf_offset = hw_data->get_pf2vf_offset(0); + lock = &accel_dev->vf.vf2pf_lock; + local_in_use_mask = ADF_VF2PF_IN_USE_BY_VF_MASK; + local_in_use_pattern = ADF_VF2PF_IN_USE_BY_VF; + remote_in_use_mask = ADF_PF2VF_IN_USE_BY_PF_MASK; + remote_in_use_pattern = ADF_PF2VF_IN_USE_BY_PF; + int_bit = ADF_VF2PF_INT; + } else { + pf2vf_offset = hw_data->get_pf2vf_offset(vf_nr); + lock = &accel_dev->pf.vf_info[vf_nr].pf2vf_lock; + local_in_use_mask = ADF_PF2VF_IN_USE_BY_PF_MASK; + local_in_use_pattern = ADF_PF2VF_IN_USE_BY_PF; + remote_in_use_mask = ADF_VF2PF_IN_USE_BY_VF_MASK; + remote_in_use_pattern = ADF_VF2PF_IN_USE_BY_VF; + int_bit = ADF_PF2VF_INT; + } + + mutex_lock(lock); + + /* Check if PF2VF CSR is in use by remote function */ + val = ADF_CSR_RD(pmisc_bar_addr, pf2vf_offset); + if ((val & remote_in_use_mask) == remote_in_use_pattern) { + dev_dbg(&GET_DEV(accel_dev), + "PF2VF CSR in use by remote function\n"); + ret = -EBUSY; + goto out; + } + + /* Attempt to get ownership of PF2VF CSR */ + msg &= ~local_in_use_mask; + msg |= local_in_use_pattern; + ADF_CSR_WR(pmisc_bar_addr, pf2vf_offset, msg); + + /* Wait in case remote func also attempting to get ownership */ + msleep(ADF_IOV_MSG_COLLISION_DETECT_DELAY); + + val = ADF_CSR_RD(pmisc_bar_addr, pf2vf_offset); + if ((val & local_in_use_mask) != local_in_use_pattern) { + dev_dbg(&GET_DEV(accel_dev), + "PF2VF CSR in use by remote - collision detected\n"); + ret = -EBUSY; + goto out; + } + + /* + * This function now owns the PV2VF CSR. The IN_USE_BY pattern must + * remain in the PF2VF CSR for all writes including ACK from remote + * until this local function relinquishes the CSR. Send the message + * by interrupting the remote. + */ + ADF_CSR_WR(pmisc_bar_addr, pf2vf_offset, msg | int_bit); + + /* Wait for confirmation from remote func it received the message */ + do { + msleep(ADF_IOV_MSG_ACK_DELAY); + val = ADF_CSR_RD(pmisc_bar_addr, pf2vf_offset); + } while ((val & int_bit) && (count++ < ADF_IOV_MSG_ACK_MAX_RETRY)); + + if (val & int_bit) { + dev_dbg(&GET_DEV(accel_dev), "ACK not received from remote\n"); + val &= ~int_bit; + ret = -EIO; + } + + /* Finished with PF2VF CSR; relinquish it and leave msg in CSR */ + ADF_CSR_WR(pmisc_bar_addr, pf2vf_offset, val & ~local_in_use_mask); +out: + mutex_unlock(lock); + return ret; +} + +/** + * adf_iov_putmsg() - send PF2VF message + * @accel_dev: Pointer to acceleration device. + * @msg: Message to send + * @vf_nr: VF number to which the message will be sent + * + * Function sends a messge from the PF to a VF + * + * Return: 0 on success, error code otherwise. + */ +int adf_iov_putmsg(struct adf_accel_dev *accel_dev, u32 msg, u8 vf_nr) +{ + u32 count = 0; + int ret; + + do { + ret = __adf_iov_putmsg(accel_dev, msg, vf_nr); + if (ret) + msleep(ADF_IOV_MSG_RETRY_DELAY); + } while (ret && (count++ < ADF_IOV_MSG_MAX_RETRIES)); + + return ret; +} +EXPORT_SYMBOL_GPL(adf_iov_putmsg); + +void adf_vf2pf_req_hndl(struct adf_accel_vf_info *vf_info) +{ + struct adf_accel_dev *accel_dev = vf_info->accel_dev; + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + int bar_id = hw_data->get_misc_bar_id(hw_data); + struct adf_bar *pmisc = &GET_BARS(accel_dev)[bar_id]; + void __iomem *pmisc_addr = pmisc->virt_addr; + u32 msg, resp = 0, vf_nr = vf_info->vf_nr; + + /* Read message from the VF */ + msg = ADF_CSR_RD(pmisc_addr, hw_data->get_pf2vf_offset(vf_nr)); + + /* To ACK, clear the VF2PFINT bit */ + msg &= ~ADF_VF2PF_INT; + ADF_CSR_WR(pmisc_addr, hw_data->get_pf2vf_offset(vf_nr), msg); + + if (!(msg & ADF_VF2PF_MSGORIGIN_SYSTEM)) + /* Ignore legacy non-system (non-kernel) VF2PF messages */ + goto err; + + switch ((msg & ADF_VF2PF_MSGTYPE_MASK) >> ADF_VF2PF_MSGTYPE_SHIFT) { + case ADF_VF2PF_MSGTYPE_COMPAT_VER_REQ: + { + u8 vf_compat_ver = msg >> ADF_VF2PF_COMPAT_VER_REQ_SHIFT; + + resp = (ADF_PF2VF_MSGORIGIN_SYSTEM | + (ADF_PF2VF_MSGTYPE_VERSION_RESP << + ADF_PF2VF_MSGTYPE_SHIFT) | + (ADF_PFVF_COMPATIBILITY_VERSION << + ADF_PF2VF_VERSION_RESP_VERS_SHIFT)); + + dev_dbg(&GET_DEV(accel_dev), + "Compatibility Version Request from VF%d vers=%u\n", + vf_nr + 1, vf_compat_ver); + + if (vf_compat_ver < hw_data->min_iov_compat_ver) { + dev_err(&GET_DEV(accel_dev), + "VF (vers %d) incompatible with PF (vers %d)\n", + vf_compat_ver, ADF_PFVF_COMPATIBILITY_VERSION); + resp |= ADF_PF2VF_VF_INCOMPATIBLE << + ADF_PF2VF_VERSION_RESP_RESULT_SHIFT; + } else if (vf_compat_ver > ADF_PFVF_COMPATIBILITY_VERSION) { + dev_err(&GET_DEV(accel_dev), + "VF (vers %d) compat with PF (vers %d) unkn.\n", + vf_compat_ver, ADF_PFVF_COMPATIBILITY_VERSION); + resp |= ADF_PF2VF_VF_COMPAT_UNKNOWN << + ADF_PF2VF_VERSION_RESP_RESULT_SHIFT; + } else { + dev_dbg(&GET_DEV(accel_dev), + "VF (vers %d) compatible with PF (vers %d)\n", + vf_compat_ver, ADF_PFVF_COMPATIBILITY_VERSION); + resp |= ADF_PF2VF_VF_COMPATIBLE << + ADF_PF2VF_VERSION_RESP_RESULT_SHIFT; + } + } + break; + case ADF_VF2PF_MSGTYPE_VERSION_REQ: + dev_dbg(&GET_DEV(accel_dev), + "Legacy VersionRequest received from VF%d 0x%x\n", + vf_nr + 1, msg); + resp = (ADF_PF2VF_MSGORIGIN_SYSTEM | + (ADF_PF2VF_MSGTYPE_VERSION_RESP << + ADF_PF2VF_MSGTYPE_SHIFT) | + (ADF_PFVF_COMPATIBILITY_VERSION << + ADF_PF2VF_VERSION_RESP_VERS_SHIFT)); + resp |= ADF_PF2VF_VF_COMPATIBLE << + ADF_PF2VF_VERSION_RESP_RESULT_SHIFT; + /* Set legacy major and minor version num */ + resp |= 1 << ADF_PF2VF_MAJORVERSION_SHIFT | + 1 << ADF_PF2VF_MINORVERSION_SHIFT; + break; + case ADF_VF2PF_MSGTYPE_INIT: + { + dev_dbg(&GET_DEV(accel_dev), + "Init message received from VF%d 0x%x\n", + vf_nr + 1, msg); + vf_info->init = true; + } + break; + case ADF_VF2PF_MSGTYPE_SHUTDOWN: + { + dev_dbg(&GET_DEV(accel_dev), + "Shutdown message received from VF%d 0x%x\n", + vf_nr + 1, msg); + vf_info->init = false; + } + break; + default: + goto err; + } + + if (resp && adf_iov_putmsg(accel_dev, resp, vf_nr)) + dev_err(&GET_DEV(accel_dev), "Failed to send response to VF\n"); + + /* re-enable interrupt on PF from this VF */ + adf_enable_vf2pf_interrupts(accel_dev, (1 << vf_nr)); + return; +err: + dev_dbg(&GET_DEV(accel_dev), "Unknown message from VF%d (0x%x);\n", + vf_nr + 1, msg); +} + +void adf_pf2vf_notify_restarting(struct adf_accel_dev *accel_dev) +{ + struct adf_accel_vf_info *vf; + u32 msg = (ADF_PF2VF_MSGORIGIN_SYSTEM | + (ADF_PF2VF_MSGTYPE_RESTARTING << ADF_PF2VF_MSGTYPE_SHIFT)); + int i, num_vfs = pci_num_vf(accel_to_pci_dev(accel_dev)); + + for (i = 0, vf = accel_dev->pf.vf_info; i < num_vfs; i++, vf++) { + if (vf->init && adf_iov_putmsg(accel_dev, msg, i)) + dev_err(&GET_DEV(accel_dev), + "Failed to send restarting msg to VF%d\n", i); + } +} + +static int adf_vf2pf_request_version(struct adf_accel_dev *accel_dev) +{ + unsigned long timeout = msecs_to_jiffies(ADF_IOV_MSG_RESP_TIMEOUT); + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + u32 msg = 0; + int ret; + + msg = ADF_VF2PF_MSGORIGIN_SYSTEM; + msg |= ADF_VF2PF_MSGTYPE_COMPAT_VER_REQ << ADF_VF2PF_MSGTYPE_SHIFT; + msg |= ADF_PFVF_COMPATIBILITY_VERSION << ADF_VF2PF_COMPAT_VER_REQ_SHIFT; + BUILD_BUG_ON(ADF_PFVF_COMPATIBILITY_VERSION > 255); + + /* Send request from VF to PF */ + ret = adf_iov_putmsg(accel_dev, msg, 0); + if (ret) { + dev_err(&GET_DEV(accel_dev), + "Failed to send Compatibility Version Request.\n"); + return ret; + } + + /* Wait for response */ + if (!wait_for_completion_timeout(&accel_dev->vf.iov_msg_completion, + timeout)) { + dev_err(&GET_DEV(accel_dev), + "IOV request/response message timeout expired\n"); + return -EIO; + } + + /* Response from PF received, check compatibility */ + switch (accel_dev->vf.compatible) { + case ADF_PF2VF_VF_COMPATIBLE: + break; + case ADF_PF2VF_VF_COMPAT_UNKNOWN: + /* VF is newer than PF and decides whether it is compatible */ + if (accel_dev->vf.pf_version >= hw_data->min_iov_compat_ver) + break; + /* fall through */ + case ADF_PF2VF_VF_INCOMPATIBLE: + dev_err(&GET_DEV(accel_dev), + "PF (vers %d) and VF (vers %d) are not compatible\n", + accel_dev->vf.pf_version, + ADF_PFVF_COMPATIBILITY_VERSION); + return -EINVAL; + default: + dev_err(&GET_DEV(accel_dev), + "Invalid response from PF; assume not compatible\n"); + return -EINVAL; + } + return ret; +} + +/** + * adf_enable_vf2pf_comms() - Function enables communication from vf to pf + * + * @accel_dev: Pointer to acceleration device virtual function. + * + * Return: 0 on success, error code otherwise. + */ +int adf_enable_vf2pf_comms(struct adf_accel_dev *accel_dev) +{ + adf_enable_pf2vf_interrupts(accel_dev); + return adf_vf2pf_request_version(accel_dev); +} +EXPORT_SYMBOL_GPL(adf_enable_vf2pf_comms); diff --git a/drivers/crypto/qat/qat_common/adf_pf2vf_msg.h b/drivers/crypto/qat/qat_common/adf_pf2vf_msg.h new file mode 100644 index 000000000000..5acd531a11ff --- /dev/null +++ b/drivers/crypto/qat/qat_common/adf_pf2vf_msg.h @@ -0,0 +1,146 @@ +/* + This file is provided under a dual BSD/GPLv2 license. When using or + redistributing this file, you may do so under either license. + + GPL LICENSE SUMMARY + Copyright(c) 2015 Intel Corporation. + This program is free software; you can redistribute it and/or modify + it under the terms of version 2 of the GNU General Public License as + published by the Free Software Foundation. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + Contact Information: + qat-linux@intel.com + + BSD LICENSE + Copyright(c) 2015 Intel Corporation. + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#ifndef ADF_PF2VF_MSG_H +#define ADF_PF2VF_MSG_H + +/* + * PF<->VF Messaging + * The PF has an array of 32-bit PF2VF registers, one for each VF. The + * PF can access all these registers; each VF can access only the one + * register associated with that particular VF. + * + * The register functionally is split into two parts: + * The bottom half is for PF->VF messages. In particular when the first + * bit of this register (bit 0) gets set an interrupt will be triggered + * in the respective VF. + * The top half is for VF->PF messages. In particular when the first bit + * of this half of register (bit 16) gets set an interrupt will be triggered + * in the PF. + * + * The remaining bits within this register are available to encode messages. + * and implement a collision control mechanism to prevent concurrent use of + * the PF2VF register by both the PF and VF. + * + * 31 30 29 28 27 26 25 24 23 22 21 20 19 18 17 16 + * _______________________________________________ + * | | | | | | | | | | | | | | | | | + * +-----------------------------------------------+ + * \___________________________/ \_________/ ^ ^ + * ^ ^ | | + * | | | VF2PF Int + * | | Message Origin + * | Message Type + * Message-specific Data/Reserved + * + * 15 14 13 12 11 10 9 8 7 6 5 4 3 2 1 0 + * _______________________________________________ + * | | | | | | | | | | | | | | | | | + * +-----------------------------------------------+ + * \___________________________/ \_________/ ^ ^ + * ^ ^ | | + * | | | PF2VF Int + * | | Message Origin + * | Message Type + * Message-specific Data/Reserved + * + * Message Origin (Should always be 1) + * A legacy out-of-tree QAT driver allowed for a set of messages not supported + * by this driver; these had a Msg Origin of 0 and are ignored by this driver. + * + * When a PF or VF attempts to send a message in the lower or upper 16 bits, + * respectively, the other 16 bits are written to first with a defined + * IN_USE_BY pattern as part of a collision control scheme (see adf_iov_putmsg). + */ + +#define ADF_PFVF_COMPATIBILITY_VERSION 0x1 /* PF<->VF compat */ + +/* PF->VF messages */ +#define ADF_PF2VF_INT BIT(0) +#define ADF_PF2VF_MSGORIGIN_SYSTEM BIT(1) +#define ADF_PF2VF_MSGTYPE_MASK 0x0000003C +#define ADF_PF2VF_MSGTYPE_SHIFT 2 +#define ADF_PF2VF_MSGTYPE_RESTARTING 0x01 +#define ADF_PF2VF_MSGTYPE_VERSION_RESP 0x02 +#define ADF_PF2VF_IN_USE_BY_PF 0x6AC20000 +#define ADF_PF2VF_IN_USE_BY_PF_MASK 0xFFFE0000 + +/* PF->VF Version Response */ +#define ADF_PF2VF_VERSION_RESP_VERS_MASK 0x00003FC0 +#define ADF_PF2VF_VERSION_RESP_VERS_SHIFT 6 +#define ADF_PF2VF_VERSION_RESP_RESULT_MASK 0x0000C000 +#define ADF_PF2VF_VERSION_RESP_RESULT_SHIFT 14 +#define ADF_PF2VF_MINORVERSION_SHIFT 6 +#define ADF_PF2VF_MAJORVERSION_SHIFT 10 +#define ADF_PF2VF_VF_COMPATIBLE 1 +#define ADF_PF2VF_VF_INCOMPATIBLE 2 +#define ADF_PF2VF_VF_COMPAT_UNKNOWN 3 + +/* VF->PF messages */ +#define ADF_VF2PF_IN_USE_BY_VF 0x00006AC2 +#define ADF_VF2PF_IN_USE_BY_VF_MASK 0x0000FFFE +#define ADF_VF2PF_INT BIT(16) +#define ADF_VF2PF_MSGORIGIN_SYSTEM BIT(17) +#define ADF_VF2PF_MSGTYPE_MASK 0x003C0000 +#define ADF_VF2PF_MSGTYPE_SHIFT 18 +#define ADF_VF2PF_MSGTYPE_INIT 0x3 +#define ADF_VF2PF_MSGTYPE_SHUTDOWN 0x4 +#define ADF_VF2PF_MSGTYPE_VERSION_REQ 0x5 +#define ADF_VF2PF_MSGTYPE_COMPAT_VER_REQ 0x6 + +/* VF->PF Compatible Version Request */ +#define ADF_VF2PF_COMPAT_VER_REQ_SHIFT 22 + +/* Collision detection */ +#define ADF_IOV_MSG_COLLISION_DETECT_DELAY 10 +#define ADF_IOV_MSG_ACK_DELAY 2 +#define ADF_IOV_MSG_ACK_MAX_RETRY 100 +#define ADF_IOV_MSG_RETRY_DELAY 5 +#define ADF_IOV_MSG_MAX_RETRIES 3 +#define ADF_IOV_MSG_RESP_TIMEOUT (ADF_IOV_MSG_ACK_DELAY * \ + ADF_IOV_MSG_ACK_MAX_RETRY + \ + ADF_IOV_MSG_COLLISION_DETECT_DELAY) +#endif /* ADF_IOV_MSG_H */ diff --git a/drivers/crypto/qat/qat_common/adf_sriov.c b/drivers/crypto/qat/qat_common/adf_sriov.c new file mode 100644 index 000000000000..2f77a4a8cecb --- /dev/null +++ b/drivers/crypto/qat/qat_common/adf_sriov.c @@ -0,0 +1,309 @@ +/* + This file is provided under a dual BSD/GPLv2 license. When using or + redistributing this file, you may do so under either license. + + GPL LICENSE SUMMARY + Copyright(c) 2015 Intel Corporation. + This program is free software; you can redistribute it and/or modify + it under the terms of version 2 of the GNU General Public License as + published by the Free Software Foundation. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + Contact Information: + qat-linux@intel.com + + BSD LICENSE + Copyright(c) 2015 Intel Corporation. + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#include <linux/workqueue.h> +#include <linux/pci.h> +#include <linux/device.h> +#include <linux/iommu.h> +#include "adf_common_drv.h" +#include "adf_cfg.h" +#include "adf_pf2vf_msg.h" + +static struct workqueue_struct *pf2vf_resp_wq; + +#define ME2FUNCTION_MAP_A_OFFSET (0x3A400 + 0x190) +#define ME2FUNCTION_MAP_A_NUM_REGS 96 + +#define ME2FUNCTION_MAP_B_OFFSET (0x3A400 + 0x310) +#define ME2FUNCTION_MAP_B_NUM_REGS 12 + +#define ME2FUNCTION_MAP_REG_SIZE 4 +#define ME2FUNCTION_MAP_VALID BIT(7) + +#define READ_CSR_ME2FUNCTION_MAP_A(pmisc_bar_addr, index) \ + ADF_CSR_RD(pmisc_bar_addr, ME2FUNCTION_MAP_A_OFFSET + \ + ME2FUNCTION_MAP_REG_SIZE * index) + +#define WRITE_CSR_ME2FUNCTION_MAP_A(pmisc_bar_addr, index, value) \ + ADF_CSR_WR(pmisc_bar_addr, ME2FUNCTION_MAP_A_OFFSET + \ + ME2FUNCTION_MAP_REG_SIZE * index, value) + +#define READ_CSR_ME2FUNCTION_MAP_B(pmisc_bar_addr, index) \ + ADF_CSR_RD(pmisc_bar_addr, ME2FUNCTION_MAP_B_OFFSET + \ + ME2FUNCTION_MAP_REG_SIZE * index) + +#define WRITE_CSR_ME2FUNCTION_MAP_B(pmisc_bar_addr, index, value) \ + ADF_CSR_WR(pmisc_bar_addr, ME2FUNCTION_MAP_B_OFFSET + \ + ME2FUNCTION_MAP_REG_SIZE * index, value) + +struct adf_pf2vf_resp { + struct work_struct pf2vf_resp_work; + struct adf_accel_vf_info *vf_info; +}; + +static void adf_iov_send_resp(struct work_struct *work) +{ + struct adf_pf2vf_resp *pf2vf_resp = + container_of(work, struct adf_pf2vf_resp, pf2vf_resp_work); + + adf_vf2pf_req_hndl(pf2vf_resp->vf_info); + kfree(pf2vf_resp); +} + +static void adf_vf2pf_bh_handler(void *data) +{ + struct adf_accel_vf_info *vf_info = (struct adf_accel_vf_info *)data; + struct adf_pf2vf_resp *pf2vf_resp; + + pf2vf_resp = kzalloc(sizeof(*pf2vf_resp), GFP_ATOMIC); + if (!pf2vf_resp) + return; + + pf2vf_resp->vf_info = vf_info; + INIT_WORK(&pf2vf_resp->pf2vf_resp_work, adf_iov_send_resp); + queue_work(pf2vf_resp_wq, &pf2vf_resp->pf2vf_resp_work); +} + +static int adf_enable_sriov(struct adf_accel_dev *accel_dev) +{ + struct pci_dev *pdev = accel_to_pci_dev(accel_dev); + int totalvfs = pci_sriov_get_totalvfs(pdev); + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + struct adf_bar *pmisc = + &GET_BARS(accel_dev)[hw_data->get_misc_bar_id(hw_data)]; + void __iomem *pmisc_addr = pmisc->virt_addr; + struct adf_accel_vf_info *vf_info; + int i; + u32 reg; + + /* Workqueue for PF2VF responses */ + pf2vf_resp_wq = create_workqueue("qat_pf2vf_resp_wq"); + if (!pf2vf_resp_wq) + return -ENOMEM; + + for (i = 0, vf_info = accel_dev->pf.vf_info; i < totalvfs; + i++, vf_info++) { + /* This ptr will be populated when VFs will be created */ + vf_info->accel_dev = accel_dev; + vf_info->vf_nr = i; + + tasklet_init(&vf_info->vf2pf_bh_tasklet, + (void *)adf_vf2pf_bh_handler, + (unsigned long)vf_info); + mutex_init(&vf_info->pf2vf_lock); + ratelimit_state_init(&vf_info->vf2pf_ratelimit, + DEFAULT_RATELIMIT_INTERVAL, + DEFAULT_RATELIMIT_BURST); + } + + /* Set Valid bits in ME Thread to PCIe Function Mapping Group A */ + for (i = 0; i < ME2FUNCTION_MAP_A_NUM_REGS; i++) { + reg = READ_CSR_ME2FUNCTION_MAP_A(pmisc_addr, i); + reg |= ME2FUNCTION_MAP_VALID; + WRITE_CSR_ME2FUNCTION_MAP_A(pmisc_addr, i, reg); + } + + /* Set Valid bits in ME Thread to PCIe Function Mapping Group B */ + for (i = 0; i < ME2FUNCTION_MAP_B_NUM_REGS; i++) { + reg = READ_CSR_ME2FUNCTION_MAP_B(pmisc_addr, i); + reg |= ME2FUNCTION_MAP_VALID; + WRITE_CSR_ME2FUNCTION_MAP_B(pmisc_addr, i, reg); + } + + /* Enable VF to PF interrupts for all VFs */ + adf_enable_vf2pf_interrupts(accel_dev, GENMASK_ULL(totalvfs - 1, 0)); + + /* + * Due to the hardware design, when SR-IOV and the ring arbiter + * are enabled all the VFs supported in hardware must be enabled in + * order for all the hardware resources (i.e. bundles) to be usable. + * When SR-IOV is enabled, each of the VFs will own one bundle. + */ + return pci_enable_sriov(pdev, totalvfs); +} + +/** + * adf_disable_sriov() - Disable SRIOV for the device + * @pdev: Pointer to pci device. + * + * Function disables SRIOV for the pci device. + * + * Return: 0 on success, error code otherwise. + */ +void adf_disable_sriov(struct adf_accel_dev *accel_dev) +{ + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + struct adf_bar *pmisc = + &GET_BARS(accel_dev)[hw_data->get_misc_bar_id(hw_data)]; + void __iomem *pmisc_addr = pmisc->virt_addr; + int totalvfs = pci_sriov_get_totalvfs(accel_to_pci_dev(accel_dev)); + struct adf_accel_vf_info *vf; + u32 reg; + int i; + + if (!accel_dev->pf.vf_info) + return; + + adf_pf2vf_notify_restarting(accel_dev); + + pci_disable_sriov(accel_to_pci_dev(accel_dev)); + + /* Disable VF to PF interrupts */ + adf_disable_vf2pf_interrupts(accel_dev, 0xFFFFFFFF); + + /* Clear Valid bits in ME Thread to PCIe Function Mapping Group A */ + for (i = 0; i < ME2FUNCTION_MAP_A_NUM_REGS; i++) { + reg = READ_CSR_ME2FUNCTION_MAP_A(pmisc_addr, i); + reg &= ~ME2FUNCTION_MAP_VALID; + WRITE_CSR_ME2FUNCTION_MAP_A(pmisc_addr, i, reg); + } + + /* Clear Valid bits in ME Thread to PCIe Function Mapping Group B */ + for (i = 0; i < ME2FUNCTION_MAP_B_NUM_REGS; i++) { + reg = READ_CSR_ME2FUNCTION_MAP_B(pmisc_addr, i); + reg &= ~ME2FUNCTION_MAP_VALID; + WRITE_CSR_ME2FUNCTION_MAP_B(pmisc_addr, i, reg); + } + + for (i = 0, vf = accel_dev->pf.vf_info; i < totalvfs; i++, vf++) { + tasklet_disable(&vf->vf2pf_bh_tasklet); + tasklet_kill(&vf->vf2pf_bh_tasklet); + mutex_destroy(&vf->pf2vf_lock); + } + + kfree(accel_dev->pf.vf_info); + accel_dev->pf.vf_info = NULL; + + if (pf2vf_resp_wq) { + destroy_workqueue(pf2vf_resp_wq); + pf2vf_resp_wq = NULL; + } +} +EXPORT_SYMBOL_GPL(adf_disable_sriov); + +/** + * adf_sriov_configure() - Enable SRIOV for the device + * @pdev: Pointer to pci device. + * + * Function enables SRIOV for the pci device. + * + * Return: 0 on success, error code otherwise. + */ +int adf_sriov_configure(struct pci_dev *pdev, int numvfs) +{ + struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev); + int totalvfs = pci_sriov_get_totalvfs(pdev); + unsigned long val; + int ret; + + if (!accel_dev) { + dev_err(&pdev->dev, "Failed to find accel_dev\n"); + return -EFAULT; + } + + if (!iommu_present(&pci_bus_type)) { + dev_err(&pdev->dev, + "IOMMU must be enabled for SR-IOV to work\n"); + return -EINVAL; + } + + if (accel_dev->pf.vf_info) { + dev_info(&pdev->dev, "Already enabled for this device\n"); + return -EINVAL; + } + + if (adf_dev_started(accel_dev)) { + if (adf_devmgr_in_reset(accel_dev) || + adf_dev_in_use(accel_dev)) { + dev_err(&GET_DEV(accel_dev), "Device busy\n"); + return -EBUSY; + } + + if (adf_dev_stop(accel_dev)) { + dev_err(&GET_DEV(accel_dev), + "Failed to stop qat_dev%d\n", + accel_dev->accel_id); + return -EFAULT; + } + + adf_dev_shutdown(accel_dev); + } + + if (adf_cfg_section_add(accel_dev, ADF_KERNEL_SEC)) + return -EFAULT; + val = 0; + if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + ADF_NUM_CY, (void *)&val, ADF_DEC)) + return -EFAULT; + + set_bit(ADF_STATUS_CONFIGURED, &accel_dev->status); + + /* Allocate memory for VF info structs */ + accel_dev->pf.vf_info = kcalloc(totalvfs, + sizeof(struct adf_accel_vf_info), + GFP_KERNEL); + if (!accel_dev->pf.vf_info) + return -ENOMEM; + + if (adf_dev_init(accel_dev)) { + dev_err(&GET_DEV(accel_dev), "Failed to init qat_dev%d\n", + accel_dev->accel_id); + return -EFAULT; + } + + if (adf_dev_start(accel_dev)) { + dev_err(&GET_DEV(accel_dev), "Failed to start qat_dev%d\n", + accel_dev->accel_id); + return -EFAULT; + } + + ret = adf_enable_sriov(accel_dev); + if (ret) + return ret; + + return numvfs; +} +EXPORT_SYMBOL_GPL(adf_sriov_configure); diff --git a/drivers/crypto/qat/qat_common/adf_transport.c b/drivers/crypto/qat/qat_common/adf_transport.c index db2926bff8a5..3865ae8d96d9 100644 --- a/drivers/crypto/qat/qat_common/adf_transport.c +++ b/drivers/crypto/qat/qat_common/adf_transport.c @@ -264,6 +264,10 @@ int adf_create_ring(struct adf_accel_dev *accel_dev, const char *section, dev_err(&GET_DEV(accel_dev), "Can't get ring number\n"); return -EFAULT; } + if (ring_num >= ADF_ETR_MAX_RINGS_PER_BANK) { + dev_err(&GET_DEV(accel_dev), "Invalid ring number\n"); + return -EFAULT; + } bank = &transport_data->banks[bank_num]; if (adf_reserve_ring(bank, ring_num)) { @@ -285,7 +289,7 @@ int adf_create_ring(struct adf_accel_dev *accel_dev, const char *section, goto err; /* Enable HW arbitration for the given ring */ - accel_dev->hw_device->hw_arb_ring_enable(ring); + adf_update_ring_arb(ring); if (adf_ring_debugfs_add(ring, ring_name)) { dev_err(&GET_DEV(accel_dev), @@ -302,14 +306,13 @@ int adf_create_ring(struct adf_accel_dev *accel_dev, const char *section, err: adf_cleanup_ring(ring); adf_unreserve_ring(bank, ring_num); - accel_dev->hw_device->hw_arb_ring_disable(ring); + adf_update_ring_arb(ring); return ret; } void adf_remove_ring(struct adf_etr_ring_data *ring) { struct adf_etr_bank_data *bank = ring->bank; - struct adf_accel_dev *accel_dev = bank->accel_dev; /* Disable interrupts for the given ring */ adf_disable_ring_irq(bank, ring->ring_number); @@ -322,7 +325,7 @@ void adf_remove_ring(struct adf_etr_ring_data *ring) adf_ring_debugfs_rm(ring); adf_unreserve_ring(bank, ring->ring_number); /* Disable HW arbitration for the given ring */ - accel_dev->hw_device->hw_arb_ring_disable(ring); + adf_update_ring_arb(ring); adf_cleanup_ring(ring); } @@ -463,7 +466,7 @@ err: * acceleration device accel_dev. * To be used by QAT device specific drivers. * - * Return: 0 on success, error code othewise. + * Return: 0 on success, error code otherwise. */ int adf_init_etr_data(struct adf_accel_dev *accel_dev) { diff --git a/drivers/crypto/qat/qat_common/adf_transport_access_macros.h b/drivers/crypto/qat/qat_common/adf_transport_access_macros.h index 160c9a36c919..6ad7e4e1edca 100644 --- a/drivers/crypto/qat/qat_common/adf_transport_access_macros.h +++ b/drivers/crypto/qat/qat_common/adf_transport_access_macros.h @@ -97,8 +97,9 @@ #define ADF_RING_SIZE_IN_BYTES_TO_SIZE(SIZE) ((1 << (SIZE - 1)) >> 7) /* Minimum ring bufer size for memory allocation */ -#define ADF_RING_SIZE_BYTES_MIN(SIZE) ((SIZE < ADF_RING_SIZE_4K) ? \ - ADF_RING_SIZE_4K : SIZE) +#define ADF_RING_SIZE_BYTES_MIN(SIZE) \ + ((SIZE < ADF_SIZE_TO_RING_SIZE_IN_BYTES(ADF_RING_SIZE_4K)) ? \ + ADF_SIZE_TO_RING_SIZE_IN_BYTES(ADF_RING_SIZE_4K) : SIZE) #define ADF_RING_SIZE_MODULO(SIZE) (SIZE + 0x6) #define ADF_SIZE_TO_POW(SIZE) ((((SIZE & 0x4) >> 1) | ((SIZE & 0x4) >> 2) | \ SIZE) & ~0x4) diff --git a/drivers/crypto/qat/qat_common/icp_qat_fw.h b/drivers/crypto/qat/qat_common/icp_qat_fw.h index f1e30e24a419..46747f01b1d1 100644 --- a/drivers/crypto/qat/qat_common/icp_qat_fw.h +++ b/drivers/crypto/qat/qat_common/icp_qat_fw.h @@ -249,6 +249,8 @@ struct icp_qat_fw_comn_resp { #define QAT_COMN_RESP_CRYPTO_STATUS_BITPOS 7 #define QAT_COMN_RESP_CRYPTO_STATUS_MASK 0x1 +#define QAT_COMN_RESP_PKE_STATUS_BITPOS 6 +#define QAT_COMN_RESP_PKE_STATUS_MASK 0x1 #define QAT_COMN_RESP_CMP_STATUS_BITPOS 5 #define QAT_COMN_RESP_CMP_STATUS_MASK 0x1 #define QAT_COMN_RESP_XLAT_STATUS_BITPOS 4 diff --git a/drivers/crypto/qat/qat_common/icp_qat_fw_pke.h b/drivers/crypto/qat/qat_common/icp_qat_fw_pke.h new file mode 100644 index 000000000000..0d7a9b51ce9f --- /dev/null +++ b/drivers/crypto/qat/qat_common/icp_qat_fw_pke.h @@ -0,0 +1,112 @@ +/* + This file is provided under a dual BSD/GPLv2 license. When using or + redistributing this file, you may do so under either license. + + GPL LICENSE SUMMARY + Copyright(c) 2014 Intel Corporation. + This program is free software; you can redistribute it and/or modify + it under the terms of version 2 of the GNU General Public License as + published by the Free Software Foundation. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + Contact Information: + qat-linux@intel.com + + BSD LICENSE + Copyright(c) 2014 Intel Corporation. + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#ifndef _ICP_QAT_FW_PKE_ +#define _ICP_QAT_FW_PKE_ + +#include "icp_qat_fw.h" + +struct icp_qat_fw_req_hdr_pke_cd_pars { + u64 content_desc_addr; + u32 content_desc_resrvd; + u32 func_id; +}; + +struct icp_qat_fw_req_pke_mid { + u64 opaque; + u64 src_data_addr; + u64 dest_data_addr; +}; + +struct icp_qat_fw_req_pke_hdr { + u8 resrvd1; + u8 resrvd2; + u8 service_type; + u8 hdr_flags; + u16 comn_req_flags; + u16 resrvd4; + struct icp_qat_fw_req_hdr_pke_cd_pars cd_pars; +}; + +struct icp_qat_fw_pke_request { + struct icp_qat_fw_req_pke_hdr pke_hdr; + struct icp_qat_fw_req_pke_mid pke_mid; + u8 output_param_count; + u8 input_param_count; + u16 resrvd1; + u32 resrvd2; + u64 next_req_adr; +}; + +struct icp_qat_fw_resp_pke_hdr { + u8 resrvd1; + u8 resrvd2; + u8 response_type; + u8 hdr_flags; + u16 comn_resp_flags; + u16 resrvd4; +}; + +struct icp_qat_fw_pke_resp { + struct icp_qat_fw_resp_pke_hdr pke_resp_hdr; + u64 opaque; + u64 src_data_addr; + u64 dest_data_addr; +}; + +#define ICP_QAT_FW_PKE_HDR_VALID_FLAG_BITPOS 7 +#define ICP_QAT_FW_PKE_HDR_VALID_FLAG_MASK 0x1 +#define ICP_QAT_FW_PKE_RESP_PKE_STAT_GET(status_word) \ + QAT_FIELD_GET(((status_word >> ICP_QAT_FW_COMN_ONE_BYTE_SHIFT) & \ + ICP_QAT_FW_COMN_SINGLE_BYTE_MASK), \ + QAT_COMN_RESP_PKE_STATUS_BITPOS, \ + QAT_COMN_RESP_PKE_STATUS_MASK) + +#define ICP_QAT_FW_PKE_HDR_VALID_FLAG_SET(hdr_t, val) \ + QAT_FIELD_SET((hdr_t.hdr_flags), (val), \ + ICP_QAT_FW_PKE_HDR_VALID_FLAG_BITPOS, \ + ICP_QAT_FW_PKE_HDR_VALID_FLAG_MASK) +#endif diff --git a/drivers/crypto/qat/qat_common/qat_algs.c b/drivers/crypto/qat/qat_common/qat_algs.c index df427c0e9e7b..2bd913aceaeb 100644 --- a/drivers/crypto/qat/qat_common/qat_algs.c +++ b/drivers/crypto/qat/qat_common/qat_algs.c @@ -53,7 +53,6 @@ #include <crypto/hash.h> #include <crypto/algapi.h> #include <crypto/authenc.h> -#include <crypto/rng.h> #include <linux/dma-mapping.h> #include "adf_accel_devices.h" #include "adf_transport.h" @@ -113,9 +112,6 @@ struct qat_alg_aead_ctx { struct crypto_shash *hash_tfm; enum icp_qat_hw_auth_algo qat_hash_alg; struct qat_crypto_instance *inst; - struct crypto_tfm *tfm; - uint8_t salt[AES_BLOCK_SIZE]; - spinlock_t lock; /* protects qat_alg_aead_ctx struct */ }; struct qat_alg_ablkcipher_ctx { @@ -130,11 +126,6 @@ struct qat_alg_ablkcipher_ctx { spinlock_t lock; /* protects qat_alg_ablkcipher_ctx struct */ }; -static int get_current_node(void) -{ - return cpu_data(current_thread_info()->cpu).phys_proc_id; -} - static int qat_get_inter_state_size(enum icp_qat_hw_auth_algo qat_hash_alg) { switch (qat_hash_alg) { @@ -278,12 +269,12 @@ static void qat_alg_init_common_hdr(struct icp_qat_fw_comn_req_hdr *header) ICP_QAT_FW_LA_NO_UPDATE_STATE); } -static int qat_alg_aead_init_enc_session(struct qat_alg_aead_ctx *ctx, +static int qat_alg_aead_init_enc_session(struct crypto_aead *aead_tfm, int alg, struct crypto_authenc_keys *keys) { - struct crypto_aead *aead_tfm = __crypto_aead_cast(ctx->tfm); - unsigned int digestsize = crypto_aead_crt(aead_tfm)->authsize; + struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(aead_tfm); + unsigned int digestsize = crypto_aead_authsize(aead_tfm); struct qat_enc *enc_ctx = &ctx->enc_cd->qat_enc_cd; struct icp_qat_hw_cipher_algo_blk *cipher = &enc_ctx->cipher; struct icp_qat_hw_auth_algo_blk *hash = @@ -358,12 +349,12 @@ static int qat_alg_aead_init_enc_session(struct qat_alg_aead_ctx *ctx, return 0; } -static int qat_alg_aead_init_dec_session(struct qat_alg_aead_ctx *ctx, +static int qat_alg_aead_init_dec_session(struct crypto_aead *aead_tfm, int alg, struct crypto_authenc_keys *keys) { - struct crypto_aead *aead_tfm = __crypto_aead_cast(ctx->tfm); - unsigned int digestsize = crypto_aead_crt(aead_tfm)->authsize; + struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(aead_tfm); + unsigned int digestsize = crypto_aead_authsize(aead_tfm); struct qat_dec *dec_ctx = &ctx->dec_cd->qat_dec_cd; struct icp_qat_hw_auth_algo_blk *hash = &dec_ctx->hash; struct icp_qat_hw_cipher_algo_blk *cipher = @@ -515,30 +506,27 @@ static int qat_alg_validate_key(int key_len, int *alg) return 0; } -static int qat_alg_aead_init_sessions(struct qat_alg_aead_ctx *ctx, +static int qat_alg_aead_init_sessions(struct crypto_aead *tfm, const uint8_t *key, unsigned int keylen) { struct crypto_authenc_keys keys; int alg; - if (crypto_rng_get_bytes(crypto_default_rng, ctx->salt, AES_BLOCK_SIZE)) - return -EFAULT; - if (crypto_authenc_extractkeys(&keys, key, keylen)) goto bad_key; if (qat_alg_validate_key(keys.enckeylen, &alg)) goto bad_key; - if (qat_alg_aead_init_enc_session(ctx, alg, &keys)) + if (qat_alg_aead_init_enc_session(tfm, alg, &keys)) goto error; - if (qat_alg_aead_init_dec_session(ctx, alg, &keys)) + if (qat_alg_aead_init_dec_session(tfm, alg, &keys)) goto error; return 0; bad_key: - crypto_tfm_set_flags(ctx->tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); + crypto_aead_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); return -EINVAL; error: return -EFAULT; @@ -567,7 +555,6 @@ static int qat_alg_aead_setkey(struct crypto_aead *tfm, const uint8_t *key, struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(tfm); struct device *dev; - spin_lock(&ctx->lock); if (ctx->enc_cd) { /* rekeying */ dev = &GET_DEV(ctx->inst->accel_dev); @@ -581,7 +568,6 @@ static int qat_alg_aead_setkey(struct crypto_aead *tfm, const uint8_t *key, struct qat_crypto_instance *inst = qat_crypto_get_instance_node(node); if (!inst) { - spin_unlock(&ctx->lock); return -EINVAL; } @@ -591,19 +577,16 @@ static int qat_alg_aead_setkey(struct crypto_aead *tfm, const uint8_t *key, &ctx->enc_cd_paddr, GFP_ATOMIC); if (!ctx->enc_cd) { - spin_unlock(&ctx->lock); return -ENOMEM; } ctx->dec_cd = dma_zalloc_coherent(dev, sizeof(*ctx->dec_cd), &ctx->dec_cd_paddr, GFP_ATOMIC); if (!ctx->dec_cd) { - spin_unlock(&ctx->lock); goto out_free_enc; } } - spin_unlock(&ctx->lock); - if (qat_alg_aead_init_sessions(ctx, key, keylen)) + if (qat_alg_aead_init_sessions(tfm, key, keylen)) goto out_free_all; return 0; @@ -654,22 +637,20 @@ static void qat_alg_free_bufl(struct qat_crypto_instance *inst, } static int qat_alg_sgl_to_bufl(struct qat_crypto_instance *inst, - struct scatterlist *assoc, int assoclen, struct scatterlist *sgl, - struct scatterlist *sglout, uint8_t *iv, - uint8_t ivlen, + struct scatterlist *sglout, struct qat_crypto_request *qat_req) { struct device *dev = &GET_DEV(inst->accel_dev); - int i, bufs = 0, sg_nctr = 0; - int n = sg_nents(sgl), assoc_n = sg_nents(assoc); + int i, sg_nctr = 0; + int n = sg_nents(sgl); struct qat_alg_buf_list *bufl; struct qat_alg_buf_list *buflout = NULL; dma_addr_t blp; dma_addr_t bloutp = 0; struct scatterlist *sg; size_t sz_out, sz = sizeof(struct qat_alg_buf_list) + - ((1 + n + assoc_n) * sizeof(struct qat_alg_buf)); + ((1 + n) * sizeof(struct qat_alg_buf)); if (unlikely(!n)) return -EINVAL; @@ -683,35 +664,8 @@ static int qat_alg_sgl_to_bufl(struct qat_crypto_instance *inst, if (unlikely(dma_mapping_error(dev, blp))) goto err; - for_each_sg(assoc, sg, assoc_n, i) { - if (!sg->length) - continue; - - if (!(assoclen > 0)) - break; - - bufl->bufers[bufs].addr = - dma_map_single(dev, sg_virt(sg), - min_t(int, assoclen, sg->length), - DMA_BIDIRECTIONAL); - bufl->bufers[bufs].len = min_t(int, assoclen, sg->length); - if (unlikely(dma_mapping_error(dev, bufl->bufers[bufs].addr))) - goto err; - bufs++; - assoclen -= sg->length; - } - - if (ivlen) { - bufl->bufers[bufs].addr = dma_map_single(dev, iv, ivlen, - DMA_BIDIRECTIONAL); - bufl->bufers[bufs].len = ivlen; - if (unlikely(dma_mapping_error(dev, bufl->bufers[bufs].addr))) - goto err; - bufs++; - } - for_each_sg(sgl, sg, n, i) { - int y = sg_nctr + bufs; + int y = sg_nctr; if (!sg->length) continue; @@ -724,7 +678,7 @@ static int qat_alg_sgl_to_bufl(struct qat_crypto_instance *inst, goto err; sg_nctr++; } - bufl->num_bufs = sg_nctr + bufs; + bufl->num_bufs = sg_nctr; qat_req->buf.bl = bufl; qat_req->buf.blp = blp; qat_req->buf.sz = sz; @@ -734,7 +688,7 @@ static int qat_alg_sgl_to_bufl(struct qat_crypto_instance *inst, n = sg_nents(sglout); sz_out = sizeof(struct qat_alg_buf_list) + - ((1 + n + assoc_n) * sizeof(struct qat_alg_buf)); + ((1 + n) * sizeof(struct qat_alg_buf)); sg_nctr = 0; buflout = kzalloc_node(sz_out, GFP_ATOMIC, dev_to_node(&GET_DEV(inst->accel_dev))); @@ -744,14 +698,8 @@ static int qat_alg_sgl_to_bufl(struct qat_crypto_instance *inst, if (unlikely(dma_mapping_error(dev, bloutp))) goto err; bufers = buflout->bufers; - /* For out of place operation dma map only data and - * reuse assoc mapping and iv */ - for (i = 0; i < bufs; i++) { - bufers[i].len = bufl->bufers[i].len; - bufers[i].addr = bufl->bufers[i].addr; - } for_each_sg(sglout, sg, n, i) { - int y = sg_nctr + bufs; + int y = sg_nctr; if (!sg->length) continue; @@ -764,7 +712,7 @@ static int qat_alg_sgl_to_bufl(struct qat_crypto_instance *inst, bufers[y].len = sg->length; sg_nctr++; } - buflout->num_bufs = sg_nctr + bufs; + buflout->num_bufs = sg_nctr; buflout->num_mapped_bufs = sg_nctr; qat_req->buf.blout = buflout; qat_req->buf.bloutp = bloutp; @@ -778,7 +726,7 @@ static int qat_alg_sgl_to_bufl(struct qat_crypto_instance *inst, err: dev_err(dev, "Failed to map buf for dma\n"); sg_nctr = 0; - for (i = 0; i < n + bufs; i++) + for (i = 0; i < n; i++) if (!dma_mapping_error(dev, bufl->bufers[i].addr)) dma_unmap_single(dev, bufl->bufers[i].addr, bufl->bufers[i].len, @@ -789,7 +737,7 @@ err: kfree(bufl); if (sgl != sglout && buflout) { n = sg_nents(sglout); - for (i = bufs; i < n + bufs; i++) + for (i = 0; i < n; i++) if (!dma_mapping_error(dev, buflout->bufers[i].addr)) dma_unmap_single(dev, buflout->bufers[i].addr, buflout->bufers[i].len, @@ -849,12 +797,10 @@ static int qat_alg_aead_dec(struct aead_request *areq) struct icp_qat_fw_la_cipher_req_params *cipher_param; struct icp_qat_fw_la_auth_req_params *auth_param; struct icp_qat_fw_la_bulk_req *msg; - int digst_size = crypto_aead_crt(aead_tfm)->authsize; + int digst_size = crypto_aead_authsize(aead_tfm); int ret, ctr = 0; - ret = qat_alg_sgl_to_bufl(ctx->inst, areq->assoc, areq->assoclen, - areq->src, areq->dst, areq->iv, - AES_BLOCK_SIZE, qat_req); + ret = qat_alg_sgl_to_bufl(ctx->inst, areq->src, areq->dst, qat_req); if (unlikely(ret)) return ret; @@ -868,12 +814,11 @@ static int qat_alg_aead_dec(struct aead_request *areq) qat_req->req.comn_mid.dest_data_addr = qat_req->buf.bloutp; cipher_param = (void *)&qat_req->req.serv_specif_rqpars; cipher_param->cipher_length = areq->cryptlen - digst_size; - cipher_param->cipher_offset = areq->assoclen + AES_BLOCK_SIZE; + cipher_param->cipher_offset = areq->assoclen; memcpy(cipher_param->u.cipher_IV_array, areq->iv, AES_BLOCK_SIZE); auth_param = (void *)((uint8_t *)cipher_param + sizeof(*cipher_param)); auth_param->auth_off = 0; - auth_param->auth_len = areq->assoclen + - cipher_param->cipher_length + AES_BLOCK_SIZE; + auth_param->auth_len = areq->assoclen + cipher_param->cipher_length; do { ret = adf_send_message(ctx->inst->sym_tx, (uint32_t *)msg); } while (ret == -EAGAIN && ctr++ < 10); @@ -885,8 +830,7 @@ static int qat_alg_aead_dec(struct aead_request *areq) return -EINPROGRESS; } -static int qat_alg_aead_enc_internal(struct aead_request *areq, uint8_t *iv, - int enc_iv) +static int qat_alg_aead_enc(struct aead_request *areq) { struct crypto_aead *aead_tfm = crypto_aead_reqtfm(areq); struct crypto_tfm *tfm = crypto_aead_tfm(aead_tfm); @@ -895,11 +839,10 @@ static int qat_alg_aead_enc_internal(struct aead_request *areq, uint8_t *iv, struct icp_qat_fw_la_cipher_req_params *cipher_param; struct icp_qat_fw_la_auth_req_params *auth_param; struct icp_qat_fw_la_bulk_req *msg; + uint8_t *iv = areq->iv; int ret, ctr = 0; - ret = qat_alg_sgl_to_bufl(ctx->inst, areq->assoc, areq->assoclen, - areq->src, areq->dst, iv, AES_BLOCK_SIZE, - qat_req); + ret = qat_alg_sgl_to_bufl(ctx->inst, areq->src, areq->dst, qat_req); if (unlikely(ret)) return ret; @@ -914,16 +857,12 @@ static int qat_alg_aead_enc_internal(struct aead_request *areq, uint8_t *iv, cipher_param = (void *)&qat_req->req.serv_specif_rqpars; auth_param = (void *)((uint8_t *)cipher_param + sizeof(*cipher_param)); - if (enc_iv) { - cipher_param->cipher_length = areq->cryptlen + AES_BLOCK_SIZE; - cipher_param->cipher_offset = areq->assoclen; - } else { - memcpy(cipher_param->u.cipher_IV_array, iv, AES_BLOCK_SIZE); - cipher_param->cipher_length = areq->cryptlen; - cipher_param->cipher_offset = areq->assoclen + AES_BLOCK_SIZE; - } + memcpy(cipher_param->u.cipher_IV_array, iv, AES_BLOCK_SIZE); + cipher_param->cipher_length = areq->cryptlen; + cipher_param->cipher_offset = areq->assoclen; + auth_param->auth_off = 0; - auth_param->auth_len = areq->assoclen + areq->cryptlen + AES_BLOCK_SIZE; + auth_param->auth_len = areq->assoclen + areq->cryptlen; do { ret = adf_send_message(ctx->inst->sym_tx, (uint32_t *)msg); @@ -936,25 +875,6 @@ static int qat_alg_aead_enc_internal(struct aead_request *areq, uint8_t *iv, return -EINPROGRESS; } -static int qat_alg_aead_enc(struct aead_request *areq) -{ - return qat_alg_aead_enc_internal(areq, areq->iv, 0); -} - -static int qat_alg_aead_genivenc(struct aead_givcrypt_request *req) -{ - struct crypto_aead *aead_tfm = crypto_aead_reqtfm(&req->areq); - struct crypto_tfm *tfm = crypto_aead_tfm(aead_tfm); - struct qat_alg_aead_ctx *ctx = crypto_tfm_ctx(tfm); - __be64 seq; - - memcpy(req->giv, ctx->salt, AES_BLOCK_SIZE); - seq = cpu_to_be64(req->seq); - memcpy(req->giv + AES_BLOCK_SIZE - sizeof(uint64_t), - &seq, sizeof(uint64_t)); - return qat_alg_aead_enc_internal(&req->areq, req->giv, 1); -} - static int qat_alg_ablkcipher_setkey(struct crypto_ablkcipher *tfm, const uint8_t *key, unsigned int keylen) @@ -1026,8 +946,7 @@ static int qat_alg_ablkcipher_encrypt(struct ablkcipher_request *req) struct icp_qat_fw_la_bulk_req *msg; int ret, ctr = 0; - ret = qat_alg_sgl_to_bufl(ctx->inst, NULL, 0, req->src, req->dst, - NULL, 0, qat_req); + ret = qat_alg_sgl_to_bufl(ctx->inst, req->src, req->dst, qat_req); if (unlikely(ret)) return ret; @@ -1064,8 +983,7 @@ static int qat_alg_ablkcipher_decrypt(struct ablkcipher_request *req) struct icp_qat_fw_la_bulk_req *msg; int ret, ctr = 0; - ret = qat_alg_sgl_to_bufl(ctx->inst, NULL, 0, req->src, req->dst, - NULL, 0, qat_req); + ret = qat_alg_sgl_to_bufl(ctx->inst, req->src, req->dst, qat_req); if (unlikely(ret)) return ret; @@ -1092,47 +1010,43 @@ static int qat_alg_ablkcipher_decrypt(struct ablkcipher_request *req) return -EINPROGRESS; } -static int qat_alg_aead_init(struct crypto_tfm *tfm, +static int qat_alg_aead_init(struct crypto_aead *tfm, enum icp_qat_hw_auth_algo hash, const char *hash_name) { - struct qat_alg_aead_ctx *ctx = crypto_tfm_ctx(tfm); + struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(tfm); ctx->hash_tfm = crypto_alloc_shash(hash_name, 0, 0); if (IS_ERR(ctx->hash_tfm)) - return -EFAULT; - spin_lock_init(&ctx->lock); + return PTR_ERR(ctx->hash_tfm); ctx->qat_hash_alg = hash; - crypto_aead_set_reqsize(__crypto_aead_cast(tfm), - sizeof(struct aead_request) + - sizeof(struct qat_crypto_request)); - ctx->tfm = tfm; + crypto_aead_set_reqsize(tfm, sizeof(struct aead_request) + + sizeof(struct qat_crypto_request)); return 0; } -static int qat_alg_aead_sha1_init(struct crypto_tfm *tfm) +static int qat_alg_aead_sha1_init(struct crypto_aead *tfm) { return qat_alg_aead_init(tfm, ICP_QAT_HW_AUTH_ALGO_SHA1, "sha1"); } -static int qat_alg_aead_sha256_init(struct crypto_tfm *tfm) +static int qat_alg_aead_sha256_init(struct crypto_aead *tfm) { return qat_alg_aead_init(tfm, ICP_QAT_HW_AUTH_ALGO_SHA256, "sha256"); } -static int qat_alg_aead_sha512_init(struct crypto_tfm *tfm) +static int qat_alg_aead_sha512_init(struct crypto_aead *tfm) { return qat_alg_aead_init(tfm, ICP_QAT_HW_AUTH_ALGO_SHA512, "sha512"); } -static void qat_alg_aead_exit(struct crypto_tfm *tfm) +static void qat_alg_aead_exit(struct crypto_aead *tfm) { - struct qat_alg_aead_ctx *ctx = crypto_tfm_ctx(tfm); + struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(tfm); struct qat_crypto_instance *inst = ctx->inst; struct device *dev; - if (!IS_ERR(ctx->hash_tfm)) - crypto_free_shash(ctx->hash_tfm); + crypto_free_shash(ctx->hash_tfm); if (!inst) return; @@ -1189,73 +1103,61 @@ static void qat_alg_ablkcipher_exit(struct crypto_tfm *tfm) qat_crypto_put_instance(inst); } -static struct crypto_alg qat_algs[] = { { - .cra_name = "authenc(hmac(sha1),cbc(aes))", - .cra_driver_name = "qat_aes_cbc_hmac_sha1", - .cra_priority = 4001, - .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC, - .cra_blocksize = AES_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct qat_alg_aead_ctx), - .cra_alignmask = 0, - .cra_type = &crypto_aead_type, - .cra_module = THIS_MODULE, - .cra_init = qat_alg_aead_sha1_init, - .cra_exit = qat_alg_aead_exit, - .cra_u = { - .aead = { - .setkey = qat_alg_aead_setkey, - .decrypt = qat_alg_aead_dec, - .encrypt = qat_alg_aead_enc, - .givencrypt = qat_alg_aead_genivenc, - .ivsize = AES_BLOCK_SIZE, - .maxauthsize = SHA1_DIGEST_SIZE, - }, + +static struct aead_alg qat_aeads[] = { { + .base = { + .cra_name = "authenc(hmac(sha1),cbc(aes))", + .cra_driver_name = "qat_aes_cbc_hmac_sha1", + .cra_priority = 4001, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct qat_alg_aead_ctx), + .cra_module = THIS_MODULE, }, + .init = qat_alg_aead_sha1_init, + .exit = qat_alg_aead_exit, + .setkey = qat_alg_aead_setkey, + .decrypt = qat_alg_aead_dec, + .encrypt = qat_alg_aead_enc, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, }, { - .cra_name = "authenc(hmac(sha256),cbc(aes))", - .cra_driver_name = "qat_aes_cbc_hmac_sha256", - .cra_priority = 4001, - .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC, - .cra_blocksize = AES_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct qat_alg_aead_ctx), - .cra_alignmask = 0, - .cra_type = &crypto_aead_type, - .cra_module = THIS_MODULE, - .cra_init = qat_alg_aead_sha256_init, - .cra_exit = qat_alg_aead_exit, - .cra_u = { - .aead = { - .setkey = qat_alg_aead_setkey, - .decrypt = qat_alg_aead_dec, - .encrypt = qat_alg_aead_enc, - .givencrypt = qat_alg_aead_genivenc, - .ivsize = AES_BLOCK_SIZE, - .maxauthsize = SHA256_DIGEST_SIZE, - }, + .base = { + .cra_name = "authenc(hmac(sha256),cbc(aes))", + .cra_driver_name = "qat_aes_cbc_hmac_sha256", + .cra_priority = 4001, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct qat_alg_aead_ctx), + .cra_module = THIS_MODULE, }, + .init = qat_alg_aead_sha256_init, + .exit = qat_alg_aead_exit, + .setkey = qat_alg_aead_setkey, + .decrypt = qat_alg_aead_dec, + .encrypt = qat_alg_aead_enc, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, }, { - .cra_name = "authenc(hmac(sha512),cbc(aes))", - .cra_driver_name = "qat_aes_cbc_hmac_sha512", - .cra_priority = 4001, - .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC, - .cra_blocksize = AES_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct qat_alg_aead_ctx), - .cra_alignmask = 0, - .cra_type = &crypto_aead_type, - .cra_module = THIS_MODULE, - .cra_init = qat_alg_aead_sha512_init, - .cra_exit = qat_alg_aead_exit, - .cra_u = { - .aead = { - .setkey = qat_alg_aead_setkey, - .decrypt = qat_alg_aead_dec, - .encrypt = qat_alg_aead_enc, - .givencrypt = qat_alg_aead_genivenc, - .ivsize = AES_BLOCK_SIZE, - .maxauthsize = SHA512_DIGEST_SIZE, - }, + .base = { + .cra_name = "authenc(hmac(sha512),cbc(aes))", + .cra_driver_name = "qat_aes_cbc_hmac_sha512", + .cra_priority = 4001, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct qat_alg_aead_ctx), + .cra_module = THIS_MODULE, }, -}, { + .init = qat_alg_aead_sha512_init, + .exit = qat_alg_aead_exit, + .setkey = qat_alg_aead_setkey, + .decrypt = qat_alg_aead_dec, + .encrypt = qat_alg_aead_enc, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA512_DIGEST_SIZE, +} }; + +static struct crypto_alg qat_algs[] = { { .cra_name = "cbc(aes)", .cra_driver_name = "qat_aes_cbc", .cra_priority = 4001, @@ -1281,42 +1183,54 @@ static struct crypto_alg qat_algs[] = { { int qat_algs_register(void) { - int ret = 0; + int ret = 0, i; mutex_lock(&algs_lock); - if (++active_devs == 1) { - int i; + if (++active_devs != 1) + goto unlock; - for (i = 0; i < ARRAY_SIZE(qat_algs); i++) - qat_algs[i].cra_flags = - (qat_algs[i].cra_type == &crypto_aead_type) ? - CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC : - CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC; + for (i = 0; i < ARRAY_SIZE(qat_algs); i++) + qat_algs[i].cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC; - ret = crypto_register_algs(qat_algs, ARRAY_SIZE(qat_algs)); - } + ret = crypto_register_algs(qat_algs, ARRAY_SIZE(qat_algs)); + if (ret) + goto unlock; + + for (i = 0; i < ARRAY_SIZE(qat_aeads); i++) + qat_aeads[i].base.cra_flags = CRYPTO_ALG_ASYNC; + + ret = crypto_register_aeads(qat_aeads, ARRAY_SIZE(qat_aeads)); + if (ret) + goto unreg_algs; + +unlock: mutex_unlock(&algs_lock); return ret; + +unreg_algs: + crypto_unregister_algs(qat_algs, ARRAY_SIZE(qat_algs)); + goto unlock; } int qat_algs_unregister(void) { - int ret = 0; - mutex_lock(&algs_lock); - if (--active_devs == 0) - ret = crypto_unregister_algs(qat_algs, ARRAY_SIZE(qat_algs)); + if (--active_devs != 0) + goto unlock; + + crypto_unregister_aeads(qat_aeads, ARRAY_SIZE(qat_aeads)); + crypto_unregister_algs(qat_algs, ARRAY_SIZE(qat_algs)); + +unlock: mutex_unlock(&algs_lock); - return ret; + return 0; } int qat_algs_init(void) { - crypto_get_default_rng(); return 0; } void qat_algs_exit(void) { - crypto_put_default_rng(); } diff --git a/drivers/crypto/qat/qat_common/qat_asym_algs.c b/drivers/crypto/qat/qat_common/qat_asym_algs.c new file mode 100644 index 000000000000..e87f51023ba4 --- /dev/null +++ b/drivers/crypto/qat/qat_common/qat_asym_algs.c @@ -0,0 +1,652 @@ +/* + This file is provided under a dual BSD/GPLv2 license. When using or + redistributing this file, you may do so under either license. + + GPL LICENSE SUMMARY + Copyright(c) 2014 Intel Corporation. + This program is free software; you can redistribute it and/or modify + it under the terms of version 2 of the GNU General Public License as + published by the Free Software Foundation. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + Contact Information: + qat-linux@intel.com + + BSD LICENSE + Copyright(c) 2014 Intel Corporation. + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#include <linux/module.h> +#include <crypto/internal/rsa.h> +#include <crypto/internal/akcipher.h> +#include <crypto/akcipher.h> +#include <linux/dma-mapping.h> +#include <linux/fips.h> +#include "qat_rsakey-asn1.h" +#include "icp_qat_fw_pke.h" +#include "adf_accel_devices.h" +#include "adf_transport.h" +#include "adf_common_drv.h" +#include "qat_crypto.h" + +static DEFINE_MUTEX(algs_lock); +static unsigned int active_devs; + +struct qat_rsa_input_params { + union { + struct { + dma_addr_t m; + dma_addr_t e; + dma_addr_t n; + } enc; + struct { + dma_addr_t c; + dma_addr_t d; + dma_addr_t n; + } dec; + u64 in_tab[8]; + }; +} __packed __aligned(64); + +struct qat_rsa_output_params { + union { + struct { + dma_addr_t c; + } enc; + struct { + dma_addr_t m; + } dec; + u64 out_tab[8]; + }; +} __packed __aligned(64); + +struct qat_rsa_ctx { + char *n; + char *e; + char *d; + dma_addr_t dma_n; + dma_addr_t dma_e; + dma_addr_t dma_d; + unsigned int key_sz; + struct qat_crypto_instance *inst; +} __packed __aligned(64); + +struct qat_rsa_request { + struct qat_rsa_input_params in; + struct qat_rsa_output_params out; + dma_addr_t phy_in; + dma_addr_t phy_out; + char *src_align; + struct icp_qat_fw_pke_request req; + struct qat_rsa_ctx *ctx; + int err; +} __aligned(64); + +static void qat_rsa_cb(struct icp_qat_fw_pke_resp *resp) +{ + struct akcipher_request *areq = (void *)(__force long)resp->opaque; + struct qat_rsa_request *req = PTR_ALIGN(akcipher_request_ctx(areq), 64); + struct device *dev = &GET_DEV(req->ctx->inst->accel_dev); + int err = ICP_QAT_FW_PKE_RESP_PKE_STAT_GET( + resp->pke_resp_hdr.comn_resp_flags); + char *ptr = areq->dst; + + err = (err == ICP_QAT_FW_COMN_STATUS_FLAG_OK) ? 0 : -EINVAL; + + if (req->src_align) + dma_free_coherent(dev, req->ctx->key_sz, req->src_align, + req->in.enc.m); + else + dma_unmap_single(dev, req->in.enc.m, req->ctx->key_sz, + DMA_TO_DEVICE); + + dma_unmap_single(dev, req->out.enc.c, req->ctx->key_sz, + DMA_FROM_DEVICE); + dma_unmap_single(dev, req->phy_in, sizeof(struct qat_rsa_input_params), + DMA_TO_DEVICE); + dma_unmap_single(dev, req->phy_out, + sizeof(struct qat_rsa_output_params), + DMA_TO_DEVICE); + + areq->dst_len = req->ctx->key_sz; + /* Need to set the corect length of the output */ + while (!(*ptr) && areq->dst_len) { + areq->dst_len--; + ptr++; + } + + if (areq->dst_len != req->ctx->key_sz) + memmove(areq->dst, ptr, areq->dst_len); + + akcipher_request_complete(areq, err); +} + +void qat_alg_asym_callback(void *_resp) +{ + struct icp_qat_fw_pke_resp *resp = _resp; + + qat_rsa_cb(resp); +} + +#define PKE_RSA_EP_512 0x1c161b21 +#define PKE_RSA_EP_1024 0x35111bf7 +#define PKE_RSA_EP_1536 0x4d111cdc +#define PKE_RSA_EP_2048 0x6e111dba +#define PKE_RSA_EP_3072 0x7d111ea3 +#define PKE_RSA_EP_4096 0xa5101f7e + +static unsigned long qat_rsa_enc_fn_id(unsigned int len) +{ + unsigned int bitslen = len << 3; + + switch (bitslen) { + case 512: + return PKE_RSA_EP_512; + case 1024: + return PKE_RSA_EP_1024; + case 1536: + return PKE_RSA_EP_1536; + case 2048: + return PKE_RSA_EP_2048; + case 3072: + return PKE_RSA_EP_3072; + case 4096: + return PKE_RSA_EP_4096; + default: + return 0; + }; +} + +#define PKE_RSA_DP1_512 0x1c161b3c +#define PKE_RSA_DP1_1024 0x35111c12 +#define PKE_RSA_DP1_1536 0x4d111cf7 +#define PKE_RSA_DP1_2048 0x6e111dda +#define PKE_RSA_DP1_3072 0x7d111ebe +#define PKE_RSA_DP1_4096 0xa5101f98 + +static unsigned long qat_rsa_dec_fn_id(unsigned int len) +{ + unsigned int bitslen = len << 3; + + switch (bitslen) { + case 512: + return PKE_RSA_DP1_512; + case 1024: + return PKE_RSA_DP1_1024; + case 1536: + return PKE_RSA_DP1_1536; + case 2048: + return PKE_RSA_DP1_2048; + case 3072: + return PKE_RSA_DP1_3072; + case 4096: + return PKE_RSA_DP1_4096; + default: + return 0; + }; +} + +static int qat_rsa_enc(struct akcipher_request *req) +{ + struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); + struct qat_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); + struct qat_crypto_instance *inst = ctx->inst; + struct device *dev = &GET_DEV(inst->accel_dev); + struct qat_rsa_request *qat_req = + PTR_ALIGN(akcipher_request_ctx(req), 64); + struct icp_qat_fw_pke_request *msg = &qat_req->req; + int ret, ctr = 0; + + if (unlikely(!ctx->n || !ctx->e)) + return -EINVAL; + + if (req->dst_len < ctx->key_sz) { + req->dst_len = ctx->key_sz; + return -EOVERFLOW; + } + memset(msg, '\0', sizeof(*msg)); + ICP_QAT_FW_PKE_HDR_VALID_FLAG_SET(msg->pke_hdr, + ICP_QAT_FW_COMN_REQ_FLAG_SET); + msg->pke_hdr.cd_pars.func_id = qat_rsa_enc_fn_id(ctx->key_sz); + if (unlikely(!msg->pke_hdr.cd_pars.func_id)) + return -EINVAL; + + qat_req->ctx = ctx; + msg->pke_hdr.service_type = ICP_QAT_FW_COMN_REQ_CPM_FW_PKE; + msg->pke_hdr.comn_req_flags = + ICP_QAT_FW_COMN_FLAGS_BUILD(QAT_COMN_PTR_TYPE_FLAT, + QAT_COMN_CD_FLD_TYPE_64BIT_ADR); + + qat_req->in.enc.e = ctx->dma_e; + qat_req->in.enc.n = ctx->dma_n; + ret = -ENOMEM; + + /* + * src can be of any size in valid range, but HW expects it to be the + * same as modulo n so in case it is different we need to allocate a + * new buf and copy src data. + * In other case we just need to map the user provided buffer. + */ + if (req->src_len < ctx->key_sz) { + int shift = ctx->key_sz - req->src_len; + + qat_req->src_align = dma_zalloc_coherent(dev, ctx->key_sz, + &qat_req->in.enc.m, + GFP_KERNEL); + if (unlikely(!qat_req->src_align)) + return ret; + + memcpy(qat_req->src_align + shift, req->src, req->src_len); + } else { + qat_req->src_align = NULL; + qat_req->in.enc.m = dma_map_single(dev, req->src, req->src_len, + DMA_TO_DEVICE); + } + qat_req->in.in_tab[3] = 0; + qat_req->out.enc.c = dma_map_single(dev, req->dst, req->dst_len, + DMA_FROM_DEVICE); + qat_req->out.out_tab[1] = 0; + qat_req->phy_in = dma_map_single(dev, &qat_req->in.enc.m, + sizeof(struct qat_rsa_input_params), + DMA_TO_DEVICE); + qat_req->phy_out = dma_map_single(dev, &qat_req->out.enc.c, + sizeof(struct qat_rsa_output_params), + DMA_TO_DEVICE); + + if (unlikely((!qat_req->src_align && + dma_mapping_error(dev, qat_req->in.enc.m)) || + dma_mapping_error(dev, qat_req->out.enc.c) || + dma_mapping_error(dev, qat_req->phy_in) || + dma_mapping_error(dev, qat_req->phy_out))) + goto unmap; + + msg->pke_mid.src_data_addr = qat_req->phy_in; + msg->pke_mid.dest_data_addr = qat_req->phy_out; + msg->pke_mid.opaque = (uint64_t)(__force long)req; + msg->input_param_count = 3; + msg->output_param_count = 1; + do { + ret = adf_send_message(ctx->inst->pke_tx, (uint32_t *)msg); + } while (ret == -EBUSY && ctr++ < 100); + + if (!ret) + return -EINPROGRESS; +unmap: + if (qat_req->src_align) + dma_free_coherent(dev, ctx->key_sz, qat_req->src_align, + qat_req->in.enc.m); + else + if (!dma_mapping_error(dev, qat_req->in.enc.m)) + dma_unmap_single(dev, qat_req->in.enc.m, ctx->key_sz, + DMA_TO_DEVICE); + if (!dma_mapping_error(dev, qat_req->out.enc.c)) + dma_unmap_single(dev, qat_req->out.enc.c, ctx->key_sz, + DMA_FROM_DEVICE); + if (!dma_mapping_error(dev, qat_req->phy_in)) + dma_unmap_single(dev, qat_req->phy_in, + sizeof(struct qat_rsa_input_params), + DMA_TO_DEVICE); + if (!dma_mapping_error(dev, qat_req->phy_out)) + dma_unmap_single(dev, qat_req->phy_out, + sizeof(struct qat_rsa_output_params), + DMA_TO_DEVICE); + return ret; +} + +static int qat_rsa_dec(struct akcipher_request *req) +{ + struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); + struct qat_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); + struct qat_crypto_instance *inst = ctx->inst; + struct device *dev = &GET_DEV(inst->accel_dev); + struct qat_rsa_request *qat_req = + PTR_ALIGN(akcipher_request_ctx(req), 64); + struct icp_qat_fw_pke_request *msg = &qat_req->req; + int ret, ctr = 0; + + if (unlikely(!ctx->n || !ctx->d)) + return -EINVAL; + + if (req->dst_len < ctx->key_sz) { + req->dst_len = ctx->key_sz; + return -EOVERFLOW; + } + memset(msg, '\0', sizeof(*msg)); + ICP_QAT_FW_PKE_HDR_VALID_FLAG_SET(msg->pke_hdr, + ICP_QAT_FW_COMN_REQ_FLAG_SET); + msg->pke_hdr.cd_pars.func_id = qat_rsa_dec_fn_id(ctx->key_sz); + if (unlikely(!msg->pke_hdr.cd_pars.func_id)) + return -EINVAL; + + qat_req->ctx = ctx; + msg->pke_hdr.service_type = ICP_QAT_FW_COMN_REQ_CPM_FW_PKE; + msg->pke_hdr.comn_req_flags = + ICP_QAT_FW_COMN_FLAGS_BUILD(QAT_COMN_PTR_TYPE_FLAT, + QAT_COMN_CD_FLD_TYPE_64BIT_ADR); + + qat_req->in.dec.d = ctx->dma_d; + qat_req->in.dec.n = ctx->dma_n; + ret = -ENOMEM; + + /* + * src can be of any size in valid range, but HW expects it to be the + * same as modulo n so in case it is different we need to allocate a + * new buf and copy src data. + * In other case we just need to map the user provided buffer. + */ + if (req->src_len < ctx->key_sz) { + int shift = ctx->key_sz - req->src_len; + + qat_req->src_align = dma_zalloc_coherent(dev, ctx->key_sz, + &qat_req->in.dec.c, + GFP_KERNEL); + if (unlikely(!qat_req->src_align)) + return ret; + + memcpy(qat_req->src_align + shift, req->src, req->src_len); + } else { + qat_req->src_align = NULL; + qat_req->in.dec.c = dma_map_single(dev, req->src, req->src_len, + DMA_TO_DEVICE); + } + qat_req->in.in_tab[3] = 0; + qat_req->out.dec.m = dma_map_single(dev, req->dst, req->dst_len, + DMA_FROM_DEVICE); + qat_req->out.out_tab[1] = 0; + qat_req->phy_in = dma_map_single(dev, &qat_req->in.dec.c, + sizeof(struct qat_rsa_input_params), + DMA_TO_DEVICE); + qat_req->phy_out = dma_map_single(dev, &qat_req->out.dec.m, + sizeof(struct qat_rsa_output_params), + DMA_TO_DEVICE); + + if (unlikely((!qat_req->src_align && + dma_mapping_error(dev, qat_req->in.dec.c)) || + dma_mapping_error(dev, qat_req->out.dec.m) || + dma_mapping_error(dev, qat_req->phy_in) || + dma_mapping_error(dev, qat_req->phy_out))) + goto unmap; + + msg->pke_mid.src_data_addr = qat_req->phy_in; + msg->pke_mid.dest_data_addr = qat_req->phy_out; + msg->pke_mid.opaque = (uint64_t)(__force long)req; + msg->input_param_count = 3; + msg->output_param_count = 1; + do { + ret = adf_send_message(ctx->inst->pke_tx, (uint32_t *)msg); + } while (ret == -EBUSY && ctr++ < 100); + + if (!ret) + return -EINPROGRESS; +unmap: + if (qat_req->src_align) + dma_free_coherent(dev, ctx->key_sz, qat_req->src_align, + qat_req->in.dec.c); + else + if (!dma_mapping_error(dev, qat_req->in.dec.c)) + dma_unmap_single(dev, qat_req->in.dec.c, ctx->key_sz, + DMA_TO_DEVICE); + if (!dma_mapping_error(dev, qat_req->out.dec.m)) + dma_unmap_single(dev, qat_req->out.dec.m, ctx->key_sz, + DMA_FROM_DEVICE); + if (!dma_mapping_error(dev, qat_req->phy_in)) + dma_unmap_single(dev, qat_req->phy_in, + sizeof(struct qat_rsa_input_params), + DMA_TO_DEVICE); + if (!dma_mapping_error(dev, qat_req->phy_out)) + dma_unmap_single(dev, qat_req->phy_out, + sizeof(struct qat_rsa_output_params), + DMA_TO_DEVICE); + return ret; +} + +int qat_rsa_get_n(void *context, size_t hdrlen, unsigned char tag, + const void *value, size_t vlen) +{ + struct qat_rsa_ctx *ctx = context; + struct qat_crypto_instance *inst = ctx->inst; + struct device *dev = &GET_DEV(inst->accel_dev); + const char *ptr = value; + int ret; + + while (!*ptr && vlen) { + ptr++; + vlen--; + } + + ctx->key_sz = vlen; + ret = -EINVAL; + /* In FIPS mode only allow key size 2K & 3K */ + if (fips_enabled && (ctx->key_sz != 256 && ctx->key_sz != 384)) { + pr_err("QAT: RSA: key size not allowed in FIPS mode\n"); + goto err; + } + /* invalid key size provided */ + if (!qat_rsa_enc_fn_id(ctx->key_sz)) + goto err; + + ret = -ENOMEM; + ctx->n = dma_zalloc_coherent(dev, ctx->key_sz, &ctx->dma_n, GFP_KERNEL); + if (!ctx->n) + goto err; + + memcpy(ctx->n, ptr, ctx->key_sz); + return 0; +err: + ctx->key_sz = 0; + ctx->n = NULL; + return ret; +} + +int qat_rsa_get_e(void *context, size_t hdrlen, unsigned char tag, + const void *value, size_t vlen) +{ + struct qat_rsa_ctx *ctx = context; + struct qat_crypto_instance *inst = ctx->inst; + struct device *dev = &GET_DEV(inst->accel_dev); + const char *ptr = value; + + while (!*ptr && vlen) { + ptr++; + vlen--; + } + + if (!ctx->key_sz || !vlen || vlen > ctx->key_sz) { + ctx->e = NULL; + return -EINVAL; + } + + ctx->e = dma_zalloc_coherent(dev, ctx->key_sz, &ctx->dma_e, GFP_KERNEL); + if (!ctx->e) { + ctx->e = NULL; + return -ENOMEM; + } + memcpy(ctx->e + (ctx->key_sz - vlen), ptr, vlen); + return 0; +} + +int qat_rsa_get_d(void *context, size_t hdrlen, unsigned char tag, + const void *value, size_t vlen) +{ + struct qat_rsa_ctx *ctx = context; + struct qat_crypto_instance *inst = ctx->inst; + struct device *dev = &GET_DEV(inst->accel_dev); + const char *ptr = value; + int ret; + + while (!*ptr && vlen) { + ptr++; + vlen--; + } + + ret = -EINVAL; + if (!ctx->key_sz || !vlen || vlen > ctx->key_sz) + goto err; + + /* In FIPS mode only allow key size 2K & 3K */ + if (fips_enabled && (vlen != 256 && vlen != 384)) { + pr_err("QAT: RSA: key size not allowed in FIPS mode\n"); + goto err; + } + + ret = -ENOMEM; + ctx->d = dma_zalloc_coherent(dev, ctx->key_sz, &ctx->dma_d, GFP_KERNEL); + if (!ctx->n) + goto err; + + memcpy(ctx->d + (ctx->key_sz - vlen), ptr, vlen); + return 0; +err: + ctx->d = NULL; + return ret; +} + +static int qat_rsa_setkey(struct crypto_akcipher *tfm, const void *key, + unsigned int keylen) +{ + struct qat_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); + struct device *dev = &GET_DEV(ctx->inst->accel_dev); + int ret; + + /* Free the old key if any */ + if (ctx->n) + dma_free_coherent(dev, ctx->key_sz, ctx->n, ctx->dma_n); + if (ctx->e) + dma_free_coherent(dev, ctx->key_sz, ctx->e, ctx->dma_e); + if (ctx->d) { + memset(ctx->d, '\0', ctx->key_sz); + dma_free_coherent(dev, ctx->key_sz, ctx->d, ctx->dma_d); + } + + ctx->n = NULL; + ctx->e = NULL; + ctx->d = NULL; + ret = asn1_ber_decoder(&qat_rsakey_decoder, ctx, key, keylen); + if (ret < 0) + goto free; + + if (!ctx->n || !ctx->e) { + /* invalid key provided */ + ret = -EINVAL; + goto free; + } + + return 0; +free: + if (ctx->d) { + memset(ctx->d, '\0', ctx->key_sz); + dma_free_coherent(dev, ctx->key_sz, ctx->d, ctx->dma_d); + ctx->d = NULL; + } + if (ctx->e) { + dma_free_coherent(dev, ctx->key_sz, ctx->e, ctx->dma_e); + ctx->e = NULL; + } + if (ctx->n) { + dma_free_coherent(dev, ctx->key_sz, ctx->n, ctx->dma_n); + ctx->n = NULL; + ctx->key_sz = 0; + } + return ret; +} + +static int qat_rsa_init_tfm(struct crypto_akcipher *tfm) +{ + struct qat_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); + struct qat_crypto_instance *inst = + qat_crypto_get_instance_node(get_current_node()); + + if (!inst) + return -EINVAL; + + ctx->key_sz = 0; + ctx->inst = inst; + return 0; +} + +static void qat_rsa_exit_tfm(struct crypto_akcipher *tfm) +{ + struct qat_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); + struct device *dev = &GET_DEV(ctx->inst->accel_dev); + + if (ctx->n) + dma_free_coherent(dev, ctx->key_sz, ctx->n, ctx->dma_n); + if (ctx->e) + dma_free_coherent(dev, ctx->key_sz, ctx->e, ctx->dma_e); + if (ctx->d) { + memset(ctx->d, '\0', ctx->key_sz); + dma_free_coherent(dev, ctx->key_sz, ctx->d, ctx->dma_d); + } + qat_crypto_put_instance(ctx->inst); + ctx->n = NULL; + ctx->d = NULL; + ctx->d = NULL; +} + +static struct akcipher_alg rsa = { + .encrypt = qat_rsa_enc, + .decrypt = qat_rsa_dec, + .sign = qat_rsa_dec, + .verify = qat_rsa_enc, + .setkey = qat_rsa_setkey, + .init = qat_rsa_init_tfm, + .exit = qat_rsa_exit_tfm, + .reqsize = sizeof(struct qat_rsa_request) + 64, + .base = { + .cra_name = "rsa", + .cra_driver_name = "qat-rsa", + .cra_priority = 1000, + .cra_module = THIS_MODULE, + .cra_ctxsize = sizeof(struct qat_rsa_ctx), + }, +}; + +int qat_asym_algs_register(void) +{ + int ret = 0; + + mutex_lock(&algs_lock); + if (++active_devs == 1) { + rsa.base.cra_flags = 0; + ret = crypto_register_akcipher(&rsa); + } + mutex_unlock(&algs_lock); + return ret; +} + +void qat_asym_algs_unregister(void) +{ + mutex_lock(&algs_lock); + if (--active_devs == 0) + crypto_unregister_akcipher(&rsa); + mutex_unlock(&algs_lock); +} diff --git a/drivers/crypto/qat/qat_common/qat_crypto.c b/drivers/crypto/qat/qat_common/qat_crypto.c index 3bd705ca5973..07c2f9f9d1fc 100644 --- a/drivers/crypto/qat/qat_common/qat_crypto.c +++ b/drivers/crypto/qat/qat_common/qat_crypto.c @@ -88,12 +88,6 @@ static int qat_crypto_free_instances(struct adf_accel_dev *accel_dev) if (inst->pke_rx) adf_remove_ring(inst->pke_rx); - if (inst->rnd_tx) - adf_remove_ring(inst->rnd_tx); - - if (inst->rnd_rx) - adf_remove_ring(inst->rnd_rx); - list_del(list_ptr); kfree(inst); } @@ -109,9 +103,11 @@ struct qat_crypto_instance *qat_crypto_get_instance_node(int node) list_for_each(itr, adf_devmgr_get_head()) { accel_dev = list_entry(itr, struct adf_accel_dev, list); + if ((node == dev_to_node(&GET_DEV(accel_dev)) || dev_to_node(&GET_DEV(accel_dev)) < 0) && - adf_dev_started(accel_dev)) + adf_dev_started(accel_dev) && + !list_empty(&accel_dev->crypto_list)) break; accel_dev = NULL; } @@ -158,7 +154,6 @@ static int qat_crypto_create_instances(struct adf_accel_dev *accel_dev) INIT_LIST_HEAD(&accel_dev->crypto_list); strlcpy(key, ADF_NUM_CY, sizeof(key)); - if (adf_cfg_get_param_value(accel_dev, SEC, key, val)) return -EFAULT; @@ -187,7 +182,9 @@ static int qat_crypto_create_instances(struct adf_accel_dev *accel_dev) if (kstrtoul(val, 10, &num_msg_sym)) goto err; + num_msg_sym = num_msg_sym >> 1; + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_SIZE, i); if (adf_cfg_get_param_value(accel_dev, SEC, key, val)) goto err; @@ -202,11 +199,6 @@ static int qat_crypto_create_instances(struct adf_accel_dev *accel_dev) msg_size, key, NULL, 0, &inst->sym_tx)) goto err; - snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_RND_TX, i); - if (adf_create_ring(accel_dev, SEC, bank, num_msg_asym, - msg_size, key, NULL, 0, &inst->rnd_tx)) - goto err; - msg_size = msg_size >> 1; snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_TX, i); if (adf_create_ring(accel_dev, SEC, bank, num_msg_asym, @@ -220,15 +212,9 @@ static int qat_crypto_create_instances(struct adf_accel_dev *accel_dev) &inst->sym_rx)) goto err; - snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_RND_RX, i); - if (adf_create_ring(accel_dev, SEC, bank, num_msg_asym, - msg_size, key, qat_alg_callback, 0, - &inst->rnd_rx)) - goto err; - snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_RX, i); if (adf_create_ring(accel_dev, SEC, bank, num_msg_asym, - msg_size, key, qat_alg_callback, 0, + msg_size, key, qat_alg_asym_callback, 0, &inst->pke_rx)) goto err; } diff --git a/drivers/crypto/qat/qat_common/qat_crypto.h b/drivers/crypto/qat/qat_common/qat_crypto.h index d503007b49e6..dc0273fe3620 100644 --- a/drivers/crypto/qat/qat_common/qat_crypto.h +++ b/drivers/crypto/qat/qat_common/qat_crypto.h @@ -57,8 +57,6 @@ struct qat_crypto_instance { struct adf_etr_ring_data *sym_rx; struct adf_etr_ring_data *pke_tx; struct adf_etr_ring_data *pke_rx; - struct adf_etr_ring_data *rnd_tx; - struct adf_etr_ring_data *rnd_rx; struct adf_accel_dev *accel_dev; struct list_head list; unsigned long state; diff --git a/drivers/crypto/qat/qat_common/qat_hal.c b/drivers/crypto/qat/qat_common/qat_hal.c index 274ff7e9de6e..8e711d1c3084 100644 --- a/drivers/crypto/qat/qat_common/qat_hal.c +++ b/drivers/crypto/qat/qat_common/qat_hal.c @@ -671,7 +671,6 @@ static int qat_hal_clear_gpr(struct icp_qat_fw_loader_handle *handle) #define ICP_DH895XCC_CAP_OFFSET (ICP_DH895XCC_AE_OFFSET + 0x10000) #define LOCAL_TO_XFER_REG_OFFSET 0x800 #define ICP_DH895XCC_EP_OFFSET 0x3a000 -#define ICP_DH895XCC_PMISC_BAR 1 int qat_hal_init(struct adf_accel_dev *accel_dev) { unsigned char ae; @@ -679,21 +678,24 @@ int qat_hal_init(struct adf_accel_dev *accel_dev) struct icp_qat_fw_loader_handle *handle; struct adf_accel_pci *pci_info = &accel_dev->accel_pci_dev; struct adf_hw_device_data *hw_data = accel_dev->hw_device; - struct adf_bar *bar = + struct adf_bar *misc_bar = &pci_info->pci_bars[hw_data->get_misc_bar_id(hw_data)]; + struct adf_bar *sram_bar = + &pci_info->pci_bars[hw_data->get_sram_bar_id(hw_data)]; handle = kzalloc(sizeof(*handle), GFP_KERNEL); if (!handle) return -ENOMEM; - handle->hal_cap_g_ctl_csr_addr_v = bar->virt_addr + + handle->hal_cap_g_ctl_csr_addr_v = misc_bar->virt_addr + ICP_DH895XCC_CAP_OFFSET; - handle->hal_cap_ae_xfer_csr_addr_v = bar->virt_addr + + handle->hal_cap_ae_xfer_csr_addr_v = misc_bar->virt_addr + ICP_DH895XCC_AE_OFFSET; - handle->hal_ep_csr_addr_v = bar->virt_addr + ICP_DH895XCC_EP_OFFSET; + handle->hal_ep_csr_addr_v = misc_bar->virt_addr + + ICP_DH895XCC_EP_OFFSET; handle->hal_cap_ae_local_csr_addr_v = handle->hal_cap_ae_xfer_csr_addr_v + LOCAL_TO_XFER_REG_OFFSET; - + handle->hal_sram_addr_v = sram_bar->virt_addr; handle->hal_handle = kzalloc(sizeof(*handle->hal_handle), GFP_KERNEL); if (!handle->hal_handle) goto out_hal_handle; diff --git a/drivers/crypto/qat/qat_common/qat_rsakey.asn1 b/drivers/crypto/qat/qat_common/qat_rsakey.asn1 new file mode 100644 index 000000000000..97b0e02b600a --- /dev/null +++ b/drivers/crypto/qat/qat_common/qat_rsakey.asn1 @@ -0,0 +1,5 @@ +RsaKey ::= SEQUENCE { + n INTEGER ({ qat_rsa_get_n }), + e INTEGER ({ qat_rsa_get_e }), + d INTEGER ({ qat_rsa_get_d }) +} diff --git a/drivers/crypto/qat/qat_common/qat_uclo.c b/drivers/crypto/qat/qat_common/qat_uclo.c index 1e27f9f7fddf..c48f181e8941 100644 --- a/drivers/crypto/qat/qat_common/qat_uclo.c +++ b/drivers/crypto/qat/qat_common/qat_uclo.c @@ -359,28 +359,7 @@ static int qat_uclo_init_umem_seg(struct icp_qat_fw_loader_handle *handle, static int qat_uclo_init_ae_memory(struct icp_qat_fw_loader_handle *handle, struct icp_qat_uof_initmem *init_mem) { - unsigned int i; - struct icp_qat_uof_memvar_attr *mem_val_attr; - - mem_val_attr = - (struct icp_qat_uof_memvar_attr *)((unsigned long)init_mem + - sizeof(struct icp_qat_uof_initmem)); - switch (init_mem->region) { - case ICP_QAT_UOF_SRAM_REGION: - if ((init_mem->addr + init_mem->num_in_bytes) > - ICP_DH895XCC_PESRAM_BAR_SIZE) { - pr_err("QAT: initmem on SRAM is out of range"); - return -EINVAL; - } - for (i = 0; i < init_mem->val_attr_num; i++) { - qat_uclo_wr_sram_by_words(handle, - init_mem->addr + - mem_val_attr->offset_in_byte, - &mem_val_attr->value, 4); - mem_val_attr++; - } - break; case ICP_QAT_UOF_LMEM_REGION: if (qat_uclo_init_lmem_seg(handle, init_mem)) return -EINVAL; @@ -990,6 +969,12 @@ out_err: return -EFAULT; } +void qat_uclo_wr_mimage(struct icp_qat_fw_loader_handle *handle, + void *addr_ptr, int mem_size) +{ + qat_uclo_wr_sram_by_words(handle, 0, addr_ptr, ALIGN(mem_size, 4)); +} + int qat_uclo_map_uof_obj(struct icp_qat_fw_loader_handle *handle, void *addr_ptr, int mem_size) { diff --git a/drivers/crypto/qat/qat_dh895xcc/Makefile b/drivers/crypto/qat/qat_dh895xcc/Makefile index 25171c557043..8c79c543740f 100644 --- a/drivers/crypto/qat/qat_dh895xcc/Makefile +++ b/drivers/crypto/qat/qat_dh895xcc/Makefile @@ -2,7 +2,4 @@ ccflags-y := -I$(src)/../qat_common obj-$(CONFIG_CRYPTO_DEV_QAT_DH895xCC) += qat_dh895xcc.o qat_dh895xcc-objs := adf_drv.o \ adf_isr.o \ - adf_dh895xcc_hw_data.o \ - adf_hw_arbiter.o \ - qat_admin.o \ - adf_admin.o + adf_dh895xcc_hw_data.o diff --git a/drivers/crypto/qat/qat_dh895xcc/adf_admin.c b/drivers/crypto/qat/qat_dh895xcc/adf_admin.c deleted file mode 100644 index e4666065c399..000000000000 --- a/drivers/crypto/qat/qat_dh895xcc/adf_admin.c +++ /dev/null @@ -1,145 +0,0 @@ -/* - This file is provided under a dual BSD/GPLv2 license. When using or - redistributing this file, you may do so under either license. - - GPL LICENSE SUMMARY - Copyright(c) 2014 Intel Corporation. - This program is free software; you can redistribute it and/or modify - it under the terms of version 2 of the GNU General Public License as - published by the Free Software Foundation. - - This program is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - General Public License for more details. - - Contact Information: - qat-linux@intel.com - - BSD LICENSE - Copyright(c) 2014 Intel Corporation. - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in - the documentation and/or other materials provided with the - distribution. - * Neither the name of Intel Corporation nor the names of its - contributors may be used to endorse or promote products derived - from this software without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ -#include <linux/types.h> -#include <linux/mutex.h> -#include <linux/slab.h> -#include <linux/delay.h> -#include <linux/pci.h> -#include <linux/dma-mapping.h> -#include <adf_accel_devices.h> -#include "adf_drv.h" -#include "adf_dh895xcc_hw_data.h" - -#define ADF_ADMINMSG_LEN 32 - -struct adf_admin_comms { - dma_addr_t phy_addr; - void *virt_addr; - void __iomem *mailbox_addr; - struct mutex lock; /* protects adf_admin_comms struct */ -}; - -int adf_put_admin_msg_sync(struct adf_accel_dev *accel_dev, - uint32_t ae, void *in, void *out) -{ - struct adf_admin_comms *admin = accel_dev->admin; - int offset = ae * ADF_ADMINMSG_LEN * 2; - void __iomem *mailbox = admin->mailbox_addr; - int mb_offset = ae * ADF_DH895XCC_MAILBOX_STRIDE; - int times, received; - - mutex_lock(&admin->lock); - - if (ADF_CSR_RD(mailbox, mb_offset) == 1) { - mutex_unlock(&admin->lock); - return -EAGAIN; - } - - memcpy(admin->virt_addr + offset, in, ADF_ADMINMSG_LEN); - ADF_CSR_WR(mailbox, mb_offset, 1); - received = 0; - for (times = 0; times < 50; times++) { - msleep(20); - if (ADF_CSR_RD(mailbox, mb_offset) == 0) { - received = 1; - break; - } - } - if (received) - memcpy(out, admin->virt_addr + offset + - ADF_ADMINMSG_LEN, ADF_ADMINMSG_LEN); - else - dev_err(&GET_DEV(accel_dev), - "Failed to send admin msg to accelerator\n"); - - mutex_unlock(&admin->lock); - return received ? 0 : -EFAULT; -} - -int adf_init_admin_comms(struct adf_accel_dev *accel_dev) -{ - struct adf_admin_comms *admin; - struct adf_bar *pmisc = &GET_BARS(accel_dev)[ADF_DH895XCC_PMISC_BAR]; - void __iomem *csr = pmisc->virt_addr; - void __iomem *mailbox = csr + ADF_DH895XCC_MAILBOX_BASE_OFFSET; - uint64_t reg_val; - - admin = kzalloc_node(sizeof(*accel_dev->admin), GFP_KERNEL, - dev_to_node(&GET_DEV(accel_dev))); - if (!admin) - return -ENOMEM; - admin->virt_addr = dma_zalloc_coherent(&GET_DEV(accel_dev), PAGE_SIZE, - &admin->phy_addr, GFP_KERNEL); - if (!admin->virt_addr) { - dev_err(&GET_DEV(accel_dev), "Failed to allocate dma buff\n"); - kfree(admin); - return -ENOMEM; - } - reg_val = (uint64_t)admin->phy_addr; - ADF_CSR_WR(csr, ADF_DH895XCC_ADMINMSGUR_OFFSET, reg_val >> 32); - ADF_CSR_WR(csr, ADF_DH895XCC_ADMINMSGLR_OFFSET, reg_val); - mutex_init(&admin->lock); - admin->mailbox_addr = mailbox; - accel_dev->admin = admin; - return 0; -} - -void adf_exit_admin_comms(struct adf_accel_dev *accel_dev) -{ - struct adf_admin_comms *admin = accel_dev->admin; - - if (!admin) - return; - - if (admin->virt_addr) - dma_free_coherent(&GET_DEV(accel_dev), PAGE_SIZE, - admin->virt_addr, admin->phy_addr); - - mutex_destroy(&admin->lock); - kfree(admin); - accel_dev->admin = NULL; -} diff --git a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c index b1386922d7a2..ff54257eced4 100644 --- a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c +++ b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c @@ -45,8 +45,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include <adf_accel_devices.h> +#include <adf_pf2vf_msg.h> +#include <adf_common_drv.h> #include "adf_dh895xcc_hw_data.h" -#include "adf_common_drv.h" #include "adf_drv.h" /* Worker thread to service arbiter mappings based on dev SKUs */ @@ -117,6 +118,11 @@ static uint32_t get_etr_bar_id(struct adf_hw_device_data *self) return ADF_DH895XCC_ETR_BAR; } +static uint32_t get_sram_bar_id(struct adf_hw_device_data *self) +{ + return ADF_DH895XCC_SRAM_BAR; +} + static enum dev_sku_info get_sku(struct adf_hw_device_data *self) { int sku = (self->fuses & ADF_DH895XCC_FUSECTL_SKU_MASK) @@ -156,6 +162,16 @@ void adf_get_arbiter_mapping(struct adf_accel_dev *accel_dev, } } +static uint32_t get_pf2vf_offset(uint32_t i) +{ + return ADF_DH895XCC_PF2VF_OFFSET(i); +} + +static uint32_t get_vintmsk_offset(uint32_t i) +{ + return ADF_DH895XCC_VINTMSK_OFFSET(i); +} + static void adf_enable_error_correction(struct adf_accel_dev *accel_dev) { struct adf_hw_device_data *hw_device = accel_dev->hw_device; @@ -192,18 +208,23 @@ static void adf_enable_ints(struct adf_accel_dev *accel_dev) /* Enable bundle and misc interrupts */ ADF_CSR_WR(addr, ADF_DH895XCC_SMIAPF0_MASK_OFFSET, - ADF_DH895XCC_SMIA0_MASK); + accel_dev->pf.vf_info ? 0 : + GENMASK_ULL(GET_MAX_BANKS(accel_dev) - 1, 0)); ADF_CSR_WR(addr, ADF_DH895XCC_SMIAPF1_MASK_OFFSET, ADF_DH895XCC_SMIA1_MASK); } +static int adf_pf_enable_vf2pf_comms(struct adf_accel_dev *accel_dev) +{ + return 0; +} + void adf_init_hw_data_dh895xcc(struct adf_hw_device_data *hw_data) { hw_data->dev_class = &dh895xcc_class; hw_data->instance_id = dh895xcc_class.instances++; hw_data->num_banks = ADF_DH895XCC_ETR_MAX_BANKS; hw_data->num_accel = ADF_DH895XCC_MAX_ACCELERATORS; - hw_data->pci_dev_id = ADF_DH895XCC_PCI_DEVICE_ID; hw_data->num_logical_accel = 1; hw_data->num_engines = ADF_DH895XCC_MAX_ACCELENGINES; hw_data->tx_rx_gap = ADF_DH895XCC_RX_RINGS_OFFSET; @@ -211,21 +232,28 @@ void adf_init_hw_data_dh895xcc(struct adf_hw_device_data *hw_data) hw_data->alloc_irq = adf_isr_resource_alloc; hw_data->free_irq = adf_isr_resource_free; hw_data->enable_error_correction = adf_enable_error_correction; - hw_data->hw_arb_ring_enable = adf_update_ring_arb_enable; - hw_data->hw_arb_ring_disable = adf_update_ring_arb_enable; hw_data->get_accel_mask = get_accel_mask; hw_data->get_ae_mask = get_ae_mask; hw_data->get_num_accels = get_num_accels; hw_data->get_num_aes = get_num_aes; hw_data->get_etr_bar_id = get_etr_bar_id; hw_data->get_misc_bar_id = get_misc_bar_id; + hw_data->get_pf2vf_offset = get_pf2vf_offset; + hw_data->get_vintmsk_offset = get_vintmsk_offset; + hw_data->get_sram_bar_id = get_sram_bar_id; hw_data->get_sku = get_sku; hw_data->fw_name = ADF_DH895XCC_FW; + hw_data->fw_mmp_name = ADF_DH895XCC_MMP; hw_data->init_admin_comms = adf_init_admin_comms; hw_data->exit_admin_comms = adf_exit_admin_comms; + hw_data->disable_iov = adf_disable_sriov; + hw_data->send_admin_init = adf_send_admin_init; hw_data->init_arb = adf_init_arb; hw_data->exit_arb = adf_exit_arb; + hw_data->get_arb_mapping = adf_get_arbiter_mapping; hw_data->enable_ints = adf_enable_ints; + hw_data->enable_vf2pf_comms = adf_pf_enable_vf2pf_comms; + hw_data->min_iov_compat_ver = ADF_PFVF_COMPATIBILITY_VERSION; } void adf_clean_hw_data_dh895xcc(struct adf_hw_device_data *hw_data) diff --git a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.h b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.h index 25269a9f24a2..88dffb297346 100644 --- a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.h +++ b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.h @@ -48,6 +48,7 @@ #define ADF_DH895x_HW_DATA_H_ /* PCIe configuration space */ +#define ADF_DH895XCC_SRAM_BAR 0 #define ADF_DH895XCC_PMISC_BAR 1 #define ADF_DH895XCC_ETR_BAR 2 #define ADF_DH895XCC_RX_RINGS_OFFSET 8 @@ -79,10 +80,11 @@ #define ADF_DH895XCC_CERRSSMSH(i) (i * 0x4000 + 0x10) #define ADF_DH895XCC_ERRSSMSH_EN BIT(3) -/* Admin Messages Registers */ -#define ADF_DH895XCC_ADMINMSGUR_OFFSET (0x3A000 + 0x574) -#define ADF_DH895XCC_ADMINMSGLR_OFFSET (0x3A000 + 0x578) -#define ADF_DH895XCC_MAILBOX_BASE_OFFSET 0x20970 -#define ADF_DH895XCC_MAILBOX_STRIDE 0x1000 +#define ADF_DH895XCC_ERRSOU3 (0x3A000 + 0x00C) +#define ADF_DH895XCC_ERRSOU5 (0x3A000 + 0x0D8) +#define ADF_DH895XCC_PF2VF_OFFSET(i) (0x3A000 + 0x280 + ((i) * 0x04)) +#define ADF_DH895XCC_VINTMSK_OFFSET(i) (0x3A000 + 0x200 + ((i) * 0x04)) +/* FW names */ #define ADF_DH895XCC_FW "qat_895xcc.bin" +#define ADF_DH895XCC_MMP "qat_mmp.bin" #endif diff --git a/drivers/crypto/qat/qat_dh895xcc/adf_drv.c b/drivers/crypto/qat/qat_dh895xcc/adf_drv.c index 1bde45b7a3c5..f8dd14f232c8 100644 --- a/drivers/crypto/qat/qat_dh895xcc/adf_drv.c +++ b/drivers/crypto/qat/qat_dh895xcc/adf_drv.c @@ -82,16 +82,21 @@ static struct pci_driver adf_driver = { .id_table = adf_pci_tbl, .name = adf_driver_name, .probe = adf_probe, - .remove = adf_remove + .remove = adf_remove, + .sriov_configure = adf_sriov_configure, }; +static void adf_cleanup_pci_dev(struct adf_accel_dev *accel_dev) +{ + pci_release_regions(accel_dev->accel_pci_dev.pci_dev); + pci_disable_device(accel_dev->accel_pci_dev.pci_dev); +} + static void adf_cleanup_accel(struct adf_accel_dev *accel_dev) { struct adf_accel_pci *accel_pci_dev = &accel_dev->accel_pci_dev; int i; - adf_dev_shutdown(accel_dev); - for (i = 0; i < ADF_PCI_MAX_BARS; i++) { struct adf_bar *bar = &accel_pci_dev->pci_bars[i]; @@ -100,7 +105,7 @@ static void adf_cleanup_accel(struct adf_accel_dev *accel_dev) } if (accel_dev->hw_device) { - switch (accel_dev->hw_device->pci_dev_id) { + switch (accel_pci_dev->pci_dev->device) { case ADF_DH895XCC_PCI_DEVICE_ID: adf_clean_hw_data_dh895xcc(accel_dev->hw_device); break; @@ -108,13 +113,11 @@ static void adf_cleanup_accel(struct adf_accel_dev *accel_dev) break; } kfree(accel_dev->hw_device); + accel_dev->hw_device = NULL; } adf_cfg_dev_remove(accel_dev); debugfs_remove(accel_dev->debugfs_dir); - adf_devmgr_rm_dev(accel_dev); - pci_release_regions(accel_pci_dev->pci_dev); - pci_disable_device(accel_pci_dev->pci_dev); - kfree(accel_dev); + adf_devmgr_rm_dev(accel_dev, NULL); } static int adf_dev_configure(struct adf_accel_dev *accel_dev) @@ -167,12 +170,6 @@ static int adf_dev_configure(struct adf_accel_dev *accel_dev) key, (void *)&val, ADF_DEC)) goto err; - val = 4; - snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_RND_TX, i); - if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, (void *)&val, ADF_DEC)) - goto err; - val = 8; snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_RX, i); if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, @@ -185,12 +182,6 @@ static int adf_dev_configure(struct adf_accel_dev *accel_dev) key, (void *)&val, ADF_DEC)) goto err; - val = 12; - snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_RND_RX, i); - if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, (void *)&val, ADF_DEC)) - goto err; - val = ADF_COALESCING_DEF_TIME; snprintf(key, sizeof(key), ADF_ETRMGR_COALESCE_TIMER_FORMAT, i); if (adf_cfg_add_key_value_param(accel_dev, "Accelerator0", @@ -217,7 +208,7 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) struct adf_hw_device_data *hw_data; char name[ADF_DEVICE_NAME_LENGTH]; unsigned int i, bar_nr; - int ret; + int ret, bar_mask; switch (ent->device) { case ADF_DH895XCC_PCI_DEVICE_ID: @@ -241,10 +232,12 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) return -ENOMEM; INIT_LIST_HEAD(&accel_dev->crypto_list); + accel_pci_dev = &accel_dev->accel_pci_dev; + accel_pci_dev->pci_dev = pdev; /* Add accel device to accel table. * This should be called before adf_cleanup_accel is called */ - if (adf_devmgr_add_dev(accel_dev)) { + if (adf_devmgr_add_dev(accel_dev, NULL)) { dev_err(&pdev->dev, "Failed to add new accelerator device.\n"); kfree(accel_dev); return -EFAULT; @@ -267,7 +260,6 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) default: return -ENODEV; } - accel_pci_dev = &accel_dev->accel_pci_dev; pci_read_config_byte(pdev, PCI_REVISION_ID, &accel_pci_dev->revid); pci_read_config_dword(pdev, ADF_DH895XCC_FUSECTL_OFFSET, &hw_data->fuses); @@ -276,7 +268,6 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) hw_data->accel_mask = hw_data->get_accel_mask(hw_data->fuses); hw_data->ae_mask = hw_data->get_ae_mask(hw_data->fuses); accel_pci_dev->sku = hw_data->get_sku(hw_data); - accel_pci_dev->pci_dev = pdev; /* If the device has no acceleration engines then ignore it. */ if (!hw_data->accel_mask || !hw_data->ae_mask || ((~hw_data->ae_mask) & 0x01)) { @@ -286,11 +277,14 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) } /* Create dev top level debugfs entry */ - snprintf(name, sizeof(name), "%s%s_dev%d", ADF_DEVICE_NAME_PREFIX, - hw_data->dev_class->name, hw_data->instance_id); + snprintf(name, sizeof(name), "%s%s_%02x:%02d.%02d", + ADF_DEVICE_NAME_PREFIX, hw_data->dev_class->name, + pdev->bus->number, PCI_SLOT(pdev->devfn), + PCI_FUNC(pdev->devfn)); + accel_dev->debugfs_dir = debugfs_create_dir(name, NULL); if (!accel_dev->debugfs_dir) { - dev_err(&pdev->dev, "Could not create debugfs dir\n"); + dev_err(&pdev->dev, "Could not create debugfs dir %s\n", name); ret = -EINVAL; goto out_err; } @@ -313,7 +307,7 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) if ((pci_set_dma_mask(pdev, DMA_BIT_MASK(32)))) { dev_err(&pdev->dev, "No usable DMA configuration\n"); ret = -EFAULT; - goto out_err; + goto out_err_disable; } else { pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)); } @@ -324,7 +318,7 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) if (pci_request_regions(pdev, adf_driver_name)) { ret = -EFAULT; - goto out_err; + goto out_err_disable; } /* Read accelerator capabilities mask */ @@ -332,19 +326,21 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) &hw_data->accel_capabilities_mask); /* Find and map all the device's BARS */ - for (i = 0; i < ADF_PCI_MAX_BARS; i++) { - struct adf_bar *bar = &accel_pci_dev->pci_bars[i]; + i = 0; + bar_mask = pci_select_bars(pdev, IORESOURCE_MEM); + for_each_set_bit(bar_nr, (const unsigned long *)&bar_mask, + ADF_PCI_MAX_BARS * 2) { + struct adf_bar *bar = &accel_pci_dev->pci_bars[i++]; - bar_nr = i * 2; bar->base_addr = pci_resource_start(pdev, bar_nr); if (!bar->base_addr) break; bar->size = pci_resource_len(pdev, bar_nr); bar->virt_addr = pci_iomap(accel_pci_dev->pci_dev, bar_nr, 0); if (!bar->virt_addr) { - dev_err(&pdev->dev, "Failed to map BAR %d\n", i); + dev_err(&pdev->dev, "Failed to map BAR %d\n", bar_nr); ret = -EFAULT; - goto out_err; + goto out_err_free_reg; } } pci_set_master(pdev); @@ -352,32 +348,40 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) if (adf_enable_aer(accel_dev, &adf_driver)) { dev_err(&pdev->dev, "Failed to enable aer\n"); ret = -EFAULT; - goto out_err; + goto out_err_free_reg; } if (pci_save_state(pdev)) { dev_err(&pdev->dev, "Failed to save pci state\n"); ret = -ENOMEM; - goto out_err; + goto out_err_free_reg; } ret = adf_dev_configure(accel_dev); if (ret) - goto out_err; + goto out_err_free_reg; ret = adf_dev_init(accel_dev); if (ret) - goto out_err; + goto out_err_dev_shutdown; ret = adf_dev_start(accel_dev); - if (ret) { - adf_dev_stop(accel_dev); - goto out_err; - } + if (ret) + goto out_err_dev_stop; - return 0; + return ret; + +out_err_dev_stop: + adf_dev_stop(accel_dev); +out_err_dev_shutdown: + adf_dev_shutdown(accel_dev); +out_err_free_reg: + pci_release_regions(accel_pci_dev->pci_dev); +out_err_disable: + pci_disable_device(accel_pci_dev->pci_dev); out_err: adf_cleanup_accel(accel_dev); + kfree(accel_dev); return ret; } @@ -391,15 +395,17 @@ static void adf_remove(struct pci_dev *pdev) } if (adf_dev_stop(accel_dev)) dev_err(&GET_DEV(accel_dev), "Failed to stop QAT accel dev\n"); + + adf_dev_shutdown(accel_dev); adf_disable_aer(accel_dev); adf_cleanup_accel(accel_dev); + adf_cleanup_pci_dev(accel_dev); + kfree(accel_dev); } static int __init adfdrv_init(void) { request_module("intel_qat"); - if (qat_admin_register()) - return -EFAULT; if (pci_register_driver(&adf_driver)) { pr_err("QAT: Driver initialization failed\n"); @@ -411,7 +417,6 @@ static int __init adfdrv_init(void) static void __exit adfdrv_release(void) { pci_unregister_driver(&adf_driver); - qat_admin_unregister(); } module_init(adfdrv_init); diff --git a/drivers/crypto/qat/qat_dh895xcc/adf_drv.h b/drivers/crypto/qat/qat_dh895xcc/adf_drv.h index a2fbb6ce75cd..85ff245bd1d8 100644 --- a/drivers/crypto/qat/qat_dh895xcc/adf_drv.h +++ b/drivers/crypto/qat/qat_dh895xcc/adf_drv.h @@ -53,15 +53,6 @@ void adf_init_hw_data_dh895xcc(struct adf_hw_device_data *hw_data); void adf_clean_hw_data_dh895xcc(struct adf_hw_device_data *hw_data); int adf_isr_resource_alloc(struct adf_accel_dev *accel_dev); void adf_isr_resource_free(struct adf_accel_dev *accel_dev); -void adf_update_ring_arb_enable(struct adf_etr_ring_data *ring); void adf_get_arbiter_mapping(struct adf_accel_dev *accel_dev, uint32_t const **arb_map_config); -int adf_init_admin_comms(struct adf_accel_dev *accel_dev); -void adf_exit_admin_comms(struct adf_accel_dev *accel_dev); -int adf_put_admin_msg_sync(struct adf_accel_dev *accel_dev, - uint32_t ae, void *in, void *out); -int qat_admin_register(void); -int qat_admin_unregister(void); -int adf_init_arb(struct adf_accel_dev *accel_dev); -void adf_exit_arb(struct adf_accel_dev *accel_dev); #endif diff --git a/drivers/crypto/qat/qat_dh895xcc/adf_isr.c b/drivers/crypto/qat/qat_dh895xcc/adf_isr.c index 0d03c109c2d3..5570f78795c1 100644 --- a/drivers/crypto/qat/qat_dh895xcc/adf_isr.c +++ b/drivers/crypto/qat/qat_dh895xcc/adf_isr.c @@ -59,21 +59,30 @@ #include <adf_transport_access_macros.h> #include <adf_transport_internal.h> #include "adf_drv.h" +#include "adf_dh895xcc_hw_data.h" static int adf_enable_msix(struct adf_accel_dev *accel_dev) { struct adf_accel_pci *pci_dev_info = &accel_dev->accel_pci_dev; struct adf_hw_device_data *hw_data = accel_dev->hw_device; - uint32_t msix_num_entries = hw_data->num_banks + 1; - int i; - - for (i = 0; i < msix_num_entries; i++) - pci_dev_info->msix_entries.entries[i].entry = i; + u32 msix_num_entries = 1; + + /* If SR-IOV is disabled, add entries for each bank */ + if (!accel_dev->pf.vf_info) { + int i; + + msix_num_entries += hw_data->num_banks; + for (i = 0; i < msix_num_entries; i++) + pci_dev_info->msix_entries.entries[i].entry = i; + } else { + pci_dev_info->msix_entries.entries[0].entry = + hw_data->num_banks; + } if (pci_enable_msix_exact(pci_dev_info->pci_dev, pci_dev_info->msix_entries.entries, msix_num_entries)) { - dev_err(&GET_DEV(accel_dev), "Failed to enable MSIX IRQ\n"); + dev_err(&GET_DEV(accel_dev), "Failed to enable MSI-X IRQ(s)\n"); return -EFAULT; } return 0; @@ -97,9 +106,58 @@ static irqreturn_t adf_msix_isr_ae(int irq, void *dev_ptr) { struct adf_accel_dev *accel_dev = dev_ptr; - dev_info(&GET_DEV(accel_dev), "qat_dev%d spurious AE interrupt\n", - accel_dev->accel_id); - return IRQ_HANDLED; +#ifdef CONFIG_PCI_IOV + /* If SR-IOV is enabled (vf_info is non-NULL), check for VF->PF ints */ + if (accel_dev->pf.vf_info) { + void __iomem *pmisc_bar_addr = + (&GET_BARS(accel_dev)[ADF_DH895XCC_PMISC_BAR])->virt_addr; + u32 vf_mask; + + /* Get the interrupt sources triggered by VFs */ + vf_mask = ((ADF_CSR_RD(pmisc_bar_addr, ADF_DH895XCC_ERRSOU5) & + 0x0000FFFF) << 16) | + ((ADF_CSR_RD(pmisc_bar_addr, ADF_DH895XCC_ERRSOU3) & + 0x01FFFE00) >> 9); + + if (vf_mask) { + struct adf_accel_vf_info *vf_info; + bool irq_handled = false; + int i; + + /* Disable VF2PF interrupts for VFs with pending ints */ + adf_disable_vf2pf_interrupts(accel_dev, vf_mask); + + /* + * Schedule tasklets to handle VF2PF interrupt BHs + * unless the VF is malicious and is attempting to + * flood the host OS with VF2PF interrupts. + */ + for_each_set_bit(i, (const unsigned long *)&vf_mask, + (sizeof(vf_mask) * BITS_PER_BYTE)) { + vf_info = accel_dev->pf.vf_info + i; + + if (!__ratelimit(&vf_info->vf2pf_ratelimit)) { + dev_info(&GET_DEV(accel_dev), + "Too many ints from VF%d\n", + vf_info->vf_nr + 1); + continue; + } + + /* Tasklet will re-enable ints from this VF */ + tasklet_hi_schedule(&vf_info->vf2pf_bh_tasklet); + irq_handled = true; + } + + if (irq_handled) + return IRQ_HANDLED; + } + } +#endif /* CONFIG_PCI_IOV */ + + dev_dbg(&GET_DEV(accel_dev), "qat_dev%d spurious AE interrupt\n", + accel_dev->accel_id); + + return IRQ_NONE; } static int adf_request_irqs(struct adf_accel_dev *accel_dev) @@ -108,28 +166,32 @@ static int adf_request_irqs(struct adf_accel_dev *accel_dev) struct adf_hw_device_data *hw_data = accel_dev->hw_device; struct msix_entry *msixe = pci_dev_info->msix_entries.entries; struct adf_etr_data *etr_data = accel_dev->transport; - int ret, i; + int ret, i = 0; char *name; - /* Request msix irq for all banks */ - for (i = 0; i < hw_data->num_banks; i++) { - struct adf_etr_bank_data *bank = &etr_data->banks[i]; - unsigned int cpu, cpus = num_online_cpus(); - - name = *(pci_dev_info->msix_entries.names + i); - snprintf(name, ADF_MAX_MSIX_VECTOR_NAME, - "qat%d-bundle%d", accel_dev->accel_id, i); - ret = request_irq(msixe[i].vector, - adf_msix_isr_bundle, 0, name, bank); - if (ret) { - dev_err(&GET_DEV(accel_dev), - "failed to enable irq %d for %s\n", - msixe[i].vector, name); - return ret; + /* Request msix irq for all banks unless SR-IOV enabled */ + if (!accel_dev->pf.vf_info) { + for (i = 0; i < hw_data->num_banks; i++) { + struct adf_etr_bank_data *bank = &etr_data->banks[i]; + unsigned int cpu, cpus = num_online_cpus(); + + name = *(pci_dev_info->msix_entries.names + i); + snprintf(name, ADF_MAX_MSIX_VECTOR_NAME, + "qat%d-bundle%d", accel_dev->accel_id, i); + ret = request_irq(msixe[i].vector, + adf_msix_isr_bundle, 0, name, bank); + if (ret) { + dev_err(&GET_DEV(accel_dev), + "failed to enable irq %d for %s\n", + msixe[i].vector, name); + return ret; + } + + cpu = ((accel_dev->accel_id * hw_data->num_banks) + + i) % cpus; + irq_set_affinity_hint(msixe[i].vector, + get_cpu_mask(cpu)); } - - cpu = ((accel_dev->accel_id * hw_data->num_banks) + i) % cpus; - irq_set_affinity_hint(msixe[i].vector, get_cpu_mask(cpu)); } /* Request msix irq for AE */ @@ -152,11 +214,13 @@ static void adf_free_irqs(struct adf_accel_dev *accel_dev) struct adf_hw_device_data *hw_data = accel_dev->hw_device; struct msix_entry *msixe = pci_dev_info->msix_entries.entries; struct adf_etr_data *etr_data = accel_dev->transport; - int i; + int i = 0; - for (i = 0; i < hw_data->num_banks; i++) { - irq_set_affinity_hint(msixe[i].vector, NULL); - free_irq(msixe[i].vector, &etr_data->banks[i]); + if (pci_dev_info->msix_entries.num_entries > 1) { + for (i = 0; i < hw_data->num_banks; i++) { + irq_set_affinity_hint(msixe[i].vector, NULL); + free_irq(msixe[i].vector, &etr_data->banks[i]); + } } irq_set_affinity_hint(msixe[i].vector, NULL); free_irq(msixe[i].vector, accel_dev); @@ -168,7 +232,11 @@ static int adf_isr_alloc_msix_entry_table(struct adf_accel_dev *accel_dev) char **names; struct msix_entry *entries; struct adf_hw_device_data *hw_data = accel_dev->hw_device; - uint32_t msix_num_entries = hw_data->num_banks + 1; + u32 msix_num_entries = 1; + + /* If SR-IOV is disabled (vf_info is NULL), add entries for each bank */ + if (!accel_dev->pf.vf_info) + msix_num_entries += hw_data->num_banks; entries = kzalloc_node(msix_num_entries * sizeof(*entries), GFP_KERNEL, dev_to_node(&GET_DEV(accel_dev))); @@ -185,6 +253,7 @@ static int adf_isr_alloc_msix_entry_table(struct adf_accel_dev *accel_dev) if (!(*(names + i))) goto err; } + accel_dev->accel_pci_dev.msix_entries.num_entries = msix_num_entries; accel_dev->accel_pci_dev.msix_entries.entries = entries; accel_dev->accel_pci_dev.msix_entries.names = names; return 0; @@ -198,13 +267,11 @@ err: static void adf_isr_free_msix_entry_table(struct adf_accel_dev *accel_dev) { - struct adf_hw_device_data *hw_data = accel_dev->hw_device; - uint32_t msix_num_entries = hw_data->num_banks + 1; char **names = accel_dev->accel_pci_dev.msix_entries.names; int i; kfree(accel_dev->accel_pci_dev.msix_entries.entries); - for (i = 0; i < msix_num_entries; i++) + for (i = 0; i < accel_dev->accel_pci_dev.msix_entries.num_entries; i++) kfree(*(names + i)); kfree(names); } diff --git a/drivers/crypto/qat/qat_dh895xccvf/Makefile b/drivers/crypto/qat/qat_dh895xccvf/Makefile new file mode 100644 index 000000000000..85399fcbbad4 --- /dev/null +++ b/drivers/crypto/qat/qat_dh895xccvf/Makefile @@ -0,0 +1,5 @@ +ccflags-y := -I$(src)/../qat_common +obj-$(CONFIG_CRYPTO_DEV_QAT_DH895xCCVF) += qat_dh895xccvf.o +qat_dh895xccvf-objs := adf_drv.o \ + adf_isr.o \ + adf_dh895xccvf_hw_data.o diff --git a/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c b/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c new file mode 100644 index 000000000000..a9a27eff41fb --- /dev/null +++ b/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c @@ -0,0 +1,172 @@ +/* + This file is provided under a dual BSD/GPLv2 license. When using or + redistributing this file, you may do so under either license. + + GPL LICENSE SUMMARY + Copyright(c) 2015 Intel Corporation. + This program is free software; you can redistribute it and/or modify + it under the terms of version 2 of the GNU General Public License as + published by the Free Software Foundation. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + Contact Information: + qat-linux@intel.com + + BSD LICENSE + Copyright(c) 2015 Intel Corporation. + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#include <adf_accel_devices.h> +#include <adf_pf2vf_msg.h> +#include <adf_common_drv.h> +#include "adf_dh895xccvf_hw_data.h" +#include "adf_drv.h" + +static struct adf_hw_device_class dh895xcciov_class = { + .name = ADF_DH895XCCVF_DEVICE_NAME, + .type = DEV_DH895XCCVF, + .instances = 0 +}; + +static u32 get_accel_mask(u32 fuse) +{ + return ADF_DH895XCCIOV_ACCELERATORS_MASK; +} + +static u32 get_ae_mask(u32 fuse) +{ + return ADF_DH895XCCIOV_ACCELENGINES_MASK; +} + +static u32 get_num_accels(struct adf_hw_device_data *self) +{ + return ADF_DH895XCCIOV_MAX_ACCELERATORS; +} + +static u32 get_num_aes(struct adf_hw_device_data *self) +{ + return ADF_DH895XCCIOV_MAX_ACCELENGINES; +} + +static u32 get_misc_bar_id(struct adf_hw_device_data *self) +{ + return ADF_DH895XCCIOV_PMISC_BAR; +} + +static u32 get_etr_bar_id(struct adf_hw_device_data *self) +{ + return ADF_DH895XCCIOV_ETR_BAR; +} + +static enum dev_sku_info get_sku(struct adf_hw_device_data *self) +{ + return DEV_SKU_VF; +} + +static u32 get_pf2vf_offset(u32 i) +{ + return ADF_DH895XCCIOV_PF2VF_OFFSET; +} + +static u32 get_vintmsk_offset(u32 i) +{ + return ADF_DH895XCCIOV_VINTMSK_OFFSET; +} + +static int adf_vf_int_noop(struct adf_accel_dev *accel_dev) +{ + return 0; +} + +static void adf_vf_void_noop(struct adf_accel_dev *accel_dev) +{ +} + +static int adf_vf2pf_init(struct adf_accel_dev *accel_dev) +{ + u32 msg = (ADF_VF2PF_MSGORIGIN_SYSTEM | + (ADF_VF2PF_MSGTYPE_INIT << ADF_VF2PF_MSGTYPE_SHIFT)); + + if (adf_iov_putmsg(accel_dev, msg, 0)) { + dev_err(&GET_DEV(accel_dev), + "Failed to send Init event to PF\n"); + return -EFAULT; + } + return 0; +} + +static void adf_vf2pf_shutdown(struct adf_accel_dev *accel_dev) +{ + u32 msg = (ADF_VF2PF_MSGORIGIN_SYSTEM | + (ADF_VF2PF_MSGTYPE_SHUTDOWN << ADF_VF2PF_MSGTYPE_SHIFT)); + + if (adf_iov_putmsg(accel_dev, msg, 0)) + dev_err(&GET_DEV(accel_dev), + "Failed to send Shutdown event to PF\n"); +} + +void adf_init_hw_data_dh895xcciov(struct adf_hw_device_data *hw_data) +{ + hw_data->dev_class = &dh895xcciov_class; + hw_data->instance_id = dh895xcciov_class.instances++; + hw_data->num_banks = ADF_DH895XCCIOV_ETR_MAX_BANKS; + hw_data->num_accel = ADF_DH895XCCIOV_MAX_ACCELERATORS; + hw_data->num_logical_accel = 1; + hw_data->num_engines = ADF_DH895XCCIOV_MAX_ACCELENGINES; + hw_data->tx_rx_gap = ADF_DH895XCCIOV_RX_RINGS_OFFSET; + hw_data->tx_rings_mask = ADF_DH895XCCIOV_TX_RINGS_MASK; + hw_data->alloc_irq = adf_vf_isr_resource_alloc; + hw_data->free_irq = adf_vf_isr_resource_free; + hw_data->enable_error_correction = adf_vf_void_noop; + hw_data->init_admin_comms = adf_vf_int_noop; + hw_data->exit_admin_comms = adf_vf_void_noop; + hw_data->send_admin_init = adf_vf2pf_init; + hw_data->init_arb = adf_vf_int_noop; + hw_data->exit_arb = adf_vf_void_noop; + hw_data->disable_iov = adf_vf2pf_shutdown; + hw_data->get_accel_mask = get_accel_mask; + hw_data->get_ae_mask = get_ae_mask; + hw_data->get_num_accels = get_num_accels; + hw_data->get_num_aes = get_num_aes; + hw_data->get_etr_bar_id = get_etr_bar_id; + hw_data->get_misc_bar_id = get_misc_bar_id; + hw_data->get_pf2vf_offset = get_pf2vf_offset; + hw_data->get_vintmsk_offset = get_vintmsk_offset; + hw_data->get_sku = get_sku; + hw_data->enable_ints = adf_vf_void_noop; + hw_data->enable_vf2pf_comms = adf_enable_vf2pf_comms; + hw_data->min_iov_compat_ver = ADF_PFVF_COMPATIBILITY_VERSION; +} + +void adf_clean_hw_data_dh895xcciov(struct adf_hw_device_data *hw_data) +{ + hw_data->dev_class->instances--; +} diff --git a/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.h b/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.h new file mode 100644 index 000000000000..8f6babfef629 --- /dev/null +++ b/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.h @@ -0,0 +1,68 @@ +/* + This file is provided under a dual BSD/GPLv2 license. When using or + redistributing this file, you may do so under either license. + + GPL LICENSE SUMMARY + Copyright(c) 2015 Intel Corporation. + This program is free software; you can redistribute it and/or modify + it under the terms of version 2 of the GNU General Public License as + published by the Free Software Foundation. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + Contact Information: + qat-linux@intel.com + + BSD LICENSE + Copyright(c) 2015 Intel Corporation. + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#ifndef ADF_DH895XVF_HW_DATA_H_ +#define ADF_DH895XVF_HW_DATA_H_ + +#define ADF_DH895XCCIOV_PMISC_BAR 1 +#define ADF_DH895XCCIOV_ACCELERATORS_MASK 0x1 +#define ADF_DH895XCCIOV_ACCELENGINES_MASK 0x1 +#define ADF_DH895XCCIOV_MAX_ACCELERATORS 1 +#define ADF_DH895XCCIOV_MAX_ACCELENGINES 1 +#define ADF_DH895XCCIOV_RX_RINGS_OFFSET 8 +#define ADF_DH895XCCIOV_TX_RINGS_MASK 0xFF +#define ADF_DH895XCCIOV_ETR_BAR 0 +#define ADF_DH895XCCIOV_ETR_MAX_BANKS 1 + +#define ADF_DH895XCCIOV_PF2VF_OFFSET 0x200 +#define ADF_DH895XCC_PF2VF_PF2VFINT BIT(0) + +#define ADF_DH895XCCIOV_VINTSOU_OFFSET 0x204 +#define ADF_DH895XCC_VINTSOU_BUN BIT(0) +#define ADF_DH895XCC_VINTSOU_PF2VF BIT(1) + +#define ADF_DH895XCCIOV_VINTMSK_OFFSET 0x208 +#endif diff --git a/drivers/crypto/qat/qat_dh895xccvf/adf_drv.c b/drivers/crypto/qat/qat_dh895xccvf/adf_drv.c new file mode 100644 index 000000000000..789426f21882 --- /dev/null +++ b/drivers/crypto/qat/qat_dh895xccvf/adf_drv.c @@ -0,0 +1,393 @@ +/* + This file is provided under a dual BSD/GPLv2 license. When using or + redistributing this file, you may do so under either license. + + GPL LICENSE SUMMARY + Copyright(c) 2014 Intel Corporation. + This program is free software; you can redistribute it and/or modify + it under the terms of version 2 of the GNU General Public License as + published by the Free Software Foundation. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + Contact Information: + qat-linux@intel.com + + BSD LICENSE + Copyright(c) 2014 Intel Corporation. + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/pci.h> +#include <linux/init.h> +#include <linux/types.h> +#include <linux/fs.h> +#include <linux/slab.h> +#include <linux/errno.h> +#include <linux/device.h> +#include <linux/dma-mapping.h> +#include <linux/platform_device.h> +#include <linux/workqueue.h> +#include <linux/io.h> +#include <adf_accel_devices.h> +#include <adf_common_drv.h> +#include <adf_cfg.h> +#include <adf_transport_access_macros.h> +#include "adf_dh895xccvf_hw_data.h" +#include "adf_drv.h" + +static const char adf_driver_name[] = ADF_DH895XCCVF_DEVICE_NAME; + +#define ADF_SYSTEM_DEVICE(device_id) \ + {PCI_DEVICE(PCI_VENDOR_ID_INTEL, device_id)} + +static const struct pci_device_id adf_pci_tbl[] = { + ADF_SYSTEM_DEVICE(ADF_DH895XCCIOV_PCI_DEVICE_ID), + {0,} +}; +MODULE_DEVICE_TABLE(pci, adf_pci_tbl); + +static int adf_probe(struct pci_dev *dev, const struct pci_device_id *ent); +static void adf_remove(struct pci_dev *dev); + +static struct pci_driver adf_driver = { + .id_table = adf_pci_tbl, + .name = adf_driver_name, + .probe = adf_probe, + .remove = adf_remove, +}; + +static void adf_cleanup_pci_dev(struct adf_accel_dev *accel_dev) +{ + pci_release_regions(accel_dev->accel_pci_dev.pci_dev); + pci_disable_device(accel_dev->accel_pci_dev.pci_dev); +} + +static void adf_cleanup_accel(struct adf_accel_dev *accel_dev) +{ + struct adf_accel_pci *accel_pci_dev = &accel_dev->accel_pci_dev; + struct adf_accel_dev *pf; + int i; + + for (i = 0; i < ADF_PCI_MAX_BARS; i++) { + struct adf_bar *bar = &accel_pci_dev->pci_bars[i]; + + if (bar->virt_addr) + pci_iounmap(accel_pci_dev->pci_dev, bar->virt_addr); + } + + if (accel_dev->hw_device) { + switch (accel_pci_dev->pci_dev->device) { + case ADF_DH895XCCIOV_PCI_DEVICE_ID: + adf_clean_hw_data_dh895xcciov(accel_dev->hw_device); + break; + default: + break; + } + kfree(accel_dev->hw_device); + accel_dev->hw_device = NULL; + } + adf_cfg_dev_remove(accel_dev); + debugfs_remove(accel_dev->debugfs_dir); + pf = adf_devmgr_pci_to_accel_dev(accel_pci_dev->pci_dev->physfn); + adf_devmgr_rm_dev(accel_dev, pf); +} + +static int adf_dev_configure(struct adf_accel_dev *accel_dev) +{ + char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES]; + unsigned long val, bank = 0; + + if (adf_cfg_section_add(accel_dev, ADF_KERNEL_SEC)) + goto err; + if (adf_cfg_section_add(accel_dev, "Accelerator0")) + goto err; + + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_BANK_NUM, 0); + if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, key, + (void *)&bank, ADF_DEC)) + goto err; + + val = bank; + snprintf(key, sizeof(key), ADF_CY "%d" ADF_ETRMGR_CORE_AFFINITY, 0); + if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, key, + (void *)&val, ADF_DEC)) + goto err; + + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_SIZE, 0); + + val = 128; + if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, key, + (void *)&val, ADF_DEC)) + goto err; + + val = 512; + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_SIZE, 0); + if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, (void *)&val, ADF_DEC)) + goto err; + + val = 0; + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_TX, 0); + if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, (void *)&val, ADF_DEC)) + goto err; + + val = 2; + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_TX, 0); + if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, (void *)&val, ADF_DEC)) + goto err; + + val = 8; + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_RX, 0); + if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, (void *)&val, ADF_DEC)) + goto err; + + val = 10; + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_RX, 0); + if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, (void *)&val, ADF_DEC)) + goto err; + + val = ADF_COALESCING_DEF_TIME; + snprintf(key, sizeof(key), ADF_ETRMGR_COALESCE_TIMER_FORMAT, + (int)bank); + if (adf_cfg_add_key_value_param(accel_dev, "Accelerator0", + key, (void *)&val, ADF_DEC)) + goto err; + + val = 1; + if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + ADF_NUM_CY, (void *)&val, ADF_DEC)) + goto err; + + set_bit(ADF_STATUS_CONFIGURED, &accel_dev->status); + return 0; +err: + dev_err(&GET_DEV(accel_dev), "Failed to configure QAT accel dev\n"); + return -EINVAL; +} + +static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) +{ + struct adf_accel_dev *accel_dev; + struct adf_accel_dev *pf; + struct adf_accel_pci *accel_pci_dev; + struct adf_hw_device_data *hw_data; + char name[ADF_DEVICE_NAME_LENGTH]; + unsigned int i, bar_nr; + int ret, bar_mask; + + switch (ent->device) { + case ADF_DH895XCCIOV_PCI_DEVICE_ID: + break; + default: + dev_err(&pdev->dev, "Invalid device 0x%x.\n", ent->device); + return -ENODEV; + } + + accel_dev = kzalloc_node(sizeof(*accel_dev), GFP_KERNEL, + dev_to_node(&pdev->dev)); + if (!accel_dev) + return -ENOMEM; + + accel_dev->is_vf = true; + pf = adf_devmgr_pci_to_accel_dev(pdev->physfn); + accel_pci_dev = &accel_dev->accel_pci_dev; + accel_pci_dev->pci_dev = pdev; + + /* Add accel device to accel table */ + if (adf_devmgr_add_dev(accel_dev, pf)) { + dev_err(&pdev->dev, "Failed to add new accelerator device.\n"); + kfree(accel_dev); + return -EFAULT; + } + INIT_LIST_HEAD(&accel_dev->crypto_list); + + accel_dev->owner = THIS_MODULE; + /* Allocate and configure device configuration structure */ + hw_data = kzalloc_node(sizeof(*hw_data), GFP_KERNEL, + dev_to_node(&pdev->dev)); + if (!hw_data) { + ret = -ENOMEM; + goto out_err; + } + accel_dev->hw_device = hw_data; + switch (ent->device) { + case ADF_DH895XCCIOV_PCI_DEVICE_ID: + adf_init_hw_data_dh895xcciov(accel_dev->hw_device); + break; + default: + ret = -ENODEV; + goto out_err; + } + + /* Get Accelerators and Accelerators Engines masks */ + hw_data->accel_mask = hw_data->get_accel_mask(hw_data->fuses); + hw_data->ae_mask = hw_data->get_ae_mask(hw_data->fuses); + accel_pci_dev->sku = hw_data->get_sku(hw_data); + + /* Create dev top level debugfs entry */ + snprintf(name, sizeof(name), "%s%s_%02x:%02d.%02d", + ADF_DEVICE_NAME_PREFIX, hw_data->dev_class->name, + pdev->bus->number, PCI_SLOT(pdev->devfn), + PCI_FUNC(pdev->devfn)); + + accel_dev->debugfs_dir = debugfs_create_dir(name, NULL); + if (!accel_dev->debugfs_dir) { + dev_err(&pdev->dev, "Could not create debugfs dir %s\n", name); + ret = -EINVAL; + goto out_err; + } + + /* Create device configuration table */ + ret = adf_cfg_dev_add(accel_dev); + if (ret) + goto out_err; + + /* enable PCI device */ + if (pci_enable_device(pdev)) { + ret = -EFAULT; + goto out_err; + } + + /* set dma identifier */ + if (pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) { + if ((pci_set_dma_mask(pdev, DMA_BIT_MASK(32)))) { + dev_err(&pdev->dev, "No usable DMA configuration\n"); + ret = -EFAULT; + goto out_err_disable; + } else { + pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)); + } + + } else { + pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)); + } + + if (pci_request_regions(pdev, adf_driver_name)) { + ret = -EFAULT; + goto out_err_disable; + } + + /* Find and map all the device's BARS */ + i = 0; + bar_mask = pci_select_bars(pdev, IORESOURCE_MEM); + for_each_set_bit(bar_nr, (const unsigned long *)&bar_mask, + ADF_PCI_MAX_BARS * 2) { + struct adf_bar *bar = &accel_pci_dev->pci_bars[i++]; + + bar->base_addr = pci_resource_start(pdev, bar_nr); + if (!bar->base_addr) + break; + bar->size = pci_resource_len(pdev, bar_nr); + bar->virt_addr = pci_iomap(accel_pci_dev->pci_dev, bar_nr, 0); + if (!bar->virt_addr) { + dev_err(&pdev->dev, "Failed to map BAR %d\n", bar_nr); + ret = -EFAULT; + goto out_err_free_reg; + } + } + pci_set_master(pdev); + /* Completion for VF2PF request/response message exchange */ + init_completion(&accel_dev->vf.iov_msg_completion); + + ret = adf_dev_configure(accel_dev); + if (ret) + goto out_err_free_reg; + + ret = adf_dev_init(accel_dev); + if (ret) + goto out_err_dev_shutdown; + + ret = adf_dev_start(accel_dev); + if (ret) + goto out_err_dev_stop; + + return ret; + +out_err_dev_stop: + adf_dev_stop(accel_dev); +out_err_dev_shutdown: + adf_dev_shutdown(accel_dev); +out_err_free_reg: + pci_release_regions(accel_pci_dev->pci_dev); +out_err_disable: + pci_disable_device(accel_pci_dev->pci_dev); +out_err: + adf_cleanup_accel(accel_dev); + kfree(accel_dev); + return ret; +} + +static void adf_remove(struct pci_dev *pdev) +{ + struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev); + + if (!accel_dev) { + pr_err("QAT: Driver removal failed\n"); + return; + } + if (adf_dev_stop(accel_dev)) + dev_err(&GET_DEV(accel_dev), "Failed to stop QAT accel dev\n"); + + adf_dev_shutdown(accel_dev); + adf_cleanup_accel(accel_dev); + adf_cleanup_pci_dev(accel_dev); + kfree(accel_dev); +} + +static int __init adfdrv_init(void) +{ + request_module("intel_qat"); + + if (pci_register_driver(&adf_driver)) { + pr_err("QAT: Driver initialization failed\n"); + return -EFAULT; + } + return 0; +} + +static void __exit adfdrv_release(void) +{ + pci_unregister_driver(&adf_driver); + adf_clean_vf_map(true); +} + +module_init(adfdrv_init); +module_exit(adfdrv_release); + +MODULE_LICENSE("Dual BSD/GPL"); +MODULE_AUTHOR("Intel"); +MODULE_DESCRIPTION("Intel(R) QuickAssist Technology"); +MODULE_VERSION(ADF_DRV_VERSION); diff --git a/drivers/crypto/qat/qat_dh895xcc/qat_admin.c b/drivers/crypto/qat/qat_dh895xccvf/adf_drv.h index 55b7a8e48bad..e270e4a63d14 100644 --- a/drivers/crypto/qat/qat_dh895xcc/qat_admin.c +++ b/drivers/crypto/qat/qat_dh895xccvf/adf_drv.h @@ -44,64 +44,14 @@ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -#include <icp_qat_fw_init_admin.h> +#ifndef ADF_DH895xVF_DRV_H_ +#define ADF_DH895xVF_DRV_H_ #include <adf_accel_devices.h> -#include <adf_common_drv.h> -#include "adf_drv.h" - -static struct service_hndl qat_admin; - -static int qat_send_admin_cmd(struct adf_accel_dev *accel_dev, int cmd) -{ - struct adf_hw_device_data *hw_device = accel_dev->hw_device; - struct icp_qat_fw_init_admin_req req; - struct icp_qat_fw_init_admin_resp resp; - int i; - - memset(&req, 0, sizeof(struct icp_qat_fw_init_admin_req)); - req.init_admin_cmd_id = cmd; - for (i = 0; i < hw_device->get_num_aes(hw_device); i++) { - memset(&resp, 0, sizeof(struct icp_qat_fw_init_admin_resp)); - if (adf_put_admin_msg_sync(accel_dev, i, &req, &resp) || - resp.init_resp_hdr.status) - return -EFAULT; - } - return 0; -} - -static int qat_admin_start(struct adf_accel_dev *accel_dev) -{ - return qat_send_admin_cmd(accel_dev, ICP_QAT_FW_INIT_ME); -} - -static int qat_admin_event_handler(struct adf_accel_dev *accel_dev, - enum adf_event event) -{ - int ret; - - switch (event) { - case ADF_EVENT_START: - ret = qat_admin_start(accel_dev); - break; - case ADF_EVENT_STOP: - case ADF_EVENT_INIT: - case ADF_EVENT_SHUTDOWN: - default: - ret = 0; - } - return ret; -} - -int qat_admin_register(void) -{ - memset(&qat_admin, 0, sizeof(struct service_hndl)); - qat_admin.event_hld = qat_admin_event_handler; - qat_admin.name = "qat_admin"; - qat_admin.admin = 1; - return adf_service_register(&qat_admin); -} - -int qat_admin_unregister(void) -{ - return adf_service_unregister(&qat_admin); -} +#include <adf_transport.h> + +void adf_init_hw_data_dh895xcciov(struct adf_hw_device_data *hw_data); +void adf_clean_hw_data_dh895xcciov(struct adf_hw_device_data *hw_data); +int adf_vf_isr_resource_alloc(struct adf_accel_dev *accel_dev); +void adf_vf_isr_resource_free(struct adf_accel_dev *accel_dev); +void adf_update_ring_arb_enable(struct adf_etr_ring_data *ring); +#endif diff --git a/drivers/crypto/qat/qat_dh895xccvf/adf_isr.c b/drivers/crypto/qat/qat_dh895xccvf/adf_isr.c new file mode 100644 index 000000000000..87c5d8adb125 --- /dev/null +++ b/drivers/crypto/qat/qat_dh895xccvf/adf_isr.c @@ -0,0 +1,258 @@ +/* + This file is provided under a dual BSD/GPLv2 license. When using or + redistributing this file, you may do so under either license. + + GPL LICENSE SUMMARY + Copyright(c) 2014 Intel Corporation. + This program is free software; you can redistribute it and/or modify + it under the terms of version 2 of the GNU General Public License as + published by the Free Software Foundation. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + Contact Information: + qat-linux@intel.com + + BSD LICENSE + Copyright(c) 2014 Intel Corporation. + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#include <linux/kernel.h> +#include <linux/init.h> +#include <linux/types.h> +#include <linux/pci.h> +#include <linux/slab.h> +#include <linux/errno.h> +#include <linux/interrupt.h> +#include <adf_accel_devices.h> +#include <adf_common_drv.h> +#include <adf_cfg.h> +#include <adf_cfg_strings.h> +#include <adf_cfg_common.h> +#include <adf_transport_access_macros.h> +#include <adf_transport_internal.h> +#include <adf_pf2vf_msg.h> +#include "adf_drv.h" +#include "adf_dh895xccvf_hw_data.h" + +static int adf_enable_msi(struct adf_accel_dev *accel_dev) +{ + struct adf_accel_pci *pci_dev_info = &accel_dev->accel_pci_dev; + int stat = pci_enable_msi(pci_dev_info->pci_dev); + + if (stat) { + dev_err(&GET_DEV(accel_dev), + "Failed to enable MSI interrupts\n"); + return stat; + } + + accel_dev->vf.irq_name = kzalloc(ADF_MAX_MSIX_VECTOR_NAME, GFP_KERNEL); + if (!accel_dev->vf.irq_name) + return -ENOMEM; + + return stat; +} + +static void adf_disable_msi(struct adf_accel_dev *accel_dev) +{ + struct pci_dev *pdev = accel_to_pci_dev(accel_dev); + + kfree(accel_dev->vf.irq_name); + pci_disable_msi(pdev); +} + +static void adf_pf2vf_bh_handler(void *data) +{ + struct adf_accel_dev *accel_dev = data; + void __iomem *pmisc_bar_addr = + (&GET_BARS(accel_dev)[ADF_DH895XCCIOV_PMISC_BAR])->virt_addr; + u32 msg; + + /* Read the message from PF */ + msg = ADF_CSR_RD(pmisc_bar_addr, ADF_DH895XCCIOV_PF2VF_OFFSET); + + if (!(msg & ADF_PF2VF_MSGORIGIN_SYSTEM)) + /* Ignore legacy non-system (non-kernel) PF2VF messages */ + goto err; + + switch ((msg & ADF_PF2VF_MSGTYPE_MASK) >> ADF_PF2VF_MSGTYPE_SHIFT) { + case ADF_PF2VF_MSGTYPE_RESTARTING: + dev_dbg(&GET_DEV(accel_dev), + "Restarting msg received from PF 0x%x\n", msg); + adf_dev_stop(accel_dev); + break; + case ADF_PF2VF_MSGTYPE_VERSION_RESP: + dev_dbg(&GET_DEV(accel_dev), + "Version resp received from PF 0x%x\n", msg); + accel_dev->vf.pf_version = + (msg & ADF_PF2VF_VERSION_RESP_VERS_MASK) >> + ADF_PF2VF_VERSION_RESP_VERS_SHIFT; + accel_dev->vf.compatible = + (msg & ADF_PF2VF_VERSION_RESP_RESULT_MASK) >> + ADF_PF2VF_VERSION_RESP_RESULT_SHIFT; + complete(&accel_dev->vf.iov_msg_completion); + break; + default: + goto err; + } + + /* To ack, clear the PF2VFINT bit */ + msg &= ~ADF_DH895XCC_PF2VF_PF2VFINT; + ADF_CSR_WR(pmisc_bar_addr, ADF_DH895XCCIOV_PF2VF_OFFSET, msg); + + /* Re-enable PF2VF interrupts */ + adf_enable_pf2vf_interrupts(accel_dev); + return; +err: + dev_err(&GET_DEV(accel_dev), + "Unknown message from PF (0x%x); leaving PF2VF ints disabled\n", + msg); +} + +static int adf_setup_pf2vf_bh(struct adf_accel_dev *accel_dev) +{ + tasklet_init(&accel_dev->vf.pf2vf_bh_tasklet, + (void *)adf_pf2vf_bh_handler, (unsigned long)accel_dev); + + mutex_init(&accel_dev->vf.vf2pf_lock); + return 0; +} + +static void adf_cleanup_pf2vf_bh(struct adf_accel_dev *accel_dev) +{ + tasklet_disable(&accel_dev->vf.pf2vf_bh_tasklet); + tasklet_kill(&accel_dev->vf.pf2vf_bh_tasklet); + mutex_destroy(&accel_dev->vf.vf2pf_lock); +} + +static irqreturn_t adf_isr(int irq, void *privdata) +{ + struct adf_accel_dev *accel_dev = privdata; + void __iomem *pmisc_bar_addr = + (&GET_BARS(accel_dev)[ADF_DH895XCCIOV_PMISC_BAR])->virt_addr; + u32 v_int; + + /* Read VF INT source CSR to determine the source of VF interrupt */ + v_int = ADF_CSR_RD(pmisc_bar_addr, ADF_DH895XCCIOV_VINTSOU_OFFSET); + + /* Check for PF2VF interrupt */ + if (v_int & ADF_DH895XCC_VINTSOU_PF2VF) { + /* Disable PF to VF interrupt */ + adf_disable_pf2vf_interrupts(accel_dev); + + /* Schedule tasklet to handle interrupt BH */ + tasklet_hi_schedule(&accel_dev->vf.pf2vf_bh_tasklet); + return IRQ_HANDLED; + } + + /* Check bundle interrupt */ + if (v_int & ADF_DH895XCC_VINTSOU_BUN) { + struct adf_etr_data *etr_data = accel_dev->transport; + struct adf_etr_bank_data *bank = &etr_data->banks[0]; + + /* Disable Flag and Coalesce Ring Interrupts */ + WRITE_CSR_INT_FLAG_AND_COL(bank->csr_addr, bank->bank_number, + 0); + tasklet_hi_schedule(&bank->resp_handler); + return IRQ_HANDLED; + } + + return IRQ_NONE; +} + +static int adf_request_msi_irq(struct adf_accel_dev *accel_dev) +{ + struct pci_dev *pdev = accel_to_pci_dev(accel_dev); + unsigned int cpu; + int ret; + + snprintf(accel_dev->vf.irq_name, ADF_MAX_MSIX_VECTOR_NAME, + "qat_%02x:%02d.%02d", pdev->bus->number, PCI_SLOT(pdev->devfn), + PCI_FUNC(pdev->devfn)); + ret = request_irq(pdev->irq, adf_isr, 0, accel_dev->vf.irq_name, + (void *)accel_dev); + if (ret) { + dev_err(&GET_DEV(accel_dev), "failed to enable irq for %s\n", + accel_dev->vf.irq_name); + return ret; + } + cpu = accel_dev->accel_id % num_online_cpus(); + irq_set_affinity_hint(pdev->irq, get_cpu_mask(cpu)); + + return ret; +} + +static int adf_setup_bh(struct adf_accel_dev *accel_dev) +{ + struct adf_etr_data *priv_data = accel_dev->transport; + + tasklet_init(&priv_data->banks[0].resp_handler, adf_response_handler, + (unsigned long)priv_data->banks); + return 0; +} + +static void adf_cleanup_bh(struct adf_accel_dev *accel_dev) +{ + struct adf_etr_data *priv_data = accel_dev->transport; + + tasklet_disable(&priv_data->banks[0].resp_handler); + tasklet_kill(&priv_data->banks[0].resp_handler); +} + +void adf_vf_isr_resource_free(struct adf_accel_dev *accel_dev) +{ + struct pci_dev *pdev = accel_to_pci_dev(accel_dev); + + irq_set_affinity_hint(pdev->irq, NULL); + free_irq(pdev->irq, (void *)accel_dev); + adf_cleanup_bh(accel_dev); + adf_cleanup_pf2vf_bh(accel_dev); + adf_disable_msi(accel_dev); +} + +int adf_vf_isr_resource_alloc(struct adf_accel_dev *accel_dev) +{ + if (adf_enable_msi(accel_dev)) + goto err_out; + + if (adf_setup_pf2vf_bh(accel_dev)) + goto err_out; + + if (adf_setup_bh(accel_dev)) + goto err_out; + + if (adf_request_msi_irq(accel_dev)) + goto err_out; + + return 0; +err_out: + adf_vf_isr_resource_free(accel_dev); + return -EFAULT; +} diff --git a/drivers/crypto/sahara.c b/drivers/crypto/sahara.c index 397a500b3d8a..1c19e44c3146 100644 --- a/drivers/crypto/sahara.c +++ b/drivers/crypto/sahara.c @@ -1516,7 +1516,7 @@ static int sahara_probe(struct platform_device *pdev) } /* Allocate HW descriptors */ - dev->hw_desc[0] = dma_alloc_coherent(&pdev->dev, + dev->hw_desc[0] = dmam_alloc_coherent(&pdev->dev, SAHARA_MAX_HW_DESC * sizeof(struct sahara_hw_desc), &dev->hw_phys_desc[0], GFP_KERNEL); if (!dev->hw_desc[0]) { @@ -1528,34 +1528,31 @@ static int sahara_probe(struct platform_device *pdev) sizeof(struct sahara_hw_desc); /* Allocate space for iv and key */ - dev->key_base = dma_alloc_coherent(&pdev->dev, 2 * AES_KEYSIZE_128, + dev->key_base = dmam_alloc_coherent(&pdev->dev, 2 * AES_KEYSIZE_128, &dev->key_phys_base, GFP_KERNEL); if (!dev->key_base) { dev_err(&pdev->dev, "Could not allocate memory for key\n"); - err = -ENOMEM; - goto err_key; + return -ENOMEM; } dev->iv_base = dev->key_base + AES_KEYSIZE_128; dev->iv_phys_base = dev->key_phys_base + AES_KEYSIZE_128; /* Allocate space for context: largest digest + message length field */ - dev->context_base = dma_alloc_coherent(&pdev->dev, + dev->context_base = dmam_alloc_coherent(&pdev->dev, SHA256_DIGEST_SIZE + 4, &dev->context_phys_base, GFP_KERNEL); if (!dev->context_base) { dev_err(&pdev->dev, "Could not allocate memory for MDHA context\n"); - err = -ENOMEM; - goto err_key; + return -ENOMEM; } /* Allocate space for HW links */ - dev->hw_link[0] = dma_alloc_coherent(&pdev->dev, + dev->hw_link[0] = dmam_alloc_coherent(&pdev->dev, SAHARA_MAX_HW_LINK * sizeof(struct sahara_hw_link), &dev->hw_phys_link[0], GFP_KERNEL); if (!dev->hw_link[0]) { dev_err(&pdev->dev, "Could not allocate hw links\n"); - err = -ENOMEM; - goto err_link; + return -ENOMEM; } for (i = 1; i < SAHARA_MAX_HW_LINK; i++) { dev->hw_phys_link[i] = dev->hw_phys_link[i - 1] + @@ -1572,15 +1569,14 @@ static int sahara_probe(struct platform_device *pdev) dev->kthread = kthread_run(sahara_queue_manage, dev, "sahara_crypto"); if (IS_ERR(dev->kthread)) { - err = PTR_ERR(dev->kthread); - goto err_link; + return PTR_ERR(dev->kthread); } init_completion(&dev->dma_completion); err = clk_prepare_enable(dev->clk_ipg); if (err) - goto err_link; + return err; err = clk_prepare_enable(dev->clk_ahb); if (err) goto clk_ipg_disable; @@ -1620,25 +1616,11 @@ static int sahara_probe(struct platform_device *pdev) return 0; err_algs: - dma_free_coherent(&pdev->dev, - SAHARA_MAX_HW_LINK * sizeof(struct sahara_hw_link), - dev->hw_link[0], dev->hw_phys_link[0]); kthread_stop(dev->kthread); dev_ptr = NULL; clk_disable_unprepare(dev->clk_ahb); clk_ipg_disable: clk_disable_unprepare(dev->clk_ipg); -err_link: - dma_free_coherent(&pdev->dev, - 2 * AES_KEYSIZE_128, - dev->key_base, dev->key_phys_base); - dma_free_coherent(&pdev->dev, - SHA256_DIGEST_SIZE, - dev->context_base, dev->context_phys_base); -err_key: - dma_free_coherent(&pdev->dev, - SAHARA_MAX_HW_DESC * sizeof(struct sahara_hw_desc), - dev->hw_desc[0], dev->hw_phys_desc[0]); return err; } @@ -1647,16 +1629,6 @@ static int sahara_remove(struct platform_device *pdev) { struct sahara_dev *dev = platform_get_drvdata(pdev); - dma_free_coherent(&pdev->dev, - SAHARA_MAX_HW_LINK * sizeof(struct sahara_hw_link), - dev->hw_link[0], dev->hw_phys_link[0]); - dma_free_coherent(&pdev->dev, - 2 * AES_KEYSIZE_128, - dev->key_base, dev->key_phys_base); - dma_free_coherent(&pdev->dev, - SAHARA_MAX_HW_DESC * sizeof(struct sahara_hw_desc), - dev->hw_desc[0], dev->hw_phys_desc[0]); - kthread_stop(dev->kthread); sahara_unregister_algs(dev); diff --git a/drivers/crypto/sunxi-ss/Makefile b/drivers/crypto/sunxi-ss/Makefile new file mode 100644 index 000000000000..8f4c7a273141 --- /dev/null +++ b/drivers/crypto/sunxi-ss/Makefile @@ -0,0 +1,2 @@ +obj-$(CONFIG_CRYPTO_DEV_SUN4I_SS) += sun4i-ss.o +sun4i-ss-y += sun4i-ss-core.o sun4i-ss-hash.o sun4i-ss-cipher.o diff --git a/drivers/crypto/sunxi-ss/sun4i-ss-cipher.c b/drivers/crypto/sunxi-ss/sun4i-ss-cipher.c new file mode 100644 index 000000000000..e070c316e8b7 --- /dev/null +++ b/drivers/crypto/sunxi-ss/sun4i-ss-cipher.c @@ -0,0 +1,542 @@ +/* + * sun4i-ss-cipher.c - hardware cryptographic accelerator for Allwinner A20 SoC + * + * Copyright (C) 2013-2015 Corentin LABBE <clabbe.montjoie@gmail.com> + * + * This file add support for AES cipher with 128,192,256 bits + * keysize in CBC and ECB mode. + * Add support also for DES and 3DES in CBC and ECB mode. + * + * You could find the datasheet in Documentation/arm/sunxi/README + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + */ +#include "sun4i-ss.h" + +static int sun4i_ss_opti_poll(struct ablkcipher_request *areq) +{ + struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(areq); + struct sun4i_tfm_ctx *op = crypto_ablkcipher_ctx(tfm); + struct sun4i_ss_ctx *ss = op->ss; + unsigned int ivsize = crypto_ablkcipher_ivsize(tfm); + struct sun4i_cipher_req_ctx *ctx = ablkcipher_request_ctx(areq); + u32 mode = ctx->mode; + /* when activating SS, the default FIFO space is SS_RX_DEFAULT(32) */ + u32 rx_cnt = SS_RX_DEFAULT; + u32 tx_cnt = 0; + u32 spaces; + u32 v; + int i, err = 0; + unsigned int ileft = areq->nbytes; + unsigned int oleft = areq->nbytes; + unsigned int todo; + struct sg_mapping_iter mi, mo; + unsigned int oi, oo; /* offset for in and out */ + + if (areq->nbytes == 0) + return 0; + + if (!areq->info) { + dev_err_ratelimited(ss->dev, "ERROR: Empty IV\n"); + return -EINVAL; + } + + if (!areq->src || !areq->dst) { + dev_err_ratelimited(ss->dev, "ERROR: Some SGs are NULL\n"); + return -EINVAL; + } + + spin_lock_bh(&ss->slock); + + for (i = 0; i < op->keylen; i += 4) + writel(*(op->key + i / 4), ss->base + SS_KEY0 + i); + + if (areq->info) { + for (i = 0; i < 4 && i < ivsize / 4; i++) { + v = *(u32 *)(areq->info + i * 4); + writel(v, ss->base + SS_IV0 + i * 4); + } + } + writel(mode, ss->base + SS_CTL); + + sg_miter_start(&mi, areq->src, sg_nents(areq->src), + SG_MITER_FROM_SG | SG_MITER_ATOMIC); + sg_miter_start(&mo, areq->dst, sg_nents(areq->dst), + SG_MITER_TO_SG | SG_MITER_ATOMIC); + sg_miter_next(&mi); + sg_miter_next(&mo); + if (!mi.addr || !mo.addr) { + dev_err_ratelimited(ss->dev, "ERROR: sg_miter return null\n"); + err = -EINVAL; + goto release_ss; + } + + ileft = areq->nbytes / 4; + oleft = areq->nbytes / 4; + oi = 0; + oo = 0; + do { + todo = min3(rx_cnt, ileft, (mi.length - oi) / 4); + if (todo > 0) { + ileft -= todo; + writesl(ss->base + SS_RXFIFO, mi.addr + oi, todo); + oi += todo * 4; + } + if (oi == mi.length) { + sg_miter_next(&mi); + oi = 0; + } + + spaces = readl(ss->base + SS_FCSR); + rx_cnt = SS_RXFIFO_SPACES(spaces); + tx_cnt = SS_TXFIFO_SPACES(spaces); + + todo = min3(tx_cnt, oleft, (mo.length - oo) / 4); + if (todo > 0) { + oleft -= todo; + readsl(ss->base + SS_TXFIFO, mo.addr + oo, todo); + oo += todo * 4; + } + if (oo == mo.length) { + sg_miter_next(&mo); + oo = 0; + } + } while (mo.length > 0); + + if (areq->info) { + for (i = 0; i < 4 && i < ivsize / 4; i++) { + v = readl(ss->base + SS_IV0 + i * 4); + *(u32 *)(areq->info + i * 4) = v; + } + } + +release_ss: + sg_miter_stop(&mi); + sg_miter_stop(&mo); + writel(0, ss->base + SS_CTL); + spin_unlock_bh(&ss->slock); + return err; +} + +/* Generic function that support SG with size not multiple of 4 */ +static int sun4i_ss_cipher_poll(struct ablkcipher_request *areq) +{ + struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(areq); + struct sun4i_tfm_ctx *op = crypto_ablkcipher_ctx(tfm); + struct sun4i_ss_ctx *ss = op->ss; + int no_chunk = 1; + struct scatterlist *in_sg = areq->src; + struct scatterlist *out_sg = areq->dst; + unsigned int ivsize = crypto_ablkcipher_ivsize(tfm); + struct sun4i_cipher_req_ctx *ctx = ablkcipher_request_ctx(areq); + u32 mode = ctx->mode; + /* when activating SS, the default FIFO space is SS_RX_DEFAULT(32) */ + u32 rx_cnt = SS_RX_DEFAULT; + u32 tx_cnt = 0; + u32 v; + u32 spaces; + int i, err = 0; + unsigned int ileft = areq->nbytes; + unsigned int oleft = areq->nbytes; + unsigned int todo; + struct sg_mapping_iter mi, mo; + unsigned int oi, oo; /* offset for in and out */ + char buf[4 * SS_RX_MAX];/* buffer for linearize SG src */ + char bufo[4 * SS_TX_MAX]; /* buffer for linearize SG dst */ + unsigned int ob = 0; /* offset in buf */ + unsigned int obo = 0; /* offset in bufo*/ + unsigned int obl = 0; /* length of data in bufo */ + + if (areq->nbytes == 0) + return 0; + + if (!areq->info) { + dev_err_ratelimited(ss->dev, "ERROR: Empty IV\n"); + return -EINVAL; + } + + if (!areq->src || !areq->dst) { + dev_err_ratelimited(ss->dev, "ERROR: Some SGs are NULL\n"); + return -EINVAL; + } + + /* + * if we have only SGs with size multiple of 4, + * we can use the SS optimized function + */ + while (in_sg && no_chunk == 1) { + if ((in_sg->length % 4) != 0) + no_chunk = 0; + in_sg = sg_next(in_sg); + } + while (out_sg && no_chunk == 1) { + if ((out_sg->length % 4) != 0) + no_chunk = 0; + out_sg = sg_next(out_sg); + } + + if (no_chunk == 1) + return sun4i_ss_opti_poll(areq); + + spin_lock_bh(&ss->slock); + + for (i = 0; i < op->keylen; i += 4) + writel(*(op->key + i / 4), ss->base + SS_KEY0 + i); + + if (areq->info) { + for (i = 0; i < 4 && i < ivsize / 4; i++) { + v = *(u32 *)(areq->info + i * 4); + writel(v, ss->base + SS_IV0 + i * 4); + } + } + writel(mode, ss->base + SS_CTL); + + sg_miter_start(&mi, areq->src, sg_nents(areq->src), + SG_MITER_FROM_SG | SG_MITER_ATOMIC); + sg_miter_start(&mo, areq->dst, sg_nents(areq->dst), + SG_MITER_TO_SG | SG_MITER_ATOMIC); + sg_miter_next(&mi); + sg_miter_next(&mo); + if (!mi.addr || !mo.addr) { + dev_err_ratelimited(ss->dev, "ERROR: sg_miter return null\n"); + err = -EINVAL; + goto release_ss; + } + ileft = areq->nbytes; + oleft = areq->nbytes; + oi = 0; + oo = 0; + + while (oleft > 0) { + if (ileft > 0) { + /* + * todo is the number of consecutive 4byte word that we + * can read from current SG + */ + todo = min3(rx_cnt, ileft / 4, (mi.length - oi) / 4); + if (todo > 0 && ob == 0) { + writesl(ss->base + SS_RXFIFO, mi.addr + oi, + todo); + ileft -= todo * 4; + oi += todo * 4; + } else { + /* + * not enough consecutive bytes, so we need to + * linearize in buf. todo is in bytes + * After that copy, if we have a multiple of 4 + * we need to be able to write all buf in one + * pass, so it is why we min() with rx_cnt + */ + todo = min3(rx_cnt * 4 - ob, ileft, + mi.length - oi); + memcpy(buf + ob, mi.addr + oi, todo); + ileft -= todo; + oi += todo; + ob += todo; + if (ob % 4 == 0) { + writesl(ss->base + SS_RXFIFO, buf, + ob / 4); + ob = 0; + } + } + if (oi == mi.length) { + sg_miter_next(&mi); + oi = 0; + } + } + + spaces = readl(ss->base + SS_FCSR); + rx_cnt = SS_RXFIFO_SPACES(spaces); + tx_cnt = SS_TXFIFO_SPACES(spaces); + dev_dbg(ss->dev, "%x %u/%u %u/%u cnt=%u %u/%u %u/%u cnt=%u %u %u\n", + mode, + oi, mi.length, ileft, areq->nbytes, rx_cnt, + oo, mo.length, oleft, areq->nbytes, tx_cnt, + todo, ob); + + if (tx_cnt == 0) + continue; + /* todo in 4bytes word */ + todo = min3(tx_cnt, oleft / 4, (mo.length - oo) / 4); + if (todo > 0) { + readsl(ss->base + SS_TXFIFO, mo.addr + oo, todo); + oleft -= todo * 4; + oo += todo * 4; + if (oo == mo.length) { + sg_miter_next(&mo); + oo = 0; + } + } else { + /* + * read obl bytes in bufo, we read at maximum for + * emptying the device + */ + readsl(ss->base + SS_TXFIFO, bufo, tx_cnt); + obl = tx_cnt * 4; + obo = 0; + do { + /* + * how many bytes we can copy ? + * no more than remaining SG size + * no more than remaining buffer + * no need to test against oleft + */ + todo = min(mo.length - oo, obl - obo); + memcpy(mo.addr + oo, bufo + obo, todo); + oleft -= todo; + obo += todo; + oo += todo; + if (oo == mo.length) { + sg_miter_next(&mo); + oo = 0; + } + } while (obo < obl); + /* bufo must be fully used here */ + } + } + if (areq->info) { + for (i = 0; i < 4 && i < ivsize / 4; i++) { + v = readl(ss->base + SS_IV0 + i * 4); + *(u32 *)(areq->info + i * 4) = v; + } + } + +release_ss: + sg_miter_stop(&mi); + sg_miter_stop(&mo); + writel(0, ss->base + SS_CTL); + spin_unlock_bh(&ss->slock); + + return err; +} + +/* CBC AES */ +int sun4i_ss_cbc_aes_encrypt(struct ablkcipher_request *areq) +{ + struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(areq); + struct sun4i_tfm_ctx *op = crypto_ablkcipher_ctx(tfm); + struct sun4i_cipher_req_ctx *rctx = ablkcipher_request_ctx(areq); + + rctx->mode = SS_OP_AES | SS_CBC | SS_ENABLED | SS_ENCRYPTION | + op->keymode; + return sun4i_ss_cipher_poll(areq); +} + +int sun4i_ss_cbc_aes_decrypt(struct ablkcipher_request *areq) +{ + struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(areq); + struct sun4i_tfm_ctx *op = crypto_ablkcipher_ctx(tfm); + struct sun4i_cipher_req_ctx *rctx = ablkcipher_request_ctx(areq); + + rctx->mode = SS_OP_AES | SS_CBC | SS_ENABLED | SS_DECRYPTION | + op->keymode; + return sun4i_ss_cipher_poll(areq); +} + +/* ECB AES */ +int sun4i_ss_ecb_aes_encrypt(struct ablkcipher_request *areq) +{ + struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(areq); + struct sun4i_tfm_ctx *op = crypto_ablkcipher_ctx(tfm); + struct sun4i_cipher_req_ctx *rctx = ablkcipher_request_ctx(areq); + + rctx->mode = SS_OP_AES | SS_ECB | SS_ENABLED | SS_ENCRYPTION | + op->keymode; + return sun4i_ss_cipher_poll(areq); +} + +int sun4i_ss_ecb_aes_decrypt(struct ablkcipher_request *areq) +{ + struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(areq); + struct sun4i_tfm_ctx *op = crypto_ablkcipher_ctx(tfm); + struct sun4i_cipher_req_ctx *rctx = ablkcipher_request_ctx(areq); + + rctx->mode = SS_OP_AES | SS_ECB | SS_ENABLED | SS_DECRYPTION | + op->keymode; + return sun4i_ss_cipher_poll(areq); +} + +/* CBC DES */ +int sun4i_ss_cbc_des_encrypt(struct ablkcipher_request *areq) +{ + struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(areq); + struct sun4i_tfm_ctx *op = crypto_ablkcipher_ctx(tfm); + struct sun4i_cipher_req_ctx *rctx = ablkcipher_request_ctx(areq); + + rctx->mode = SS_OP_DES | SS_CBC | SS_ENABLED | SS_ENCRYPTION | + op->keymode; + return sun4i_ss_cipher_poll(areq); +} + +int sun4i_ss_cbc_des_decrypt(struct ablkcipher_request *areq) +{ + struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(areq); + struct sun4i_tfm_ctx *op = crypto_ablkcipher_ctx(tfm); + struct sun4i_cipher_req_ctx *rctx = ablkcipher_request_ctx(areq); + + rctx->mode = SS_OP_DES | SS_CBC | SS_ENABLED | SS_DECRYPTION | + op->keymode; + return sun4i_ss_cipher_poll(areq); +} + +/* ECB DES */ +int sun4i_ss_ecb_des_encrypt(struct ablkcipher_request *areq) +{ + struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(areq); + struct sun4i_tfm_ctx *op = crypto_ablkcipher_ctx(tfm); + struct sun4i_cipher_req_ctx *rctx = ablkcipher_request_ctx(areq); + + rctx->mode = SS_OP_DES | SS_ECB | SS_ENABLED | SS_ENCRYPTION | + op->keymode; + return sun4i_ss_cipher_poll(areq); +} + +int sun4i_ss_ecb_des_decrypt(struct ablkcipher_request *areq) +{ + struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(areq); + struct sun4i_tfm_ctx *op = crypto_ablkcipher_ctx(tfm); + struct sun4i_cipher_req_ctx *rctx = ablkcipher_request_ctx(areq); + + rctx->mode = SS_OP_DES | SS_ECB | SS_ENABLED | SS_DECRYPTION | + op->keymode; + return sun4i_ss_cipher_poll(areq); +} + +/* CBC 3DES */ +int sun4i_ss_cbc_des3_encrypt(struct ablkcipher_request *areq) +{ + struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(areq); + struct sun4i_tfm_ctx *op = crypto_ablkcipher_ctx(tfm); + struct sun4i_cipher_req_ctx *rctx = ablkcipher_request_ctx(areq); + + rctx->mode = SS_OP_3DES | SS_CBC | SS_ENABLED | SS_ENCRYPTION | + op->keymode; + return sun4i_ss_cipher_poll(areq); +} + +int sun4i_ss_cbc_des3_decrypt(struct ablkcipher_request *areq) +{ + struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(areq); + struct sun4i_tfm_ctx *op = crypto_ablkcipher_ctx(tfm); + struct sun4i_cipher_req_ctx *rctx = ablkcipher_request_ctx(areq); + + rctx->mode = SS_OP_3DES | SS_CBC | SS_ENABLED | SS_DECRYPTION | + op->keymode; + return sun4i_ss_cipher_poll(areq); +} + +/* ECB 3DES */ +int sun4i_ss_ecb_des3_encrypt(struct ablkcipher_request *areq) +{ + struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(areq); + struct sun4i_tfm_ctx *op = crypto_ablkcipher_ctx(tfm); + struct sun4i_cipher_req_ctx *rctx = ablkcipher_request_ctx(areq); + + rctx->mode = SS_OP_3DES | SS_ECB | SS_ENABLED | SS_ENCRYPTION | + op->keymode; + return sun4i_ss_cipher_poll(areq); +} + +int sun4i_ss_ecb_des3_decrypt(struct ablkcipher_request *areq) +{ + struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(areq); + struct sun4i_tfm_ctx *op = crypto_ablkcipher_ctx(tfm); + struct sun4i_cipher_req_ctx *rctx = ablkcipher_request_ctx(areq); + + rctx->mode = SS_OP_3DES | SS_ECB | SS_ENABLED | SS_DECRYPTION | + op->keymode; + return sun4i_ss_cipher_poll(areq); +} + +int sun4i_ss_cipher_init(struct crypto_tfm *tfm) +{ + struct sun4i_tfm_ctx *op = crypto_tfm_ctx(tfm); + struct crypto_alg *alg = tfm->__crt_alg; + struct sun4i_ss_alg_template *algt; + + memset(op, 0, sizeof(struct sun4i_tfm_ctx)); + + algt = container_of(alg, struct sun4i_ss_alg_template, alg.crypto); + op->ss = algt->ss; + + tfm->crt_ablkcipher.reqsize = sizeof(struct sun4i_cipher_req_ctx); + + return 0; +} + +/* check and set the AES key, prepare the mode to be used */ +int sun4i_ss_aes_setkey(struct crypto_ablkcipher *tfm, const u8 *key, + unsigned int keylen) +{ + struct sun4i_tfm_ctx *op = crypto_ablkcipher_ctx(tfm); + struct sun4i_ss_ctx *ss = op->ss; + + switch (keylen) { + case 128 / 8: + op->keymode = SS_AES_128BITS; + break; + case 192 / 8: + op->keymode = SS_AES_192BITS; + break; + case 256 / 8: + op->keymode = SS_AES_256BITS; + break; + default: + dev_err(ss->dev, "ERROR: Invalid keylen %u\n", keylen); + crypto_ablkcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); + return -EINVAL; + } + op->keylen = keylen; + memcpy(op->key, key, keylen); + return 0; +} + +/* check and set the DES key, prepare the mode to be used */ +int sun4i_ss_des_setkey(struct crypto_ablkcipher *tfm, const u8 *key, + unsigned int keylen) +{ + struct sun4i_tfm_ctx *op = crypto_ablkcipher_ctx(tfm); + struct sun4i_ss_ctx *ss = op->ss; + u32 flags; + u32 tmp[DES_EXPKEY_WORDS]; + int ret; + + if (unlikely(keylen != DES_KEY_SIZE)) { + dev_err(ss->dev, "Invalid keylen %u\n", keylen); + crypto_ablkcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); + return -EINVAL; + } + + flags = crypto_ablkcipher_get_flags(tfm); + + ret = des_ekey(tmp, key); + if (unlikely(ret == 0) && (flags & CRYPTO_TFM_REQ_WEAK_KEY)) { + crypto_ablkcipher_set_flags(tfm, CRYPTO_TFM_RES_WEAK_KEY); + dev_dbg(ss->dev, "Weak key %u\n", keylen); + return -EINVAL; + } + + op->keylen = keylen; + memcpy(op->key, key, keylen); + return 0; +} + +/* check and set the 3DES key, prepare the mode to be used */ +int sun4i_ss_des3_setkey(struct crypto_ablkcipher *tfm, const u8 *key, + unsigned int keylen) +{ + struct sun4i_tfm_ctx *op = crypto_ablkcipher_ctx(tfm); + struct sun4i_ss_ctx *ss = op->ss; + + if (unlikely(keylen != 3 * DES_KEY_SIZE)) { + dev_err(ss->dev, "Invalid keylen %u\n", keylen); + crypto_ablkcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); + return -EINVAL; + } + op->keylen = keylen; + memcpy(op->key, key, keylen); + return 0; +} diff --git a/drivers/crypto/sunxi-ss/sun4i-ss-core.c b/drivers/crypto/sunxi-ss/sun4i-ss-core.c new file mode 100644 index 000000000000..eab6fe227fa0 --- /dev/null +++ b/drivers/crypto/sunxi-ss/sun4i-ss-core.c @@ -0,0 +1,425 @@ +/* + * sun4i-ss-core.c - hardware cryptographic accelerator for Allwinner A20 SoC + * + * Copyright (C) 2013-2015 Corentin LABBE <clabbe.montjoie@gmail.com> + * + * Core file which registers crypto algorithms supported by the SS. + * + * You could find a link for the datasheet in Documentation/arm/sunxi/README + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + */ +#include <linux/clk.h> +#include <linux/crypto.h> +#include <linux/io.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/platform_device.h> +#include <crypto/scatterwalk.h> +#include <linux/scatterlist.h> +#include <linux/interrupt.h> +#include <linux/delay.h> +#include <linux/reset.h> + +#include "sun4i-ss.h" + +static struct sun4i_ss_alg_template ss_algs[] = { +{ .type = CRYPTO_ALG_TYPE_AHASH, + .mode = SS_OP_MD5, + .alg.hash = { + .init = sun4i_hash_init, + .update = sun4i_hash_update, + .final = sun4i_hash_final, + .finup = sun4i_hash_finup, + .digest = sun4i_hash_digest, + .export = sun4i_hash_export_md5, + .import = sun4i_hash_import_md5, + .halg = { + .digestsize = MD5_DIGEST_SIZE, + .base = { + .cra_name = "md5", + .cra_driver_name = "md5-sun4i-ss", + .cra_priority = 300, + .cra_alignmask = 3, + .cra_flags = CRYPTO_ALG_TYPE_AHASH, + .cra_blocksize = MD5_HMAC_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct sun4i_req_ctx), + .cra_module = THIS_MODULE, + .cra_type = &crypto_ahash_type, + .cra_init = sun4i_hash_crainit + } + } + } +}, +{ .type = CRYPTO_ALG_TYPE_AHASH, + .mode = SS_OP_SHA1, + .alg.hash = { + .init = sun4i_hash_init, + .update = sun4i_hash_update, + .final = sun4i_hash_final, + .finup = sun4i_hash_finup, + .digest = sun4i_hash_digest, + .export = sun4i_hash_export_sha1, + .import = sun4i_hash_import_sha1, + .halg = { + .digestsize = SHA1_DIGEST_SIZE, + .base = { + .cra_name = "sha1", + .cra_driver_name = "sha1-sun4i-ss", + .cra_priority = 300, + .cra_alignmask = 3, + .cra_flags = CRYPTO_ALG_TYPE_AHASH, + .cra_blocksize = SHA1_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct sun4i_req_ctx), + .cra_module = THIS_MODULE, + .cra_type = &crypto_ahash_type, + .cra_init = sun4i_hash_crainit + } + } + } +}, +{ .type = CRYPTO_ALG_TYPE_ABLKCIPHER, + .alg.crypto = { + .cra_name = "cbc(aes)", + .cra_driver_name = "cbc-aes-sun4i-ss", + .cra_priority = 300, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER, + .cra_ctxsize = sizeof(struct sun4i_tfm_ctx), + .cra_module = THIS_MODULE, + .cra_alignmask = 3, + .cra_type = &crypto_ablkcipher_type, + .cra_init = sun4i_ss_cipher_init, + .cra_ablkcipher = { + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = sun4i_ss_aes_setkey, + .encrypt = sun4i_ss_cbc_aes_encrypt, + .decrypt = sun4i_ss_cbc_aes_decrypt, + } + } +}, +{ .type = CRYPTO_ALG_TYPE_ABLKCIPHER, + .alg.crypto = { + .cra_name = "ecb(aes)", + .cra_driver_name = "ecb-aes-sun4i-ss", + .cra_priority = 300, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER, + .cra_ctxsize = sizeof(struct sun4i_tfm_ctx), + .cra_module = THIS_MODULE, + .cra_alignmask = 3, + .cra_type = &crypto_ablkcipher_type, + .cra_init = sun4i_ss_cipher_init, + .cra_ablkcipher = { + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = sun4i_ss_aes_setkey, + .encrypt = sun4i_ss_ecb_aes_encrypt, + .decrypt = sun4i_ss_ecb_aes_decrypt, + } + } +}, +{ .type = CRYPTO_ALG_TYPE_ABLKCIPHER, + .alg.crypto = { + .cra_name = "cbc(des)", + .cra_driver_name = "cbc-des-sun4i-ss", + .cra_priority = 300, + .cra_blocksize = DES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER, + .cra_ctxsize = sizeof(struct sun4i_req_ctx), + .cra_module = THIS_MODULE, + .cra_alignmask = 3, + .cra_type = &crypto_ablkcipher_type, + .cra_init = sun4i_ss_cipher_init, + .cra_u.ablkcipher = { + .min_keysize = DES_KEY_SIZE, + .max_keysize = DES_KEY_SIZE, + .ivsize = DES_BLOCK_SIZE, + .setkey = sun4i_ss_des_setkey, + .encrypt = sun4i_ss_cbc_des_encrypt, + .decrypt = sun4i_ss_cbc_des_decrypt, + } + } +}, +{ .type = CRYPTO_ALG_TYPE_ABLKCIPHER, + .alg.crypto = { + .cra_name = "ecb(des)", + .cra_driver_name = "ecb-des-sun4i-ss", + .cra_priority = 300, + .cra_blocksize = DES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER, + .cra_ctxsize = sizeof(struct sun4i_req_ctx), + .cra_module = THIS_MODULE, + .cra_alignmask = 3, + .cra_type = &crypto_ablkcipher_type, + .cra_init = sun4i_ss_cipher_init, + .cra_u.ablkcipher = { + .min_keysize = DES_KEY_SIZE, + .max_keysize = DES_KEY_SIZE, + .setkey = sun4i_ss_des_setkey, + .encrypt = sun4i_ss_ecb_des_encrypt, + .decrypt = sun4i_ss_ecb_des_decrypt, + } + } +}, +{ .type = CRYPTO_ALG_TYPE_ABLKCIPHER, + .alg.crypto = { + .cra_name = "cbc(des3_ede)", + .cra_driver_name = "cbc-des3-sun4i-ss", + .cra_priority = 300, + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER, + .cra_ctxsize = sizeof(struct sun4i_req_ctx), + .cra_module = THIS_MODULE, + .cra_alignmask = 3, + .cra_type = &crypto_ablkcipher_type, + .cra_init = sun4i_ss_cipher_init, + .cra_u.ablkcipher = { + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .ivsize = DES3_EDE_BLOCK_SIZE, + .setkey = sun4i_ss_des3_setkey, + .encrypt = sun4i_ss_cbc_des3_encrypt, + .decrypt = sun4i_ss_cbc_des3_decrypt, + } + } +}, +{ .type = CRYPTO_ALG_TYPE_ABLKCIPHER, + .alg.crypto = { + .cra_name = "ecb(des3_ede)", + .cra_driver_name = "ecb-des3-sun4i-ss", + .cra_priority = 300, + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER, + .cra_ctxsize = sizeof(struct sun4i_req_ctx), + .cra_module = THIS_MODULE, + .cra_alignmask = 3, + .cra_type = &crypto_ablkcipher_type, + .cra_init = sun4i_ss_cipher_init, + .cra_u.ablkcipher = { + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .ivsize = DES3_EDE_BLOCK_SIZE, + .setkey = sun4i_ss_des3_setkey, + .encrypt = sun4i_ss_ecb_des3_encrypt, + .decrypt = sun4i_ss_ecb_des3_decrypt, + } + } +}, +}; + +static int sun4i_ss_probe(struct platform_device *pdev) +{ + struct resource *res; + u32 v; + int err, i; + unsigned long cr; + const unsigned long cr_ahb = 24 * 1000 * 1000; + const unsigned long cr_mod = 150 * 1000 * 1000; + struct sun4i_ss_ctx *ss; + + if (!pdev->dev.of_node) + return -ENODEV; + + ss = devm_kzalloc(&pdev->dev, sizeof(*ss), GFP_KERNEL); + if (!ss) + return -ENOMEM; + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + ss->base = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(ss->base)) { + dev_err(&pdev->dev, "Cannot request MMIO\n"); + return PTR_ERR(ss->base); + } + + ss->ssclk = devm_clk_get(&pdev->dev, "mod"); + if (IS_ERR(ss->ssclk)) { + err = PTR_ERR(ss->ssclk); + dev_err(&pdev->dev, "Cannot get SS clock err=%d\n", err); + return err; + } + dev_dbg(&pdev->dev, "clock ss acquired\n"); + + ss->busclk = devm_clk_get(&pdev->dev, "ahb"); + if (IS_ERR(ss->busclk)) { + err = PTR_ERR(ss->busclk); + dev_err(&pdev->dev, "Cannot get AHB SS clock err=%d\n", err); + return err; + } + dev_dbg(&pdev->dev, "clock ahb_ss acquired\n"); + + ss->reset = devm_reset_control_get_optional(&pdev->dev, "ahb"); + if (IS_ERR(ss->reset)) { + if (PTR_ERR(ss->reset) == -EPROBE_DEFER) + return PTR_ERR(ss->reset); + dev_info(&pdev->dev, "no reset control found\n"); + ss->reset = NULL; + } + + /* Enable both clocks */ + err = clk_prepare_enable(ss->busclk); + if (err != 0) { + dev_err(&pdev->dev, "Cannot prepare_enable busclk\n"); + return err; + } + err = clk_prepare_enable(ss->ssclk); + if (err != 0) { + dev_err(&pdev->dev, "Cannot prepare_enable ssclk\n"); + goto error_ssclk; + } + + /* + * Check that clock have the correct rates given in the datasheet + * Try to set the clock to the maximum allowed + */ + err = clk_set_rate(ss->ssclk, cr_mod); + if (err != 0) { + dev_err(&pdev->dev, "Cannot set clock rate to ssclk\n"); + goto error_clk; + } + + /* Deassert reset if we have a reset control */ + if (ss->reset) { + err = reset_control_deassert(ss->reset); + if (err) { + dev_err(&pdev->dev, "Cannot deassert reset control\n"); + goto error_clk; + } + } + + /* + * The only impact on clocks below requirement are bad performance, + * so do not print "errors" + * warn on Overclocked clocks + */ + cr = clk_get_rate(ss->busclk); + if (cr >= cr_ahb) + dev_dbg(&pdev->dev, "Clock bus %lu (%lu MHz) (must be >= %lu)\n", + cr, cr / 1000000, cr_ahb); + else + dev_warn(&pdev->dev, "Clock bus %lu (%lu MHz) (must be >= %lu)\n", + cr, cr / 1000000, cr_ahb); + + cr = clk_get_rate(ss->ssclk); + if (cr <= cr_mod) + if (cr < cr_mod) + dev_warn(&pdev->dev, "Clock ss %lu (%lu MHz) (must be <= %lu)\n", + cr, cr / 1000000, cr_mod); + else + dev_dbg(&pdev->dev, "Clock ss %lu (%lu MHz) (must be <= %lu)\n", + cr, cr / 1000000, cr_mod); + else + dev_warn(&pdev->dev, "Clock ss is at %lu (%lu MHz) (must be <= %lu)\n", + cr, cr / 1000000, cr_mod); + + /* + * Datasheet named it "Die Bonding ID" + * I expect to be a sort of Security System Revision number. + * Since the A80 seems to have an other version of SS + * this info could be useful + */ + writel(SS_ENABLED, ss->base + SS_CTL); + v = readl(ss->base + SS_CTL); + v >>= 16; + v &= 0x07; + dev_info(&pdev->dev, "Die ID %d\n", v); + writel(0, ss->base + SS_CTL); + + ss->dev = &pdev->dev; + + spin_lock_init(&ss->slock); + + for (i = 0; i < ARRAY_SIZE(ss_algs); i++) { + ss_algs[i].ss = ss; + switch (ss_algs[i].type) { + case CRYPTO_ALG_TYPE_ABLKCIPHER: + err = crypto_register_alg(&ss_algs[i].alg.crypto); + if (err != 0) { + dev_err(ss->dev, "Fail to register %s\n", + ss_algs[i].alg.crypto.cra_name); + goto error_alg; + } + break; + case CRYPTO_ALG_TYPE_AHASH: + err = crypto_register_ahash(&ss_algs[i].alg.hash); + if (err != 0) { + dev_err(ss->dev, "Fail to register %s\n", + ss_algs[i].alg.hash.halg.base.cra_name); + goto error_alg; + } + break; + } + } + platform_set_drvdata(pdev, ss); + return 0; +error_alg: + i--; + for (; i >= 0; i--) { + switch (ss_algs[i].type) { + case CRYPTO_ALG_TYPE_ABLKCIPHER: + crypto_unregister_alg(&ss_algs[i].alg.crypto); + break; + case CRYPTO_ALG_TYPE_AHASH: + crypto_unregister_ahash(&ss_algs[i].alg.hash); + break; + } + } + if (ss->reset) + reset_control_assert(ss->reset); +error_clk: + clk_disable_unprepare(ss->ssclk); +error_ssclk: + clk_disable_unprepare(ss->busclk); + return err; +} + +static int sun4i_ss_remove(struct platform_device *pdev) +{ + int i; + struct sun4i_ss_ctx *ss = platform_get_drvdata(pdev); + + for (i = 0; i < ARRAY_SIZE(ss_algs); i++) { + switch (ss_algs[i].type) { + case CRYPTO_ALG_TYPE_ABLKCIPHER: + crypto_unregister_alg(&ss_algs[i].alg.crypto); + break; + case CRYPTO_ALG_TYPE_AHASH: + crypto_unregister_ahash(&ss_algs[i].alg.hash); + break; + } + } + + writel(0, ss->base + SS_CTL); + if (ss->reset) + reset_control_assert(ss->reset); + clk_disable_unprepare(ss->busclk); + clk_disable_unprepare(ss->ssclk); + return 0; +} + +static const struct of_device_id a20ss_crypto_of_match_table[] = { + { .compatible = "allwinner,sun4i-a10-crypto" }, + {} +}; +MODULE_DEVICE_TABLE(of, a20ss_crypto_of_match_table); + +static struct platform_driver sun4i_ss_driver = { + .probe = sun4i_ss_probe, + .remove = sun4i_ss_remove, + .driver = { + .name = "sun4i-ss", + .of_match_table = a20ss_crypto_of_match_table, + }, +}; + +module_platform_driver(sun4i_ss_driver); + +MODULE_DESCRIPTION("Allwinner Security System cryptographic accelerator"); +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Corentin LABBE <clabbe.montjoie@gmail.com>"); diff --git a/drivers/crypto/sunxi-ss/sun4i-ss-hash.c b/drivers/crypto/sunxi-ss/sun4i-ss-hash.c new file mode 100644 index 000000000000..ff8031498809 --- /dev/null +++ b/drivers/crypto/sunxi-ss/sun4i-ss-hash.c @@ -0,0 +1,492 @@ +/* + * sun4i-ss-hash.c - hardware cryptographic accelerator for Allwinner A20 SoC + * + * Copyright (C) 2013-2015 Corentin LABBE <clabbe.montjoie@gmail.com> + * + * This file add support for MD5 and SHA1. + * + * You could find the datasheet in Documentation/arm/sunxi/README + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + */ +#include "sun4i-ss.h" +#include <linux/scatterlist.h> + +/* This is a totally arbitrary value */ +#define SS_TIMEOUT 100 + +int sun4i_hash_crainit(struct crypto_tfm *tfm) +{ + crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), + sizeof(struct sun4i_req_ctx)); + return 0; +} + +/* sun4i_hash_init: initialize request context */ +int sun4i_hash_init(struct ahash_request *areq) +{ + struct sun4i_req_ctx *op = ahash_request_ctx(areq); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct ahash_alg *alg = __crypto_ahash_alg(tfm->base.__crt_alg); + struct sun4i_ss_alg_template *algt; + struct sun4i_ss_ctx *ss; + + memset(op, 0, sizeof(struct sun4i_req_ctx)); + + algt = container_of(alg, struct sun4i_ss_alg_template, alg.hash); + ss = algt->ss; + op->ss = algt->ss; + op->mode = algt->mode; + + return 0; +} + +int sun4i_hash_export_md5(struct ahash_request *areq, void *out) +{ + struct sun4i_req_ctx *op = ahash_request_ctx(areq); + struct md5_state *octx = out; + int i; + + octx->byte_count = op->byte_count + op->len; + + memcpy(octx->block, op->buf, op->len); + + if (op->byte_count > 0) { + for (i = 0; i < 4; i++) + octx->hash[i] = op->hash[i]; + } else { + octx->hash[0] = SHA1_H0; + octx->hash[1] = SHA1_H1; + octx->hash[2] = SHA1_H2; + octx->hash[3] = SHA1_H3; + } + + return 0; +} + +int sun4i_hash_import_md5(struct ahash_request *areq, const void *in) +{ + struct sun4i_req_ctx *op = ahash_request_ctx(areq); + const struct md5_state *ictx = in; + int i; + + sun4i_hash_init(areq); + + op->byte_count = ictx->byte_count & ~0x3F; + op->len = ictx->byte_count & 0x3F; + + memcpy(op->buf, ictx->block, op->len); + + for (i = 0; i < 4; i++) + op->hash[i] = ictx->hash[i]; + + return 0; +} + +int sun4i_hash_export_sha1(struct ahash_request *areq, void *out) +{ + struct sun4i_req_ctx *op = ahash_request_ctx(areq); + struct sha1_state *octx = out; + int i; + + octx->count = op->byte_count + op->len; + + memcpy(octx->buffer, op->buf, op->len); + + if (op->byte_count > 0) { + for (i = 0; i < 5; i++) + octx->state[i] = op->hash[i]; + } else { + octx->state[0] = SHA1_H0; + octx->state[1] = SHA1_H1; + octx->state[2] = SHA1_H2; + octx->state[3] = SHA1_H3; + octx->state[4] = SHA1_H4; + } + + return 0; +} + +int sun4i_hash_import_sha1(struct ahash_request *areq, const void *in) +{ + struct sun4i_req_ctx *op = ahash_request_ctx(areq); + const struct sha1_state *ictx = in; + int i; + + sun4i_hash_init(areq); + + op->byte_count = ictx->count & ~0x3F; + op->len = ictx->count & 0x3F; + + memcpy(op->buf, ictx->buffer, op->len); + + for (i = 0; i < 5; i++) + op->hash[i] = ictx->state[i]; + + return 0; +} + +/* + * sun4i_hash_update: update hash engine + * + * Could be used for both SHA1 and MD5 + * Write data by step of 32bits and put then in the SS. + * + * Since we cannot leave partial data and hash state in the engine, + * we need to get the hash state at the end of this function. + * We can get the hash state every 64 bytes + * + * So the first work is to get the number of bytes to write to SS modulo 64 + * The extra bytes will go to a temporary buffer op->buf storing op->len bytes + * + * So at the begin of update() + * if op->len + areq->nbytes < 64 + * => all data will be written to wait buffer (op->buf) and end=0 + * if not, write all data from op->buf to the device and position end to + * complete to 64bytes + * + * example 1: + * update1 60o => op->len=60 + * update2 60o => need one more word to have 64 bytes + * end=4 + * so write all data from op->buf and one word of SGs + * write remaining data in op->buf + * final state op->len=56 + */ +int sun4i_hash_update(struct ahash_request *areq) +{ + u32 v, ivmode = 0; + unsigned int i = 0; + /* + * i is the total bytes read from SGs, to be compared to areq->nbytes + * i is important because we cannot rely on SG length since the sum of + * SG->length could be greater than areq->nbytes + */ + + struct sun4i_req_ctx *op = ahash_request_ctx(areq); + struct sun4i_ss_ctx *ss = op->ss; + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + unsigned int in_i = 0; /* advancement in the current SG */ + unsigned int end; + /* + * end is the position when we need to stop writing to the device, + * to be compared to i + */ + int in_r, err = 0; + unsigned int todo; + u32 spaces, rx_cnt = SS_RX_DEFAULT; + size_t copied = 0; + struct sg_mapping_iter mi; + + dev_dbg(ss->dev, "%s %s bc=%llu len=%u mode=%x wl=%u h0=%0x", + __func__, crypto_tfm_alg_name(areq->base.tfm), + op->byte_count, areq->nbytes, op->mode, + op->len, op->hash[0]); + + if (areq->nbytes == 0) + return 0; + + /* protect against overflow */ + if (areq->nbytes > UINT_MAX - op->len) { + dev_err(ss->dev, "Cannot process too large request\n"); + return -EINVAL; + } + + if (op->len + areq->nbytes < 64) { + /* linearize data to op->buf */ + copied = sg_pcopy_to_buffer(areq->src, sg_nents(areq->src), + op->buf + op->len, areq->nbytes, 0); + op->len += copied; + return 0; + } + + end = ((areq->nbytes + op->len) / 64) * 64 - op->len; + + if (end > areq->nbytes || areq->nbytes - end > 63) { + dev_err(ss->dev, "ERROR: Bound error %u %u\n", + end, areq->nbytes); + return -EINVAL; + } + + spin_lock_bh(&ss->slock); + + /* + * if some data have been processed before, + * we need to restore the partial hash state + */ + if (op->byte_count > 0) { + ivmode = SS_IV_ARBITRARY; + for (i = 0; i < 5; i++) + writel(op->hash[i], ss->base + SS_IV0 + i * 4); + } + /* Enable the device */ + writel(op->mode | SS_ENABLED | ivmode, ss->base + SS_CTL); + + i = 0; + sg_miter_start(&mi, areq->src, sg_nents(areq->src), + SG_MITER_FROM_SG | SG_MITER_ATOMIC); + sg_miter_next(&mi); + in_i = 0; + + do { + /* + * we need to linearize in two case: + * - the buffer is already used + * - the SG does not have enough byte remaining ( < 4) + */ + if (op->len > 0 || (mi.length - in_i) < 4) { + /* + * if we have entered here we have two reason to stop + * - the buffer is full + * - reach the end + */ + while (op->len < 64 && i < end) { + /* how many bytes we can read from current SG */ + in_r = min3(mi.length - in_i, end - i, + 64 - op->len); + memcpy(op->buf + op->len, mi.addr + in_i, in_r); + op->len += in_r; + i += in_r; + in_i += in_r; + if (in_i == mi.length) { + sg_miter_next(&mi); + in_i = 0; + } + } + if (op->len > 3 && (op->len % 4) == 0) { + /* write buf to the device */ + writesl(ss->base + SS_RXFIFO, op->buf, + op->len / 4); + op->byte_count += op->len; + op->len = 0; + } + } + if (mi.length - in_i > 3 && i < end) { + /* how many bytes we can read from current SG */ + in_r = min3(mi.length - in_i, areq->nbytes - i, + ((mi.length - in_i) / 4) * 4); + /* how many bytes we can write in the device*/ + todo = min3((u32)(end - i) / 4, rx_cnt, (u32)in_r / 4); + writesl(ss->base + SS_RXFIFO, mi.addr + in_i, todo); + op->byte_count += todo * 4; + i += todo * 4; + in_i += todo * 4; + rx_cnt -= todo; + if (rx_cnt == 0) { + spaces = readl(ss->base + SS_FCSR); + rx_cnt = SS_RXFIFO_SPACES(spaces); + } + if (in_i == mi.length) { + sg_miter_next(&mi); + in_i = 0; + } + } + } while (i < end); + /* final linear */ + if ((areq->nbytes - i) < 64) { + while (i < areq->nbytes && in_i < mi.length && op->len < 64) { + /* how many bytes we can read from current SG */ + in_r = min3(mi.length - in_i, areq->nbytes - i, + 64 - op->len); + memcpy(op->buf + op->len, mi.addr + in_i, in_r); + op->len += in_r; + i += in_r; + in_i += in_r; + if (in_i == mi.length) { + sg_miter_next(&mi); + in_i = 0; + } + } + } + + sg_miter_stop(&mi); + + writel(op->mode | SS_ENABLED | SS_DATA_END, ss->base + SS_CTL); + i = 0; + do { + v = readl(ss->base + SS_CTL); + i++; + } while (i < SS_TIMEOUT && (v & SS_DATA_END) > 0); + if (i >= SS_TIMEOUT) { + dev_err_ratelimited(ss->dev, + "ERROR: hash end timeout %d>%d ctl=%x len=%u\n", + i, SS_TIMEOUT, v, areq->nbytes); + err = -EIO; + goto release_ss; + } + + /* get the partial hash only if something was written */ + for (i = 0; i < crypto_ahash_digestsize(tfm) / 4; i++) + op->hash[i] = readl(ss->base + SS_MD0 + i * 4); + +release_ss: + writel(0, ss->base + SS_CTL); + spin_unlock_bh(&ss->slock); + return err; +} + +/* + * sun4i_hash_final: finalize hashing operation + * + * If we have some remaining bytes, we write them. + * Then ask the SS for finalizing the hashing operation + * + * I do not check RX FIFO size in this function since the size is 32 + * after each enabling and this function neither write more than 32 words. + */ +int sun4i_hash_final(struct ahash_request *areq) +{ + u32 v, ivmode = 0; + unsigned int i; + unsigned int j = 0; + int zeros, err = 0; + unsigned int index, padlen; + __be64 bits; + struct sun4i_req_ctx *op = ahash_request_ctx(areq); + struct sun4i_ss_ctx *ss = op->ss; + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + u32 bf[32]; + u32 wb = 0; + unsigned int nwait, nbw = 0; + + dev_dbg(ss->dev, "%s: byte=%llu len=%u mode=%x wl=%u h=%x", + __func__, op->byte_count, areq->nbytes, op->mode, + op->len, op->hash[0]); + + spin_lock_bh(&ss->slock); + + /* + * if we have already written something, + * restore the partial hash state + */ + if (op->byte_count > 0) { + ivmode = SS_IV_ARBITRARY; + for (i = 0; i < crypto_ahash_digestsize(tfm) / 4; i++) + writel(op->hash[i], ss->base + SS_IV0 + i * 4); + } + writel(op->mode | SS_ENABLED | ivmode, ss->base + SS_CTL); + + /* write the remaining words of the wait buffer */ + if (op->len > 0) { + nwait = op->len / 4; + if (nwait > 0) { + writesl(ss->base + SS_RXFIFO, op->buf, nwait); + op->byte_count += 4 * nwait; + } + nbw = op->len - 4 * nwait; + wb = *(u32 *)(op->buf + nwait * 4); + wb &= (0xFFFFFFFF >> (4 - nbw) * 8); + } + + /* write the remaining bytes of the nbw buffer */ + if (nbw > 0) { + wb |= ((1 << 7) << (nbw * 8)); + bf[j++] = wb; + } else { + bf[j++] = 1 << 7; + } + + /* + * number of space to pad to obtain 64o minus 8(size) minus 4 (final 1) + * I take the operations from other MD5/SHA1 implementations + */ + + /* we have already send 4 more byte of which nbw data */ + if (op->mode == SS_OP_MD5) { + index = (op->byte_count + 4) & 0x3f; + op->byte_count += nbw; + if (index > 56) + zeros = (120 - index) / 4; + else + zeros = (56 - index) / 4; + } else { + op->byte_count += nbw; + index = op->byte_count & 0x3f; + padlen = (index < 56) ? (56 - index) : ((64 + 56) - index); + zeros = (padlen - 1) / 4; + } + + memset(bf + j, 0, 4 * zeros); + j += zeros; + + /* write the length of data */ + if (op->mode == SS_OP_SHA1) { + bits = cpu_to_be64(op->byte_count << 3); + bf[j++] = bits & 0xffffffff; + bf[j++] = (bits >> 32) & 0xffffffff; + } else { + bf[j++] = (op->byte_count << 3) & 0xffffffff; + bf[j++] = (op->byte_count >> 29) & 0xffffffff; + } + writesl(ss->base + SS_RXFIFO, bf, j); + + /* Tell the SS to stop the hashing */ + writel(op->mode | SS_ENABLED | SS_DATA_END, ss->base + SS_CTL); + + /* + * Wait for SS to finish the hash. + * The timeout could happen only in case of bad overcloking + * or driver bug. + */ + i = 0; + do { + v = readl(ss->base + SS_CTL); + i++; + } while (i < SS_TIMEOUT && (v & SS_DATA_END) > 0); + if (i >= SS_TIMEOUT) { + dev_err_ratelimited(ss->dev, + "ERROR: hash end timeout %d>%d ctl=%x len=%u\n", + i, SS_TIMEOUT, v, areq->nbytes); + err = -EIO; + goto release_ss; + } + + /* Get the hash from the device */ + if (op->mode == SS_OP_SHA1) { + for (i = 0; i < 5; i++) { + v = cpu_to_be32(readl(ss->base + SS_MD0 + i * 4)); + memcpy(areq->result + i * 4, &v, 4); + } + } else { + for (i = 0; i < 4; i++) { + v = readl(ss->base + SS_MD0 + i * 4); + memcpy(areq->result + i * 4, &v, 4); + } + } + +release_ss: + writel(0, ss->base + SS_CTL); + spin_unlock_bh(&ss->slock); + return err; +} + +/* sun4i_hash_finup: finalize hashing operation after an update */ +int sun4i_hash_finup(struct ahash_request *areq) +{ + int err; + + err = sun4i_hash_update(areq); + if (err != 0) + return err; + + return sun4i_hash_final(areq); +} + +/* combo of init/update/final functions */ +int sun4i_hash_digest(struct ahash_request *areq) +{ + int err; + + err = sun4i_hash_init(areq); + if (err != 0) + return err; + + err = sun4i_hash_update(areq); + if (err != 0) + return err; + + return sun4i_hash_final(areq); +} diff --git a/drivers/crypto/sunxi-ss/sun4i-ss.h b/drivers/crypto/sunxi-ss/sun4i-ss.h new file mode 100644 index 000000000000..8e9c05f6e4d4 --- /dev/null +++ b/drivers/crypto/sunxi-ss/sun4i-ss.h @@ -0,0 +1,201 @@ +/* + * sun4i-ss.h - hardware cryptographic accelerator for Allwinner A20 SoC + * + * Copyright (C) 2013-2015 Corentin LABBE <clabbe.montjoie@gmail.com> + * + * Support AES cipher with 128,192,256 bits keysize. + * Support MD5 and SHA1 hash algorithms. + * Support DES and 3DES + * + * You could find the datasheet in Documentation/arm/sunxi/README + * + * Licensed under the GPL-2. + */ + +#include <linux/clk.h> +#include <linux/crypto.h> +#include <linux/io.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/platform_device.h> +#include <linux/reset.h> +#include <crypto/scatterwalk.h> +#include <linux/scatterlist.h> +#include <linux/interrupt.h> +#include <linux/delay.h> +#include <crypto/md5.h> +#include <crypto/sha.h> +#include <crypto/hash.h> +#include <crypto/internal/hash.h> +#include <crypto/aes.h> +#include <crypto/des.h> +#include <crypto/internal/rng.h> + +#define SS_CTL 0x00 +#define SS_KEY0 0x04 +#define SS_KEY1 0x08 +#define SS_KEY2 0x0C +#define SS_KEY3 0x10 +#define SS_KEY4 0x14 +#define SS_KEY5 0x18 +#define SS_KEY6 0x1C +#define SS_KEY7 0x20 + +#define SS_IV0 0x24 +#define SS_IV1 0x28 +#define SS_IV2 0x2C +#define SS_IV3 0x30 + +#define SS_FCSR 0x44 + +#define SS_MD0 0x4C +#define SS_MD1 0x50 +#define SS_MD2 0x54 +#define SS_MD3 0x58 +#define SS_MD4 0x5C + +#define SS_RXFIFO 0x200 +#define SS_TXFIFO 0x204 + +/* SS_CTL configuration values */ + +/* PRNG generator mode - bit 15 */ +#define SS_PRNG_ONESHOT (0 << 15) +#define SS_PRNG_CONTINUE (1 << 15) + +/* IV mode for hash */ +#define SS_IV_ARBITRARY (1 << 14) + +/* SS operation mode - bits 12-13 */ +#define SS_ECB (0 << 12) +#define SS_CBC (1 << 12) +#define SS_CTS (3 << 12) + +/* Counter width for CNT mode - bits 10-11 */ +#define SS_CNT_16BITS (0 << 10) +#define SS_CNT_32BITS (1 << 10) +#define SS_CNT_64BITS (2 << 10) + +/* Key size for AES - bits 8-9 */ +#define SS_AES_128BITS (0 << 8) +#define SS_AES_192BITS (1 << 8) +#define SS_AES_256BITS (2 << 8) + +/* Operation direction - bit 7 */ +#define SS_ENCRYPTION (0 << 7) +#define SS_DECRYPTION (1 << 7) + +/* SS Method - bits 4-6 */ +#define SS_OP_AES (0 << 4) +#define SS_OP_DES (1 << 4) +#define SS_OP_3DES (2 << 4) +#define SS_OP_SHA1 (3 << 4) +#define SS_OP_MD5 (4 << 4) +#define SS_OP_PRNG (5 << 4) + +/* Data end bit - bit 2 */ +#define SS_DATA_END (1 << 2) + +/* PRNG start bit - bit 1 */ +#define SS_PRNG_START (1 << 1) + +/* SS Enable bit - bit 0 */ +#define SS_DISABLED (0 << 0) +#define SS_ENABLED (1 << 0) + +/* SS_FCSR configuration values */ +/* RX FIFO status - bit 30 */ +#define SS_RXFIFO_FREE (1 << 30) + +/* RX FIFO empty spaces - bits 24-29 */ +#define SS_RXFIFO_SPACES(val) (((val) >> 24) & 0x3f) + +/* TX FIFO status - bit 22 */ +#define SS_TXFIFO_AVAILABLE (1 << 22) + +/* TX FIFO available spaces - bits 16-21 */ +#define SS_TXFIFO_SPACES(val) (((val) >> 16) & 0x3f) + +#define SS_RX_MAX 32 +#define SS_RX_DEFAULT SS_RX_MAX +#define SS_TX_MAX 33 + +#define SS_RXFIFO_EMP_INT_PENDING (1 << 10) +#define SS_TXFIFO_AVA_INT_PENDING (1 << 8) +#define SS_RXFIFO_EMP_INT_ENABLE (1 << 2) +#define SS_TXFIFO_AVA_INT_ENABLE (1 << 0) + +struct sun4i_ss_ctx { + void __iomem *base; + int irq; + struct clk *busclk; + struct clk *ssclk; + struct reset_control *reset; + struct device *dev; + struct resource *res; + spinlock_t slock; /* control the use of the device */ +}; + +struct sun4i_ss_alg_template { + u32 type; + u32 mode; + union { + struct crypto_alg crypto; + struct ahash_alg hash; + } alg; + struct sun4i_ss_ctx *ss; +}; + +struct sun4i_tfm_ctx { + u32 key[AES_MAX_KEY_SIZE / 4];/* divided by sizeof(u32) */ + u32 keylen; + u32 keymode; + struct sun4i_ss_ctx *ss; +}; + +struct sun4i_cipher_req_ctx { + u32 mode; +}; + +struct sun4i_req_ctx { + u32 mode; + u64 byte_count; /* number of bytes "uploaded" to the device */ + u32 hash[5]; /* for storing SS_IVx register */ + char buf[64]; + unsigned int len; + struct sun4i_ss_ctx *ss; +}; + +int sun4i_hash_crainit(struct crypto_tfm *tfm); +int sun4i_hash_init(struct ahash_request *areq); +int sun4i_hash_update(struct ahash_request *areq); +int sun4i_hash_final(struct ahash_request *areq); +int sun4i_hash_finup(struct ahash_request *areq); +int sun4i_hash_digest(struct ahash_request *areq); +int sun4i_hash_export_md5(struct ahash_request *areq, void *out); +int sun4i_hash_import_md5(struct ahash_request *areq, const void *in); +int sun4i_hash_export_sha1(struct ahash_request *areq, void *out); +int sun4i_hash_import_sha1(struct ahash_request *areq, const void *in); + +int sun4i_ss_cbc_aes_encrypt(struct ablkcipher_request *areq); +int sun4i_ss_cbc_aes_decrypt(struct ablkcipher_request *areq); +int sun4i_ss_ecb_aes_encrypt(struct ablkcipher_request *areq); +int sun4i_ss_ecb_aes_decrypt(struct ablkcipher_request *areq); + +int sun4i_ss_cbc_des_encrypt(struct ablkcipher_request *areq); +int sun4i_ss_cbc_des_decrypt(struct ablkcipher_request *areq); +int sun4i_ss_ecb_des_encrypt(struct ablkcipher_request *areq); +int sun4i_ss_ecb_des_decrypt(struct ablkcipher_request *areq); + +int sun4i_ss_cbc_des3_encrypt(struct ablkcipher_request *areq); +int sun4i_ss_cbc_des3_decrypt(struct ablkcipher_request *areq); +int sun4i_ss_ecb_des3_encrypt(struct ablkcipher_request *areq); +int sun4i_ss_ecb_des3_decrypt(struct ablkcipher_request *areq); + +int sun4i_ss_cipher_init(struct crypto_tfm *tfm); +int sun4i_ss_aes_setkey(struct crypto_ablkcipher *tfm, const u8 *key, + unsigned int keylen); +int sun4i_ss_des_setkey(struct crypto_ablkcipher *tfm, const u8 *key, + unsigned int keylen); +int sun4i_ss_des3_setkey(struct crypto_ablkcipher *tfm, const u8 *key, + unsigned int keylen); diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c index 83aca95a95bc..cd774534d987 100644 --- a/drivers/crypto/talitos.c +++ b/drivers/crypto/talitos.c @@ -766,6 +766,7 @@ static int talitos_rng_init(struct hwrng *rng) static int talitos_register_rng(struct device *dev) { struct talitos_private *priv = dev_get_drvdata(dev); + int err; priv->rng.name = dev_driver_string(dev), priv->rng.init = talitos_rng_init, @@ -773,14 +774,22 @@ static int talitos_register_rng(struct device *dev) priv->rng.data_read = talitos_rng_data_read, priv->rng.priv = (unsigned long)dev; - return hwrng_register(&priv->rng); + err = hwrng_register(&priv->rng); + if (!err) + priv->rng_registered = true; + + return err; } static void talitos_unregister_rng(struct device *dev) { struct talitos_private *priv = dev_get_drvdata(dev); + if (!priv->rng_registered) + return; + hwrng_unregister(&priv->rng); + priv->rng_registered = false; } /* @@ -799,7 +808,6 @@ struct talitos_ctx { unsigned int keylen; unsigned int enckeylen; unsigned int authkeylen; - unsigned int authsize; }; #define HASH_MAX_BLOCK_SIZE SHA512_BLOCK_SIZE @@ -819,16 +827,6 @@ struct talitos_ahash_req_ctx { struct scatterlist *psrc; }; -static int aead_setauthsize(struct crypto_aead *authenc, - unsigned int authsize) -{ - struct talitos_ctx *ctx = crypto_aead_ctx(authenc); - - ctx->authsize = authsize; - - return 0; -} - static int aead_setkey(struct crypto_aead *authenc, const u8 *key, unsigned int keylen) { @@ -857,12 +855,11 @@ badkey: /* * talitos_edesc - s/w-extended descriptor - * @assoc_nents: number of segments in associated data scatterlist * @src_nents: number of segments in input scatterlist * @dst_nents: number of segments in output scatterlist - * @assoc_chained: whether assoc is chained or not * @src_chained: whether src is chained or not * @dst_chained: whether dst is chained or not + * @icv_ool: whether ICV is out-of-line * @iv_dma: dma address of iv for checking continuity and link table * @dma_len: length of dma mapped link_tbl space * @dma_link_tbl: bus physical address of link_tbl/buf @@ -875,12 +872,11 @@ badkey: * of link_tbl data */ struct talitos_edesc { - int assoc_nents; int src_nents; int dst_nents; - bool assoc_chained; bool src_chained; bool dst_chained; + bool icv_ool; dma_addr_t iv_dma; int dma_len; dma_addr_t dma_link_tbl; @@ -952,14 +948,6 @@ static void ipsec_esp_unmap(struct device *dev, unmap_single_talitos_ptr(dev, &edesc->desc.ptr[2], DMA_TO_DEVICE); unmap_single_talitos_ptr(dev, &edesc->desc.ptr[0], DMA_TO_DEVICE); - if (edesc->assoc_chained) - talitos_unmap_sg_chain(dev, areq->assoc, DMA_TO_DEVICE); - else if (areq->assoclen) - /* assoc_nents counts also for IV in non-contiguous cases */ - dma_unmap_sg(dev, areq->assoc, - edesc->assoc_nents ? edesc->assoc_nents - 1 : 1, - DMA_TO_DEVICE); - talitos_sg_unmap(dev, edesc, areq->src, areq->dst); if (edesc->dma_len) @@ -976,7 +964,7 @@ static void ipsec_esp_encrypt_done(struct device *dev, { struct aead_request *areq = context; struct crypto_aead *authenc = crypto_aead_reqtfm(areq); - struct talitos_ctx *ctx = crypto_aead_ctx(authenc); + unsigned int authsize = crypto_aead_authsize(authenc); struct talitos_edesc *edesc; struct scatterlist *sg; void *icvdata; @@ -986,13 +974,12 @@ static void ipsec_esp_encrypt_done(struct device *dev, ipsec_esp_unmap(dev, edesc, areq); /* copy the generated ICV to dst */ - if (edesc->dst_nents) { + if (edesc->icv_ool) { icvdata = &edesc->link_tbl[edesc->src_nents + - edesc->dst_nents + 2 + - edesc->assoc_nents]; + edesc->dst_nents + 2]; sg = sg_last(areq->dst, edesc->dst_nents); - memcpy((char *)sg_virt(sg) + sg->length - ctx->authsize, - icvdata, ctx->authsize); + memcpy((char *)sg_virt(sg) + sg->length - authsize, + icvdata, authsize); } kfree(edesc); @@ -1006,10 +993,10 @@ static void ipsec_esp_decrypt_swauth_done(struct device *dev, { struct aead_request *req = context; struct crypto_aead *authenc = crypto_aead_reqtfm(req); - struct talitos_ctx *ctx = crypto_aead_ctx(authenc); + unsigned int authsize = crypto_aead_authsize(authenc); struct talitos_edesc *edesc; struct scatterlist *sg; - void *icvdata; + char *oicv, *icv; edesc = container_of(desc, struct talitos_edesc, desc); @@ -1017,16 +1004,18 @@ static void ipsec_esp_decrypt_swauth_done(struct device *dev, if (!err) { /* auth check */ - if (edesc->dma_len) - icvdata = &edesc->link_tbl[edesc->src_nents + - edesc->dst_nents + 2 + - edesc->assoc_nents]; - else - icvdata = &edesc->link_tbl[0]; - sg = sg_last(req->dst, edesc->dst_nents ? : 1); - err = memcmp(icvdata, (char *)sg_virt(sg) + sg->length - - ctx->authsize, ctx->authsize) ? -EBADMSG : 0; + icv = (char *)sg_virt(sg) + sg->length - authsize; + + if (edesc->dma_len) { + oicv = (char *)&edesc->link_tbl[edesc->src_nents + + edesc->dst_nents + 2]; + if (edesc->icv_ool) + icv = oicv + authsize; + } else + oicv = (char *)&edesc->link_tbl[0]; + + err = memcmp(oicv, icv, authsize) ? -EBADMSG : 0; } kfree(edesc); @@ -1059,53 +1048,69 @@ static void ipsec_esp_decrypt_hwauth_done(struct device *dev, * convert scatterlist to SEC h/w link table format * stop at cryptlen bytes */ -static int sg_to_link_tbl(struct scatterlist *sg, int sg_count, - int cryptlen, struct talitos_ptr *link_tbl_ptr) +static int sg_to_link_tbl_offset(struct scatterlist *sg, int sg_count, + unsigned int offset, int cryptlen, + struct talitos_ptr *link_tbl_ptr) { int n_sg = sg_count; + int count = 0; - while (sg && n_sg--) { - to_talitos_ptr(link_tbl_ptr, sg_dma_address(sg), 0); - link_tbl_ptr->len = cpu_to_be16(sg_dma_len(sg)); - link_tbl_ptr->j_extent = 0; - link_tbl_ptr++; - cryptlen -= sg_dma_len(sg); - sg = sg_next(sg); - } + while (cryptlen && sg && n_sg--) { + unsigned int len = sg_dma_len(sg); + + if (offset >= len) { + offset -= len; + goto next; + } + + len -= offset; + + if (len > cryptlen) + len = cryptlen; - /* adjust (decrease) last one (or two) entry's len to cryptlen */ - link_tbl_ptr--; - while (be16_to_cpu(link_tbl_ptr->len) <= (-cryptlen)) { - /* Empty this entry, and move to previous one */ - cryptlen += be16_to_cpu(link_tbl_ptr->len); - link_tbl_ptr->len = 0; - sg_count--; - link_tbl_ptr--; + to_talitos_ptr(link_tbl_ptr + count, + sg_dma_address(sg) + offset, 0); + link_tbl_ptr[count].len = cpu_to_be16(len); + link_tbl_ptr[count].j_extent = 0; + count++; + cryptlen -= len; + offset = 0; + +next: + sg = sg_next(sg); } - link_tbl_ptr->len = cpu_to_be16(be16_to_cpu(link_tbl_ptr->len) - + cryptlen); /* tag end of link table */ - link_tbl_ptr->j_extent = DESC_PTR_LNKTBL_RETURN; + if (count > 0) + link_tbl_ptr[count - 1].j_extent = DESC_PTR_LNKTBL_RETURN; - return sg_count; + return count; +} + +static inline int sg_to_link_tbl(struct scatterlist *sg, int sg_count, + int cryptlen, + struct talitos_ptr *link_tbl_ptr) +{ + return sg_to_link_tbl_offset(sg, sg_count, 0, cryptlen, + link_tbl_ptr); } /* * fill in and submit ipsec_esp descriptor */ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq, - u64 seq, void (*callback) (struct device *dev, - struct talitos_desc *desc, - void *context, int error)) + void (*callback)(struct device *dev, + struct talitos_desc *desc, + void *context, int error)) { struct crypto_aead *aead = crypto_aead_reqtfm(areq); + unsigned int authsize = crypto_aead_authsize(aead); struct talitos_ctx *ctx = crypto_aead_ctx(aead); struct device *dev = ctx->dev; struct talitos_desc *desc = &edesc->desc; unsigned int cryptlen = areq->cryptlen; - unsigned int authsize = ctx->authsize; unsigned int ivsize = crypto_aead_ivsize(aead); + int tbl_off = 0; int sg_count, ret; int sg_link_tbl_len; @@ -1113,36 +1118,27 @@ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq, map_single_talitos_ptr(dev, &desc->ptr[0], ctx->authkeylen, &ctx->key, DMA_TO_DEVICE); + sg_count = talitos_map_sg(dev, areq->src, edesc->src_nents ?: 1, + (areq->src == areq->dst) ? DMA_BIDIRECTIONAL + : DMA_TO_DEVICE, + edesc->src_chained); + /* hmac data */ - desc->ptr[1].len = cpu_to_be16(areq->assoclen + ivsize); - if (edesc->assoc_nents) { - int tbl_off = edesc->src_nents + edesc->dst_nents + 2; - struct talitos_ptr *tbl_ptr = &edesc->link_tbl[tbl_off]; + desc->ptr[1].len = cpu_to_be16(areq->assoclen); + if (sg_count > 1 && + (ret = sg_to_link_tbl_offset(areq->src, sg_count, 0, + areq->assoclen, + &edesc->link_tbl[tbl_off])) > 1) { + tbl_off += ret; to_talitos_ptr(&desc->ptr[1], edesc->dma_link_tbl + tbl_off * sizeof(struct talitos_ptr), 0); desc->ptr[1].j_extent = DESC_PTR_LNKTBL_JUMP; - /* assoc_nents - 1 entries for assoc, 1 for IV */ - sg_count = sg_to_link_tbl(areq->assoc, edesc->assoc_nents - 1, - areq->assoclen, tbl_ptr); - - /* add IV to link table */ - tbl_ptr += sg_count - 1; - tbl_ptr->j_extent = 0; - tbl_ptr++; - to_talitos_ptr(tbl_ptr, edesc->iv_dma, 0); - tbl_ptr->len = cpu_to_be16(ivsize); - tbl_ptr->j_extent = DESC_PTR_LNKTBL_RETURN; - dma_sync_single_for_device(dev, edesc->dma_link_tbl, edesc->dma_len, DMA_BIDIRECTIONAL); } else { - if (areq->assoclen) - to_talitos_ptr(&desc->ptr[1], - sg_dma_address(areq->assoc), 0); - else - to_talitos_ptr(&desc->ptr[1], edesc->iv_dma, 0); + to_talitos_ptr(&desc->ptr[1], sg_dma_address(areq->src), 0); desc->ptr[1].j_extent = 0; } @@ -1150,8 +1146,6 @@ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq, to_talitos_ptr(&desc->ptr[2], edesc->iv_dma, 0); desc->ptr[2].len = cpu_to_be16(ivsize); desc->ptr[2].j_extent = 0; - /* Sync needed for the aead_givencrypt case */ - dma_sync_single_for_device(dev, edesc->iv_dma, ivsize, DMA_TO_DEVICE); /* cipher key */ map_single_talitos_ptr(dev, &desc->ptr[3], ctx->enckeylen, @@ -1167,33 +1161,24 @@ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq, desc->ptr[4].len = cpu_to_be16(cryptlen); desc->ptr[4].j_extent = authsize; - sg_count = talitos_map_sg(dev, areq->src, edesc->src_nents ? : 1, - (areq->src == areq->dst) ? DMA_BIDIRECTIONAL - : DMA_TO_DEVICE, - edesc->src_chained); - - if (sg_count == 1) { + sg_link_tbl_len = cryptlen; + if (edesc->desc.hdr & DESC_HDR_MODE1_MDEU_CICV) + sg_link_tbl_len += authsize; + + if (sg_count > 1 && + (ret = sg_to_link_tbl_offset(areq->src, sg_count, areq->assoclen, + sg_link_tbl_len, + &edesc->link_tbl[tbl_off])) > 1) { + tbl_off += ret; + desc->ptr[4].j_extent |= DESC_PTR_LNKTBL_JUMP; + to_talitos_ptr(&desc->ptr[4], edesc->dma_link_tbl + + tbl_off * + sizeof(struct talitos_ptr), 0); + dma_sync_single_for_device(dev, edesc->dma_link_tbl, + edesc->dma_len, + DMA_BIDIRECTIONAL); + } else to_talitos_ptr(&desc->ptr[4], sg_dma_address(areq->src), 0); - } else { - sg_link_tbl_len = cryptlen; - - if (edesc->desc.hdr & DESC_HDR_MODE1_MDEU_CICV) - sg_link_tbl_len = cryptlen + authsize; - - sg_count = sg_to_link_tbl(areq->src, sg_count, sg_link_tbl_len, - &edesc->link_tbl[0]); - if (sg_count > 1) { - desc->ptr[4].j_extent |= DESC_PTR_LNKTBL_JUMP; - to_talitos_ptr(&desc->ptr[4], edesc->dma_link_tbl, 0); - dma_sync_single_for_device(dev, edesc->dma_link_tbl, - edesc->dma_len, - DMA_BIDIRECTIONAL); - } else { - /* Only one segment now, so no link tbl needed */ - to_talitos_ptr(&desc->ptr[4], - sg_dma_address(areq->src), 0); - } - } /* cipher out */ desc->ptr[5].len = cpu_to_be16(cryptlen); @@ -1204,16 +1189,17 @@ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq, edesc->dst_nents ? : 1, DMA_FROM_DEVICE, edesc->dst_chained); - if (sg_count == 1) { - to_talitos_ptr(&desc->ptr[5], sg_dma_address(areq->dst), 0); - } else { - int tbl_off = edesc->src_nents + 1; + edesc->icv_ool = false; + + if (sg_count > 1 && + (sg_count = sg_to_link_tbl_offset(areq->dst, sg_count, + areq->assoclen, cryptlen, + &edesc->link_tbl[tbl_off])) > + 1) { struct talitos_ptr *tbl_ptr = &edesc->link_tbl[tbl_off]; to_talitos_ptr(&desc->ptr[5], edesc->dma_link_tbl + tbl_off * sizeof(struct talitos_ptr), 0); - sg_count = sg_to_link_tbl(areq->dst, sg_count, cryptlen, - tbl_ptr); /* Add an entry to the link table for ICV data */ tbl_ptr += sg_count - 1; @@ -1224,13 +1210,16 @@ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq, /* icv data follows link tables */ to_talitos_ptr(tbl_ptr, edesc->dma_link_tbl + - (tbl_off + edesc->dst_nents + 1 + - edesc->assoc_nents) * - sizeof(struct talitos_ptr), 0); + (edesc->src_nents + edesc->dst_nents + + 2) * sizeof(struct talitos_ptr) + + authsize, 0); desc->ptr[5].j_extent |= DESC_PTR_LNKTBL_JUMP; dma_sync_single_for_device(ctx->dev, edesc->dma_link_tbl, edesc->dma_len, DMA_BIDIRECTIONAL); - } + + edesc->icv_ool = true; + } else + to_talitos_ptr(&desc->ptr[5], sg_dma_address(areq->dst), 0); /* iv out */ map_single_talitos_ptr(dev, &desc->ptr[6], ivsize, ctx->iv, @@ -1268,7 +1257,6 @@ static int sg_count(struct scatterlist *sg_list, int nbytes, bool *chained) * allocate and map the extended descriptor */ static struct talitos_edesc *talitos_edesc_alloc(struct device *dev, - struct scatterlist *assoc, struct scatterlist *src, struct scatterlist *dst, u8 *iv, @@ -1281,8 +1269,8 @@ static struct talitos_edesc *talitos_edesc_alloc(struct device *dev, bool encrypt) { struct talitos_edesc *edesc; - int assoc_nents = 0, src_nents, dst_nents, alloc_len, dma_len; - bool assoc_chained = false, src_chained = false, dst_chained = false; + int src_nents, dst_nents, alloc_len, dma_len; + bool src_chained = false, dst_chained = false; dma_addr_t iv_dma = 0; gfp_t flags = cryptoflags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : GFP_ATOMIC; @@ -1298,48 +1286,35 @@ static struct talitos_edesc *talitos_edesc_alloc(struct device *dev, if (ivsize) iv_dma = dma_map_single(dev, iv, ivsize, DMA_TO_DEVICE); - if (assoclen) { - /* - * Currently it is assumed that iv is provided whenever assoc - * is. - */ - BUG_ON(!iv); - - assoc_nents = sg_count(assoc, assoclen, &assoc_chained); - talitos_map_sg(dev, assoc, assoc_nents, DMA_TO_DEVICE, - assoc_chained); - assoc_nents = (assoc_nents == 1) ? 0 : assoc_nents; - - if (assoc_nents || sg_dma_address(assoc) + assoclen != iv_dma) - assoc_nents = assoc_nents ? assoc_nents + 1 : 2; - } - if (!dst || dst == src) { - src_nents = sg_count(src, cryptlen + authsize, &src_chained); + src_nents = sg_count(src, assoclen + cryptlen + authsize, + &src_chained); src_nents = (src_nents == 1) ? 0 : src_nents; dst_nents = dst ? src_nents : 0; } else { /* dst && dst != src*/ - src_nents = sg_count(src, cryptlen + (encrypt ? 0 : authsize), + src_nents = sg_count(src, assoclen + cryptlen + + (encrypt ? 0 : authsize), &src_chained); src_nents = (src_nents == 1) ? 0 : src_nents; - dst_nents = sg_count(dst, cryptlen + (encrypt ? authsize : 0), + dst_nents = sg_count(dst, assoclen + cryptlen + + (encrypt ? authsize : 0), &dst_chained); dst_nents = (dst_nents == 1) ? 0 : dst_nents; } /* * allocate space for base edesc plus the link tables, - * allowing for two separate entries for ICV and generated ICV (+ 2), - * and the ICV data itself + * allowing for two separate entries for AD and generated ICV (+ 2), + * and space for two sets of ICVs (stashed and generated) */ alloc_len = sizeof(struct talitos_edesc); - if (assoc_nents || src_nents || dst_nents) { + if (src_nents || dst_nents) { if (is_sec1) dma_len = (src_nents ? cryptlen : 0) + (dst_nents ? cryptlen : 0); else - dma_len = (src_nents + dst_nents + 2 + assoc_nents) * - sizeof(struct talitos_ptr) + authsize; + dma_len = (src_nents + dst_nents + 2) * + sizeof(struct talitos_ptr) + authsize * 2; alloc_len += dma_len; } else { dma_len = 0; @@ -1348,13 +1323,6 @@ static struct talitos_edesc *talitos_edesc_alloc(struct device *dev, edesc = kmalloc(alloc_len, GFP_DMA | flags); if (!edesc) { - if (assoc_chained) - talitos_unmap_sg_chain(dev, assoc, DMA_TO_DEVICE); - else if (assoclen) - dma_unmap_sg(dev, assoc, - assoc_nents ? assoc_nents - 1 : 1, - DMA_TO_DEVICE); - if (iv_dma) dma_unmap_single(dev, iv_dma, ivsize, DMA_TO_DEVICE); @@ -1362,10 +1330,8 @@ static struct talitos_edesc *talitos_edesc_alloc(struct device *dev, return ERR_PTR(-ENOMEM); } - edesc->assoc_nents = assoc_nents; edesc->src_nents = src_nents; edesc->dst_nents = dst_nents; - edesc->assoc_chained = assoc_chained; edesc->src_chained = src_chained; edesc->dst_chained = dst_chained; edesc->iv_dma = iv_dma; @@ -1382,12 +1348,13 @@ static struct talitos_edesc *aead_edesc_alloc(struct aead_request *areq, u8 *iv, int icv_stashing, bool encrypt) { struct crypto_aead *authenc = crypto_aead_reqtfm(areq); + unsigned int authsize = crypto_aead_authsize(authenc); struct talitos_ctx *ctx = crypto_aead_ctx(authenc); unsigned int ivsize = crypto_aead_ivsize(authenc); - return talitos_edesc_alloc(ctx->dev, areq->assoc, areq->src, areq->dst, + return talitos_edesc_alloc(ctx->dev, areq->src, areq->dst, iv, areq->assoclen, areq->cryptlen, - ctx->authsize, ivsize, icv_stashing, + authsize, ivsize, icv_stashing, areq->base.flags, encrypt); } @@ -1405,14 +1372,14 @@ static int aead_encrypt(struct aead_request *req) /* set encrypt */ edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT; - return ipsec_esp(edesc, req, 0, ipsec_esp_encrypt_done); + return ipsec_esp(edesc, req, ipsec_esp_encrypt_done); } static int aead_decrypt(struct aead_request *req) { struct crypto_aead *authenc = crypto_aead_reqtfm(req); + unsigned int authsize = crypto_aead_authsize(authenc); struct talitos_ctx *ctx = crypto_aead_ctx(authenc); - unsigned int authsize = ctx->authsize; struct talitos_private *priv = dev_get_drvdata(ctx->dev); struct talitos_edesc *edesc; struct scatterlist *sg; @@ -1437,7 +1404,7 @@ static int aead_decrypt(struct aead_request *req) /* reset integrity check result bits */ edesc->desc.hdr_lo = 0; - return ipsec_esp(edesc, req, 0, ipsec_esp_decrypt_hwauth_done); + return ipsec_esp(edesc, req, ipsec_esp_decrypt_hwauth_done); } /* Have to check the ICV with software */ @@ -1445,40 +1412,16 @@ static int aead_decrypt(struct aead_request *req) /* stash incoming ICV for later cmp with ICV generated by the h/w */ if (edesc->dma_len) - icvdata = &edesc->link_tbl[edesc->src_nents + - edesc->dst_nents + 2 + - edesc->assoc_nents]; + icvdata = (char *)&edesc->link_tbl[edesc->src_nents + + edesc->dst_nents + 2]; else icvdata = &edesc->link_tbl[0]; sg = sg_last(req->src, edesc->src_nents ? : 1); - memcpy(icvdata, (char *)sg_virt(sg) + sg->length - ctx->authsize, - ctx->authsize); + memcpy(icvdata, (char *)sg_virt(sg) + sg->length - authsize, authsize); - return ipsec_esp(edesc, req, 0, ipsec_esp_decrypt_swauth_done); -} - -static int aead_givencrypt(struct aead_givcrypt_request *req) -{ - struct aead_request *areq = &req->areq; - struct crypto_aead *authenc = crypto_aead_reqtfm(areq); - struct talitos_ctx *ctx = crypto_aead_ctx(authenc); - struct talitos_edesc *edesc; - - /* allocate extended descriptor */ - edesc = aead_edesc_alloc(areq, req->giv, 0, true); - if (IS_ERR(edesc)) - return PTR_ERR(edesc); - - /* set encrypt */ - edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT; - - memcpy(req->giv, ctx->iv, crypto_aead_ivsize(authenc)); - /* avoid consecutive packets going out with same IV */ - *(__be64 *)req->giv ^= cpu_to_be64(req->seq); - - return ipsec_esp(edesc, areq, req->seq, ipsec_esp_encrypt_done); + return ipsec_esp(edesc, req, ipsec_esp_decrypt_swauth_done); } static int ablkcipher_setkey(struct crypto_ablkcipher *cipher, @@ -1710,7 +1653,7 @@ static struct talitos_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request * struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher); unsigned int ivsize = crypto_ablkcipher_ivsize(cipher); - return talitos_edesc_alloc(ctx->dev, NULL, areq->src, areq->dst, + return talitos_edesc_alloc(ctx->dev, areq->src, areq->dst, areq->info, 0, areq->nbytes, 0, ivsize, 0, areq->base.flags, encrypt); } @@ -1895,7 +1838,7 @@ static struct talitos_edesc *ahash_edesc_alloc(struct ahash_request *areq, struct talitos_ctx *ctx = crypto_ahash_ctx(tfm); struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); - return talitos_edesc_alloc(ctx->dev, NULL, req_ctx->psrc, NULL, NULL, 0, + return talitos_edesc_alloc(ctx->dev, req_ctx->psrc, NULL, NULL, 0, nbytes, 0, 0, 0, areq->base.flags, false); } @@ -2161,6 +2104,7 @@ struct talitos_alg_template { union { struct crypto_alg crypto; struct ahash_alg hash; + struct aead_alg aead; } alg; __be32 desc_hdr_template; }; @@ -2168,15 +2112,16 @@ struct talitos_alg_template { static struct talitos_alg_template driver_algs[] = { /* AEAD algorithms. These use a single-pass ipsec_esp descriptor */ { .type = CRYPTO_ALG_TYPE_AEAD, - .alg.crypto = { - .cra_name = "authenc(hmac(sha1),cbc(aes))", - .cra_driver_name = "authenc-hmac-sha1-cbc-aes-talitos", - .cra_blocksize = AES_BLOCK_SIZE, - .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC, - .cra_aead = { - .ivsize = AES_BLOCK_SIZE, - .maxauthsize = SHA1_DIGEST_SIZE, - } + .alg.aead = { + .base = { + .cra_name = "authenc(hmac(sha1),cbc(aes))", + .cra_driver_name = "authenc-hmac-sha1-" + "cbc-aes-talitos", + .cra_blocksize = AES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_ASYNC, + }, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, }, .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | DESC_HDR_SEL0_AESU | @@ -2187,15 +2132,17 @@ static struct talitos_alg_template driver_algs[] = { DESC_HDR_MODE1_MDEU_SHA1_HMAC, }, { .type = CRYPTO_ALG_TYPE_AEAD, - .alg.crypto = { - .cra_name = "authenc(hmac(sha1),cbc(des3_ede))", - .cra_driver_name = "authenc-hmac-sha1-cbc-3des-talitos", - .cra_blocksize = DES3_EDE_BLOCK_SIZE, - .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC, - .cra_aead = { - .ivsize = DES3_EDE_BLOCK_SIZE, - .maxauthsize = SHA1_DIGEST_SIZE, - } + .alg.aead = { + .base = { + .cra_name = "authenc(hmac(sha1)," + "cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-sha1-" + "cbc-3des-talitos", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_ASYNC, + }, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, }, .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | DESC_HDR_SEL0_DEU | @@ -2207,15 +2154,16 @@ static struct talitos_alg_template driver_algs[] = { DESC_HDR_MODE1_MDEU_SHA1_HMAC, }, { .type = CRYPTO_ALG_TYPE_AEAD, - .alg.crypto = { - .cra_name = "authenc(hmac(sha224),cbc(aes))", - .cra_driver_name = "authenc-hmac-sha224-cbc-aes-talitos", - .cra_blocksize = AES_BLOCK_SIZE, - .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC, - .cra_aead = { - .ivsize = AES_BLOCK_SIZE, - .maxauthsize = SHA224_DIGEST_SIZE, - } + .alg.aead = { + .base = { + .cra_name = "authenc(hmac(sha224),cbc(aes))", + .cra_driver_name = "authenc-hmac-sha224-" + "cbc-aes-talitos", + .cra_blocksize = AES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_ASYNC, + }, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA224_DIGEST_SIZE, }, .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | DESC_HDR_SEL0_AESU | @@ -2226,15 +2174,17 @@ static struct talitos_alg_template driver_algs[] = { DESC_HDR_MODE1_MDEU_SHA224_HMAC, }, { .type = CRYPTO_ALG_TYPE_AEAD, - .alg.crypto = { - .cra_name = "authenc(hmac(sha224),cbc(des3_ede))", - .cra_driver_name = "authenc-hmac-sha224-cbc-3des-talitos", - .cra_blocksize = DES3_EDE_BLOCK_SIZE, - .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC, - .cra_aead = { - .ivsize = DES3_EDE_BLOCK_SIZE, - .maxauthsize = SHA224_DIGEST_SIZE, - } + .alg.aead = { + .base = { + .cra_name = "authenc(hmac(sha224)," + "cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-sha224-" + "cbc-3des-talitos", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_ASYNC, + }, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA224_DIGEST_SIZE, }, .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | DESC_HDR_SEL0_DEU | @@ -2246,15 +2196,16 @@ static struct talitos_alg_template driver_algs[] = { DESC_HDR_MODE1_MDEU_SHA224_HMAC, }, { .type = CRYPTO_ALG_TYPE_AEAD, - .alg.crypto = { - .cra_name = "authenc(hmac(sha256),cbc(aes))", - .cra_driver_name = "authenc-hmac-sha256-cbc-aes-talitos", - .cra_blocksize = AES_BLOCK_SIZE, - .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC, - .cra_aead = { - .ivsize = AES_BLOCK_SIZE, - .maxauthsize = SHA256_DIGEST_SIZE, - } + .alg.aead = { + .base = { + .cra_name = "authenc(hmac(sha256),cbc(aes))", + .cra_driver_name = "authenc-hmac-sha256-" + "cbc-aes-talitos", + .cra_blocksize = AES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_ASYNC, + }, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, }, .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | DESC_HDR_SEL0_AESU | @@ -2265,15 +2216,17 @@ static struct talitos_alg_template driver_algs[] = { DESC_HDR_MODE1_MDEU_SHA256_HMAC, }, { .type = CRYPTO_ALG_TYPE_AEAD, - .alg.crypto = { - .cra_name = "authenc(hmac(sha256),cbc(des3_ede))", - .cra_driver_name = "authenc-hmac-sha256-cbc-3des-talitos", - .cra_blocksize = DES3_EDE_BLOCK_SIZE, - .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC, - .cra_aead = { - .ivsize = DES3_EDE_BLOCK_SIZE, - .maxauthsize = SHA256_DIGEST_SIZE, - } + .alg.aead = { + .base = { + .cra_name = "authenc(hmac(sha256)," + "cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-sha256-" + "cbc-3des-talitos", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_ASYNC, + }, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, }, .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | DESC_HDR_SEL0_DEU | @@ -2285,15 +2238,16 @@ static struct talitos_alg_template driver_algs[] = { DESC_HDR_MODE1_MDEU_SHA256_HMAC, }, { .type = CRYPTO_ALG_TYPE_AEAD, - .alg.crypto = { - .cra_name = "authenc(hmac(sha384),cbc(aes))", - .cra_driver_name = "authenc-hmac-sha384-cbc-aes-talitos", - .cra_blocksize = AES_BLOCK_SIZE, - .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC, - .cra_aead = { - .ivsize = AES_BLOCK_SIZE, - .maxauthsize = SHA384_DIGEST_SIZE, - } + .alg.aead = { + .base = { + .cra_name = "authenc(hmac(sha384),cbc(aes))", + .cra_driver_name = "authenc-hmac-sha384-" + "cbc-aes-talitos", + .cra_blocksize = AES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_ASYNC, + }, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA384_DIGEST_SIZE, }, .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | DESC_HDR_SEL0_AESU | @@ -2304,15 +2258,17 @@ static struct talitos_alg_template driver_algs[] = { DESC_HDR_MODE1_MDEUB_SHA384_HMAC, }, { .type = CRYPTO_ALG_TYPE_AEAD, - .alg.crypto = { - .cra_name = "authenc(hmac(sha384),cbc(des3_ede))", - .cra_driver_name = "authenc-hmac-sha384-cbc-3des-talitos", - .cra_blocksize = DES3_EDE_BLOCK_SIZE, - .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC, - .cra_aead = { - .ivsize = DES3_EDE_BLOCK_SIZE, - .maxauthsize = SHA384_DIGEST_SIZE, - } + .alg.aead = { + .base = { + .cra_name = "authenc(hmac(sha384)," + "cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-sha384-" + "cbc-3des-talitos", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_ASYNC, + }, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA384_DIGEST_SIZE, }, .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | DESC_HDR_SEL0_DEU | @@ -2324,15 +2280,16 @@ static struct talitos_alg_template driver_algs[] = { DESC_HDR_MODE1_MDEUB_SHA384_HMAC, }, { .type = CRYPTO_ALG_TYPE_AEAD, - .alg.crypto = { - .cra_name = "authenc(hmac(sha512),cbc(aes))", - .cra_driver_name = "authenc-hmac-sha512-cbc-aes-talitos", - .cra_blocksize = AES_BLOCK_SIZE, - .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC, - .cra_aead = { - .ivsize = AES_BLOCK_SIZE, - .maxauthsize = SHA512_DIGEST_SIZE, - } + .alg.aead = { + .base = { + .cra_name = "authenc(hmac(sha512),cbc(aes))", + .cra_driver_name = "authenc-hmac-sha512-" + "cbc-aes-talitos", + .cra_blocksize = AES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_ASYNC, + }, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA512_DIGEST_SIZE, }, .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | DESC_HDR_SEL0_AESU | @@ -2343,15 +2300,17 @@ static struct talitos_alg_template driver_algs[] = { DESC_HDR_MODE1_MDEUB_SHA512_HMAC, }, { .type = CRYPTO_ALG_TYPE_AEAD, - .alg.crypto = { - .cra_name = "authenc(hmac(sha512),cbc(des3_ede))", - .cra_driver_name = "authenc-hmac-sha512-cbc-3des-talitos", - .cra_blocksize = DES3_EDE_BLOCK_SIZE, - .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC, - .cra_aead = { - .ivsize = DES3_EDE_BLOCK_SIZE, - .maxauthsize = SHA512_DIGEST_SIZE, - } + .alg.aead = { + .base = { + .cra_name = "authenc(hmac(sha512)," + "cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-sha512-" + "cbc-3des-talitos", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_ASYNC, + }, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA512_DIGEST_SIZE, }, .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | DESC_HDR_SEL0_DEU | @@ -2363,15 +2322,16 @@ static struct talitos_alg_template driver_algs[] = { DESC_HDR_MODE1_MDEUB_SHA512_HMAC, }, { .type = CRYPTO_ALG_TYPE_AEAD, - .alg.crypto = { - .cra_name = "authenc(hmac(md5),cbc(aes))", - .cra_driver_name = "authenc-hmac-md5-cbc-aes-talitos", - .cra_blocksize = AES_BLOCK_SIZE, - .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC, - .cra_aead = { - .ivsize = AES_BLOCK_SIZE, - .maxauthsize = MD5_DIGEST_SIZE, - } + .alg.aead = { + .base = { + .cra_name = "authenc(hmac(md5),cbc(aes))", + .cra_driver_name = "authenc-hmac-md5-" + "cbc-aes-talitos", + .cra_blocksize = AES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_ASYNC, + }, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = MD5_DIGEST_SIZE, }, .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | DESC_HDR_SEL0_AESU | @@ -2382,15 +2342,16 @@ static struct talitos_alg_template driver_algs[] = { DESC_HDR_MODE1_MDEU_MD5_HMAC, }, { .type = CRYPTO_ALG_TYPE_AEAD, - .alg.crypto = { - .cra_name = "authenc(hmac(md5),cbc(des3_ede))", - .cra_driver_name = "authenc-hmac-md5-cbc-3des-talitos", - .cra_blocksize = DES3_EDE_BLOCK_SIZE, - .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC, - .cra_aead = { - .ivsize = DES3_EDE_BLOCK_SIZE, - .maxauthsize = MD5_DIGEST_SIZE, - } + .alg.aead = { + .base = { + .cra_name = "authenc(hmac(md5),cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-md5-" + "cbc-3des-talitos", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_ASYNC, + }, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = MD5_DIGEST_SIZE, }, .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | DESC_HDR_SEL0_DEU | @@ -2658,15 +2619,9 @@ static int talitos_cra_init(struct crypto_tfm *tfm) return 0; } -static int talitos_cra_init_aead(struct crypto_tfm *tfm) +static int talitos_cra_init_aead(struct crypto_aead *tfm) { - struct talitos_ctx *ctx = crypto_tfm_ctx(tfm); - - talitos_cra_init(tfm); - - /* random first IV */ - get_random_bytes(ctx->iv, TALITOS_MAX_IV_LENGTH); - + talitos_cra_init(crypto_aead_tfm(tfm)); return 0; } @@ -2713,9 +2668,9 @@ static int talitos_remove(struct platform_device *ofdev) list_for_each_entry_safe(t_alg, n, &priv->alg_list, entry) { switch (t_alg->algt.type) { case CRYPTO_ALG_TYPE_ABLKCIPHER: - case CRYPTO_ALG_TYPE_AEAD: - crypto_unregister_alg(&t_alg->algt.alg.crypto); break; + case CRYPTO_ALG_TYPE_AEAD: + crypto_unregister_aead(&t_alg->algt.alg.aead); case CRYPTO_ALG_TYPE_AHASH: crypto_unregister_ahash(&t_alg->algt.alg.hash); break; @@ -2727,7 +2682,7 @@ static int talitos_remove(struct platform_device *ofdev) if (hw_supports(dev, DESC_HDR_SEL0_RNG)) talitos_unregister_rng(dev); - for (i = 0; i < priv->num_channels; i++) + for (i = 0; priv->chan && i < priv->num_channels; i++) kfree(priv->chan[i].fifo); kfree(priv->chan); @@ -2774,15 +2729,11 @@ static struct talitos_crypto_alg *talitos_alg_alloc(struct device *dev, alg->cra_ablkcipher.geniv = "eseqiv"; break; case CRYPTO_ALG_TYPE_AEAD: - alg = &t_alg->algt.alg.crypto; - alg->cra_init = talitos_cra_init_aead; - alg->cra_type = &crypto_aead_type; - alg->cra_aead.setkey = aead_setkey; - alg->cra_aead.setauthsize = aead_setauthsize; - alg->cra_aead.encrypt = aead_encrypt; - alg->cra_aead.decrypt = aead_decrypt; - alg->cra_aead.givencrypt = aead_givencrypt; - alg->cra_aead.geniv = "<built-in>"; + alg = &t_alg->algt.alg.aead.base; + t_alg->algt.alg.aead.init = talitos_cra_init_aead; + t_alg->algt.alg.aead.setkey = aead_setkey; + t_alg->algt.alg.aead.encrypt = aead_encrypt; + t_alg->algt.alg.aead.decrypt = aead_decrypt; break; case CRYPTO_ALG_TYPE_AHASH: alg = &t_alg->algt.alg.hash.halg.base; @@ -3041,7 +2992,7 @@ static int talitos_probe(struct platform_device *ofdev) for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { if (hw_supports(dev, driver_algs[i].desc_hdr_template)) { struct talitos_crypto_alg *t_alg; - char *name = NULL; + struct crypto_alg *alg = NULL; t_alg = talitos_alg_alloc(dev, &driver_algs[i]); if (IS_ERR(t_alg)) { @@ -3053,21 +3004,26 @@ static int talitos_probe(struct platform_device *ofdev) switch (t_alg->algt.type) { case CRYPTO_ALG_TYPE_ABLKCIPHER: - case CRYPTO_ALG_TYPE_AEAD: err = crypto_register_alg( &t_alg->algt.alg.crypto); - name = t_alg->algt.alg.crypto.cra_driver_name; + alg = &t_alg->algt.alg.crypto; break; + + case CRYPTO_ALG_TYPE_AEAD: + err = crypto_register_aead( + &t_alg->algt.alg.aead); + alg = &t_alg->algt.alg.aead.base; + break; + case CRYPTO_ALG_TYPE_AHASH: err = crypto_register_ahash( &t_alg->algt.alg.hash); - name = - t_alg->algt.alg.hash.halg.base.cra_driver_name; + alg = &t_alg->algt.alg.hash.halg.base; break; } if (err) { dev_err(dev, "%s alg registration failed\n", - name); + alg->cra_driver_name); kfree(t_alg); } else list_add_tail(&t_alg->entry, &priv->alg_list); diff --git a/drivers/crypto/talitos.h b/drivers/crypto/talitos.h index 314daf55e7f7..0090f3211d68 100644 --- a/drivers/crypto/talitos.h +++ b/drivers/crypto/talitos.h @@ -52,12 +52,7 @@ struct talitos_ptr { __be32 ptr; /* address */ }; -static const struct talitos_ptr zero_entry = { - .len = 0, - .j_extent = 0, - .eptr = 0, - .ptr = 0 -}; +static const struct talitos_ptr zero_entry; /* descriptor */ struct talitos_desc { @@ -154,6 +149,7 @@ struct talitos_private { /* hwrng device */ struct hwrng rng; + bool rng_registered; }; extern int talitos_submit(struct device *dev, int ch, struct talitos_desc *desc, diff --git a/drivers/crypto/vmx/aes.c b/drivers/crypto/vmx/aes.c index e79e567e43aa..263af709e536 100644 --- a/drivers/crypto/vmx/aes.c +++ b/drivers/crypto/vmx/aes.c @@ -84,6 +84,7 @@ static int p8_aes_setkey(struct crypto_tfm *tfm, const u8 *key, preempt_disable(); pagefault_disable(); enable_kernel_altivec(); + enable_kernel_vsx(); ret = aes_p8_set_encrypt_key(key, keylen * 8, &ctx->enc_key); ret += aes_p8_set_decrypt_key(key, keylen * 8, &ctx->dec_key); pagefault_enable(); @@ -103,6 +104,7 @@ static void p8_aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) preempt_disable(); pagefault_disable(); enable_kernel_altivec(); + enable_kernel_vsx(); aes_p8_encrypt(src, dst, &ctx->enc_key); pagefault_enable(); preempt_enable(); @@ -119,6 +121,7 @@ static void p8_aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) preempt_disable(); pagefault_disable(); enable_kernel_altivec(); + enable_kernel_vsx(); aes_p8_decrypt(src, dst, &ctx->dec_key); pagefault_enable(); preempt_enable(); diff --git a/drivers/crypto/vmx/aes_cbc.c b/drivers/crypto/vmx/aes_cbc.c index 7299995c78ec..0b8fe2ec5315 100644 --- a/drivers/crypto/vmx/aes_cbc.c +++ b/drivers/crypto/vmx/aes_cbc.c @@ -85,6 +85,7 @@ static int p8_aes_cbc_setkey(struct crypto_tfm *tfm, const u8 *key, preempt_disable(); pagefault_disable(); enable_kernel_altivec(); + enable_kernel_vsx(); ret = aes_p8_set_encrypt_key(key, keylen * 8, &ctx->enc_key); ret += aes_p8_set_decrypt_key(key, keylen * 8, &ctx->dec_key); pagefault_enable(); @@ -115,6 +116,7 @@ static int p8_aes_cbc_encrypt(struct blkcipher_desc *desc, preempt_disable(); pagefault_disable(); enable_kernel_altivec(); + enable_kernel_vsx(); blkcipher_walk_init(&walk, dst, src, nbytes); ret = blkcipher_walk_virt(desc, &walk); @@ -155,6 +157,7 @@ static int p8_aes_cbc_decrypt(struct blkcipher_desc *desc, preempt_disable(); pagefault_disable(); enable_kernel_altivec(); + enable_kernel_vsx(); blkcipher_walk_init(&walk, dst, src, nbytes); ret = blkcipher_walk_virt(desc, &walk); diff --git a/drivers/crypto/vmx/aes_ctr.c b/drivers/crypto/vmx/aes_ctr.c index 7adae42a7b79..ee1306cd8f59 100644 --- a/drivers/crypto/vmx/aes_ctr.c +++ b/drivers/crypto/vmx/aes_ctr.c @@ -82,6 +82,7 @@ static int p8_aes_ctr_setkey(struct crypto_tfm *tfm, const u8 *key, pagefault_disable(); enable_kernel_altivec(); + enable_kernel_vsx(); ret = aes_p8_set_encrypt_key(key, keylen * 8, &ctx->enc_key); pagefault_enable(); @@ -100,6 +101,7 @@ static void p8_aes_ctr_final(struct p8_aes_ctr_ctx *ctx, pagefault_disable(); enable_kernel_altivec(); + enable_kernel_vsx(); aes_p8_encrypt(ctrblk, keystream, &ctx->enc_key); pagefault_enable(); @@ -113,6 +115,7 @@ static int p8_aes_ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *src, unsigned int nbytes) { int ret; + u64 inc; struct blkcipher_walk walk; struct p8_aes_ctr_ctx *ctx = crypto_tfm_ctx(crypto_blkcipher_tfm(desc->tfm)); @@ -131,6 +134,7 @@ static int p8_aes_ctr_crypt(struct blkcipher_desc *desc, while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) { pagefault_disable(); enable_kernel_altivec(); + enable_kernel_vsx(); aes_p8_ctr32_encrypt_blocks(walk.src.virt.addr, walk.dst.virt.addr, (nbytes & @@ -140,7 +144,12 @@ static int p8_aes_ctr_crypt(struct blkcipher_desc *desc, walk.iv); pagefault_enable(); - crypto_inc(walk.iv, AES_BLOCK_SIZE); + /* We need to update IV mostly for last bytes/round */ + inc = (nbytes & AES_BLOCK_MASK) / AES_BLOCK_SIZE; + if (inc > 0) + while (inc--) + crypto_inc(walk.iv, AES_BLOCK_SIZE); + nbytes &= AES_BLOCK_SIZE - 1; ret = blkcipher_walk_done(desc, &walk, nbytes); } diff --git a/drivers/crypto/vmx/aesp8-ppc.pl b/drivers/crypto/vmx/aesp8-ppc.pl index 6c5c20c6108e..228053921b3f 100644 --- a/drivers/crypto/vmx/aesp8-ppc.pl +++ b/drivers/crypto/vmx/aesp8-ppc.pl @@ -1437,28 +1437,28 @@ Load_ctr32_enc_key: ?vperm v31,v31,$out0,$keyperm lvx v25,$x10,$key_ # pre-load round[2] - vadduwm $two,$one,$one + vadduqm $two,$one,$one subi $inp,$inp,15 # undo "caller" $SHL $len,$len,4 - vadduwm $out1,$ivec,$one # counter values ... - vadduwm $out2,$ivec,$two + vadduqm $out1,$ivec,$one # counter values ... + vadduqm $out2,$ivec,$two vxor $out0,$ivec,$rndkey0 # ... xored with rndkey[0] le?li $idx,8 - vadduwm $out3,$out1,$two + vadduqm $out3,$out1,$two vxor $out1,$out1,$rndkey0 le?lvsl $inpperm,0,$idx - vadduwm $out4,$out2,$two + vadduqm $out4,$out2,$two vxor $out2,$out2,$rndkey0 le?vspltisb $tmp,0x0f - vadduwm $out5,$out3,$two + vadduqm $out5,$out3,$two vxor $out3,$out3,$rndkey0 le?vxor $inpperm,$inpperm,$tmp # transform for lvx_u/stvx_u - vadduwm $out6,$out4,$two + vadduqm $out6,$out4,$two vxor $out4,$out4,$rndkey0 - vadduwm $out7,$out5,$two + vadduqm $out7,$out5,$two vxor $out5,$out5,$rndkey0 - vadduwm $ivec,$out6,$two # next counter value + vadduqm $ivec,$out6,$two # next counter value vxor $out6,$out6,$rndkey0 vxor $out7,$out7,$rndkey0 @@ -1594,27 +1594,27 @@ Loop_ctr32_enc8x_middle: vcipherlast $in0,$out0,$in0 vcipherlast $in1,$out1,$in1 - vadduwm $out1,$ivec,$one # counter values ... + vadduqm $out1,$ivec,$one # counter values ... vcipherlast $in2,$out2,$in2 - vadduwm $out2,$ivec,$two + vadduqm $out2,$ivec,$two vxor $out0,$ivec,$rndkey0 # ... xored with rndkey[0] vcipherlast $in3,$out3,$in3 - vadduwm $out3,$out1,$two + vadduqm $out3,$out1,$two vxor $out1,$out1,$rndkey0 vcipherlast $in4,$out4,$in4 - vadduwm $out4,$out2,$two + vadduqm $out4,$out2,$two vxor $out2,$out2,$rndkey0 vcipherlast $in5,$out5,$in5 - vadduwm $out5,$out3,$two + vadduqm $out5,$out3,$two vxor $out3,$out3,$rndkey0 vcipherlast $in6,$out6,$in6 - vadduwm $out6,$out4,$two + vadduqm $out6,$out4,$two vxor $out4,$out4,$rndkey0 vcipherlast $in7,$out7,$in7 - vadduwm $out7,$out5,$two + vadduqm $out7,$out5,$two vxor $out5,$out5,$rndkey0 le?vperm $in0,$in0,$in0,$inpperm - vadduwm $ivec,$out6,$two # next counter value + vadduqm $ivec,$out6,$two # next counter value vxor $out6,$out6,$rndkey0 le?vperm $in1,$in1,$in1,$inpperm vxor $out7,$out7,$rndkey0 diff --git a/drivers/crypto/vmx/ghash.c b/drivers/crypto/vmx/ghash.c index b5e29002b666..2183a2e77641 100644 --- a/drivers/crypto/vmx/ghash.c +++ b/drivers/crypto/vmx/ghash.c @@ -119,6 +119,7 @@ static int p8_ghash_setkey(struct crypto_shash *tfm, const u8 *key, preempt_disable(); pagefault_disable(); enable_kernel_altivec(); + enable_kernel_vsx(); enable_kernel_fp(); gcm_init_p8(ctx->htable, (const u64 *) key); pagefault_enable(); @@ -149,6 +150,7 @@ static int p8_ghash_update(struct shash_desc *desc, preempt_disable(); pagefault_disable(); enable_kernel_altivec(); + enable_kernel_vsx(); enable_kernel_fp(); gcm_ghash_p8(dctx->shash, ctx->htable, dctx->buffer, GHASH_DIGEST_SIZE); @@ -163,6 +165,7 @@ static int p8_ghash_update(struct shash_desc *desc, preempt_disable(); pagefault_disable(); enable_kernel_altivec(); + enable_kernel_vsx(); enable_kernel_fp(); gcm_ghash_p8(dctx->shash, ctx->htable, src, len); pagefault_enable(); @@ -193,6 +196,7 @@ static int p8_ghash_final(struct shash_desc *desc, u8 *out) preempt_disable(); pagefault_disable(); enable_kernel_altivec(); + enable_kernel_vsx(); enable_kernel_fp(); gcm_ghash_p8(dctx->shash, ctx->htable, dctx->buffer, GHASH_DIGEST_SIZE); diff --git a/drivers/crypto/vmx/ghashp8-ppc.pl b/drivers/crypto/vmx/ghashp8-ppc.pl index 0a6f899839dd..d8429cb71f02 100644 --- a/drivers/crypto/vmx/ghashp8-ppc.pl +++ b/drivers/crypto/vmx/ghashp8-ppc.pl @@ -61,6 +61,12 @@ $code=<<___; mtspr 256,r0 li r10,0x30 lvx_u $H,0,r4 # load H + le?xor r7,r7,r7 + le?addi r7,r7,0x8 # need a vperm start with 08 + le?lvsr 5,0,r7 + le?vspltisb 6,0x0f + le?vxor 5,5,6 # set a b-endian mask + le?vperm $H,$H,$H,5 vspltisb $xC2,-16 # 0xf0 vspltisb $t0,1 # one diff --git a/drivers/crypto/vmx/ppc-xlate.pl b/drivers/crypto/vmx/ppc-xlate.pl index a59188494af8..b9997335f193 100644 --- a/drivers/crypto/vmx/ppc-xlate.pl +++ b/drivers/crypto/vmx/ppc-xlate.pl @@ -169,6 +169,7 @@ my $vpmsumd = sub { vcrypto_op(@_, 1224); }; my $vpmsubh = sub { vcrypto_op(@_, 1096); }; my $vpmsumw = sub { vcrypto_op(@_, 1160); }; my $vaddudm = sub { vcrypto_op(@_, 192); }; +my $vadduqm = sub { vcrypto_op(@_, 256); }; my $mtsle = sub { my ($f, $arg) = @_; diff --git a/drivers/pci/quirks.c b/drivers/pci/quirks.c index de5f610e0810..6a30252cd79f 100644 --- a/drivers/pci/quirks.c +++ b/drivers/pci/quirks.c @@ -4048,3 +4048,88 @@ void pci_dev_specific_enable_acs(struct pci_dev *dev) } } } + +/* + * The PCI capabilities list for Intel DH895xCC VFs (device id 0x0443) with + * QuickAssist Technology (QAT) is prematurely terminated in hardware. The + * Next Capability pointer in the MSI Capability Structure should point to + * the PCIe Capability Structure but is incorrectly hardwired as 0 terminating + * the list. + */ +static void quirk_intel_qat_vf_cap(struct pci_dev *pdev) +{ + int pos, i = 0; + u8 next_cap; + u16 reg16, *cap; + struct pci_cap_saved_state *state; + + /* Bail if the hardware bug is fixed */ + if (pdev->pcie_cap || pci_find_capability(pdev, PCI_CAP_ID_EXP)) + return; + + /* Bail if MSI Capability Structure is not found for some reason */ + pos = pci_find_capability(pdev, PCI_CAP_ID_MSI); + if (!pos) + return; + + /* + * Bail if Next Capability pointer in the MSI Capability Structure + * is not the expected incorrect 0x00. + */ + pci_read_config_byte(pdev, pos + 1, &next_cap); + if (next_cap) + return; + + /* + * PCIe Capability Structure is expected to be at 0x50 and should + * terminate the list (Next Capability pointer is 0x00). Verify + * Capability Id and Next Capability pointer is as expected. + * Open-code some of set_pcie_port_type() and pci_cfg_space_size_ext() + * to correctly set kernel data structures which have already been + * set incorrectly due to the hardware bug. + */ + pos = 0x50; + pci_read_config_word(pdev, pos, ®16); + if (reg16 == (0x0000 | PCI_CAP_ID_EXP)) { + u32 status; +#ifndef PCI_EXP_SAVE_REGS +#define PCI_EXP_SAVE_REGS 7 +#endif + int size = PCI_EXP_SAVE_REGS * sizeof(u16); + + pdev->pcie_cap = pos; + pci_read_config_word(pdev, pos + PCI_EXP_FLAGS, ®16); + pdev->pcie_flags_reg = reg16; + pci_read_config_word(pdev, pos + PCI_EXP_DEVCAP, ®16); + pdev->pcie_mpss = reg16 & PCI_EXP_DEVCAP_PAYLOAD; + + pdev->cfg_size = PCI_CFG_SPACE_EXP_SIZE; + if (pci_read_config_dword(pdev, PCI_CFG_SPACE_SIZE, &status) != + PCIBIOS_SUCCESSFUL || (status == 0xffffffff)) + pdev->cfg_size = PCI_CFG_SPACE_SIZE; + + if (pci_find_saved_cap(pdev, PCI_CAP_ID_EXP)) + return; + + /* + * Save PCIE cap + */ + state = kzalloc(sizeof(*state) + size, GFP_KERNEL); + if (!state) + return; + + state->cap.cap_nr = PCI_CAP_ID_EXP; + state->cap.cap_extended = 0; + state->cap.size = size; + cap = (u16 *)&state->cap.data[0]; + pcie_capability_read_word(pdev, PCI_EXP_DEVCTL, &cap[i++]); + pcie_capability_read_word(pdev, PCI_EXP_LNKCTL, &cap[i++]); + pcie_capability_read_word(pdev, PCI_EXP_SLTCTL, &cap[i++]); + pcie_capability_read_word(pdev, PCI_EXP_RTCTL, &cap[i++]); + pcie_capability_read_word(pdev, PCI_EXP_DEVCTL2, &cap[i++]); + pcie_capability_read_word(pdev, PCI_EXP_LNKCTL2, &cap[i++]); + pcie_capability_read_word(pdev, PCI_EXP_SLTCTL2, &cap[i++]); + hlist_add_head(&state->next, &pdev->saved_cap_space); + } +} +DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_INTEL, 0x443, quirk_intel_qat_vf_cap); diff --git a/include/crypto/aead.h b/include/crypto/aead.h index 7169ad04acc0..077cae1e6b51 100644 --- a/include/crypto/aead.h +++ b/include/crypto/aead.h @@ -1,7 +1,7 @@ /* * AEAD: Authenticated Encryption with Associated Data * - * Copyright (c) 2007 Herbert Xu <herbert@gondor.apana.org.au> + * Copyright (c) 2007-2015 Herbert Xu <herbert@gondor.apana.org.au> * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the Free @@ -45,16 +45,40 @@ * a breach in the integrity of the message. In essence, that -EBADMSG error * code is the key bonus an AEAD cipher has over "standard" block chaining * modes. + * + * Memory Structure: + * + * To support the needs of the most prominent user of AEAD ciphers, namely + * IPSEC, the AEAD ciphers have a special memory layout the caller must adhere + * to. + * + * The scatter list pointing to the input data must contain: + * + * * for RFC4106 ciphers, the concatenation of + * associated authentication data || IV || plaintext or ciphertext. Note, the + * same IV (buffer) is also set with the aead_request_set_crypt call. Note, + * the API call of aead_request_set_ad must provide the length of the AAD and + * the IV. The API call of aead_request_set_crypt only points to the size of + * the input plaintext or ciphertext. + * + * * for "normal" AEAD ciphers, the concatenation of + * associated authentication data || plaintext or ciphertext. + * + * It is important to note that if multiple scatter gather list entries form + * the input data mentioned above, the first entry must not point to a NULL + * buffer. If there is any potential where the AAD buffer can be NULL, the + * calling code must contain a precaution to ensure that this does not result + * in the first scatter gather list entry pointing to a NULL buffer. */ +struct crypto_aead; + /** * struct aead_request - AEAD request * @base: Common attributes for async crypto requests - * @old: Boolean whether the old or new AEAD API is used * @assoclen: Length in bytes of associated data for authentication * @cryptlen: Length of data to be encrypted or decrypted * @iv: Initialisation vector - * @assoc: Associated data * @src: Source data * @dst: Destination data * @__ctx: Start of private context data @@ -62,14 +86,11 @@ struct aead_request { struct crypto_async_request base; - bool old; - unsigned int assoclen; unsigned int cryptlen; u8 *iv; - struct scatterlist *assoc; struct scatterlist *src; struct scatterlist *dst; @@ -77,19 +98,6 @@ struct aead_request { }; /** - * struct aead_givcrypt_request - AEAD request with IV generation - * @seq: Sequence number for IV generation - * @giv: Space for generated IV - * @areq: The AEAD request itself - */ -struct aead_givcrypt_request { - u64 seq; - u8 *giv; - - struct aead_request areq; -}; - -/** * struct aead_alg - AEAD cipher definition * @maxauthsize: Set the maximum authentication tag size supported by the * transformation. A transformation may support smaller tag sizes. @@ -141,16 +149,6 @@ struct aead_alg { }; struct crypto_aead { - int (*setkey)(struct crypto_aead *tfm, const u8 *key, - unsigned int keylen); - int (*setauthsize)(struct crypto_aead *tfm, unsigned int authsize); - int (*encrypt)(struct aead_request *req); - int (*decrypt)(struct aead_request *req); - int (*givencrypt)(struct aead_givcrypt_request *req); - int (*givdecrypt)(struct aead_givcrypt_request *req); - - struct crypto_aead *child; - unsigned int authsize; unsigned int reqsize; @@ -192,16 +190,6 @@ static inline void crypto_free_aead(struct crypto_aead *tfm) crypto_destroy_tfm(tfm, crypto_aead_tfm(tfm)); } -static inline struct crypto_aead *crypto_aead_crt(struct crypto_aead *tfm) -{ - return tfm; -} - -static inline struct old_aead_alg *crypto_old_aead_alg(struct crypto_aead *tfm) -{ - return &crypto_aead_tfm(tfm)->__crt_alg->cra_aead; -} - static inline struct aead_alg *crypto_aead_alg(struct crypto_aead *tfm) { return container_of(crypto_aead_tfm(tfm)->__crt_alg, @@ -210,8 +198,7 @@ static inline struct aead_alg *crypto_aead_alg(struct crypto_aead *tfm) static inline unsigned int crypto_aead_alg_ivsize(struct aead_alg *alg) { - return alg->base.cra_aead.encrypt ? alg->base.cra_aead.ivsize : - alg->ivsize; + return alg->ivsize; } /** @@ -337,7 +324,7 @@ static inline struct crypto_aead *crypto_aead_reqtfm(struct aead_request *req) */ static inline int crypto_aead_encrypt(struct aead_request *req) { - return crypto_aead_reqtfm(req)->encrypt(req); + return crypto_aead_alg(crypto_aead_reqtfm(req))->encrypt(req); } /** @@ -364,10 +351,12 @@ static inline int crypto_aead_encrypt(struct aead_request *req) */ static inline int crypto_aead_decrypt(struct aead_request *req) { - if (req->cryptlen < crypto_aead_authsize(crypto_aead_reqtfm(req))) + struct crypto_aead *aead = crypto_aead_reqtfm(req); + + if (req->cryptlen < crypto_aead_authsize(aead)) return -EINVAL; - return crypto_aead_reqtfm(req)->decrypt(req); + return crypto_aead_alg(aead)->decrypt(req); } /** @@ -387,7 +376,10 @@ static inline int crypto_aead_decrypt(struct aead_request *req) * * Return: number of bytes */ -unsigned int crypto_aead_reqsize(struct crypto_aead *tfm); +static inline unsigned int crypto_aead_reqsize(struct crypto_aead *tfm) +{ + return tfm->reqsize; +} /** * aead_request_set_tfm() - update cipher handle reference in request @@ -400,7 +392,7 @@ unsigned int crypto_aead_reqsize(struct crypto_aead *tfm); static inline void aead_request_set_tfm(struct aead_request *req, struct crypto_aead *tfm) { - req->base.tfm = crypto_aead_tfm(tfm->child); + req->base.tfm = crypto_aead_tfm(tfm); } /** @@ -526,23 +518,6 @@ static inline void aead_request_set_crypt(struct aead_request *req, } /** - * aead_request_set_assoc() - set the associated data scatter / gather list - * @req: request handle - * @assoc: associated data scatter / gather list - * @assoclen: number of bytes to process from @assoc - * - * Obsolete, do not use. - */ -static inline void aead_request_set_assoc(struct aead_request *req, - struct scatterlist *assoc, - unsigned int assoclen) -{ - req->assoc = assoc; - req->assoclen = assoclen; - req->old = true; -} - -/** * aead_request_set_ad - set associated data information * @req: request handle * @assoclen: number of bytes in associated data @@ -554,77 +529,6 @@ static inline void aead_request_set_ad(struct aead_request *req, unsigned int assoclen) { req->assoclen = assoclen; - req->old = false; -} - -static inline struct crypto_aead *aead_givcrypt_reqtfm( - struct aead_givcrypt_request *req) -{ - return crypto_aead_reqtfm(&req->areq); -} - -static inline int crypto_aead_givencrypt(struct aead_givcrypt_request *req) -{ - return aead_givcrypt_reqtfm(req)->givencrypt(req); -}; - -static inline int crypto_aead_givdecrypt(struct aead_givcrypt_request *req) -{ - return aead_givcrypt_reqtfm(req)->givdecrypt(req); -}; - -static inline void aead_givcrypt_set_tfm(struct aead_givcrypt_request *req, - struct crypto_aead *tfm) -{ - req->areq.base.tfm = crypto_aead_tfm(tfm); -} - -static inline struct aead_givcrypt_request *aead_givcrypt_alloc( - struct crypto_aead *tfm, gfp_t gfp) -{ - struct aead_givcrypt_request *req; - - req = kmalloc(sizeof(struct aead_givcrypt_request) + - crypto_aead_reqsize(tfm), gfp); - - if (likely(req)) - aead_givcrypt_set_tfm(req, tfm); - - return req; -} - -static inline void aead_givcrypt_free(struct aead_givcrypt_request *req) -{ - kfree(req); -} - -static inline void aead_givcrypt_set_callback( - struct aead_givcrypt_request *req, u32 flags, - crypto_completion_t compl, void *data) -{ - aead_request_set_callback(&req->areq, flags, compl, data); -} - -static inline void aead_givcrypt_set_crypt(struct aead_givcrypt_request *req, - struct scatterlist *src, - struct scatterlist *dst, - unsigned int nbytes, void *iv) -{ - aead_request_set_crypt(&req->areq, src, dst, nbytes, iv); -} - -static inline void aead_givcrypt_set_assoc(struct aead_givcrypt_request *req, - struct scatterlist *assoc, - unsigned int assoclen) -{ - aead_request_set_assoc(&req->areq, assoc, assoclen); -} - -static inline void aead_givcrypt_set_giv(struct aead_givcrypt_request *req, - u8 *giv, u64 seq) -{ - req->giv = giv; - req->seq = seq; } #endif /* _CRYPTO_AEAD_H */ diff --git a/include/crypto/algapi.h b/include/crypto/algapi.h index d4ebf6e9af6a..c9fe145f7dd3 100644 --- a/include/crypto/algapi.h +++ b/include/crypto/algapi.h @@ -18,6 +18,7 @@ #include <linux/skbuff.h> struct crypto_aead; +struct crypto_instance; struct module; struct rtattr; struct seq_file; @@ -30,6 +31,7 @@ struct crypto_type { void (*show)(struct seq_file *m, struct crypto_alg *alg); int (*report)(struct sk_buff *skb, struct crypto_alg *alg); struct crypto_alg *(*lookup)(const char *name, u32 type, u32 mask); + void (*free)(struct crypto_instance *inst); unsigned int type; unsigned int maskclear; @@ -180,7 +182,6 @@ struct crypto_instance *crypto_alloc_instance(const char *name, void crypto_init_queue(struct crypto_queue *queue, unsigned int max_qlen); int crypto_enqueue_request(struct crypto_queue *queue, struct crypto_async_request *request); -void *__crypto_dequeue_request(struct crypto_queue *queue, unsigned int offset); struct crypto_async_request *crypto_dequeue_request(struct crypto_queue *queue); int crypto_tfm_in_queue(struct crypto_queue *queue, struct crypto_tfm *tfm); diff --git a/include/crypto/chacha20.h b/include/crypto/chacha20.h new file mode 100644 index 000000000000..274bbaeeed0f --- /dev/null +++ b/include/crypto/chacha20.h @@ -0,0 +1,25 @@ +/* + * Common values for the ChaCha20 algorithm + */ + +#ifndef _CRYPTO_CHACHA20_H +#define _CRYPTO_CHACHA20_H + +#include <linux/types.h> +#include <linux/crypto.h> + +#define CHACHA20_IV_SIZE 16 +#define CHACHA20_KEY_SIZE 32 +#define CHACHA20_BLOCK_SIZE 64 + +struct chacha20_ctx { + u32 key[8]; +}; + +void crypto_chacha20_init(u32 *state, struct chacha20_ctx *ctx, u8 *iv); +int crypto_chacha20_setkey(struct crypto_tfm *tfm, const u8 *key, + unsigned int keysize); +int crypto_chacha20_crypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes); + +#endif diff --git a/include/crypto/hash.h b/include/crypto/hash.h index 57c8a6ee33c2..8e920b44c0ac 100644 --- a/include/crypto/hash.h +++ b/include/crypto/hash.h @@ -63,6 +63,11 @@ struct ahash_request { void *__ctx[] CRYPTO_MINALIGN_ATTR; }; +#define AHASH_REQUEST_ON_STACK(name, ahash) \ + char __##name##_desc[sizeof(struct ahash_request) + \ + crypto_ahash_reqsize(ahash)] CRYPTO_MINALIGN_ATTR; \ + struct ahash_request *name = (void *)__##name##_desc + /** * struct ahash_alg - asynchronous message digest definition * @init: Initialize the transformation context. Intended only to initialize the diff --git a/include/crypto/internal/aead.h b/include/crypto/internal/aead.h index 4b2547186519..5554cdd8d6c1 100644 --- a/include/crypto/internal/aead.h +++ b/include/crypto/internal/aead.h @@ -1,7 +1,7 @@ /* * AEAD: Authenticated Encryption with Associated Data * - * Copyright (c) 2007 Herbert Xu <herbert@gondor.apana.org.au> + * Copyright (c) 2007-2015 Herbert Xu <herbert@gondor.apana.org.au> * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the Free @@ -21,6 +21,7 @@ struct rtattr; struct aead_instance { + void (*free)(struct aead_instance *inst); union { struct { char head[offsetof(struct aead_alg, base)]; @@ -34,20 +35,15 @@ struct crypto_aead_spawn { struct crypto_spawn base; }; -extern const struct crypto_type crypto_aead_type; -extern const struct crypto_type crypto_nivaead_type; +struct aead_queue { + struct crypto_queue base; +}; static inline void *crypto_aead_ctx(struct crypto_aead *tfm) { return crypto_tfm_ctx(&tfm->base); } -static inline struct crypto_instance *crypto_aead_alg_instance( - struct crypto_aead *aead) -{ - return crypto_tfm_alg_instance(&aead->base); -} - static inline struct crypto_instance *aead_crypto_instance( struct aead_instance *inst) { @@ -61,7 +57,7 @@ static inline struct aead_instance *aead_instance(struct crypto_instance *inst) static inline struct aead_instance *aead_alg_instance(struct crypto_aead *aead) { - return aead_instance(crypto_aead_alg_instance(aead)); + return aead_instance(crypto_tfm_alg_instance(&aead->base)); } static inline void *aead_instance_ctx(struct aead_instance *inst) @@ -90,8 +86,6 @@ static inline void crypto_set_aead_spawn( crypto_set_spawn(&spawn->base, inst); } -struct crypto_alg *crypto_lookup_aead(const char *name, u32 type, u32 mask); - int crypto_grab_aead(struct crypto_aead_spawn *spawn, const char *name, u32 type, u32 mask); @@ -100,12 +94,6 @@ static inline void crypto_drop_aead(struct crypto_aead_spawn *spawn) crypto_drop_spawn(&spawn->base); } -static inline struct crypto_alg *crypto_aead_spawn_alg( - struct crypto_aead_spawn *spawn) -{ - return spawn->base.alg; -} - static inline struct aead_alg *crypto_spawn_aead_alg( struct crypto_aead_spawn *spawn) { @@ -118,43 +106,51 @@ static inline struct crypto_aead *crypto_spawn_aead( return crypto_spawn_tfm2(&spawn->base); } -struct aead_instance *aead_geniv_alloc(struct crypto_template *tmpl, - struct rtattr **tb, u32 type, u32 mask); -void aead_geniv_free(struct aead_instance *inst); -int aead_geniv_init(struct crypto_tfm *tfm); -void aead_geniv_exit(struct crypto_tfm *tfm); +static inline void crypto_aead_set_reqsize(struct crypto_aead *aead, + unsigned int reqsize) +{ + aead->reqsize = reqsize; +} -static inline struct crypto_aead *aead_geniv_base(struct crypto_aead *geniv) +static inline unsigned int crypto_aead_alg_maxauthsize(struct aead_alg *alg) { - return geniv->child; + return alg->maxauthsize; } -static inline void *aead_givcrypt_reqctx(struct aead_givcrypt_request *req) +static inline unsigned int crypto_aead_maxauthsize(struct crypto_aead *aead) { - return aead_request_ctx(&req->areq); + return crypto_aead_alg_maxauthsize(crypto_aead_alg(aead)); } -static inline void aead_givcrypt_complete(struct aead_givcrypt_request *req, - int err) +static inline void aead_init_queue(struct aead_queue *queue, + unsigned int max_qlen) { - aead_request_complete(&req->areq, err); + crypto_init_queue(&queue->base, max_qlen); } -static inline void crypto_aead_set_reqsize(struct crypto_aead *aead, - unsigned int reqsize) +static inline int aead_enqueue_request(struct aead_queue *queue, + struct aead_request *request) { - crypto_aead_crt(aead)->reqsize = reqsize; + return crypto_enqueue_request(&queue->base, &request->base); } -static inline unsigned int crypto_aead_alg_maxauthsize(struct aead_alg *alg) +static inline struct aead_request *aead_dequeue_request( + struct aead_queue *queue) { - return alg->base.cra_aead.encrypt ? alg->base.cra_aead.maxauthsize : - alg->maxauthsize; + struct crypto_async_request *req; + + req = crypto_dequeue_request(&queue->base); + + return req ? container_of(req, struct aead_request, base) : NULL; } -static inline unsigned int crypto_aead_maxauthsize(struct crypto_aead *aead) +static inline struct aead_request *aead_get_backlog(struct aead_queue *queue) { - return crypto_aead_alg_maxauthsize(crypto_aead_alg(aead)); + struct crypto_async_request *req; + + req = crypto_get_backlog(&queue->base); + + return req ? container_of(req, struct aead_request, base) : NULL; } int crypto_register_aead(struct aead_alg *alg); diff --git a/include/crypto/internal/geniv.h b/include/crypto/internal/geniv.h index 9ca9b871aba5..59333635e712 100644 --- a/include/crypto/internal/geniv.h +++ b/include/crypto/internal/geniv.h @@ -15,10 +15,19 @@ #include <crypto/internal/aead.h> #include <linux/spinlock.h> +#include <linux/types.h> struct aead_geniv_ctx { spinlock_t lock; struct crypto_aead *child; + struct crypto_blkcipher *null; + u8 salt[] __attribute__ ((aligned(__alignof__(u32)))); }; +struct aead_instance *aead_geniv_alloc(struct crypto_template *tmpl, + struct rtattr **tb, u32 type, u32 mask); +void aead_geniv_free(struct aead_instance *inst); +int aead_init_geniv(struct crypto_aead *tfm); +void aead_exit_geniv(struct crypto_aead *tfm); + #endif /* _CRYPTO_INTERNAL_GENIV_H */ diff --git a/include/crypto/internal/skcipher.h b/include/crypto/internal/skcipher.h index b3a46c515d1b..2cf7a61ece59 100644 --- a/include/crypto/internal/skcipher.h +++ b/include/crypto/internal/skcipher.h @@ -107,5 +107,20 @@ static inline u32 ablkcipher_request_flags(struct ablkcipher_request *req) return req->base.flags; } +static inline void *crypto_skcipher_ctx(struct crypto_skcipher *tfm) +{ + return crypto_tfm_ctx(&tfm->base); +} + +static inline void *skcipher_request_ctx(struct skcipher_request *req) +{ + return req->__ctx; +} + +static inline u32 skcipher_request_flags(struct skcipher_request *req) +{ + return req->base.flags; +} + #endif /* _CRYPTO_INTERNAL_SKCIPHER_H */ diff --git a/include/crypto/poly1305.h b/include/crypto/poly1305.h new file mode 100644 index 000000000000..894df59b74e4 --- /dev/null +++ b/include/crypto/poly1305.h @@ -0,0 +1,41 @@ +/* + * Common values for the Poly1305 algorithm + */ + +#ifndef _CRYPTO_POLY1305_H +#define _CRYPTO_POLY1305_H + +#include <linux/types.h> +#include <linux/crypto.h> + +#define POLY1305_BLOCK_SIZE 16 +#define POLY1305_KEY_SIZE 32 +#define POLY1305_DIGEST_SIZE 16 + +struct poly1305_desc_ctx { + /* key */ + u32 r[5]; + /* finalize key */ + u32 s[4]; + /* accumulator */ + u32 h[5]; + /* partial buffer */ + u8 buf[POLY1305_BLOCK_SIZE]; + /* bytes used in partial buffer */ + unsigned int buflen; + /* r key has been set */ + bool rset; + /* s key has been set */ + bool sset; +}; + +int crypto_poly1305_init(struct shash_desc *desc); +int crypto_poly1305_setkey(struct crypto_shash *tfm, + const u8 *key, unsigned int keylen); +unsigned int crypto_poly1305_setdesckey(struct poly1305_desc_ctx *dctx, + const u8 *src, unsigned int srclen); +int crypto_poly1305_update(struct shash_desc *desc, + const u8 *src, unsigned int srclen); +int crypto_poly1305_final(struct shash_desc *desc, u8 *dst); + +#endif diff --git a/include/crypto/skcipher.h b/include/crypto/skcipher.h index 07d245f073d1..d8dd41fb034f 100644 --- a/include/crypto/skcipher.h +++ b/include/crypto/skcipher.h @@ -1,7 +1,7 @@ /* * Symmetric key ciphers. * - * Copyright (c) 2007 Herbert Xu <herbert@gondor.apana.org.au> + * Copyright (c) 2007-2015 Herbert Xu <herbert@gondor.apana.org.au> * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the Free @@ -18,6 +18,28 @@ #include <linux/slab.h> /** + * struct skcipher_request - Symmetric key cipher request + * @cryptlen: Number of bytes to encrypt or decrypt + * @iv: Initialisation Vector + * @src: Source SG list + * @dst: Destination SG list + * @base: Underlying async request request + * @__ctx: Start of private context data + */ +struct skcipher_request { + unsigned int cryptlen; + + u8 *iv; + + struct scatterlist *src; + struct scatterlist *dst; + + struct crypto_async_request base; + + void *__ctx[] CRYPTO_MINALIGN_ATTR; +}; + +/** * struct skcipher_givcrypt_request - Crypto request with IV generation * @seq: Sequence number for IV generation * @giv: Space for generated IV @@ -30,6 +52,23 @@ struct skcipher_givcrypt_request { struct ablkcipher_request creq; }; +struct crypto_skcipher { + int (*setkey)(struct crypto_skcipher *tfm, const u8 *key, + unsigned int keylen); + int (*encrypt)(struct skcipher_request *req); + int (*decrypt)(struct skcipher_request *req); + + unsigned int ivsize; + unsigned int reqsize; + + struct crypto_tfm base; +}; + +#define SKCIPHER_REQUEST_ON_STACK(name, tfm) \ + char __##name##_desc[sizeof(struct skcipher_request) + \ + crypto_skcipher_reqsize(tfm)] CRYPTO_MINALIGN_ATTR; \ + struct skcipher_request *name = (void *)__##name##_desc + static inline struct crypto_ablkcipher *skcipher_givcrypt_reqtfm( struct skcipher_givcrypt_request *req) { @@ -106,5 +145,355 @@ static inline void skcipher_givcrypt_set_giv( req->seq = seq; } +/** + * DOC: Symmetric Key Cipher API + * + * Symmetric key cipher API is used with the ciphers of type + * CRYPTO_ALG_TYPE_SKCIPHER (listed as type "skcipher" in /proc/crypto). + * + * Asynchronous cipher operations imply that the function invocation for a + * cipher request returns immediately before the completion of the operation. + * The cipher request is scheduled as a separate kernel thread and therefore + * load-balanced on the different CPUs via the process scheduler. To allow + * the kernel crypto API to inform the caller about the completion of a cipher + * request, the caller must provide a callback function. That function is + * invoked with the cipher handle when the request completes. + * + * To support the asynchronous operation, additional information than just the + * cipher handle must be supplied to the kernel crypto API. That additional + * information is given by filling in the skcipher_request data structure. + * + * For the symmetric key cipher API, the state is maintained with the tfm + * cipher handle. A single tfm can be used across multiple calls and in + * parallel. For asynchronous block cipher calls, context data supplied and + * only used by the caller can be referenced the request data structure in + * addition to the IV used for the cipher request. The maintenance of such + * state information would be important for a crypto driver implementer to + * have, because when calling the callback function upon completion of the + * cipher operation, that callback function may need some information about + * which operation just finished if it invoked multiple in parallel. This + * state information is unused by the kernel crypto API. + */ + +static inline struct crypto_skcipher *__crypto_skcipher_cast( + struct crypto_tfm *tfm) +{ + return container_of(tfm, struct crypto_skcipher, base); +} + +/** + * crypto_alloc_skcipher() - allocate symmetric key cipher handle + * @alg_name: is the cra_name / name or cra_driver_name / driver name of the + * skcipher cipher + * @type: specifies the type of the cipher + * @mask: specifies the mask for the cipher + * + * Allocate a cipher handle for an skcipher. The returned struct + * crypto_skcipher is the cipher handle that is required for any subsequent + * API invocation for that skcipher. + * + * Return: allocated cipher handle in case of success; IS_ERR() is true in case + * of an error, PTR_ERR() returns the error code. + */ +struct crypto_skcipher *crypto_alloc_skcipher(const char *alg_name, + u32 type, u32 mask); + +static inline struct crypto_tfm *crypto_skcipher_tfm( + struct crypto_skcipher *tfm) +{ + return &tfm->base; +} + +/** + * crypto_free_skcipher() - zeroize and free cipher handle + * @tfm: cipher handle to be freed + */ +static inline void crypto_free_skcipher(struct crypto_skcipher *tfm) +{ + crypto_destroy_tfm(tfm, crypto_skcipher_tfm(tfm)); +} + +/** + * crypto_has_skcipher() - Search for the availability of an skcipher. + * @alg_name: is the cra_name / name or cra_driver_name / driver name of the + * skcipher + * @type: specifies the type of the cipher + * @mask: specifies the mask for the cipher + * + * Return: true when the skcipher is known to the kernel crypto API; false + * otherwise + */ +static inline int crypto_has_skcipher(const char *alg_name, u32 type, + u32 mask) +{ + return crypto_has_alg(alg_name, crypto_skcipher_type(type), + crypto_skcipher_mask(mask)); +} + +/** + * crypto_skcipher_ivsize() - obtain IV size + * @tfm: cipher handle + * + * The size of the IV for the skcipher referenced by the cipher handle is + * returned. This IV size may be zero if the cipher does not need an IV. + * + * Return: IV size in bytes + */ +static inline unsigned int crypto_skcipher_ivsize(struct crypto_skcipher *tfm) +{ + return tfm->ivsize; +} + +/** + * crypto_skcipher_blocksize() - obtain block size of cipher + * @tfm: cipher handle + * + * The block size for the skcipher referenced with the cipher handle is + * returned. The caller may use that information to allocate appropriate + * memory for the data returned by the encryption or decryption operation + * + * Return: block size of cipher + */ +static inline unsigned int crypto_skcipher_blocksize( + struct crypto_skcipher *tfm) +{ + return crypto_tfm_alg_blocksize(crypto_skcipher_tfm(tfm)); +} + +static inline unsigned int crypto_skcipher_alignmask( + struct crypto_skcipher *tfm) +{ + return crypto_tfm_alg_alignmask(crypto_skcipher_tfm(tfm)); +} + +static inline u32 crypto_skcipher_get_flags(struct crypto_skcipher *tfm) +{ + return crypto_tfm_get_flags(crypto_skcipher_tfm(tfm)); +} + +static inline void crypto_skcipher_set_flags(struct crypto_skcipher *tfm, + u32 flags) +{ + crypto_tfm_set_flags(crypto_skcipher_tfm(tfm), flags); +} + +static inline void crypto_skcipher_clear_flags(struct crypto_skcipher *tfm, + u32 flags) +{ + crypto_tfm_clear_flags(crypto_skcipher_tfm(tfm), flags); +} + +/** + * crypto_skcipher_setkey() - set key for cipher + * @tfm: cipher handle + * @key: buffer holding the key + * @keylen: length of the key in bytes + * + * The caller provided key is set for the skcipher referenced by the cipher + * handle. + * + * Note, the key length determines the cipher type. Many block ciphers implement + * different cipher modes depending on the key size, such as AES-128 vs AES-192 + * vs. AES-256. When providing a 16 byte key for an AES cipher handle, AES-128 + * is performed. + * + * Return: 0 if the setting of the key was successful; < 0 if an error occurred + */ +static inline int crypto_skcipher_setkey(struct crypto_skcipher *tfm, + const u8 *key, unsigned int keylen) +{ + return tfm->setkey(tfm, key, keylen); +} + +/** + * crypto_skcipher_reqtfm() - obtain cipher handle from request + * @req: skcipher_request out of which the cipher handle is to be obtained + * + * Return the crypto_skcipher handle when furnishing an skcipher_request + * data structure. + * + * Return: crypto_skcipher handle + */ +static inline struct crypto_skcipher *crypto_skcipher_reqtfm( + struct skcipher_request *req) +{ + return __crypto_skcipher_cast(req->base.tfm); +} + +/** + * crypto_skcipher_encrypt() - encrypt plaintext + * @req: reference to the skcipher_request handle that holds all information + * needed to perform the cipher operation + * + * Encrypt plaintext data using the skcipher_request handle. That data + * structure and how it is filled with data is discussed with the + * skcipher_request_* functions. + * + * Return: 0 if the cipher operation was successful; < 0 if an error occurred + */ +static inline int crypto_skcipher_encrypt(struct skcipher_request *req) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + + return tfm->encrypt(req); +} + +/** + * crypto_skcipher_decrypt() - decrypt ciphertext + * @req: reference to the skcipher_request handle that holds all information + * needed to perform the cipher operation + * + * Decrypt ciphertext data using the skcipher_request handle. That data + * structure and how it is filled with data is discussed with the + * skcipher_request_* functions. + * + * Return: 0 if the cipher operation was successful; < 0 if an error occurred + */ +static inline int crypto_skcipher_decrypt(struct skcipher_request *req) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + + return tfm->decrypt(req); +} + +/** + * DOC: Symmetric Key Cipher Request Handle + * + * The skcipher_request data structure contains all pointers to data + * required for the symmetric key cipher operation. This includes the cipher + * handle (which can be used by multiple skcipher_request instances), pointer + * to plaintext and ciphertext, asynchronous callback function, etc. It acts + * as a handle to the skcipher_request_* API calls in a similar way as + * skcipher handle to the crypto_skcipher_* API calls. + */ + +/** + * crypto_skcipher_reqsize() - obtain size of the request data structure + * @tfm: cipher handle + * + * Return: number of bytes + */ +static inline unsigned int crypto_skcipher_reqsize(struct crypto_skcipher *tfm) +{ + return tfm->reqsize; +} + +/** + * skcipher_request_set_tfm() - update cipher handle reference in request + * @req: request handle to be modified + * @tfm: cipher handle that shall be added to the request handle + * + * Allow the caller to replace the existing skcipher handle in the request + * data structure with a different one. + */ +static inline void skcipher_request_set_tfm(struct skcipher_request *req, + struct crypto_skcipher *tfm) +{ + req->base.tfm = crypto_skcipher_tfm(tfm); +} + +static inline struct skcipher_request *skcipher_request_cast( + struct crypto_async_request *req) +{ + return container_of(req, struct skcipher_request, base); +} + +/** + * skcipher_request_alloc() - allocate request data structure + * @tfm: cipher handle to be registered with the request + * @gfp: memory allocation flag that is handed to kmalloc by the API call. + * + * Allocate the request data structure that must be used with the skcipher + * encrypt and decrypt API calls. During the allocation, the provided skcipher + * handle is registered in the request data structure. + * + * Return: allocated request handle in case of success; IS_ERR() is true in case + * of an error, PTR_ERR() returns the error code. + */ +static inline struct skcipher_request *skcipher_request_alloc( + struct crypto_skcipher *tfm, gfp_t gfp) +{ + struct skcipher_request *req; + + req = kmalloc(sizeof(struct skcipher_request) + + crypto_skcipher_reqsize(tfm), gfp); + + if (likely(req)) + skcipher_request_set_tfm(req, tfm); + + return req; +} + +/** + * skcipher_request_free() - zeroize and free request data structure + * @req: request data structure cipher handle to be freed + */ +static inline void skcipher_request_free(struct skcipher_request *req) +{ + kzfree(req); +} + +/** + * skcipher_request_set_callback() - set asynchronous callback function + * @req: request handle + * @flags: specify zero or an ORing of the flags + * CRYPTO_TFM_REQ_MAY_BACKLOG the request queue may back log and + * increase the wait queue beyond the initial maximum size; + * CRYPTO_TFM_REQ_MAY_SLEEP the request processing may sleep + * @compl: callback function pointer to be registered with the request handle + * @data: The data pointer refers to memory that is not used by the kernel + * crypto API, but provided to the callback function for it to use. Here, + * the caller can provide a reference to memory the callback function can + * operate on. As the callback function is invoked asynchronously to the + * related functionality, it may need to access data structures of the + * related functionality which can be referenced using this pointer. The + * callback function can access the memory via the "data" field in the + * crypto_async_request data structure provided to the callback function. + * + * This function allows setting the callback function that is triggered once the + * cipher operation completes. + * + * The callback function is registered with the skcipher_request handle and + * must comply with the following template + * + * void callback_function(struct crypto_async_request *req, int error) + */ +static inline void skcipher_request_set_callback(struct skcipher_request *req, + u32 flags, + crypto_completion_t compl, + void *data) +{ + req->base.complete = compl; + req->base.data = data; + req->base.flags = flags; +} + +/** + * skcipher_request_set_crypt() - set data buffers + * @req: request handle + * @src: source scatter / gather list + * @dst: destination scatter / gather list + * @cryptlen: number of bytes to process from @src + * @iv: IV for the cipher operation which must comply with the IV size defined + * by crypto_skcipher_ivsize + * + * This function allows setting of the source data and destination data + * scatter / gather lists. + * + * For encryption, the source is treated as the plaintext and the + * destination is the ciphertext. For a decryption operation, the use is + * reversed - the source is the ciphertext and the destination is the plaintext. + */ +static inline void skcipher_request_set_crypt( + struct skcipher_request *req, + struct scatterlist *src, struct scatterlist *dst, + unsigned int cryptlen, void *iv) +{ + req->src = src; + req->dst = dst; + req->cryptlen = cryptlen; + req->iv = iv; +} + #endif /* _CRYPTO_SKCIPHER_H */ diff --git a/include/dt-bindings/clock/imx6qdl-clock.h b/include/dt-bindings/clock/imx6qdl-clock.h index 8780868458a0..8de173ff19f3 100644 --- a/include/dt-bindings/clock/imx6qdl-clock.h +++ b/include/dt-bindings/clock/imx6qdl-clock.h @@ -251,6 +251,9 @@ #define IMX6QDL_CLK_VIDEO_27M 238 #define IMX6QDL_CLK_MIPI_CORE_CFG 239 #define IMX6QDL_CLK_MIPI_IPG 240 -#define IMX6QDL_CLK_END 241 +#define IMX6QDL_CLK_CAAM_MEM 241 +#define IMX6QDL_CLK_CAAM_ACLK 242 +#define IMX6QDL_CLK_CAAM_IPG 243 +#define IMX6QDL_CLK_END 244 #endif /* __DT_BINDINGS_CLOCK_IMX6QDL_H */ diff --git a/include/linux/crypto.h b/include/linux/crypto.h index 81ef938b0a8e..e71cb70a1ac2 100644 --- a/include/linux/crypto.h +++ b/include/linux/crypto.h @@ -102,12 +102,6 @@ #define CRYPTO_ALG_INTERNAL 0x00002000 /* - * Temporary flag used to prevent legacy AEAD implementations from - * being used by user-space. - */ -#define CRYPTO_ALG_AEAD_NEW 0x00004000 - -/* * Transform masks and values (for crt_flags). */ #define CRYPTO_TFM_REQ_MASK 0x000fff00 @@ -142,13 +136,10 @@ struct scatterlist; struct crypto_ablkcipher; struct crypto_async_request; -struct crypto_aead; struct crypto_blkcipher; struct crypto_hash; struct crypto_tfm; struct crypto_type; -struct aead_request; -struct aead_givcrypt_request; struct skcipher_givcrypt_request; typedef void (*crypto_completion_t)(struct crypto_async_request *req, int err); @@ -275,47 +266,6 @@ struct ablkcipher_alg { }; /** - * struct old_aead_alg - AEAD cipher definition - * @maxauthsize: Set the maximum authentication tag size supported by the - * transformation. A transformation may support smaller tag sizes. - * As the authentication tag is a message digest to ensure the - * integrity of the encrypted data, a consumer typically wants the - * largest authentication tag possible as defined by this - * variable. - * @setauthsize: Set authentication size for the AEAD transformation. This - * function is used to specify the consumer requested size of the - * authentication tag to be either generated by the transformation - * during encryption or the size of the authentication tag to be - * supplied during the decryption operation. This function is also - * responsible for checking the authentication tag size for - * validity. - * @setkey: see struct ablkcipher_alg - * @encrypt: see struct ablkcipher_alg - * @decrypt: see struct ablkcipher_alg - * @givencrypt: see struct ablkcipher_alg - * @givdecrypt: see struct ablkcipher_alg - * @geniv: see struct ablkcipher_alg - * @ivsize: see struct ablkcipher_alg - * - * All fields except @givencrypt , @givdecrypt , @geniv and @ivsize are - * mandatory and must be filled. - */ -struct old_aead_alg { - int (*setkey)(struct crypto_aead *tfm, const u8 *key, - unsigned int keylen); - int (*setauthsize)(struct crypto_aead *tfm, unsigned int authsize); - int (*encrypt)(struct aead_request *req); - int (*decrypt)(struct aead_request *req); - int (*givencrypt)(struct aead_givcrypt_request *req); - int (*givdecrypt)(struct aead_givcrypt_request *req); - - const char *geniv; - - unsigned int ivsize; - unsigned int maxauthsize; -}; - -/** * struct blkcipher_alg - synchronous block cipher definition * @min_keysize: see struct ablkcipher_alg * @max_keysize: see struct ablkcipher_alg @@ -409,7 +359,6 @@ struct compress_alg { #define cra_ablkcipher cra_u.ablkcipher -#define cra_aead cra_u.aead #define cra_blkcipher cra_u.blkcipher #define cra_cipher cra_u.cipher #define cra_compress cra_u.compress @@ -460,7 +409,7 @@ struct compress_alg { * struct crypto_type, which implements callbacks common for all * transformation types. There are multiple options: * &crypto_blkcipher_type, &crypto_ablkcipher_type, - * &crypto_ahash_type, &crypto_aead_type, &crypto_rng_type. + * &crypto_ahash_type, &crypto_rng_type. * This field might be empty. In that case, there are no common * callbacks. This is the case for: cipher, compress, shash. * @cra_u: Callbacks implementing the transformation. This is a union of @@ -508,7 +457,6 @@ struct crypto_alg { union { struct ablkcipher_alg ablkcipher; - struct old_aead_alg aead; struct blkcipher_alg blkcipher; struct cipher_alg cipher; struct compress_alg compress; diff --git a/lib/mpi/mpicoder.c b/lib/mpi/mpicoder.c index bc0a1da8afba..95c52a95259e 100644 --- a/lib/mpi/mpicoder.c +++ b/lib/mpi/mpicoder.c @@ -146,18 +146,25 @@ int mpi_read_buffer(MPI a, uint8_t *buf, unsigned buf_len, unsigned *nbytes, uint8_t *p; mpi_limb_t alimb; unsigned int n = mpi_get_size(a); - int i; + int i, lzeros = 0; - if (buf_len < n || !buf) + if (buf_len < n || !buf || !nbytes) return -EINVAL; if (sign) *sign = a->sign; - if (nbytes) - *nbytes = n; + p = (void *)&a->d[a->nlimbs] - 1; + + for (i = a->nlimbs * sizeof(alimb) - 1; i >= 0; i--, p--) { + if (!*p) + lzeros++; + else + break; + } p = buf; + *nbytes = n - lzeros; for (i = a->nlimbs - 1; i >= 0; i--) { alimb = a->d[i]; @@ -178,6 +185,19 @@ int mpi_read_buffer(MPI a, uint8_t *buf, unsigned buf_len, unsigned *nbytes, #else #error please implement for this limb size. #endif + + if (lzeros > 0) { + if (lzeros >= sizeof(alimb)) { + p -= sizeof(alimb); + } else { + mpi_limb_t *limb1 = (void *)p - sizeof(alimb); + mpi_limb_t *limb2 = (void *)p - sizeof(alimb) + + lzeros; + *limb1 = *limb2; + p -= lzeros; + } + lzeros -= sizeof(alimb); + } } return 0; } @@ -197,7 +217,7 @@ EXPORT_SYMBOL_GPL(mpi_read_buffer); */ void *mpi_get_buffer(MPI a, unsigned *nbytes, int *sign) { - uint8_t *buf, *p; + uint8_t *buf; unsigned int n; int ret; @@ -220,14 +240,6 @@ void *mpi_get_buffer(MPI a, unsigned *nbytes, int *sign) kfree(buf); return NULL; } - - /* this is sub-optimal but we need to do the shift operation - * because the caller has to free the returned buffer */ - for (p = buf; !*p && *nbytes; p++, --*nbytes) - ; - if (p != buf) - memmove(buf, p, *nbytes); - return buf; } EXPORT_SYMBOL_GPL(mpi_get_buffer); diff --git a/net/xfrm/xfrm_algo.c b/net/xfrm/xfrm_algo.c index 42f7c76cf853..f07224d8b88f 100644 --- a/net/xfrm/xfrm_algo.c +++ b/net/xfrm/xfrm_algo.c @@ -31,7 +31,7 @@ static struct xfrm_algo_desc aead_list[] = { .uinfo = { .aead = { - .geniv = "seqniv", + .geniv = "seqiv", .icv_truncbits = 64, } }, @@ -50,7 +50,7 @@ static struct xfrm_algo_desc aead_list[] = { .uinfo = { .aead = { - .geniv = "seqniv", + .geniv = "seqiv", .icv_truncbits = 96, } }, @@ -69,7 +69,7 @@ static struct xfrm_algo_desc aead_list[] = { .uinfo = { .aead = { - .geniv = "seqniv", + .geniv = "seqiv", .icv_truncbits = 128, } }, @@ -88,7 +88,7 @@ static struct xfrm_algo_desc aead_list[] = { .uinfo = { .aead = { - .geniv = "seqniv", + .geniv = "seqiv", .icv_truncbits = 64, } }, @@ -107,7 +107,7 @@ static struct xfrm_algo_desc aead_list[] = { .uinfo = { .aead = { - .geniv = "seqniv", + .geniv = "seqiv", .icv_truncbits = 96, } }, @@ -126,7 +126,7 @@ static struct xfrm_algo_desc aead_list[] = { .uinfo = { .aead = { - .geniv = "seqniv", + .geniv = "seqiv", .icv_truncbits = 128, } }, @@ -164,7 +164,7 @@ static struct xfrm_algo_desc aead_list[] = { .uinfo = { .aead = { - .geniv = "seqniv", + .geniv = "seqiv", .icv_truncbits = 128, } }, |