summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2022-01-11 10:21:35 -0800
committerLinus Torvalds <torvalds@linux-foundation.org>2022-01-11 10:21:35 -0800
commit5c947d0dbae8038ec1c8b538891f6475350542ee (patch)
treebd81b14e0cd2212bf885b835d9da39db51a33d43
parent6f38be8f2ccd9babf04b9b23539108542a59fcb8 (diff)
parent5f21d7d283dd82865bdb0123795b3accf0d42b67 (diff)
downloadlinux-5c947d0dbae8038ec1c8b538891f6475350542ee.tar.bz2
Merge branch 'linus' of git://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6
Pull crypto updates from Herbert Xu: "Algorithms: - Drop alignment requirement for data in aesni - Use synchronous seeding from the /dev/random in DRBG - Reseed nopr DRBGs every 5 minutes from /dev/random - Add KDF algorithms currently used by security/DH - Fix lack of entropy on some AMD CPUs with jitter RNG Drivers: - Add support for the D1 variant in sun8i-ce - Add SEV_INIT_EX support in ccp - PFVF support for GEN4 host driver in qat - Compression support for GEN4 devices in qat - Add cn10k random number generator support" * 'linus' of git://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6: (145 commits) crypto: af_alg - rewrite NULL pointer check lib/mpi: Add the return value check of kcalloc() crypto: qat - fix definition of ring reset results crypto: hisilicon - cleanup warning in qm_get_qos_value() crypto: kdf - select SHA-256 required for self-test crypto: x86/aesni - don't require alignment of data crypto: ccp - remove unneeded semicolon crypto: stm32/crc32 - Fix kernel BUG triggered in probe() crypto: s390/sha512 - Use macros instead of direct IV numbers crypto: sparc/sha - remove duplicate hash init function crypto: powerpc/sha - remove duplicate hash init function crypto: mips/sha - remove duplicate hash init function crypto: sha256 - remove duplicate generic hash init function crypto: jitter - add oversampling of noise source MAINTAINERS: update SEC2 driver maintainers list crypto: ux500 - Use platform_get_irq() to get the interrupt crypto: hisilicon/qm - disable qm clock-gating crypto: omap-aes - Fix broken pm_runtime_and_get() usage MAINTAINERS: update caam crypto driver maintainers list crypto: octeontx2 - prevent underflow in get_cores_bmap() ...
-rw-r--r--Documentation/virt/kvm/amd-memory-encryption.rst6
-rw-r--r--MAINTAINERS3
-rw-r--r--arch/mips/cavium-octeon/crypto/octeon-sha1.c17
-rw-r--r--arch/mips/cavium-octeon/crypto/octeon-sha256.c39
-rw-r--r--arch/mips/cavium-octeon/crypto/octeon-sha512.c39
-rw-r--r--arch/powerpc/crypto/sha1-spe-glue.c17
-rw-r--r--arch/powerpc/crypto/sha1.c14
-rw-r--r--arch/powerpc/crypto/sha256-spe-glue.c39
-rw-r--r--arch/s390/crypto/sha512_s390.c32
-rw-r--r--arch/sparc/crypto/sha1_glue.c14
-rw-r--r--arch/sparc/crypto/sha256_glue.c37
-rw-r--r--arch/sparc/crypto/sha512_glue.c37
-rw-r--r--arch/x86/crypto/aesni-intel_glue.c4
-rw-r--r--arch/x86/crypto/curve25519-x86_64.c767
-rw-r--r--arch/x86/crypto/des3_ede_glue.c4
-rw-r--r--crypto/Kconfig4
-rw-r--r--crypto/Makefile5
-rw-r--r--crypto/af_alg.c7
-rw-r--r--crypto/dh.c5
-rw-r--r--crypto/drbg.c150
-rw-r--r--crypto/jitterentropy-kcapi.c6
-rw-r--r--crypto/jitterentropy.c32
-rw-r--r--crypto/jitterentropy.h1
-rw-r--r--crypto/kdf_sp800108.c153
-rw-r--r--crypto/rsa.c4
-rw-r--r--crypto/sha256_generic.c16
-rw-r--r--crypto/testmgr.c9
-rw-r--r--crypto/testmgr.h6
-rw-r--r--drivers/char/hw_random/Kconfig13
-rw-r--r--drivers/char/hw_random/Makefile1
-rw-r--r--drivers/char/hw_random/cavium-rng-vf.c194
-rw-r--r--drivers/char/hw_random/cavium-rng.c11
-rw-r--r--drivers/char/hw_random/cn10k-rng.c181
-rw-r--r--drivers/crypto/allwinner/sun8i-ce/sun8i-ce-core.c21
-rw-r--r--drivers/crypto/allwinner/sun8i-ce/sun8i-ce.h1
-rw-r--r--drivers/crypto/atmel-aes.c6
-rw-r--r--drivers/crypto/caam/caamalg.c6
-rw-r--r--drivers/crypto/caam/caamalg_qi2.c2
-rw-r--r--drivers/crypto/caam/caamhash.c3
-rw-r--r--drivers/crypto/caam/caampkc.c3
-rw-r--r--drivers/crypto/cavium/cpt/cptvf_main.c5
-rw-r--r--drivers/crypto/ccp/ccp-dev.c2
-rw-r--r--drivers/crypto/ccp/sev-dev.c259
-rw-r--r--drivers/crypto/ccree/cc_request_mgr.c1
-rw-r--r--drivers/crypto/hisilicon/hpre/hpre_crypto.c7
-rw-r--r--drivers/crypto/hisilicon/hpre/hpre_main.c2
-rw-r--r--drivers/crypto/hisilicon/qm.c483
-rw-r--r--drivers/crypto/hisilicon/sec2/sec_main.c2
-rw-r--r--drivers/crypto/hisilicon/zip/zip_main.c12
-rw-r--r--drivers/crypto/keembay/keembay-ocs-ecc.c1
-rw-r--r--drivers/crypto/marvell/octeontx/otx_cptvf_main.c4
-rw-r--r--drivers/crypto/marvell/octeontx2/Makefile2
-rw-r--r--drivers/crypto/marvell/octeontx2/otx2_cpt_common.h1
-rw-r--r--drivers/crypto/marvell/octeontx2/otx2_cpt_devlink.c108
-rw-r--r--drivers/crypto/marvell/octeontx2/otx2_cpt_devlink.h20
-rw-r--r--drivers/crypto/marvell/octeontx2/otx2_cptpf.h3
-rw-r--r--drivers/crypto/marvell/octeontx2/otx2_cptpf_main.c18
-rw-r--r--drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c315
-rw-r--r--drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.h7
-rw-r--r--drivers/crypto/marvell/octeontx2/otx2_cptvf_algs.c5
-rw-r--r--drivers/crypto/omap-aes.c2
-rw-r--r--drivers/crypto/omap-des.c8
-rw-r--r--drivers/crypto/qat/Kconfig1
-rw-r--r--drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c154
-rw-r--r--drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.h2
-rw-r--r--drivers/crypto/qat/qat_4xxx/adf_drv.c33
-rw-r--r--drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c12
-rw-r--r--drivers/crypto/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c15
-rw-r--r--drivers/crypto/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.h1
-rw-r--r--drivers/crypto/qat/qat_c3xxxvf/adf_drv.c6
-rw-r--r--drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c12
-rw-r--r--drivers/crypto/qat/qat_c62xvf/adf_c62xvf_hw_data.c15
-rw-r--r--drivers/crypto/qat/qat_c62xvf/adf_c62xvf_hw_data.h1
-rw-r--r--drivers/crypto/qat/qat_c62xvf/adf_drv.c6
-rw-r--r--drivers/crypto/qat/qat_common/Makefile6
-rw-r--r--drivers/crypto/qat/qat_common/adf_accel_devices.h47
-rw-r--r--drivers/crypto/qat/qat_common/adf_accel_engine.c8
-rw-r--r--drivers/crypto/qat/qat_common/adf_admin.c47
-rw-r--r--drivers/crypto/qat/qat_common/adf_cfg.c1
-rw-r--r--drivers/crypto/qat/qat_common/adf_cfg_common.h13
-rw-r--r--drivers/crypto/qat/qat_common/adf_cfg_strings.h3
-rw-r--r--drivers/crypto/qat/qat_common/adf_common_drv.h42
-rw-r--r--drivers/crypto/qat/qat_common/adf_gen2_hw_data.c105
-rw-r--r--drivers/crypto/qat/qat_common/adf_gen2_hw_data.h22
-rw-r--r--drivers/crypto/qat/qat_common/adf_gen2_pfvf.c381
-rw-r--r--drivers/crypto/qat/qat_common/adf_gen2_pfvf.h29
-rw-r--r--drivers/crypto/qat/qat_common/adf_gen4_hw_data.c69
-rw-r--r--drivers/crypto/qat/qat_common/adf_gen4_hw_data.h17
-rw-r--r--drivers/crypto/qat/qat_common/adf_gen4_pfvf.c148
-rw-r--r--drivers/crypto/qat/qat_common/adf_gen4_pfvf.h17
-rw-r--r--drivers/crypto/qat/qat_common/adf_init.c11
-rw-r--r--drivers/crypto/qat/qat_common/adf_isr.c111
-rw-r--r--drivers/crypto/qat/qat_common/adf_pf2vf_msg.c416
-rw-r--r--drivers/crypto/qat/qat_common/adf_pf2vf_msg.h93
-rw-r--r--drivers/crypto/qat/qat_common/adf_pfvf_msg.h259
-rw-r--r--drivers/crypto/qat/qat_common/adf_pfvf_pf_msg.c52
-rw-r--r--drivers/crypto/qat/qat_common/adf_pfvf_pf_msg.h18
-rw-r--r--drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.c346
-rw-r--r--drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.h13
-rw-r--r--drivers/crypto/qat/qat_common/adf_pfvf_utils.c65
-rw-r--r--drivers/crypto/qat/qat_common/adf_pfvf_utils.h31
-rw-r--r--drivers/crypto/qat/qat_common/adf_pfvf_vf_msg.c167
-rw-r--r--drivers/crypto/qat/qat_common/adf_pfvf_vf_msg.h23
-rw-r--r--drivers/crypto/qat/qat_common/adf_pfvf_vf_proto.c368
-rw-r--r--drivers/crypto/qat/qat_common/adf_pfvf_vf_proto.h17
-rw-r--r--drivers/crypto/qat/qat_common/adf_sriov.c59
-rw-r--r--drivers/crypto/qat/qat_common/adf_vf2pf_msg.c48
-rw-r--r--drivers/crypto/qat/qat_common/adf_vf_isr.c106
-rw-r--r--drivers/crypto/qat/qat_common/icp_qat_fw_init_admin.h4
-rw-r--r--drivers/crypto/qat/qat_common/icp_qat_hw.h13
-rw-r--r--drivers/crypto/qat/qat_common/qat_crypto.c25
-rw-r--r--drivers/crypto/qat/qat_common/qat_hal.c41
-rw-r--r--drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c44
-rw-r--r--drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.h2
-rw-r--r--drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c15
-rw-r--r--drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.h1
-rw-r--r--drivers/crypto/qat/qat_dh895xccvf/adf_drv.c6
-rw-r--r--drivers/crypto/qce/aead.c2
-rw-r--r--drivers/crypto/qce/sha.c2
-rw-r--r--drivers/crypto/qce/skcipher.c2
-rw-r--r--drivers/crypto/sa2ul.c19
-rw-r--r--drivers/crypto/stm32/stm32-crc32.c4
-rw-r--r--drivers/crypto/stm32/stm32-cryp.c988
-rw-r--r--drivers/crypto/stm32/stm32-hash.c6
-rw-r--r--drivers/crypto/ux500/cryp/cryp.h2
-rw-r--r--drivers/crypto/ux500/cryp/cryp_core.c26
-rw-r--r--include/crypto/aead.h4
-rw-r--r--include/crypto/algapi.h5
-rw-r--r--include/crypto/blake2b.h1
-rw-r--r--include/crypto/blake2s.h2
-rw-r--r--include/crypto/cryptd.h3
-rw-r--r--include/crypto/drbg.h11
-rw-r--r--include/crypto/engine.h6
-rw-r--r--include/crypto/internal/kdf_selftest.h71
-rw-r--r--include/crypto/kdf_sp800108.h61
-rw-r--r--include/crypto/pcrypt.h2
-rw-r--r--include/crypto/scatterwalk.h3
-rw-r--r--include/crypto/skcipher.h6
-rw-r--r--include/linux/psp-sev.h21
-rw-r--r--lib/mpi/mpi-mod.c2
-rw-r--r--security/keys/Kconfig2
-rw-r--r--security/keys/dh.c130
142 files changed, 5482 insertions, 2642 deletions
diff --git a/Documentation/virt/kvm/amd-memory-encryption.rst b/Documentation/virt/kvm/amd-memory-encryption.rst
index 5c081c8c7164..1c6847fff304 100644
--- a/Documentation/virt/kvm/amd-memory-encryption.rst
+++ b/Documentation/virt/kvm/amd-memory-encryption.rst
@@ -85,6 +85,12 @@ guests, such as launching, running, snapshotting, migrating and decommissioning.
The KVM_SEV_INIT command is used by the hypervisor to initialize the SEV platform
context. In a typical workflow, this command should be the first command issued.
+The firmware can be initialized either by using its own non-volatile storage or
+the OS can manage the NV storage for the firmware using the module parameter
+``init_ex_path``. The file specified by ``init_ex_path`` must exist. To create
+a new NV storage file allocate the file with 32KB bytes of 0xFF as required by
+the SEV spec.
+
Returns: 0 on success, -negative on error
2. KVM_SEV_LAUNCH_START
diff --git a/MAINTAINERS b/MAINTAINERS
index 5603963bbd9e..38a0a612ccda 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -7557,6 +7557,7 @@ F: include/video/
FREESCALE CAAM (Cryptographic Acceleration and Assurance Module) DRIVER
M: Horia Geantă <horia.geanta@nxp.com>
M: Pankaj Gupta <pankaj.gupta@nxp.com>
+M: Gaurav Jain <gaurav.jain@nxp.com>
L: linux-crypto@vger.kernel.org
S: Maintained
F: Documentation/devicetree/bindings/crypto/fsl-sec4.txt
@@ -8676,6 +8677,7 @@ F: drivers/scsi/hisi_sas/
HISILICON SECURITY ENGINE V2 DRIVER (SEC2)
M: Zaibo Xu <xuzaibo@huawei.com>
+M: Kai Ye <yekai13@huawei.com>
L: linux-crypto@vger.kernel.org
S: Maintained
F: Documentation/ABI/testing/debugfs-hisi-sec
@@ -9721,7 +9723,6 @@ F: Documentation/devicetree/bindings/crypto/intel,keembay-ocs-ecc.yaml
F: drivers/crypto/keembay/Kconfig
F: drivers/crypto/keembay/Makefile
F: drivers/crypto/keembay/keembay-ocs-ecc.c
-F: drivers/crypto/keembay/ocs-ecc-curve-defs.h
INTEL KEEM BAY OCS HCU CRYPTO DRIVER
M: Daniele Alessandrelli <daniele.alessandrelli@intel.com>
diff --git a/arch/mips/cavium-octeon/crypto/octeon-sha1.c b/arch/mips/cavium-octeon/crypto/octeon-sha1.c
index 30f1d75208a5..37a07b3c4568 100644
--- a/arch/mips/cavium-octeon/crypto/octeon-sha1.c
+++ b/arch/mips/cavium-octeon/crypto/octeon-sha1.c
@@ -15,6 +15,7 @@
#include <linux/mm.h>
#include <crypto/sha1.h>
+#include <crypto/sha1_base.h>
#include <linux/init.h>
#include <linux/types.h>
#include <linux/module.h>
@@ -71,20 +72,6 @@ static void octeon_sha1_transform(const void *_block)
octeon_sha1_start(block[7]);
}
-static int octeon_sha1_init(struct shash_desc *desc)
-{
- struct sha1_state *sctx = shash_desc_ctx(desc);
-
- sctx->state[0] = SHA1_H0;
- sctx->state[1] = SHA1_H1;
- sctx->state[2] = SHA1_H2;
- sctx->state[3] = SHA1_H3;
- sctx->state[4] = SHA1_H4;
- sctx->count = 0;
-
- return 0;
-}
-
static void __octeon_sha1_update(struct sha1_state *sctx, const u8 *data,
unsigned int len)
{
@@ -200,7 +187,7 @@ static int octeon_sha1_import(struct shash_desc *desc, const void *in)
static struct shash_alg octeon_sha1_alg = {
.digestsize = SHA1_DIGEST_SIZE,
- .init = octeon_sha1_init,
+ .init = sha1_base_init,
.update = octeon_sha1_update,
.final = octeon_sha1_final,
.export = octeon_sha1_export,
diff --git a/arch/mips/cavium-octeon/crypto/octeon-sha256.c b/arch/mips/cavium-octeon/crypto/octeon-sha256.c
index 36cb92895d72..435e4a6e7f13 100644
--- a/arch/mips/cavium-octeon/crypto/octeon-sha256.c
+++ b/arch/mips/cavium-octeon/crypto/octeon-sha256.c
@@ -16,6 +16,7 @@
#include <linux/mm.h>
#include <crypto/sha2.h>
+#include <crypto/sha256_base.h>
#include <linux/init.h>
#include <linux/types.h>
#include <linux/module.h>
@@ -63,40 +64,6 @@ static void octeon_sha256_transform(const void *_block)
octeon_sha256_start(block[7]);
}
-static int octeon_sha224_init(struct shash_desc *desc)
-{
- struct sha256_state *sctx = shash_desc_ctx(desc);
-
- sctx->state[0] = SHA224_H0;
- sctx->state[1] = SHA224_H1;
- sctx->state[2] = SHA224_H2;
- sctx->state[3] = SHA224_H3;
- sctx->state[4] = SHA224_H4;
- sctx->state[5] = SHA224_H5;
- sctx->state[6] = SHA224_H6;
- sctx->state[7] = SHA224_H7;
- sctx->count = 0;
-
- return 0;
-}
-
-static int octeon_sha256_init(struct shash_desc *desc)
-{
- struct sha256_state *sctx = shash_desc_ctx(desc);
-
- sctx->state[0] = SHA256_H0;
- sctx->state[1] = SHA256_H1;
- sctx->state[2] = SHA256_H2;
- sctx->state[3] = SHA256_H3;
- sctx->state[4] = SHA256_H4;
- sctx->state[5] = SHA256_H5;
- sctx->state[6] = SHA256_H6;
- sctx->state[7] = SHA256_H7;
- sctx->count = 0;
-
- return 0;
-}
-
static void __octeon_sha256_update(struct sha256_state *sctx, const u8 *data,
unsigned int len)
{
@@ -224,7 +191,7 @@ static int octeon_sha256_import(struct shash_desc *desc, const void *in)
static struct shash_alg octeon_sha256_algs[2] = { {
.digestsize = SHA256_DIGEST_SIZE,
- .init = octeon_sha256_init,
+ .init = sha256_base_init,
.update = octeon_sha256_update,
.final = octeon_sha256_final,
.export = octeon_sha256_export,
@@ -240,7 +207,7 @@ static struct shash_alg octeon_sha256_algs[2] = { {
}
}, {
.digestsize = SHA224_DIGEST_SIZE,
- .init = octeon_sha224_init,
+ .init = sha224_base_init,
.update = octeon_sha256_update,
.final = octeon_sha224_final,
.descsize = sizeof(struct sha256_state),
diff --git a/arch/mips/cavium-octeon/crypto/octeon-sha512.c b/arch/mips/cavium-octeon/crypto/octeon-sha512.c
index 359f039820d8..2dee9354e33f 100644
--- a/arch/mips/cavium-octeon/crypto/octeon-sha512.c
+++ b/arch/mips/cavium-octeon/crypto/octeon-sha512.c
@@ -15,6 +15,7 @@
#include <linux/mm.h>
#include <crypto/sha2.h>
+#include <crypto/sha512_base.h>
#include <linux/init.h>
#include <linux/types.h>
#include <linux/module.h>
@@ -74,40 +75,6 @@ static void octeon_sha512_transform(const void *_block)
octeon_sha512_start(block[15]);
}
-static int octeon_sha512_init(struct shash_desc *desc)
-{
- struct sha512_state *sctx = shash_desc_ctx(desc);
-
- sctx->state[0] = SHA512_H0;
- sctx->state[1] = SHA512_H1;
- sctx->state[2] = SHA512_H2;
- sctx->state[3] = SHA512_H3;
- sctx->state[4] = SHA512_H4;
- sctx->state[5] = SHA512_H5;
- sctx->state[6] = SHA512_H6;
- sctx->state[7] = SHA512_H7;
- sctx->count[0] = sctx->count[1] = 0;
-
- return 0;
-}
-
-static int octeon_sha384_init(struct shash_desc *desc)
-{
- struct sha512_state *sctx = shash_desc_ctx(desc);
-
- sctx->state[0] = SHA384_H0;
- sctx->state[1] = SHA384_H1;
- sctx->state[2] = SHA384_H2;
- sctx->state[3] = SHA384_H3;
- sctx->state[4] = SHA384_H4;
- sctx->state[5] = SHA384_H5;
- sctx->state[6] = SHA384_H6;
- sctx->state[7] = SHA384_H7;
- sctx->count[0] = sctx->count[1] = 0;
-
- return 0;
-}
-
static void __octeon_sha512_update(struct sha512_state *sctx, const u8 *data,
unsigned int len)
{
@@ -223,7 +190,7 @@ static int octeon_sha384_final(struct shash_desc *desc, u8 *hash)
static struct shash_alg octeon_sha512_algs[2] = { {
.digestsize = SHA512_DIGEST_SIZE,
- .init = octeon_sha512_init,
+ .init = sha512_base_init,
.update = octeon_sha512_update,
.final = octeon_sha512_final,
.descsize = sizeof(struct sha512_state),
@@ -236,7 +203,7 @@ static struct shash_alg octeon_sha512_algs[2] = { {
}
}, {
.digestsize = SHA384_DIGEST_SIZE,
- .init = octeon_sha384_init,
+ .init = sha384_base_init,
.update = octeon_sha512_update,
.final = octeon_sha384_final,
.descsize = sizeof(struct sha512_state),
diff --git a/arch/powerpc/crypto/sha1-spe-glue.c b/arch/powerpc/crypto/sha1-spe-glue.c
index 88e8ea73bfa7..9170892a8557 100644
--- a/arch/powerpc/crypto/sha1-spe-glue.c
+++ b/arch/powerpc/crypto/sha1-spe-glue.c
@@ -13,6 +13,7 @@
#include <linux/mm.h>
#include <linux/types.h>
#include <crypto/sha1.h>
+#include <crypto/sha1_base.h>
#include <asm/byteorder.h>
#include <asm/switch_to.h>
#include <linux/hardirq.h>
@@ -55,20 +56,6 @@ static inline void ppc_sha1_clear_context(struct sha1_state *sctx)
do { *ptr++ = 0; } while (--count);
}
-static int ppc_spe_sha1_init(struct shash_desc *desc)
-{
- struct sha1_state *sctx = shash_desc_ctx(desc);
-
- sctx->state[0] = SHA1_H0;
- sctx->state[1] = SHA1_H1;
- sctx->state[2] = SHA1_H2;
- sctx->state[3] = SHA1_H3;
- sctx->state[4] = SHA1_H4;
- sctx->count = 0;
-
- return 0;
-}
-
static int ppc_spe_sha1_update(struct shash_desc *desc, const u8 *data,
unsigned int len)
{
@@ -168,7 +155,7 @@ static int ppc_spe_sha1_import(struct shash_desc *desc, const void *in)
static struct shash_alg alg = {
.digestsize = SHA1_DIGEST_SIZE,
- .init = ppc_spe_sha1_init,
+ .init = sha1_base_init,
.update = ppc_spe_sha1_update,
.final = ppc_spe_sha1_final,
.export = ppc_spe_sha1_export,
diff --git a/arch/powerpc/crypto/sha1.c b/arch/powerpc/crypto/sha1.c
index 7a55d790cdb1..f283bbd3f121 100644
--- a/arch/powerpc/crypto/sha1.c
+++ b/arch/powerpc/crypto/sha1.c
@@ -18,21 +18,11 @@
#include <linux/mm.h>
#include <linux/types.h>
#include <crypto/sha1.h>
+#include <crypto/sha1_base.h>
#include <asm/byteorder.h>
void powerpc_sha_transform(u32 *state, const u8 *src);
-static int powerpc_sha1_init(struct shash_desc *desc)
-{
- struct sha1_state *sctx = shash_desc_ctx(desc);
-
- *sctx = (struct sha1_state){
- .state = { SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4 },
- };
-
- return 0;
-}
-
static int powerpc_sha1_update(struct shash_desc *desc, const u8 *data,
unsigned int len)
{
@@ -114,7 +104,7 @@ static int powerpc_sha1_import(struct shash_desc *desc, const void *in)
static struct shash_alg alg = {
.digestsize = SHA1_DIGEST_SIZE,
- .init = powerpc_sha1_init,
+ .init = sha1_base_init,
.update = powerpc_sha1_update,
.final = powerpc_sha1_final,
.export = powerpc_sha1_export,
diff --git a/arch/powerpc/crypto/sha256-spe-glue.c b/arch/powerpc/crypto/sha256-spe-glue.c
index ffedea7e4bef..2997d13236e0 100644
--- a/arch/powerpc/crypto/sha256-spe-glue.c
+++ b/arch/powerpc/crypto/sha256-spe-glue.c
@@ -14,6 +14,7 @@
#include <linux/mm.h>
#include <linux/types.h>
#include <crypto/sha2.h>
+#include <crypto/sha256_base.h>
#include <asm/byteorder.h>
#include <asm/switch_to.h>
#include <linux/hardirq.h>
@@ -56,40 +57,6 @@ static inline void ppc_sha256_clear_context(struct sha256_state *sctx)
do { *ptr++ = 0; } while (--count);
}
-static int ppc_spe_sha256_init(struct shash_desc *desc)
-{
- struct sha256_state *sctx = shash_desc_ctx(desc);
-
- sctx->state[0] = SHA256_H0;
- sctx->state[1] = SHA256_H1;
- sctx->state[2] = SHA256_H2;
- sctx->state[3] = SHA256_H3;
- sctx->state[4] = SHA256_H4;
- sctx->state[5] = SHA256_H5;
- sctx->state[6] = SHA256_H6;
- sctx->state[7] = SHA256_H7;
- sctx->count = 0;
-
- return 0;
-}
-
-static int ppc_spe_sha224_init(struct shash_desc *desc)
-{
- struct sha256_state *sctx = shash_desc_ctx(desc);
-
- sctx->state[0] = SHA224_H0;
- sctx->state[1] = SHA224_H1;
- sctx->state[2] = SHA224_H2;
- sctx->state[3] = SHA224_H3;
- sctx->state[4] = SHA224_H4;
- sctx->state[5] = SHA224_H5;
- sctx->state[6] = SHA224_H6;
- sctx->state[7] = SHA224_H7;
- sctx->count = 0;
-
- return 0;
-}
-
static int ppc_spe_sha256_update(struct shash_desc *desc, const u8 *data,
unsigned int len)
{
@@ -214,7 +181,7 @@ static int ppc_spe_sha256_import(struct shash_desc *desc, const void *in)
static struct shash_alg algs[2] = { {
.digestsize = SHA256_DIGEST_SIZE,
- .init = ppc_spe_sha256_init,
+ .init = sha256_base_init,
.update = ppc_spe_sha256_update,
.final = ppc_spe_sha256_final,
.export = ppc_spe_sha256_export,
@@ -230,7 +197,7 @@ static struct shash_alg algs[2] = { {
}
}, {
.digestsize = SHA224_DIGEST_SIZE,
- .init = ppc_spe_sha224_init,
+ .init = sha224_base_init,
.update = ppc_spe_sha256_update,
.final = ppc_spe_sha224_final,
.export = ppc_spe_sha256_export,
diff --git a/arch/s390/crypto/sha512_s390.c b/arch/s390/crypto/sha512_s390.c
index 29a6bd404c59..43ce4956df73 100644
--- a/arch/s390/crypto/sha512_s390.c
+++ b/arch/s390/crypto/sha512_s390.c
@@ -22,14 +22,14 @@ static int sha512_init(struct shash_desc *desc)
{
struct s390_sha_ctx *ctx = shash_desc_ctx(desc);
- *(__u64 *)&ctx->state[0] = 0x6a09e667f3bcc908ULL;
- *(__u64 *)&ctx->state[2] = 0xbb67ae8584caa73bULL;
- *(__u64 *)&ctx->state[4] = 0x3c6ef372fe94f82bULL;
- *(__u64 *)&ctx->state[6] = 0xa54ff53a5f1d36f1ULL;
- *(__u64 *)&ctx->state[8] = 0x510e527fade682d1ULL;
- *(__u64 *)&ctx->state[10] = 0x9b05688c2b3e6c1fULL;
- *(__u64 *)&ctx->state[12] = 0x1f83d9abfb41bd6bULL;
- *(__u64 *)&ctx->state[14] = 0x5be0cd19137e2179ULL;
+ *(__u64 *)&ctx->state[0] = SHA512_H0;
+ *(__u64 *)&ctx->state[2] = SHA512_H1;
+ *(__u64 *)&ctx->state[4] = SHA512_H2;
+ *(__u64 *)&ctx->state[6] = SHA512_H3;
+ *(__u64 *)&ctx->state[8] = SHA512_H4;
+ *(__u64 *)&ctx->state[10] = SHA512_H5;
+ *(__u64 *)&ctx->state[12] = SHA512_H6;
+ *(__u64 *)&ctx->state[14] = SHA512_H7;
ctx->count = 0;
ctx->func = CPACF_KIMD_SHA_512;
@@ -87,14 +87,14 @@ static int sha384_init(struct shash_desc *desc)
{
struct s390_sha_ctx *ctx = shash_desc_ctx(desc);
- *(__u64 *)&ctx->state[0] = 0xcbbb9d5dc1059ed8ULL;
- *(__u64 *)&ctx->state[2] = 0x629a292a367cd507ULL;
- *(__u64 *)&ctx->state[4] = 0x9159015a3070dd17ULL;
- *(__u64 *)&ctx->state[6] = 0x152fecd8f70e5939ULL;
- *(__u64 *)&ctx->state[8] = 0x67332667ffc00b31ULL;
- *(__u64 *)&ctx->state[10] = 0x8eb44a8768581511ULL;
- *(__u64 *)&ctx->state[12] = 0xdb0c2e0d64f98fa7ULL;
- *(__u64 *)&ctx->state[14] = 0x47b5481dbefa4fa4ULL;
+ *(__u64 *)&ctx->state[0] = SHA384_H0;
+ *(__u64 *)&ctx->state[2] = SHA384_H1;
+ *(__u64 *)&ctx->state[4] = SHA384_H2;
+ *(__u64 *)&ctx->state[6] = SHA384_H3;
+ *(__u64 *)&ctx->state[8] = SHA384_H4;
+ *(__u64 *)&ctx->state[10] = SHA384_H5;
+ *(__u64 *)&ctx->state[12] = SHA384_H6;
+ *(__u64 *)&ctx->state[14] = SHA384_H7;
ctx->count = 0;
ctx->func = CPACF_KIMD_SHA_512;
diff --git a/arch/sparc/crypto/sha1_glue.c b/arch/sparc/crypto/sha1_glue.c
index 86a654cce5ab..06b7becfcb21 100644
--- a/arch/sparc/crypto/sha1_glue.c
+++ b/arch/sparc/crypto/sha1_glue.c
@@ -17,6 +17,7 @@
#include <linux/mm.h>
#include <linux/types.h>
#include <crypto/sha1.h>
+#include <crypto/sha1_base.h>
#include <asm/pstate.h>
#include <asm/elf.h>
@@ -26,17 +27,6 @@
asmlinkage void sha1_sparc64_transform(u32 *digest, const char *data,
unsigned int rounds);
-static int sha1_sparc64_init(struct shash_desc *desc)
-{
- struct sha1_state *sctx = shash_desc_ctx(desc);
-
- *sctx = (struct sha1_state){
- .state = { SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4 },
- };
-
- return 0;
-}
-
static void __sha1_sparc64_update(struct sha1_state *sctx, const u8 *data,
unsigned int len, unsigned int partial)
{
@@ -128,7 +118,7 @@ static int sha1_sparc64_import(struct shash_desc *desc, const void *in)
static struct shash_alg alg = {
.digestsize = SHA1_DIGEST_SIZE,
- .init = sha1_sparc64_init,
+ .init = sha1_base_init,
.update = sha1_sparc64_update,
.final = sha1_sparc64_final,
.export = sha1_sparc64_export,
diff --git a/arch/sparc/crypto/sha256_glue.c b/arch/sparc/crypto/sha256_glue.c
index 60ec524cf9ca..285561a1cde5 100644
--- a/arch/sparc/crypto/sha256_glue.c
+++ b/arch/sparc/crypto/sha256_glue.c
@@ -17,6 +17,7 @@
#include <linux/mm.h>
#include <linux/types.h>
#include <crypto/sha2.h>
+#include <crypto/sha256_base.h>
#include <asm/pstate.h>
#include <asm/elf.h>
@@ -26,38 +27,6 @@
asmlinkage void sha256_sparc64_transform(u32 *digest, const char *data,
unsigned int rounds);
-static int sha224_sparc64_init(struct shash_desc *desc)
-{
- struct sha256_state *sctx = shash_desc_ctx(desc);
- sctx->state[0] = SHA224_H0;
- sctx->state[1] = SHA224_H1;
- sctx->state[2] = SHA224_H2;
- sctx->state[3] = SHA224_H3;
- sctx->state[4] = SHA224_H4;
- sctx->state[5] = SHA224_H5;
- sctx->state[6] = SHA224_H6;
- sctx->state[7] = SHA224_H7;
- sctx->count = 0;
-
- return 0;
-}
-
-static int sha256_sparc64_init(struct shash_desc *desc)
-{
- struct sha256_state *sctx = shash_desc_ctx(desc);
- sctx->state[0] = SHA256_H0;
- sctx->state[1] = SHA256_H1;
- sctx->state[2] = SHA256_H2;
- sctx->state[3] = SHA256_H3;
- sctx->state[4] = SHA256_H4;
- sctx->state[5] = SHA256_H5;
- sctx->state[6] = SHA256_H6;
- sctx->state[7] = SHA256_H7;
- sctx->count = 0;
-
- return 0;
-}
-
static void __sha256_sparc64_update(struct sha256_state *sctx, const u8 *data,
unsigned int len, unsigned int partial)
{
@@ -158,7 +127,7 @@ static int sha256_sparc64_import(struct shash_desc *desc, const void *in)
static struct shash_alg sha256_alg = {
.digestsize = SHA256_DIGEST_SIZE,
- .init = sha256_sparc64_init,
+ .init = sha256_base_init,
.update = sha256_sparc64_update,
.final = sha256_sparc64_final,
.export = sha256_sparc64_export,
@@ -176,7 +145,7 @@ static struct shash_alg sha256_alg = {
static struct shash_alg sha224_alg = {
.digestsize = SHA224_DIGEST_SIZE,
- .init = sha224_sparc64_init,
+ .init = sha224_base_init,
.update = sha256_sparc64_update,
.final = sha224_sparc64_final,
.descsize = sizeof(struct sha256_state),
diff --git a/arch/sparc/crypto/sha512_glue.c b/arch/sparc/crypto/sha512_glue.c
index 273ce21918c1..d66efa4ec59a 100644
--- a/arch/sparc/crypto/sha512_glue.c
+++ b/arch/sparc/crypto/sha512_glue.c
@@ -16,6 +16,7 @@
#include <linux/mm.h>
#include <linux/types.h>
#include <crypto/sha2.h>
+#include <crypto/sha512_base.h>
#include <asm/pstate.h>
#include <asm/elf.h>
@@ -25,38 +26,6 @@
asmlinkage void sha512_sparc64_transform(u64 *digest, const char *data,
unsigned int rounds);
-static int sha512_sparc64_init(struct shash_desc *desc)
-{
- struct sha512_state *sctx = shash_desc_ctx(desc);
- sctx->state[0] = SHA512_H0;
- sctx->state[1] = SHA512_H1;
- sctx->state[2] = SHA512_H2;
- sctx->state[3] = SHA512_H3;
- sctx->state[4] = SHA512_H4;
- sctx->state[5] = SHA512_H5;
- sctx->state[6] = SHA512_H6;
- sctx->state[7] = SHA512_H7;
- sctx->count[0] = sctx->count[1] = 0;
-
- return 0;
-}
-
-static int sha384_sparc64_init(struct shash_desc *desc)
-{
- struct sha512_state *sctx = shash_desc_ctx(desc);
- sctx->state[0] = SHA384_H0;
- sctx->state[1] = SHA384_H1;
- sctx->state[2] = SHA384_H2;
- sctx->state[3] = SHA384_H3;
- sctx->state[4] = SHA384_H4;
- sctx->state[5] = SHA384_H5;
- sctx->state[6] = SHA384_H6;
- sctx->state[7] = SHA384_H7;
- sctx->count[0] = sctx->count[1] = 0;
-
- return 0;
-}
-
static void __sha512_sparc64_update(struct sha512_state *sctx, const u8 *data,
unsigned int len, unsigned int partial)
{
@@ -146,7 +115,7 @@ static int sha384_sparc64_final(struct shash_desc *desc, u8 *hash)
static struct shash_alg sha512 = {
.digestsize = SHA512_DIGEST_SIZE,
- .init = sha512_sparc64_init,
+ .init = sha512_base_init,
.update = sha512_sparc64_update,
.final = sha512_sparc64_final,
.descsize = sizeof(struct sha512_state),
@@ -161,7 +130,7 @@ static struct shash_alg sha512 = {
static struct shash_alg sha384 = {
.digestsize = SHA384_DIGEST_SIZE,
- .init = sha384_sparc64_init,
+ .init = sha384_base_init,
.update = sha512_sparc64_update,
.final = sha384_sparc64_final,
.descsize = sizeof(struct sha512_state),
diff --git a/arch/x86/crypto/aesni-intel_glue.c b/arch/x86/crypto/aesni-intel_glue.c
index e09f4672dd38..41901ba9d3a2 100644
--- a/arch/x86/crypto/aesni-intel_glue.c
+++ b/arch/x86/crypto/aesni-intel_glue.c
@@ -1107,7 +1107,7 @@ static struct aead_alg aesni_aeads[] = { {
.cra_flags = CRYPTO_ALG_INTERNAL,
.cra_blocksize = 1,
.cra_ctxsize = sizeof(struct aesni_rfc4106_gcm_ctx),
- .cra_alignmask = AESNI_ALIGN - 1,
+ .cra_alignmask = 0,
.cra_module = THIS_MODULE,
},
}, {
@@ -1124,7 +1124,7 @@ static struct aead_alg aesni_aeads[] = { {
.cra_flags = CRYPTO_ALG_INTERNAL,
.cra_blocksize = 1,
.cra_ctxsize = sizeof(struct generic_gcmaes_ctx),
- .cra_alignmask = AESNI_ALIGN - 1,
+ .cra_alignmask = 0,
.cra_module = THIS_MODULE,
},
} };
diff --git a/arch/x86/crypto/curve25519-x86_64.c b/arch/x86/crypto/curve25519-x86_64.c
index 38caf61cd5b7..d55fa9e9b9e6 100644
--- a/arch/x86/crypto/curve25519-x86_64.c
+++ b/arch/x86/crypto/curve25519-x86_64.c
@@ -64,10 +64,9 @@ static inline u64 add_scalar(u64 *out, const u64 *f1, u64 f2)
/* Return the carry bit in a register */
" adcx %%r11, %1;"
- : "+&r" (f2), "=&r" (carry_r)
- : "r" (out), "r" (f1)
- : "%r8", "%r9", "%r10", "%r11", "memory", "cc"
- );
+ : "+&r"(f2), "=&r"(carry_r)
+ : "r"(out), "r"(f1)
+ : "%r8", "%r9", "%r10", "%r11", "memory", "cc");
return carry_r;
}
@@ -108,10 +107,9 @@ static inline void fadd(u64 *out, const u64 *f1, const u64 *f2)
" cmovc %0, %%rax;"
" add %%rax, %%r8;"
" movq %%r8, 0(%1);"
- : "+&r" (f2)
- : "r" (out), "r" (f1)
- : "%rax", "%rcx", "%r8", "%r9", "%r10", "%r11", "memory", "cc"
- );
+ : "+&r"(f2)
+ : "r"(out), "r"(f1)
+ : "%rax", "%rcx", "%r8", "%r9", "%r10", "%r11", "memory", "cc");
}
/* Computes the field subtraction of two field elements */
@@ -151,10 +149,9 @@ static inline void fsub(u64 *out, const u64 *f1, const u64 *f2)
" movq %%r9, 8(%0);"
" movq %%r10, 16(%0);"
" movq %%r11, 24(%0);"
- :
- : "r" (out), "r" (f1), "r" (f2)
- : "%rax", "%rcx", "%r8", "%r9", "%r10", "%r11", "memory", "cc"
- );
+ :
+ : "r"(out), "r"(f1), "r"(f2)
+ : "%rax", "%rcx", "%r8", "%r9", "%r10", "%r11", "memory", "cc");
}
/* Computes a field multiplication: out <- f1 * f2
@@ -162,239 +159,400 @@ static inline void fsub(u64 *out, const u64 *f1, const u64 *f2)
static inline void fmul(u64 *out, const u64 *f1, const u64 *f2, u64 *tmp)
{
asm volatile(
+
/* Compute the raw multiplication: tmp <- src1 * src2 */
/* Compute src1[0] * src2 */
- " movq 0(%1), %%rdx;"
- " mulxq 0(%3), %%r8, %%r9;" " xor %%r10d, %%r10d;" " movq %%r8, 0(%0);"
- " mulxq 8(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " movq %%r10, 8(%0);"
- " mulxq 16(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;"
- " mulxq 24(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " mov $0, %%rax;"
- " adox %%rdx, %%rax;"
+ " movq 0(%0), %%rdx;"
+ " mulxq 0(%1), %%r8, %%r9;"
+ " xor %%r10d, %%r10d;"
+ " movq %%r8, 0(%2);"
+ " mulxq 8(%1), %%r10, %%r11;"
+ " adox %%r9, %%r10;"
+ " movq %%r10, 8(%2);"
+ " mulxq 16(%1), %%rbx, %%r13;"
+ " adox %%r11, %%rbx;"
+ " mulxq 24(%1), %%r14, %%rdx;"
+ " adox %%r13, %%r14;"
+ " mov $0, %%rax;"
+ " adox %%rdx, %%rax;"
+
/* Compute src1[1] * src2 */
- " movq 8(%1), %%rdx;"
- " mulxq 0(%3), %%r8, %%r9;" " xor %%r10d, %%r10d;" " adcxq 8(%0), %%r8;" " movq %%r8, 8(%0);"
- " mulxq 8(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " adcx %%rbx, %%r10;" " movq %%r10, 16(%0);"
- " mulxq 16(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" " adcx %%r14, %%rbx;" " mov $0, %%r8;"
- " mulxq 24(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " adcx %%rax, %%r14;" " mov $0, %%rax;"
- " adox %%rdx, %%rax;" " adcx %%r8, %%rax;"
+ " movq 8(%0), %%rdx;"
+ " mulxq 0(%1), %%r8, %%r9;"
+ " xor %%r10d, %%r10d;"
+ " adcxq 8(%2), %%r8;"
+ " movq %%r8, 8(%2);"
+ " mulxq 8(%1), %%r10, %%r11;"
+ " adox %%r9, %%r10;"
+ " adcx %%rbx, %%r10;"
+ " movq %%r10, 16(%2);"
+ " mulxq 16(%1), %%rbx, %%r13;"
+ " adox %%r11, %%rbx;"
+ " adcx %%r14, %%rbx;"
+ " mov $0, %%r8;"
+ " mulxq 24(%1), %%r14, %%rdx;"
+ " adox %%r13, %%r14;"
+ " adcx %%rax, %%r14;"
+ " mov $0, %%rax;"
+ " adox %%rdx, %%rax;"
+ " adcx %%r8, %%rax;"
+
/* Compute src1[2] * src2 */
- " movq 16(%1), %%rdx;"
- " mulxq 0(%3), %%r8, %%r9;" " xor %%r10d, %%r10d;" " adcxq 16(%0), %%r8;" " movq %%r8, 16(%0);"
- " mulxq 8(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " adcx %%rbx, %%r10;" " movq %%r10, 24(%0);"
- " mulxq 16(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" " adcx %%r14, %%rbx;" " mov $0, %%r8;"
- " mulxq 24(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " adcx %%rax, %%r14;" " mov $0, %%rax;"
- " adox %%rdx, %%rax;" " adcx %%r8, %%rax;"
+ " movq 16(%0), %%rdx;"
+ " mulxq 0(%1), %%r8, %%r9;"
+ " xor %%r10d, %%r10d;"
+ " adcxq 16(%2), %%r8;"
+ " movq %%r8, 16(%2);"
+ " mulxq 8(%1), %%r10, %%r11;"
+ " adox %%r9, %%r10;"
+ " adcx %%rbx, %%r10;"
+ " movq %%r10, 24(%2);"
+ " mulxq 16(%1), %%rbx, %%r13;"
+ " adox %%r11, %%rbx;"
+ " adcx %%r14, %%rbx;"
+ " mov $0, %%r8;"
+ " mulxq 24(%1), %%r14, %%rdx;"
+ " adox %%r13, %%r14;"
+ " adcx %%rax, %%r14;"
+ " mov $0, %%rax;"
+ " adox %%rdx, %%rax;"
+ " adcx %%r8, %%rax;"
+
/* Compute src1[3] * src2 */
- " movq 24(%1), %%rdx;"
- " mulxq 0(%3), %%r8, %%r9;" " xor %%r10d, %%r10d;" " adcxq 24(%0), %%r8;" " movq %%r8, 24(%0);"
- " mulxq 8(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " adcx %%rbx, %%r10;" " movq %%r10, 32(%0);"
- " mulxq 16(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" " adcx %%r14, %%rbx;" " movq %%rbx, 40(%0);" " mov $0, %%r8;"
- " mulxq 24(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " adcx %%rax, %%r14;" " movq %%r14, 48(%0);" " mov $0, %%rax;"
- " adox %%rdx, %%rax;" " adcx %%r8, %%rax;" " movq %%rax, 56(%0);"
+ " movq 24(%0), %%rdx;"
+ " mulxq 0(%1), %%r8, %%r9;"
+ " xor %%r10d, %%r10d;"
+ " adcxq 24(%2), %%r8;"
+ " movq %%r8, 24(%2);"
+ " mulxq 8(%1), %%r10, %%r11;"
+ " adox %%r9, %%r10;"
+ " adcx %%rbx, %%r10;"
+ " movq %%r10, 32(%2);"
+ " mulxq 16(%1), %%rbx, %%r13;"
+ " adox %%r11, %%rbx;"
+ " adcx %%r14, %%rbx;"
+ " movq %%rbx, 40(%2);"
+ " mov $0, %%r8;"
+ " mulxq 24(%1), %%r14, %%rdx;"
+ " adox %%r13, %%r14;"
+ " adcx %%rax, %%r14;"
+ " movq %%r14, 48(%2);"
+ " mov $0, %%rax;"
+ " adox %%rdx, %%rax;"
+ " adcx %%r8, %%rax;"
+ " movq %%rax, 56(%2);"
+
/* Line up pointers */
- " mov %0, %1;"
" mov %2, %0;"
+ " mov %3, %2;"
/* Wrap the result back into the field */
/* Step 1: Compute dst + carry == tmp_hi * 38 + tmp_lo */
" mov $38, %%rdx;"
- " mulxq 32(%1), %%r8, %%r13;"
- " xor %k3, %k3;"
- " adoxq 0(%1), %%r8;"
- " mulxq 40(%1), %%r9, %%rbx;"
+ " mulxq 32(%0), %%r8, %%r13;"
+ " xor %k1, %k1;"
+ " adoxq 0(%0), %%r8;"
+ " mulxq 40(%0), %%r9, %%rbx;"
" adcx %%r13, %%r9;"
- " adoxq 8(%1), %%r9;"
- " mulxq 48(%1), %%r10, %%r13;"
+ " adoxq 8(%0), %%r9;"
+ " mulxq 48(%0), %%r10, %%r13;"
" adcx %%rbx, %%r10;"
- " adoxq 16(%1), %%r10;"
- " mulxq 56(%1), %%r11, %%rax;"
+ " adoxq 16(%0), %%r10;"
+ " mulxq 56(%0), %%r11, %%rax;"
" adcx %%r13, %%r11;"
- " adoxq 24(%1), %%r11;"
- " adcx %3, %%rax;"
- " adox %3, %%rax;"
+ " adoxq 24(%0), %%r11;"
+ " adcx %1, %%rax;"
+ " adox %1, %%rax;"
" imul %%rdx, %%rax;"
/* Step 2: Fold the carry back into dst */
" add %%rax, %%r8;"
- " adcx %3, %%r9;"
- " movq %%r9, 8(%0);"
- " adcx %3, %%r10;"
- " movq %%r10, 16(%0);"
- " adcx %3, %%r11;"
- " movq %%r11, 24(%0);"
+ " adcx %1, %%r9;"
+ " movq %%r9, 8(%2);"
+ " adcx %1, %%r10;"
+ " movq %%r10, 16(%2);"
+ " adcx %1, %%r11;"
+ " movq %%r11, 24(%2);"
/* Step 3: Fold the carry bit back in; guaranteed not to carry at this point */
" mov $0, %%rax;"
" cmovc %%rdx, %%rax;"
" add %%rax, %%r8;"
- " movq %%r8, 0(%0);"
- : "+&r" (tmp), "+&r" (f1), "+&r" (out), "+&r" (f2)
- :
- : "%rax", "%rdx", "%r8", "%r9", "%r10", "%r11", "%rbx", "%r13", "%r14", "memory", "cc"
- );
+ " movq %%r8, 0(%2);"
+ : "+&r"(f1), "+&r"(f2), "+&r"(tmp)
+ : "r"(out)
+ : "%rax", "%rbx", "%rdx", "%r8", "%r9", "%r10", "%r11", "%r13",
+ "%r14", "memory", "cc");
}
/* Computes two field multiplications:
- * out[0] <- f1[0] * f2[0]
- * out[1] <- f1[1] * f2[1]
- * Uses the 16-element buffer tmp for intermediate results. */
+ * out[0] <- f1[0] * f2[0]
+ * out[1] <- f1[1] * f2[1]
+ * Uses the 16-element buffer tmp for intermediate results: */
static inline void fmul2(u64 *out, const u64 *f1, const u64 *f2, u64 *tmp)
{
asm volatile(
+
/* Compute the raw multiplication tmp[0] <- f1[0] * f2[0] */
/* Compute src1[0] * src2 */
- " movq 0(%1), %%rdx;"
- " mulxq 0(%3), %%r8, %%r9;" " xor %%r10d, %%r10d;" " movq %%r8, 0(%0);"
- " mulxq 8(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " movq %%r10, 8(%0);"
- " mulxq 16(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;"
- " mulxq 24(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " mov $0, %%rax;"
- " adox %%rdx, %%rax;"
+ " movq 0(%0), %%rdx;"
+ " mulxq 0(%1), %%r8, %%r9;"
+ " xor %%r10d, %%r10d;"
+ " movq %%r8, 0(%2);"
+ " mulxq 8(%1), %%r10, %%r11;"
+ " adox %%r9, %%r10;"
+ " movq %%r10, 8(%2);"
+ " mulxq 16(%1), %%rbx, %%r13;"
+ " adox %%r11, %%rbx;"
+ " mulxq 24(%1), %%r14, %%rdx;"
+ " adox %%r13, %%r14;"
+ " mov $0, %%rax;"
+ " adox %%rdx, %%rax;"
+
/* Compute src1[1] * src2 */
- " movq 8(%1), %%rdx;"
- " mulxq 0(%3), %%r8, %%r9;" " xor %%r10d, %%r10d;" " adcxq 8(%0), %%r8;" " movq %%r8, 8(%0);"
- " mulxq 8(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " adcx %%rbx, %%r10;" " movq %%r10, 16(%0);"
- " mulxq 16(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" " adcx %%r14, %%rbx;" " mov $0, %%r8;"
- " mulxq 24(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " adcx %%rax, %%r14;" " mov $0, %%rax;"
- " adox %%rdx, %%rax;" " adcx %%r8, %%rax;"
+ " movq 8(%0), %%rdx;"
+ " mulxq 0(%1), %%r8, %%r9;"
+ " xor %%r10d, %%r10d;"
+ " adcxq 8(%2), %%r8;"
+ " movq %%r8, 8(%2);"
+ " mulxq 8(%1), %%r10, %%r11;"
+ " adox %%r9, %%r10;"
+ " adcx %%rbx, %%r10;"
+ " movq %%r10, 16(%2);"
+ " mulxq 16(%1), %%rbx, %%r13;"
+ " adox %%r11, %%rbx;"
+ " adcx %%r14, %%rbx;"
+ " mov $0, %%r8;"
+ " mulxq 24(%1), %%r14, %%rdx;"
+ " adox %%r13, %%r14;"
+ " adcx %%rax, %%r14;"
+ " mov $0, %%rax;"
+ " adox %%rdx, %%rax;"
+ " adcx %%r8, %%rax;"
+
/* Compute src1[2] * src2 */
- " movq 16(%1), %%rdx;"
- " mulxq 0(%3), %%r8, %%r9;" " xor %%r10d, %%r10d;" " adcxq 16(%0), %%r8;" " movq %%r8, 16(%0);"
- " mulxq 8(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " adcx %%rbx, %%r10;" " movq %%r10, 24(%0);"
- " mulxq 16(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" " adcx %%r14, %%rbx;" " mov $0, %%r8;"
- " mulxq 24(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " adcx %%rax, %%r14;" " mov $0, %%rax;"
- " adox %%rdx, %%rax;" " adcx %%r8, %%rax;"
+ " movq 16(%0), %%rdx;"
+ " mulxq 0(%1), %%r8, %%r9;"
+ " xor %%r10d, %%r10d;"
+ " adcxq 16(%2), %%r8;"
+ " movq %%r8, 16(%2);"
+ " mulxq 8(%1), %%r10, %%r11;"
+ " adox %%r9, %%r10;"
+ " adcx %%rbx, %%r10;"
+ " movq %%r10, 24(%2);"
+ " mulxq 16(%1), %%rbx, %%r13;"
+ " adox %%r11, %%rbx;"
+ " adcx %%r14, %%rbx;"
+ " mov $0, %%r8;"
+ " mulxq 24(%1), %%r14, %%rdx;"
+ " adox %%r13, %%r14;"
+ " adcx %%rax, %%r14;"
+ " mov $0, %%rax;"
+ " adox %%rdx, %%rax;"
+ " adcx %%r8, %%rax;"
+
/* Compute src1[3] * src2 */
- " movq 24(%1), %%rdx;"
- " mulxq 0(%3), %%r8, %%r9;" " xor %%r10d, %%r10d;" " adcxq 24(%0), %%r8;" " movq %%r8, 24(%0);"
- " mulxq 8(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " adcx %%rbx, %%r10;" " movq %%r10, 32(%0);"
- " mulxq 16(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" " adcx %%r14, %%rbx;" " movq %%rbx, 40(%0);" " mov $0, %%r8;"
- " mulxq 24(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " adcx %%rax, %%r14;" " movq %%r14, 48(%0);" " mov $0, %%rax;"
- " adox %%rdx, %%rax;" " adcx %%r8, %%rax;" " movq %%rax, 56(%0);"
+ " movq 24(%0), %%rdx;"
+ " mulxq 0(%1), %%r8, %%r9;"
+ " xor %%r10d, %%r10d;"
+ " adcxq 24(%2), %%r8;"
+ " movq %%r8, 24(%2);"
+ " mulxq 8(%1), %%r10, %%r11;"
+ " adox %%r9, %%r10;"
+ " adcx %%rbx, %%r10;"
+ " movq %%r10, 32(%2);"
+ " mulxq 16(%1), %%rbx, %%r13;"
+ " adox %%r11, %%rbx;"
+ " adcx %%r14, %%rbx;"
+ " movq %%rbx, 40(%2);"
+ " mov $0, %%r8;"
+ " mulxq 24(%1), %%r14, %%rdx;"
+ " adox %%r13, %%r14;"
+ " adcx %%rax, %%r14;"
+ " movq %%r14, 48(%2);"
+ " mov $0, %%rax;"
+ " adox %%rdx, %%rax;"
+ " adcx %%r8, %%rax;"
+ " movq %%rax, 56(%2);"
/* Compute the raw multiplication tmp[1] <- f1[1] * f2[1] */
/* Compute src1[0] * src2 */
- " movq 32(%1), %%rdx;"
- " mulxq 32(%3), %%r8, %%r9;" " xor %%r10d, %%r10d;" " movq %%r8, 64(%0);"
- " mulxq 40(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " movq %%r10, 72(%0);"
- " mulxq 48(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;"
- " mulxq 56(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " mov $0, %%rax;"
- " adox %%rdx, %%rax;"
+ " movq 32(%0), %%rdx;"
+ " mulxq 32(%1), %%r8, %%r9;"
+ " xor %%r10d, %%r10d;"
+ " movq %%r8, 64(%2);"
+ " mulxq 40(%1), %%r10, %%r11;"
+ " adox %%r9, %%r10;"
+ " movq %%r10, 72(%2);"
+ " mulxq 48(%1), %%rbx, %%r13;"
+ " adox %%r11, %%rbx;"
+ " mulxq 56(%1), %%r14, %%rdx;"
+ " adox %%r13, %%r14;"
+ " mov $0, %%rax;"
+ " adox %%rdx, %%rax;"
+
/* Compute src1[1] * src2 */
- " movq 40(%1), %%rdx;"
- " mulxq 32(%3), %%r8, %%r9;" " xor %%r10d, %%r10d;" " adcxq 72(%0), %%r8;" " movq %%r8, 72(%0);"
- " mulxq 40(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " adcx %%rbx, %%r10;" " movq %%r10, 80(%0);"
- " mulxq 48(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" " adcx %%r14, %%rbx;" " mov $0, %%r8;"
- " mulxq 56(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " adcx %%rax, %%r14;" " mov $0, %%rax;"
- " adox %%rdx, %%rax;" " adcx %%r8, %%rax;"
+ " movq 40(%0), %%rdx;"
+ " mulxq 32(%1), %%r8, %%r9;"
+ " xor %%r10d, %%r10d;"
+ " adcxq 72(%2), %%r8;"
+ " movq %%r8, 72(%2);"
+ " mulxq 40(%1), %%r10, %%r11;"
+ " adox %%r9, %%r10;"
+ " adcx %%rbx, %%r10;"
+ " movq %%r10, 80(%2);"
+ " mulxq 48(%1), %%rbx, %%r13;"
+ " adox %%r11, %%rbx;"
+ " adcx %%r14, %%rbx;"
+ " mov $0, %%r8;"
+ " mulxq 56(%1), %%r14, %%rdx;"
+ " adox %%r13, %%r14;"
+ " adcx %%rax, %%r14;"
+ " mov $0, %%rax;"
+ " adox %%rdx, %%rax;"
+ " adcx %%r8, %%rax;"
+
/* Compute src1[2] * src2 */
- " movq 48(%1), %%rdx;"
- " mulxq 32(%3), %%r8, %%r9;" " xor %%r10d, %%r10d;" " adcxq 80(%0), %%r8;" " movq %%r8, 80(%0);"
- " mulxq 40(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " adcx %%rbx, %%r10;" " movq %%r10, 88(%0);"
- " mulxq 48(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" " adcx %%r14, %%rbx;" " mov $0, %%r8;"
- " mulxq 56(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " adcx %%rax, %%r14;" " mov $0, %%rax;"
- " adox %%rdx, %%rax;" " adcx %%r8, %%rax;"
+ " movq 48(%0), %%rdx;"
+ " mulxq 32(%1), %%r8, %%r9;"
+ " xor %%r10d, %%r10d;"
+ " adcxq 80(%2), %%r8;"
+ " movq %%r8, 80(%2);"
+ " mulxq 40(%1), %%r10, %%r11;"
+ " adox %%r9, %%r10;"
+ " adcx %%rbx, %%r10;"
+ " movq %%r10, 88(%2);"
+ " mulxq 48(%1), %%rbx, %%r13;"
+ " adox %%r11, %%rbx;"
+ " adcx %%r14, %%rbx;"
+ " mov $0, %%r8;"
+ " mulxq 56(%1), %%r14, %%rdx;"
+ " adox %%r13, %%r14;"
+ " adcx %%rax, %%r14;"
+ " mov $0, %%rax;"
+ " adox %%rdx, %%rax;"
+ " adcx %%r8, %%rax;"
+
/* Compute src1[3] * src2 */
- " movq 56(%1), %%rdx;"
- " mulxq 32(%3), %%r8, %%r9;" " xor %%r10d, %%r10d;" " adcxq 88(%0), %%r8;" " movq %%r8, 88(%0);"
- " mulxq 40(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " adcx %%rbx, %%r10;" " movq %%r10, 96(%0);"
- " mulxq 48(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" " adcx %%r14, %%rbx;" " movq %%rbx, 104(%0);" " mov $0, %%r8;"
- " mulxq 56(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " adcx %%rax, %%r14;" " movq %%r14, 112(%0);" " mov $0, %%rax;"
- " adox %%rdx, %%rax;" " adcx %%r8, %%rax;" " movq %%rax, 120(%0);"
+ " movq 56(%0), %%rdx;"
+ " mulxq 32(%1), %%r8, %%r9;"
+ " xor %%r10d, %%r10d;"
+ " adcxq 88(%2), %%r8;"
+ " movq %%r8, 88(%2);"
+ " mulxq 40(%1), %%r10, %%r11;"
+ " adox %%r9, %%r10;"
+ " adcx %%rbx, %%r10;"
+ " movq %%r10, 96(%2);"
+ " mulxq 48(%1), %%rbx, %%r13;"
+ " adox %%r11, %%rbx;"
+ " adcx %%r14, %%rbx;"
+ " movq %%rbx, 104(%2);"
+ " mov $0, %%r8;"
+ " mulxq 56(%1), %%r14, %%rdx;"
+ " adox %%r13, %%r14;"
+ " adcx %%rax, %%r14;"
+ " movq %%r14, 112(%2);"
+ " mov $0, %%rax;"
+ " adox %%rdx, %%rax;"
+ " adcx %%r8, %%rax;"
+ " movq %%rax, 120(%2);"
+
/* Line up pointers */
- " mov %0, %1;"
" mov %2, %0;"
+ " mov %3, %2;"
/* Wrap the results back into the field */
/* Step 1: Compute dst + carry == tmp_hi * 38 + tmp_lo */
" mov $38, %%rdx;"
- " mulxq 32(%1), %%r8, %%r13;"
- " xor %k3, %k3;"
- " adoxq 0(%1), %%r8;"
- " mulxq 40(%1), %%r9, %%rbx;"
+ " mulxq 32(%0), %%r8, %%r13;"
+ " xor %k1, %k1;"
+ " adoxq 0(%0), %%r8;"
+ " mulxq 40(%0), %%r9, %%rbx;"
" adcx %%r13, %%r9;"
- " adoxq 8(%1), %%r9;"
- " mulxq 48(%1), %%r10, %%r13;"
+ " adoxq 8(%0), %%r9;"
+ " mulxq 48(%0), %%r10, %%r13;"
" adcx %%rbx, %%r10;"
- " adoxq 16(%1), %%r10;"
- " mulxq 56(%1), %%r11, %%rax;"
+ " adoxq 16(%0), %%r10;"
+ " mulxq 56(%0), %%r11, %%rax;"
" adcx %%r13, %%r11;"
- " adoxq 24(%1), %%r11;"
- " adcx %3, %%rax;"
- " adox %3, %%rax;"
+ " adoxq 24(%0), %%r11;"
+ " adcx %1, %%rax;"
+ " adox %1, %%rax;"
" imul %%rdx, %%rax;"
/* Step 2: Fold the carry back into dst */
" add %%rax, %%r8;"
- " adcx %3, %%r9;"
- " movq %%r9, 8(%0);"
- " adcx %3, %%r10;"
- " movq %%r10, 16(%0);"
- " adcx %3, %%r11;"
- " movq %%r11, 24(%0);"
+ " adcx %1, %%r9;"
+ " movq %%r9, 8(%2);"
+ " adcx %1, %%r10;"
+ " movq %%r10, 16(%2);"
+ " adcx %1, %%r11;"
+ " movq %%r11, 24(%2);"
/* Step 3: Fold the carry bit back in; guaranteed not to carry at this point */
" mov $0, %%rax;"
" cmovc %%rdx, %%rax;"
" add %%rax, %%r8;"
- " movq %%r8, 0(%0);"
+ " movq %%r8, 0(%2);"
/* Step 1: Compute dst + carry == tmp_hi * 38 + tmp_lo */
" mov $38, %%rdx;"
- " mulxq 96(%1), %%r8, %%r13;"
- " xor %k3, %k3;"
- " adoxq 64(%1), %%r8;"
- " mulxq 104(%1), %%r9, %%rbx;"
+ " mulxq 96(%0), %%r8, %%r13;"
+ " xor %k1, %k1;"
+ " adoxq 64(%0), %%r8;"
+ " mulxq 104(%0), %%r9, %%rbx;"
" adcx %%r13, %%r9;"
- " adoxq 72(%1), %%r9;"
- " mulxq 112(%1), %%r10, %%r13;"
+ " adoxq 72(%0), %%r9;"
+ " mulxq 112(%0), %%r10, %%r13;"
" adcx %%rbx, %%r10;"
- " adoxq 80(%1), %%r10;"
- " mulxq 120(%1), %%r11, %%rax;"
+ " adoxq 80(%0), %%r10;"
+ " mulxq 120(%0), %%r11, %%rax;"
" adcx %%r13, %%r11;"
- " adoxq 88(%1), %%r11;"
- " adcx %3, %%rax;"
- " adox %3, %%rax;"
+ " adoxq 88(%0), %%r11;"
+ " adcx %1, %%rax;"
+ " adox %1, %%rax;"
" imul %%rdx, %%rax;"
/* Step 2: Fold the carry back into dst */
" add %%rax, %%r8;"
- " adcx %3, %%r9;"
- " movq %%r9, 40(%0);"
- " adcx %3, %%r10;"
- " movq %%r10, 48(%0);"
- " adcx %3, %%r11;"
- " movq %%r11, 56(%0);"
+ " adcx %1, %%r9;"
+ " movq %%r9, 40(%2);"
+ " adcx %1, %%r10;"
+ " movq %%r10, 48(%2);"
+ " adcx %1, %%r11;"
+ " movq %%r11, 56(%2);"
/* Step 3: Fold the carry bit back in; guaranteed not to carry at this point */
" mov $0, %%rax;"
" cmovc %%rdx, %%rax;"
" add %%rax, %%r8;"
- " movq %%r8, 32(%0);"
- : "+&r" (tmp), "+&r" (f1), "+&r" (out), "+&r" (f2)
- :
- : "%rax", "%rdx", "%r8", "%r9", "%r10", "%r11", "%rbx", "%r13", "%r14", "memory", "cc"
- );
+ " movq %%r8, 32(%2);"
+ : "+&r"(f1), "+&r"(f2), "+&r"(tmp)
+ : "r"(out)
+ : "%rax", "%rbx", "%rdx", "%r8", "%r9", "%r10", "%r11", "%r13",
+ "%r14", "memory", "cc");
}
-/* Computes the field multiplication of four-element f1 with value in f2 */
+/* Computes the field multiplication of four-element f1 with value in f2
+ * Requires f2 to be smaller than 2^17 */
static inline void fmul_scalar(u64 *out, const u64 *f1, u64 f2)
{
register u64 f2_r asm("rdx") = f2;
asm volatile(
/* Compute the raw multiplication of f1*f2 */
- " mulxq 0(%2), %%r8, %%rcx;" /* f1[0]*f2 */
- " mulxq 8(%2), %%r9, %%rbx;" /* f1[1]*f2 */
+ " mulxq 0(%2), %%r8, %%rcx;" /* f1[0]*f2 */
+ " mulxq 8(%2), %%r9, %%rbx;" /* f1[1]*f2 */
" add %%rcx, %%r9;"
" mov $0, %%rcx;"
- " mulxq 16(%2), %%r10, %%r13;" /* f1[2]*f2 */
+ " mulxq 16(%2), %%r10, %%r13;" /* f1[2]*f2 */
" adcx %%rbx, %%r10;"
- " mulxq 24(%2), %%r11, %%rax;" /* f1[3]*f2 */
+ " mulxq 24(%2), %%r11, %%rax;" /* f1[3]*f2 */
" adcx %%r13, %%r11;"
" adcx %%rcx, %%rax;"
@@ -418,17 +576,17 @@ static inline void fmul_scalar(u64 *out, const u64 *f1, u64 f2)
" cmovc %%rdx, %%rax;"
" add %%rax, %%r8;"
" movq %%r8, 0(%1);"
- : "+&r" (f2_r)
- : "r" (out), "r" (f1)
- : "%rax", "%rcx", "%r8", "%r9", "%r10", "%r11", "%rbx", "%r13", "memory", "cc"
- );
+ : "+&r"(f2_r)
+ : "r"(out), "r"(f1)
+ : "%rax", "%rbx", "%rcx", "%r8", "%r9", "%r10", "%r11", "%r13",
+ "memory", "cc");
}
/* Computes p1 <- bit ? p2 : p1 in constant time */
static inline void cswap2(u64 bit, const u64 *p1, const u64 *p2)
{
asm volatile(
- /* Invert the polarity of bit to match cmov expectations */
+ /* Transfer bit into CF flag */
" add $18446744073709551615, %0;"
/* cswap p1[0], p2[0] */
@@ -502,10 +660,9 @@ static inline void cswap2(u64 bit, const u64 *p1, const u64 *p2)
" cmovc %%r10, %%r9;"
" movq %%r8, 56(%1);"
" movq %%r9, 56(%2);"
- : "+&r" (bit)
- : "r" (p1), "r" (p2)
- : "%r8", "%r9", "%r10", "memory", "cc"
- );
+ : "+&r"(bit)
+ : "r"(p1), "r"(p2)
+ : "%r8", "%r9", "%r10", "memory", "cc");
}
/* Computes the square of a field element: out <- f * f
@@ -516,15 +673,22 @@ static inline void fsqr(u64 *out, const u64 *f, u64 *tmp)
/* Compute the raw multiplication: tmp <- f * f */
/* Step 1: Compute all partial products */
- " movq 0(%1), %%rdx;" /* f[0] */
- " mulxq 8(%1), %%r8, %%r14;" " xor %%r15d, %%r15d;" /* f[1]*f[0] */
- " mulxq 16(%1), %%r9, %%r10;" " adcx %%r14, %%r9;" /* f[2]*f[0] */
- " mulxq 24(%1), %%rax, %%rcx;" " adcx %%rax, %%r10;" /* f[3]*f[0] */
- " movq 24(%1), %%rdx;" /* f[3] */
- " mulxq 8(%1), %%r11, %%rbx;" " adcx %%rcx, %%r11;" /* f[1]*f[3] */
- " mulxq 16(%1), %%rax, %%r13;" " adcx %%rax, %%rbx;" /* f[2]*f[3] */
- " movq 8(%1), %%rdx;" " adcx %%r15, %%r13;" /* f1 */
- " mulxq 16(%1), %%rax, %%rcx;" " mov $0, %%r14;" /* f[2]*f[1] */
+ " movq 0(%0), %%rdx;" /* f[0] */
+ " mulxq 8(%0), %%r8, %%r14;"
+ " xor %%r15d, %%r15d;" /* f[1]*f[0] */
+ " mulxq 16(%0), %%r9, %%r10;"
+ " adcx %%r14, %%r9;" /* f[2]*f[0] */
+ " mulxq 24(%0), %%rax, %%rcx;"
+ " adcx %%rax, %%r10;" /* f[3]*f[0] */
+ " movq 24(%0), %%rdx;" /* f[3] */
+ " mulxq 8(%0), %%r11, %%rbx;"
+ " adcx %%rcx, %%r11;" /* f[1]*f[3] */
+ " mulxq 16(%0), %%rax, %%r13;"
+ " adcx %%rax, %%rbx;" /* f[2]*f[3] */
+ " movq 8(%0), %%rdx;"
+ " adcx %%r15, %%r13;" /* f1 */
+ " mulxq 16(%0), %%rax, %%rcx;"
+ " mov $0, %%r14;" /* f[2]*f[1] */
/* Step 2: Compute two parallel carry chains */
" xor %%r15d, %%r15d;"
@@ -542,39 +706,50 @@ static inline void fsqr(u64 *out, const u64 *f, u64 *tmp)
" adcx %%r14, %%r14;"
/* Step 3: Compute intermediate squares */
- " movq 0(%1), %%rdx;" " mulx %%rdx, %%rax, %%rcx;" /* f[0]^2 */
- " movq %%rax, 0(%0);"
- " add %%rcx, %%r8;" " movq %%r8, 8(%0);"
- " movq 8(%1), %%rdx;" " mulx %%rdx, %%rax, %%rcx;" /* f[1]^2 */
- " adcx %%rax, %%r9;" " movq %%r9, 16(%0);"
- " adcx %%rcx, %%r10;" " movq %%r10, 24(%0);"
- " movq 16(%1), %%rdx;" " mulx %%rdx, %%rax, %%rcx;" /* f[2]^2 */
- " adcx %%rax, %%r11;" " movq %%r11, 32(%0);"
- " adcx %%rcx, %%rbx;" " movq %%rbx, 40(%0);"
- " movq 24(%1), %%rdx;" " mulx %%rdx, %%rax, %%rcx;" /* f[3]^2 */
- " adcx %%rax, %%r13;" " movq %%r13, 48(%0);"
- " adcx %%rcx, %%r14;" " movq %%r14, 56(%0);"
+ " movq 0(%0), %%rdx;"
+ " mulx %%rdx, %%rax, %%rcx;" /* f[0]^2 */
+ " movq %%rax, 0(%1);"
+ " add %%rcx, %%r8;"
+ " movq %%r8, 8(%1);"
+ " movq 8(%0), %%rdx;"
+ " mulx %%rdx, %%rax, %%rcx;" /* f[1]^2 */
+ " adcx %%rax, %%r9;"
+ " movq %%r9, 16(%1);"
+ " adcx %%rcx, %%r10;"
+ " movq %%r10, 24(%1);"
+ " movq 16(%0), %%rdx;"
+ " mulx %%rdx, %%rax, %%rcx;" /* f[2]^2 */
+ " adcx %%rax, %%r11;"
+ " movq %%r11, 32(%1);"
+ " adcx %%rcx, %%rbx;"
+ " movq %%rbx, 40(%1);"
+ " movq 24(%0), %%rdx;"
+ " mulx %%rdx, %%rax, %%rcx;" /* f[3]^2 */
+ " adcx %%rax, %%r13;"
+ " movq %%r13, 48(%1);"
+ " adcx %%rcx, %%r14;"
+ " movq %%r14, 56(%1);"
/* Line up pointers */
- " mov %0, %1;"
- " mov %2, %0;"
+ " mov %1, %0;"
+ " mov %2, %1;"
/* Wrap the result back into the field */
/* Step 1: Compute dst + carry == tmp_hi * 38 + tmp_lo */
" mov $38, %%rdx;"
- " mulxq 32(%1), %%r8, %%r13;"
+ " mulxq 32(%0), %%r8, %%r13;"
" xor %%ecx, %%ecx;"
- " adoxq 0(%1), %%r8;"
- " mulxq 40(%1), %%r9, %%rbx;"
+ " adoxq 0(%0), %%r8;"
+ " mulxq 40(%0), %%r9, %%rbx;"
" adcx %%r13, %%r9;"
- " adoxq 8(%1), %%r9;"
- " mulxq 48(%1), %%r10, %%r13;"
+ " adoxq 8(%0), %%r9;"
+ " mulxq 48(%0), %%r10, %%r13;"
" adcx %%rbx, %%r10;"
- " adoxq 16(%1), %%r10;"
- " mulxq 56(%1), %%r11, %%rax;"
+ " adoxq 16(%0), %%r10;"
+ " mulxq 56(%0), %%r11, %%rax;"
" adcx %%r13, %%r11;"
- " adoxq 24(%1), %%r11;"
+ " adoxq 24(%0), %%r11;"
" adcx %%rcx, %%rax;"
" adox %%rcx, %%rax;"
" imul %%rdx, %%rax;"
@@ -582,40 +757,47 @@ static inline void fsqr(u64 *out, const u64 *f, u64 *tmp)
/* Step 2: Fold the carry back into dst */
" add %%rax, %%r8;"
" adcx %%rcx, %%r9;"
- " movq %%r9, 8(%0);"
+ " movq %%r9, 8(%1);"
" adcx %%rcx, %%r10;"
- " movq %%r10, 16(%0);"
+ " movq %%r10, 16(%1);"
" adcx %%rcx, %%r11;"
- " movq %%r11, 24(%0);"
+ " movq %%r11, 24(%1);"
/* Step 3: Fold the carry bit back in; guaranteed not to carry at this point */
" mov $0, %%rax;"
" cmovc %%rdx, %%rax;"
" add %%rax, %%r8;"
- " movq %%r8, 0(%0);"
- : "+&r" (tmp), "+&r" (f), "+&r" (out)
- :
- : "%rax", "%rcx", "%rdx", "%r8", "%r9", "%r10", "%r11", "%rbx", "%r13", "%r14", "%r15", "memory", "cc"
- );
+ " movq %%r8, 0(%1);"
+ : "+&r"(f), "+&r"(tmp)
+ : "r"(out)
+ : "%rax", "%rbx", "%rcx", "%rdx", "%r8", "%r9", "%r10", "%r11",
+ "%r13", "%r14", "%r15", "memory", "cc");
}
/* Computes two field squarings:
- * out[0] <- f[0] * f[0]
- * out[1] <- f[1] * f[1]
+ * out[0] <- f[0] * f[0]
+ * out[1] <- f[1] * f[1]
* Uses the 16-element buffer tmp for intermediate results */
static inline void fsqr2(u64 *out, const u64 *f, u64 *tmp)
{
asm volatile(
/* Step 1: Compute all partial products */
- " movq 0(%1), %%rdx;" /* f[0] */
- " mulxq 8(%1), %%r8, %%r14;" " xor %%r15d, %%r15d;" /* f[1]*f[0] */
- " mulxq 16(%1), %%r9, %%r10;" " adcx %%r14, %%r9;" /* f[2]*f[0] */
- " mulxq 24(%1), %%rax, %%rcx;" " adcx %%rax, %%r10;" /* f[3]*f[0] */
- " movq 24(%1), %%rdx;" /* f[3] */
- " mulxq 8(%1), %%r11, %%rbx;" " adcx %%rcx, %%r11;" /* f[1]*f[3] */
- " mulxq 16(%1), %%rax, %%r13;" " adcx %%rax, %%rbx;" /* f[2]*f[3] */
- " movq 8(%1), %%rdx;" " adcx %%r15, %%r13;" /* f1 */
- " mulxq 16(%1), %%rax, %%rcx;" " mov $0, %%r14;" /* f[2]*f[1] */
+ " movq 0(%0), %%rdx;" /* f[0] */
+ " mulxq 8(%0), %%r8, %%r14;"
+ " xor %%r15d, %%r15d;" /* f[1]*f[0] */
+ " mulxq 16(%0), %%r9, %%r10;"
+ " adcx %%r14, %%r9;" /* f[2]*f[0] */
+ " mulxq 24(%0), %%rax, %%rcx;"
+ " adcx %%rax, %%r10;" /* f[3]*f[0] */
+ " movq 24(%0), %%rdx;" /* f[3] */
+ " mulxq 8(%0), %%r11, %%rbx;"
+ " adcx %%rcx, %%r11;" /* f[1]*f[3] */
+ " mulxq 16(%0), %%rax, %%r13;"
+ " adcx %%rax, %%rbx;" /* f[2]*f[3] */
+ " movq 8(%0), %%rdx;"
+ " adcx %%r15, %%r13;" /* f1 */
+ " mulxq 16(%0), %%rax, %%rcx;"
+ " mov $0, %%r14;" /* f[2]*f[1] */
/* Step 2: Compute two parallel carry chains */
" xor %%r15d, %%r15d;"
@@ -633,29 +815,47 @@ static inline void fsqr2(u64 *out, const u64 *f, u64 *tmp)
" adcx %%r14, %%r14;"
/* Step 3: Compute intermediate squares */
- " movq 0(%1), %%rdx;" " mulx %%rdx, %%rax, %%rcx;" /* f[0]^2 */
- " movq %%rax, 0(%0);"
- " add %%rcx, %%r8;" " movq %%r8, 8(%0);"
- " movq 8(%1), %%rdx;" " mulx %%rdx, %%rax, %%rcx;" /* f[1]^2 */
- " adcx %%rax, %%r9;" " movq %%r9, 16(%0);"
- " adcx %%rcx, %%r10;" " movq %%r10, 24(%0);"
- " movq 16(%1), %%rdx;" " mulx %%rdx, %%rax, %%rcx;" /* f[2]^2 */
- " adcx %%rax, %%r11;" " movq %%r11, 32(%0);"
- " adcx %%rcx, %%rbx;" " movq %%rbx, 40(%0);"
- " movq 24(%1), %%rdx;" " mulx %%rdx, %%rax, %%rcx;" /* f[3]^2 */
- " adcx %%rax, %%r13;" " movq %%r13, 48(%0);"
- " adcx %%rcx, %%r14;" " movq %%r14, 56(%0);"
+ " movq 0(%0), %%rdx;"
+ " mulx %%rdx, %%rax, %%rcx;" /* f[0]^2 */
+ " movq %%rax, 0(%1);"
+ " add %%rcx, %%r8;"
+ " movq %%r8, 8(%1);"
+ " movq 8(%0), %%rdx;"
+ " mulx %%rdx, %%rax, %%rcx;" /* f[1]^2 */
+ " adcx %%rax, %%r9;"
+ " movq %%r9, 16(%1);"
+ " adcx %%rcx, %%r10;"
+ " movq %%r10, 24(%1);"
+ " movq 16(%0), %%rdx;"
+ " mulx %%rdx, %%rax, %%rcx;" /* f[2]^2 */
+ " adcx %%rax, %%r11;"
+ " movq %%r11, 32(%1);"
+ " adcx %%rcx, %%rbx;"
+ " movq %%rbx, 40(%1);"
+ " movq 24(%0), %%rdx;"
+ " mulx %%rdx, %%rax, %%rcx;" /* f[3]^2 */
+ " adcx %%rax, %%r13;"
+ " movq %%r13, 48(%1);"
+ " adcx %%rcx, %%r14;"
+ " movq %%r14, 56(%1);"
/* Step 1: Compute all partial products */
- " movq 32(%1), %%rdx;" /* f[0] */
- " mulxq 40(%1), %%r8, %%r14;" " xor %%r15d, %%r15d;" /* f[1]*f[0] */
- " mulxq 48(%1), %%r9, %%r10;" " adcx %%r14, %%r9;" /* f[2]*f[0] */
- " mulxq 56(%1), %%rax, %%rcx;" " adcx %%rax, %%r10;" /* f[3]*f[0] */
- " movq 56(%1), %%rdx;" /* f[3] */
- " mulxq 40(%1), %%r11, %%rbx;" " adcx %%rcx, %%r11;" /* f[1]*f[3] */
- " mulxq 48(%1), %%rax, %%r13;" " adcx %%rax, %%rbx;" /* f[2]*f[3] */
- " movq 40(%1), %%rdx;" " adcx %%r15, %%r13;" /* f1 */
- " mulxq 48(%1), %%rax, %%rcx;" " mov $0, %%r14;" /* f[2]*f[1] */
+ " movq 32(%0), %%rdx;" /* f[0] */
+ " mulxq 40(%0), %%r8, %%r14;"
+ " xor %%r15d, %%r15d;" /* f[1]*f[0] */
+ " mulxq 48(%0), %%r9, %%r10;"
+ " adcx %%r14, %%r9;" /* f[2]*f[0] */
+ " mulxq 56(%0), %%rax, %%rcx;"
+ " adcx %%rax, %%r10;" /* f[3]*f[0] */
+ " movq 56(%0), %%rdx;" /* f[3] */
+ " mulxq 40(%0), %%r11, %%rbx;"
+ " adcx %%rcx, %%r11;" /* f[1]*f[3] */
+ " mulxq 48(%0), %%rax, %%r13;"
+ " adcx %%rax, %%rbx;" /* f[2]*f[3] */
+ " movq 40(%0), %%rdx;"
+ " adcx %%r15, %%r13;" /* f1 */
+ " mulxq 48(%0), %%rax, %%rcx;"
+ " mov $0, %%r14;" /* f[2]*f[1] */
/* Step 2: Compute two parallel carry chains */
" xor %%r15d, %%r15d;"
@@ -673,37 +873,48 @@ static inline void fsqr2(u64 *out, const u64 *f, u64 *tmp)
" adcx %%r14, %%r14;"
/* Step 3: Compute intermediate squares */
- " movq 32(%1), %%rdx;" " mulx %%rdx, %%rax, %%rcx;" /* f[0]^2 */
- " movq %%rax, 64(%0);"
- " add %%rcx, %%r8;" " movq %%r8, 72(%0);"
- " movq 40(%1), %%rdx;" " mulx %%rdx, %%rax, %%rcx;" /* f[1]^2 */
- " adcx %%rax, %%r9;" " movq %%r9, 80(%0);"
- " adcx %%rcx, %%r10;" " movq %%r10, 88(%0);"
- " movq 48(%1), %%rdx;" " mulx %%rdx, %%rax, %%rcx;" /* f[2]^2 */
- " adcx %%rax, %%r11;" " movq %%r11, 96(%0);"
- " adcx %%rcx, %%rbx;" " movq %%rbx, 104(%0);"
- " movq 56(%1), %%rdx;" " mulx %%rdx, %%rax, %%rcx;" /* f[3]^2 */
- " adcx %%rax, %%r13;" " movq %%r13, 112(%0);"
- " adcx %%rcx, %%r14;" " movq %%r14, 120(%0);"
+ " movq 32(%0), %%rdx;"
+ " mulx %%rdx, %%rax, %%rcx;" /* f[0]^2 */
+ " movq %%rax, 64(%1);"
+ " add %%rcx, %%r8;"
+ " movq %%r8, 72(%1);"
+ " movq 40(%0), %%rdx;"
+ " mulx %%rdx, %%rax, %%rcx;" /* f[1]^2 */
+ " adcx %%rax, %%r9;"
+ " movq %%r9, 80(%1);"
+ " adcx %%rcx, %%r10;"
+ " movq %%r10, 88(%1);"
+ " movq 48(%0), %%rdx;"
+ " mulx %%rdx, %%rax, %%rcx;" /* f[2]^2 */
+ " adcx %%rax, %%r11;"
+ " movq %%r11, 96(%1);"
+ " adcx %%rcx, %%rbx;"
+ " movq %%rbx, 104(%1);"
+ " movq 56(%0), %%rdx;"
+ " mulx %%rdx, %%rax, %%rcx;" /* f[3]^2 */
+ " adcx %%rax, %%r13;"
+ " movq %%r13, 112(%1);"
+ " adcx %%rcx, %%r14;"
+ " movq %%r14, 120(%1);"
/* Line up pointers */
- " mov %0, %1;"
- " mov %2, %0;"
+ " mov %1, %0;"
+ " mov %2, %1;"
/* Step 1: Compute dst + carry == tmp_hi * 38 + tmp_lo */
" mov $38, %%rdx;"
- " mulxq 32(%1), %%r8, %%r13;"
+ " mulxq 32(%0), %%r8, %%r13;"
" xor %%ecx, %%ecx;"
- " adoxq 0(%1), %%r8;"
- " mulxq 40(%1), %%r9, %%rbx;"
+ " adoxq 0(%0), %%r8;"
+ " mulxq 40(%0), %%r9, %%rbx;"
" adcx %%r13, %%r9;"
- " adoxq 8(%1), %%r9;"
- " mulxq 48(%1), %%r10, %%r13;"
+ " adoxq 8(%0), %%r9;"
+ " mulxq 48(%0), %%r10, %%r13;"
" adcx %%rbx, %%r10;"
- " adoxq 16(%1), %%r10;"
- " mulxq 56(%1), %%r11, %%rax;"
+ " adoxq 16(%0), %%r10;"
+ " mulxq 56(%0), %%r11, %%rax;"
" adcx %%r13, %%r11;"
- " adoxq 24(%1), %%r11;"
+ " adoxq 24(%0), %%r11;"
" adcx %%rcx, %%rax;"
" adox %%rcx, %%rax;"
" imul %%rdx, %%rax;"
@@ -711,32 +922,32 @@ static inline void fsqr2(u64 *out, const u64 *f, u64 *tmp)
/* Step 2: Fold the carry back into dst */
" add %%rax, %%r8;"
" adcx %%rcx, %%r9;"
- " movq %%r9, 8(%0);"
+ " movq %%r9, 8(%1);"
" adcx %%rcx, %%r10;"
- " movq %%r10, 16(%0);"
+ " movq %%r10, 16(%1);"
" adcx %%rcx, %%r11;"
- " movq %%r11, 24(%0);"
+ " movq %%r11, 24(%1);"
/* Step 3: Fold the carry bit back in; guaranteed not to carry at this point */
" mov $0, %%rax;"
" cmovc %%rdx, %%rax;"
" add %%rax, %%r8;"
- " movq %%r8, 0(%0);"
+ " movq %%r8, 0(%1);"
/* Step 1: Compute dst + carry == tmp_hi * 38 + tmp_lo */
" mov $38, %%rdx;"
- " mulxq 96(%1), %%r8, %%r13;"
+ " mulxq 96(%0), %%r8, %%r13;"
" xor %%ecx, %%ecx;"
- " adoxq 64(%1), %%r8;"
- " mulxq 104(%1), %%r9, %%rbx;"
+ " adoxq 64(%0), %%r8;"
+ " mulxq 104(%0), %%r9, %%rbx;"
" adcx %%r13, %%r9;"
- " adoxq 72(%1), %%r9;"
- " mulxq 112(%1), %%r10, %%r13;"
+ " adoxq 72(%0), %%r9;"
+ " mulxq 112(%0), %%r10, %%r13;"
" adcx %%rbx, %%r10;"
- " adoxq 80(%1), %%r10;"
- " mulxq 120(%1), %%r11, %%rax;"
+ " adoxq 80(%0), %%r10;"
+ " mulxq 120(%0), %%r11, %%rax;"
" adcx %%r13, %%r11;"
- " adoxq 88(%1), %%r11;"
+ " adoxq 88(%0), %%r11;"
" adcx %%rcx, %%rax;"
" adox %%rcx, %%rax;"
" imul %%rdx, %%rax;"
@@ -744,21 +955,21 @@ static inline void fsqr2(u64 *out, const u64 *f, u64 *tmp)
/* Step 2: Fold the carry back into dst */
" add %%rax, %%r8;"
" adcx %%rcx, %%r9;"
- " movq %%r9, 40(%0);"
+ " movq %%r9, 40(%1);"
" adcx %%rcx, %%r10;"
- " movq %%r10, 48(%0);"
+ " movq %%r10, 48(%1);"
" adcx %%rcx, %%r11;"
- " movq %%r11, 56(%0);"
+ " movq %%r11, 56(%1);"
/* Step 3: Fold the carry bit back in; guaranteed not to carry at this point */
" mov $0, %%rax;"
" cmovc %%rdx, %%rax;"
" add %%rax, %%r8;"
- " movq %%r8, 32(%0);"
- : "+&r" (tmp), "+&r" (f), "+&r" (out)
- :
- : "%rax", "%rcx", "%rdx", "%r8", "%r9", "%r10", "%r11", "%rbx", "%r13", "%r14", "%r15", "memory", "cc"
- );
+ " movq %%r8, 32(%1);"
+ : "+&r"(f), "+&r"(tmp)
+ : "r"(out)
+ : "%rax", "%rbx", "%rcx", "%rdx", "%r8", "%r9", "%r10", "%r11",
+ "%r13", "%r14", "%r15", "memory", "cc");
}
static void point_add_and_double(u64 *q, u64 *p01_tmp1, u64 *tmp2)
diff --git a/arch/x86/crypto/des3_ede_glue.c b/arch/x86/crypto/des3_ede_glue.c
index e7cb68a3db3b..787c234d2469 100644
--- a/arch/x86/crypto/des3_ede_glue.c
+++ b/arch/x86/crypto/des3_ede_glue.c
@@ -164,7 +164,7 @@ static int cbc_encrypt(struct skcipher_request *req)
err = skcipher_walk_virt(&walk, req, false);
- while ((nbytes = walk.nbytes)) {
+ while (walk.nbytes) {
nbytes = __cbc_encrypt(ctx, &walk);
err = skcipher_walk_done(&walk, nbytes);
}
@@ -243,7 +243,7 @@ static int cbc_decrypt(struct skcipher_request *req)
err = skcipher_walk_virt(&walk, req, false);
- while ((nbytes = walk.nbytes)) {
+ while (walk.nbytes) {
nbytes = __cbc_decrypt(ctx, &walk);
err = skcipher_walk_done(&walk, nbytes);
}
diff --git a/crypto/Kconfig b/crypto/Kconfig
index 55718de56137..94bfa32cc6a1 100644
--- a/crypto/Kconfig
+++ b/crypto/Kconfig
@@ -1845,6 +1845,10 @@ config CRYPTO_JITTERENTROPY
random numbers. This Jitterentropy RNG registers with
the kernel crypto API and can be used by any caller.
+config CRYPTO_KDF800108_CTR
+ tristate
+ select CRYPTO_SHA256
+
config CRYPTO_USER_API
tristate
diff --git a/crypto/Makefile b/crypto/Makefile
index 429c4d57458c..d76bff8d0ffd 100644
--- a/crypto/Makefile
+++ b/crypto/Makefile
@@ -200,3 +200,8 @@ obj-$(CONFIG_ASYMMETRIC_KEY_TYPE) += asymmetric_keys/
obj-$(CONFIG_CRYPTO_HASH_INFO) += hash_info.o
crypto_simd-y := simd.o
obj-$(CONFIG_CRYPTO_SIMD) += crypto_simd.o
+
+#
+# Key derivation function
+#
+obj-$(CONFIG_CRYPTO_KDF800108_CTR) += kdf_sp800108.o
diff --git a/crypto/af_alg.c b/crypto/af_alg.c
index 3dd5a773c320..e1ea18536a5f 100644
--- a/crypto/af_alg.c
+++ b/crypto/af_alg.c
@@ -931,16 +931,19 @@ int af_alg_sendmsg(struct socket *sock, struct msghdr *msg, size_t size,
sg_unmark_end(sg + sgl->cur - 1);
do {
+ struct page *pg;
unsigned int i = sgl->cur;
plen = min_t(size_t, len, PAGE_SIZE);
- sg_assign_page(sg + i, alloc_page(GFP_KERNEL));
- if (!sg_page(sg + i)) {
+ pg = alloc_page(GFP_KERNEL);
+ if (!pg) {
err = -ENOMEM;
goto unlock;
}
+ sg_assign_page(sg + i, pg);
+
err = memcpy_from_msg(page_address(sg_page(sg + i)),
msg, plen);
if (err) {
diff --git a/crypto/dh.c b/crypto/dh.c
index cd4f32092e5c..27e62a2a8027 100644
--- a/crypto/dh.c
+++ b/crypto/dh.c
@@ -5,11 +5,11 @@
* Authors: Salvatore Benedetto <salvatore.benedetto@intel.com>
*/
+#include <linux/fips.h>
#include <linux/module.h>
#include <crypto/internal/kpp.h>
#include <crypto/kpp.h>
#include <crypto/dh.h>
-#include <linux/fips.h>
#include <linux/mpi.h>
struct dh_ctx {
@@ -47,6 +47,9 @@ static inline struct dh_ctx *dh_get_ctx(struct crypto_kpp *tfm)
static int dh_check_params_length(unsigned int p_len)
{
+ if (fips_enabled)
+ return (p_len < 2048) ? -EINVAL : 0;
+
return (p_len < 1536) ? -EINVAL : 0;
}
diff --git a/crypto/drbg.c b/crypto/drbg.c
index f72f340a1321..177983b6ae38 100644
--- a/crypto/drbg.c
+++ b/crypto/drbg.c
@@ -100,6 +100,7 @@
#include <crypto/drbg.h>
#include <crypto/internal/cipher.h>
#include <linux/kernel.h>
+#include <linux/jiffies.h>
/***************************************************************
* Backend cipher definitions available to DRBG
@@ -1036,17 +1037,39 @@ static const struct drbg_state_ops drbg_hash_ops = {
******************************************************************/
static inline int __drbg_seed(struct drbg_state *drbg, struct list_head *seed,
- int reseed)
+ int reseed, enum drbg_seed_state new_seed_state)
{
int ret = drbg->d_ops->update(drbg, seed, reseed);
if (ret)
return ret;
- drbg->seeded = true;
+ drbg->seeded = new_seed_state;
+ drbg->last_seed_time = jiffies;
/* 10.1.1.2 / 10.1.1.3 step 5 */
drbg->reseed_ctr = 1;
+ switch (drbg->seeded) {
+ case DRBG_SEED_STATE_UNSEEDED:
+ /* Impossible, but handle it to silence compiler warnings. */
+ fallthrough;
+ case DRBG_SEED_STATE_PARTIAL:
+ /*
+ * Require frequent reseeds until the seed source is
+ * fully initialized.
+ */
+ drbg->reseed_threshold = 50;
+ break;
+
+ case DRBG_SEED_STATE_FULL:
+ /*
+ * Seed source has become fully initialized, frequent
+ * reseeds no longer required.
+ */
+ drbg->reseed_threshold = drbg_max_requests(drbg);
+ break;
+ }
+
return ret;
}
@@ -1066,12 +1089,10 @@ static inline int drbg_get_random_bytes(struct drbg_state *drbg,
return 0;
}
-static void drbg_async_seed(struct work_struct *work)
+static int drbg_seed_from_random(struct drbg_state *drbg)
{
struct drbg_string data;
LIST_HEAD(seedlist);
- struct drbg_state *drbg = container_of(work, struct drbg_state,
- seed_work);
unsigned int entropylen = drbg_sec_strength(drbg->core->flags);
unsigned char entropy[32];
int ret;
@@ -1082,26 +1103,35 @@ static void drbg_async_seed(struct work_struct *work)
drbg_string_fill(&data, entropy, entropylen);
list_add_tail(&data.list, &seedlist);
- mutex_lock(&drbg->drbg_mutex);
-
ret = drbg_get_random_bytes(drbg, entropy, entropylen);
if (ret)
- goto unlock;
+ goto out;
- /* Set seeded to false so that if __drbg_seed fails the
- * next generate call will trigger a reseed.
- */
- drbg->seeded = false;
+ ret = __drbg_seed(drbg, &seedlist, true, DRBG_SEED_STATE_FULL);
- __drbg_seed(drbg, &seedlist, true);
+out:
+ memzero_explicit(entropy, entropylen);
+ return ret;
+}
- if (drbg->seeded)
- drbg->reseed_threshold = drbg_max_requests(drbg);
+static bool drbg_nopr_reseed_interval_elapsed(struct drbg_state *drbg)
+{
+ unsigned long next_reseed;
-unlock:
- mutex_unlock(&drbg->drbg_mutex);
+ /* Don't ever reseed from get_random_bytes() in test mode. */
+ if (list_empty(&drbg->test_data.list))
+ return false;
- memzero_explicit(entropy, entropylen);
+ /*
+ * Obtain fresh entropy for the nopr DRBGs after 300s have
+ * elapsed in order to still achieve sort of partial
+ * prediction resistance over the time domain at least. Note
+ * that the period of 300s has been chosen to match the
+ * CRNG_RESEED_INTERVAL of the get_random_bytes()' chacha
+ * rngs.
+ */
+ next_reseed = drbg->last_seed_time + 300 * HZ;
+ return time_after(jiffies, next_reseed);
}
/*
@@ -1123,6 +1153,7 @@ static int drbg_seed(struct drbg_state *drbg, struct drbg_string *pers,
unsigned int entropylen = drbg_sec_strength(drbg->core->flags);
struct drbg_string data1;
LIST_HEAD(seedlist);
+ enum drbg_seed_state new_seed_state = DRBG_SEED_STATE_FULL;
/* 9.1 / 9.2 / 9.3.1 step 3 */
if (pers && pers->len > (drbg_max_addtl(drbg))) {
@@ -1150,6 +1181,9 @@ static int drbg_seed(struct drbg_state *drbg, struct drbg_string *pers,
BUG_ON((entropylen * 2) > sizeof(entropy));
/* Get seed from in-kernel /dev/urandom */
+ if (!rng_is_initialized())
+ new_seed_state = DRBG_SEED_STATE_PARTIAL;
+
ret = drbg_get_random_bytes(drbg, entropy, entropylen);
if (ret)
goto out;
@@ -1159,11 +1193,14 @@ static int drbg_seed(struct drbg_state *drbg, struct drbg_string *pers,
pr_devel("DRBG: (re)seeding with %u bytes of entropy\n",
entropylen);
} else {
- /* Get seed from Jitter RNG */
+ /*
+ * Get seed from Jitter RNG, failures are
+ * fatal only in FIPS mode.
+ */
ret = crypto_rng_get_bytes(drbg->jent,
entropy + entropylen,
entropylen);
- if (ret) {
+ if (fips_enabled && ret) {
pr_devel("DRBG: jent failed with %d\n", ret);
/*
@@ -1206,7 +1243,7 @@ static int drbg_seed(struct drbg_state *drbg, struct drbg_string *pers,
memset(drbg->C, 0, drbg_statelen(drbg));
}
- ret = __drbg_seed(drbg, &seedlist, reseed);
+ ret = __drbg_seed(drbg, &seedlist, reseed, new_seed_state);
out:
memzero_explicit(entropy, entropylen * 2);
@@ -1386,19 +1423,26 @@ static int drbg_generate(struct drbg_state *drbg,
* here. The spec is a bit convoluted here, we make it simpler.
*/
if (drbg->reseed_threshold < drbg->reseed_ctr)
- drbg->seeded = false;
+ drbg->seeded = DRBG_SEED_STATE_UNSEEDED;
- if (drbg->pr || !drbg->seeded) {
+ if (drbg->pr || drbg->seeded == DRBG_SEED_STATE_UNSEEDED) {
pr_devel("DRBG: reseeding before generation (prediction "
"resistance: %s, state %s)\n",
drbg->pr ? "true" : "false",
- drbg->seeded ? "seeded" : "unseeded");
+ (drbg->seeded == DRBG_SEED_STATE_FULL ?
+ "seeded" : "unseeded"));
/* 9.3.1 steps 7.1 through 7.3 */
len = drbg_seed(drbg, addtl, true);
if (len)
goto err;
/* 9.3.1 step 7.4 */
addtl = NULL;
+ } else if (rng_is_initialized() &&
+ (drbg->seeded == DRBG_SEED_STATE_PARTIAL ||
+ drbg_nopr_reseed_interval_elapsed(drbg))) {
+ len = drbg_seed_from_random(drbg);
+ if (len)
+ goto err;
}
if (addtl && 0 < addtl->len)
@@ -1491,51 +1535,23 @@ static int drbg_generate_long(struct drbg_state *drbg,
return 0;
}
-static void drbg_schedule_async_seed(struct random_ready_callback *rdy)
-{
- struct drbg_state *drbg = container_of(rdy, struct drbg_state,
- random_ready);
-
- schedule_work(&drbg->seed_work);
-}
-
static int drbg_prepare_hrng(struct drbg_state *drbg)
{
- int err;
-
/* We do not need an HRNG in test mode. */
if (list_empty(&drbg->test_data.list))
return 0;
drbg->jent = crypto_alloc_rng("jitterentropy_rng", 0, 0);
+ if (IS_ERR(drbg->jent)) {
+ const int err = PTR_ERR(drbg->jent);
- INIT_WORK(&drbg->seed_work, drbg_async_seed);
-
- drbg->random_ready.owner = THIS_MODULE;
- drbg->random_ready.func = drbg_schedule_async_seed;
-
- err = add_random_ready_callback(&drbg->random_ready);
-
- switch (err) {
- case 0:
- break;
-
- case -EALREADY:
- err = 0;
- fallthrough;
-
- default:
- drbg->random_ready.func = NULL;
- return err;
+ drbg->jent = NULL;
+ if (fips_enabled || err != -ENOENT)
+ return err;
+ pr_info("DRBG: Continuing without Jitter RNG\n");
}
- /*
- * Require frequent reseeds until the seed source is fully
- * initialized.
- */
- drbg->reseed_threshold = 50;
-
- return err;
+ return 0;
}
/*
@@ -1578,7 +1594,8 @@ static int drbg_instantiate(struct drbg_state *drbg, struct drbg_string *pers,
if (!drbg->core) {
drbg->core = &drbg_cores[coreref];
drbg->pr = pr;
- drbg->seeded = false;
+ drbg->seeded = DRBG_SEED_STATE_UNSEEDED;
+ drbg->last_seed_time = 0;
drbg->reseed_threshold = drbg_max_requests(drbg);
ret = drbg_alloc_state(drbg);
@@ -1589,14 +1606,6 @@ static int drbg_instantiate(struct drbg_state *drbg, struct drbg_string *pers,
if (ret)
goto free_everything;
- if (IS_ERR(drbg->jent)) {
- ret = PTR_ERR(drbg->jent);
- drbg->jent = NULL;
- if (fips_enabled || ret != -ENOENT)
- goto free_everything;
- pr_info("DRBG: Continuing without Jitter RNG\n");
- }
-
reseed = false;
}
@@ -1629,11 +1638,6 @@ free_everything:
*/
static int drbg_uninstantiate(struct drbg_state *drbg)
{
- if (drbg->random_ready.func) {
- del_random_ready_callback(&drbg->random_ready);
- cancel_work_sync(&drbg->seed_work);
- }
-
if (!IS_ERR_OR_NULL(drbg->jent))
crypto_free_rng(drbg->jent);
drbg->jent = NULL;
diff --git a/crypto/jitterentropy-kcapi.c b/crypto/jitterentropy-kcapi.c
index e8a4165a1874..2d115bec15ae 100644
--- a/crypto/jitterentropy-kcapi.c
+++ b/crypto/jitterentropy-kcapi.c
@@ -40,7 +40,6 @@
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/slab.h>
-#include <linux/fips.h>
#include <linux/time.h>
#include <crypto/internal/rng.h>
@@ -60,11 +59,6 @@ void jent_zfree(void *ptr)
kfree_sensitive(ptr);
}
-int jent_fips_enabled(void)
-{
- return fips_enabled;
-}
-
void jent_panic(char *s)
{
panic("%s", s);
diff --git a/crypto/jitterentropy.c b/crypto/jitterentropy.c
index 4dc2261cdeef..93bff3213823 100644
--- a/crypto/jitterentropy.c
+++ b/crypto/jitterentropy.c
@@ -117,6 +117,22 @@ struct rand_data {
#define JENT_EHEALTH 9 /* Health test failed during initialization */
#define JENT_ERCT 10 /* RCT failed during initialization */
+/*
+ * The output n bits can receive more than n bits of min entropy, of course,
+ * but the fixed output of the conditioning function can only asymptotically
+ * approach the output size bits of min entropy, not attain that bound. Random
+ * maps will tend to have output collisions, which reduces the creditable
+ * output entropy (that is what SP 800-90B Section 3.1.5.1.2 attempts to bound).
+ *
+ * The value "64" is justified in Appendix A.4 of the current 90C draft,
+ * and aligns with NIST's in "epsilon" definition in this document, which is
+ * that a string can be considered "full entropy" if you can bound the min
+ * entropy in each bit of output to at least 1-epsilon, where epsilon is
+ * required to be <= 2^(-32).
+ */
+#define JENT_ENTROPY_SAFETY_FACTOR 64
+
+#include <linux/fips.h>
#include "jitterentropy.h"
/***************************************************************************
@@ -265,7 +281,6 @@ static int jent_stuck(struct rand_data *ec, __u64 current_delta)
{
__u64 delta2 = jent_delta(ec->last_delta, current_delta);
__u64 delta3 = jent_delta(ec->last_delta2, delta2);
- unsigned int delta_masked = current_delta & JENT_APT_WORD_MASK;
ec->last_delta = current_delta;
ec->last_delta2 = delta2;
@@ -274,7 +289,7 @@ static int jent_stuck(struct rand_data *ec, __u64 current_delta)
* Insert the result of the comparison of two back-to-back time
* deltas.
*/
- jent_apt_insert(ec, delta_masked);
+ jent_apt_insert(ec, current_delta);
if (!current_delta || !delta2 || !delta3) {
/* RCT with a stuck bit */
@@ -299,10 +314,6 @@ static int jent_stuck(struct rand_data *ec, __u64 current_delta)
*/
static int jent_health_failure(struct rand_data *ec)
{
- /* Test is only enabled in FIPS mode */
- if (!jent_fips_enabled())
- return 0;
-
return ec->health_failure;
}
@@ -547,12 +558,15 @@ static int jent_measure_jitter(struct rand_data *ec)
*/
static void jent_gen_entropy(struct rand_data *ec)
{
- unsigned int k = 0;
+ unsigned int k = 0, safety_factor = 0;
+
+ if (fips_enabled)
+ safety_factor = JENT_ENTROPY_SAFETY_FACTOR;
/* priming of the ->prev_time value */
jent_measure_jitter(ec);
- while (1) {
+ while (!jent_health_failure(ec)) {
/* If a stuck measurement is received, repeat measurement */
if (jent_measure_jitter(ec))
continue;
@@ -561,7 +575,7 @@ static void jent_gen_entropy(struct rand_data *ec)
* We multiply the loop value with ->osr to obtain the
* oversampling rate requested by the caller
*/
- if (++k >= (DATA_SIZE_BITS * ec->osr))
+ if (++k >= ((DATA_SIZE_BITS + safety_factor) * ec->osr))
break;
}
}
diff --git a/crypto/jitterentropy.h b/crypto/jitterentropy.h
index c83fff32d130..b7397b617ef0 100644
--- a/crypto/jitterentropy.h
+++ b/crypto/jitterentropy.h
@@ -2,7 +2,6 @@
extern void *jent_zalloc(unsigned int len);
extern void jent_zfree(void *ptr);
-extern int jent_fips_enabled(void);
extern void jent_panic(char *s);
extern void jent_memcpy(void *dest, const void *src, unsigned int n);
extern void jent_get_nstime(__u64 *out);
diff --git a/crypto/kdf_sp800108.c b/crypto/kdf_sp800108.c
new file mode 100644
index 000000000000..58edf7797abf
--- /dev/null
+++ b/crypto/kdf_sp800108.c
@@ -0,0 +1,153 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/*
+ * SP800-108 Key-derivation function
+ *
+ * Copyright (C) 2021, Stephan Mueller <smueller@chronox.de>
+ */
+
+#include <linux/fips.h>
+#include <linux/module.h>
+#include <crypto/kdf_sp800108.h>
+#include <crypto/internal/kdf_selftest.h>
+
+/*
+ * SP800-108 CTR KDF implementation
+ */
+int crypto_kdf108_ctr_generate(struct crypto_shash *kmd,
+ const struct kvec *info, unsigned int info_nvec,
+ u8 *dst, unsigned int dlen)
+{
+ SHASH_DESC_ON_STACK(desc, kmd);
+ __be32 counter = cpu_to_be32(1);
+ const unsigned int h = crypto_shash_digestsize(kmd), dlen_orig = dlen;
+ unsigned int i;
+ int err = 0;
+ u8 *dst_orig = dst;
+
+ desc->tfm = kmd;
+
+ while (dlen) {
+ err = crypto_shash_init(desc);
+ if (err)
+ goto out;
+
+ err = crypto_shash_update(desc, (u8 *)&counter, sizeof(__be32));
+ if (err)
+ goto out;
+
+ for (i = 0; i < info_nvec; i++) {
+ err = crypto_shash_update(desc, info[i].iov_base,
+ info[i].iov_len);
+ if (err)
+ goto out;
+ }
+
+ if (dlen < h) {
+ u8 tmpbuffer[HASH_MAX_DIGESTSIZE];
+
+ err = crypto_shash_final(desc, tmpbuffer);
+ if (err)
+ goto out;
+ memcpy(dst, tmpbuffer, dlen);
+ memzero_explicit(tmpbuffer, h);
+ goto out;
+ }
+
+ err = crypto_shash_final(desc, dst);
+ if (err)
+ goto out;
+
+ dlen -= h;
+ dst += h;
+ counter = cpu_to_be32(be32_to_cpu(counter) + 1);
+ }
+
+out:
+ if (err)
+ memzero_explicit(dst_orig, dlen_orig);
+ shash_desc_zero(desc);
+ return err;
+}
+EXPORT_SYMBOL(crypto_kdf108_ctr_generate);
+
+/*
+ * The seeding of the KDF
+ */
+int crypto_kdf108_setkey(struct crypto_shash *kmd,
+ const u8 *key, size_t keylen,
+ const u8 *ikm, size_t ikmlen)
+{
+ unsigned int ds = crypto_shash_digestsize(kmd);
+
+ /* SP800-108 does not support IKM */
+ if (ikm || ikmlen)
+ return -EINVAL;
+
+ /* Check according to SP800-108 section 7.2 */
+ if (ds > keylen)
+ return -EINVAL;
+
+ /* Set the key for the MAC used for the KDF. */
+ return crypto_shash_setkey(kmd, key, keylen);
+}
+EXPORT_SYMBOL(crypto_kdf108_setkey);
+
+/*
+ * Test vector obtained from
+ * http://csrc.nist.gov/groups/STM/cavp/documents/KBKDF800-108/CounterMode.zip
+ */
+static const struct kdf_testvec kdf_ctr_hmac_sha256_tv_template[] = {
+ {
+ .key = "\xdd\x1d\x91\xb7\xd9\x0b\x2b\xd3"
+ "\x13\x85\x33\xce\x92\xb2\x72\xfb"
+ "\xf8\xa3\x69\x31\x6a\xef\xe2\x42"
+ "\xe6\x59\xcc\x0a\xe2\x38\xaf\xe0",
+ .keylen = 32,
+ .ikm = NULL,
+ .ikmlen = 0,
+ .info = {
+ .iov_base = "\x01\x32\x2b\x96\xb3\x0a\xcd\x19"
+ "\x79\x79\x44\x4e\x46\x8e\x1c\x5c"
+ "\x68\x59\xbf\x1b\x1c\xf9\x51\xb7"
+ "\xe7\x25\x30\x3e\x23\x7e\x46\xb8"
+ "\x64\xa1\x45\xfa\xb2\x5e\x51\x7b"
+ "\x08\xf8\x68\x3d\x03\x15\xbb\x29"
+ "\x11\xd8\x0a\x0e\x8a\xba\x17\xf3"
+ "\xb4\x13\xfa\xac",
+ .iov_len = 60
+ },
+ .expected = "\x10\x62\x13\x42\xbf\xb0\xfd\x40"
+ "\x04\x6c\x0e\x29\xf2\xcf\xdb\xf0",
+ .expectedlen = 16
+ }
+};
+
+static int __init crypto_kdf108_init(void)
+{
+ int ret = kdf_test(&kdf_ctr_hmac_sha256_tv_template[0], "hmac(sha256)",
+ crypto_kdf108_setkey, crypto_kdf108_ctr_generate);
+
+ if (ret) {
+ if (fips_enabled)
+ panic("alg: self-tests for CTR-KDF (hmac(sha256)) failed (rc=%d)\n",
+ ret);
+
+ WARN(1,
+ "alg: self-tests for CTR-KDF (hmac(sha256)) failed (rc=%d)\n",
+ ret);
+ } else {
+ pr_info("alg: self-tests for CTR-KDF (hmac(sha256)) passed\n");
+ }
+
+ return ret;
+}
+
+static void __exit crypto_kdf108_exit(void) { }
+
+module_init(crypto_kdf108_init);
+module_exit(crypto_kdf108_exit);
+
+MODULE_LICENSE("GPL v2");
+MODULE_AUTHOR("Stephan Mueller <smueller@chronox.de>");
+MODULE_DESCRIPTION("Key Derivation Function conformant to SP800-108");
diff --git a/crypto/rsa.c b/crypto/rsa.c
index 4cdbec95d077..39e04176b04b 100644
--- a/crypto/rsa.c
+++ b/crypto/rsa.c
@@ -5,6 +5,7 @@
* Authors: Tadeusz Struk <tadeusz.struk@intel.com>
*/
+#include <linux/fips.h>
#include <linux/module.h>
#include <linux/mpi.h>
#include <crypto/internal/rsa.h>
@@ -144,6 +145,9 @@ static int rsa_check_key_length(unsigned int len)
case 512:
case 1024:
case 1536:
+ if (fips_enabled)
+ return -EINVAL;
+ fallthrough;
case 2048:
case 3072:
case 4096:
diff --git a/crypto/sha256_generic.c b/crypto/sha256_generic.c
index 3b377197236e..bf147b01e313 100644
--- a/crypto/sha256_generic.c
+++ b/crypto/sha256_generic.c
@@ -33,18 +33,6 @@ const u8 sha256_zero_message_hash[SHA256_DIGEST_SIZE] = {
};
EXPORT_SYMBOL_GPL(sha256_zero_message_hash);
-static int crypto_sha256_init(struct shash_desc *desc)
-{
- sha256_init(shash_desc_ctx(desc));
- return 0;
-}
-
-static int crypto_sha224_init(struct shash_desc *desc)
-{
- sha224_init(shash_desc_ctx(desc));
- return 0;
-}
-
int crypto_sha256_update(struct shash_desc *desc, const u8 *data,
unsigned int len)
{
@@ -72,7 +60,7 @@ EXPORT_SYMBOL(crypto_sha256_finup);
static struct shash_alg sha256_algs[2] = { {
.digestsize = SHA256_DIGEST_SIZE,
- .init = crypto_sha256_init,
+ .init = sha256_base_init,
.update = crypto_sha256_update,
.final = crypto_sha256_final,
.finup = crypto_sha256_finup,
@@ -86,7 +74,7 @@ static struct shash_alg sha256_algs[2] = { {
}
}, {
.digestsize = SHA224_DIGEST_SIZE,
- .init = crypto_sha224_init,
+ .init = sha224_base_init,
.update = crypto_sha256_update,
.final = crypto_sha256_final,
.finup = crypto_sha256_finup,
diff --git a/crypto/testmgr.c b/crypto/testmgr.c
index 58eee8eab4bf..5831d4bbc64f 100644
--- a/crypto/testmgr.c
+++ b/crypto/testmgr.c
@@ -4193,7 +4193,6 @@ static const struct alg_test_desc alg_test_descs[] = {
}, {
.alg = "authenc(hmac(sha1),cbc(des3_ede))",
.test = alg_test_aead,
- .fips_allowed = 1,
.suite = {
.aead = __VECS(hmac_sha1_des3_ede_cbc_tv_temp)
}
@@ -4220,7 +4219,6 @@ static const struct alg_test_desc alg_test_descs[] = {
}, {
.alg = "authenc(hmac(sha224),cbc(des3_ede))",
.test = alg_test_aead,
- .fips_allowed = 1,
.suite = {
.aead = __VECS(hmac_sha224_des3_ede_cbc_tv_temp)
}
@@ -4240,7 +4238,6 @@ static const struct alg_test_desc alg_test_descs[] = {
}, {
.alg = "authenc(hmac(sha256),cbc(des3_ede))",
.test = alg_test_aead,
- .fips_allowed = 1,
.suite = {
.aead = __VECS(hmac_sha256_des3_ede_cbc_tv_temp)
}
@@ -4261,7 +4258,6 @@ static const struct alg_test_desc alg_test_descs[] = {
}, {
.alg = "authenc(hmac(sha384),cbc(des3_ede))",
.test = alg_test_aead,
- .fips_allowed = 1,
.suite = {
.aead = __VECS(hmac_sha384_des3_ede_cbc_tv_temp)
}
@@ -4289,7 +4285,6 @@ static const struct alg_test_desc alg_test_descs[] = {
}, {
.alg = "authenc(hmac(sha512),cbc(des3_ede))",
.test = alg_test_aead,
- .fips_allowed = 1,
.suite = {
.aead = __VECS(hmac_sha512_des3_ede_cbc_tv_temp)
}
@@ -4399,7 +4394,6 @@ static const struct alg_test_desc alg_test_descs[] = {
}, {
.alg = "cbc(des3_ede)",
.test = alg_test_skcipher,
- .fips_allowed = 1,
.suite = {
.cipher = __VECS(des3_ede_cbc_tv_template)
},
@@ -4505,7 +4499,6 @@ static const struct alg_test_desc alg_test_descs[] = {
}
}, {
.alg = "cmac(des3_ede)",
- .fips_allowed = 1,
.test = alg_test_hash,
.suite = {
.hash = __VECS(des3_ede_cmac64_tv_template)
@@ -4580,7 +4573,6 @@ static const struct alg_test_desc alg_test_descs[] = {
}, {
.alg = "ctr(des3_ede)",
.test = alg_test_skcipher,
- .fips_allowed = 1,
.suite = {
.cipher = __VECS(des3_ede_ctr_tv_template)
}
@@ -4846,7 +4838,6 @@ static const struct alg_test_desc alg_test_descs[] = {
}, {
.alg = "ecb(des3_ede)",
.test = alg_test_skcipher,
- .fips_allowed = 1,
.suite = {
.cipher = __VECS(des3_ede_tv_template)
}
diff --git a/crypto/testmgr.h b/crypto/testmgr.h
index 779720bf9364..a253d66ba1c1 100644
--- a/crypto/testmgr.h
+++ b/crypto/testmgr.h
@@ -257,9 +257,9 @@ static const struct akcipher_testvec rsa_tv_template[] = {
}, {
#endif
.key =
- "\x30\x82\x02\x1F" /* sequence of 543 bytes */
+ "\x30\x82\x02\x20" /* sequence of 544 bytes */
"\x02\x01\x01" /* version - integer of 1 byte */
- "\x02\x82\x01\x00" /* modulus - integer of 256 bytes */
+ "\x02\x82\x01\x01\x00" /* modulus - integer of 256 bytes */
"\xDB\x10\x1A\xC2\xA3\xF1\xDC\xFF\x13\x6B\xED\x44\xDF\xF0\x02\x6D"
"\x13\xC7\x88\xDA\x70\x6B\x54\xF1\xE8\x27\xDC\xC3\x0F\x99\x6A\xFA"
"\xC6\x67\xFF\x1D\x1E\x3C\x1D\xC1\xB5\x5F\x6C\xC0\xB2\x07\x3A\x6D"
@@ -299,7 +299,7 @@ static const struct akcipher_testvec rsa_tv_template[] = {
"\x02\x01\x00" /* exponent1 - integer of 1 byte */
"\x02\x01\x00" /* exponent2 - integer of 1 byte */
"\x02\x01\x00", /* coefficient - integer of 1 byte */
- .key_len = 547,
+ .key_len = 548,
.m = "\x54\x85\x9b\x34\x2c\x49\xea\x2a",
.c =
"\xb2\x97\x76\xb4\xae\x3e\x38\x3c\x7e\x64\x1f\xcc\xa2\x7f\xf6\xbe"
diff --git a/drivers/char/hw_random/Kconfig b/drivers/char/hw_random/Kconfig
index 814b3d0ca7b7..001b819f5298 100644
--- a/drivers/char/hw_random/Kconfig
+++ b/drivers/char/hw_random/Kconfig
@@ -414,7 +414,7 @@ config HW_RANDOM_MESON
config HW_RANDOM_CAVIUM
tristate "Cavium ThunderX Random Number Generator support"
- depends on HW_RANDOM && PCI && (ARM64 || (COMPILE_TEST && 64BIT))
+ depends on HW_RANDOM && PCI && ARM64
default HW_RANDOM
help
This driver provides kernel-side support for the Random Number
@@ -538,6 +538,17 @@ config HW_RANDOM_ARM_SMCCC_TRNG
To compile this driver as a module, choose M here: the
module will be called arm_smccc_trng.
+config HW_RANDOM_CN10K
+ tristate "Marvell CN10K Random Number Generator support"
+ depends on HW_RANDOM && PCI && ARM64
+ default HW_RANDOM
+ help
+ This driver provides support for the True Random Number
+ generator available in Marvell CN10K SoCs.
+
+ To compile this driver as a module, choose M here.
+ The module will be called cn10k_rng. If unsure, say Y.
+
endif # HW_RANDOM
config UML_RANDOM
diff --git a/drivers/char/hw_random/Makefile b/drivers/char/hw_random/Makefile
index a5a1c765a394..a2f1ce0790d1 100644
--- a/drivers/char/hw_random/Makefile
+++ b/drivers/char/hw_random/Makefile
@@ -46,3 +46,4 @@ obj-$(CONFIG_HW_RANDOM_NPCM) += npcm-rng.o
obj-$(CONFIG_HW_RANDOM_CCTRNG) += cctrng.o
obj-$(CONFIG_HW_RANDOM_XIPHERA) += xiphera-trng.o
obj-$(CONFIG_HW_RANDOM_ARM_SMCCC_TRNG) += arm_smccc_trng.o
+obj-$(CONFIG_HW_RANDOM_CN10K) += cn10k-rng.o
diff --git a/drivers/char/hw_random/cavium-rng-vf.c b/drivers/char/hw_random/cavium-rng-vf.c
index 3de4a6a443ef..6f66919652bf 100644
--- a/drivers/char/hw_random/cavium-rng-vf.c
+++ b/drivers/char/hw_random/cavium-rng-vf.c
@@ -1,10 +1,7 @@
+// SPDX-License-Identifier: GPL-2.0
/*
- * Hardware Random Number Generator support for Cavium, Inc.
- * Thunder processor family.
- *
- * This file is subject to the terms and conditions of the GNU General Public
- * License. See the file "COPYING" in the main directory of this archive
- * for more details.
+ * Hardware Random Number Generator support.
+ * Cavium Thunder, Marvell OcteonTx/Tx2 processor families.
*
* Copyright (C) 2016 Cavium, Inc.
*/
@@ -15,16 +12,146 @@
#include <linux/pci.h>
#include <linux/pci_ids.h>
+#include <asm/arch_timer.h>
+
+/* PCI device IDs */
+#define PCI_DEVID_CAVIUM_RNG_PF 0xA018
+#define PCI_DEVID_CAVIUM_RNG_VF 0xA033
+
+#define HEALTH_STATUS_REG 0x38
+
+/* RST device info */
+#define PCI_DEVICE_ID_RST_OTX2 0xA085
+#define RST_BOOT_REG 0x1600ULL
+#define CLOCK_BASE_RATE 50000000ULL
+#define MSEC_TO_NSEC(x) (x * 1000000)
+
struct cavium_rng {
struct hwrng ops;
void __iomem *result;
+ void __iomem *pf_regbase;
+ struct pci_dev *pdev;
+ u64 clock_rate;
+ u64 prev_error;
+ u64 prev_time;
};
+static inline bool is_octeontx(struct pci_dev *pdev)
+{
+ if (midr_is_cpu_model_range(read_cpuid_id(), MIDR_THUNDERX_83XX,
+ MIDR_CPU_VAR_REV(0, 0),
+ MIDR_CPU_VAR_REV(3, 0)) ||
+ midr_is_cpu_model_range(read_cpuid_id(), MIDR_THUNDERX_81XX,
+ MIDR_CPU_VAR_REV(0, 0),
+ MIDR_CPU_VAR_REV(3, 0)) ||
+ midr_is_cpu_model_range(read_cpuid_id(), MIDR_THUNDERX,
+ MIDR_CPU_VAR_REV(0, 0),
+ MIDR_CPU_VAR_REV(3, 0)))
+ return true;
+
+ return false;
+}
+
+static u64 rng_get_coprocessor_clkrate(void)
+{
+ u64 ret = CLOCK_BASE_RATE * 16; /* Assume 800Mhz as default */
+ struct pci_dev *pdev;
+ void __iomem *base;
+
+ pdev = pci_get_device(PCI_VENDOR_ID_CAVIUM,
+ PCI_DEVICE_ID_RST_OTX2, NULL);
+ if (!pdev)
+ goto error;
+
+ base = pci_ioremap_bar(pdev, 0);
+ if (!base)
+ goto error_put_pdev;
+
+ /* RST: PNR_MUL * 50Mhz gives clockrate */
+ ret = CLOCK_BASE_RATE * ((readq(base + RST_BOOT_REG) >> 33) & 0x3F);
+
+ iounmap(base);
+
+error_put_pdev:
+ pci_dev_put(pdev);
+
+error:
+ return ret;
+}
+
+static int check_rng_health(struct cavium_rng *rng)
+{
+ u64 cur_err, cur_time;
+ u64 status, cycles;
+ u64 time_elapsed;
+
+
+ /* Skip checking health for OcteonTx */
+ if (!rng->pf_regbase)
+ return 0;
+
+ status = readq(rng->pf_regbase + HEALTH_STATUS_REG);
+ if (status & BIT_ULL(0)) {
+ dev_err(&rng->pdev->dev, "HWRNG: Startup health test failed\n");
+ return -EIO;
+ }
+
+ cycles = status >> 1;
+ if (!cycles)
+ return 0;
+
+ cur_time = arch_timer_read_counter();
+
+ /* RNM_HEALTH_STATUS[CYCLES_SINCE_HEALTH_FAILURE]
+ * Number of coprocessor cycles times 2 since the last failure.
+ * This field doesn't get cleared/updated until another failure.
+ */
+ cycles = cycles / 2;
+ cur_err = (cycles * 1000000000) / rng->clock_rate; /* In nanosec */
+
+ /* Ignore errors that happenned a long time ago, these
+ * are most likely false positive errors.
+ */
+ if (cur_err > MSEC_TO_NSEC(10)) {
+ rng->prev_error = 0;
+ rng->prev_time = 0;
+ return 0;
+ }
+
+ if (rng->prev_error) {
+ /* Calculate time elapsed since last error
+ * '1' tick of CNTVCT is 10ns, since it runs at 100Mhz.
+ */
+ time_elapsed = (cur_time - rng->prev_time) * 10;
+ time_elapsed += rng->prev_error;
+
+ /* Check if current error is a new one or the old one itself.
+ * If error is a new one then consider there is a persistent
+ * issue with entropy, declare hardware failure.
+ */
+ if (cur_err < time_elapsed) {
+ dev_err(&rng->pdev->dev, "HWRNG failure detected\n");
+ rng->prev_error = cur_err;
+ rng->prev_time = cur_time;
+ return -EIO;
+ }
+ }
+
+ rng->prev_error = cur_err;
+ rng->prev_time = cur_time;
+ return 0;
+}
+
/* Read data from the RNG unit */
static int cavium_rng_read(struct hwrng *rng, void *dat, size_t max, bool wait)
{
struct cavium_rng *p = container_of(rng, struct cavium_rng, ops);
unsigned int size = max;
+ int err = 0;
+
+ err = check_rng_health(p);
+ if (err)
+ return err;
while (size >= 8) {
*((u64 *)dat) = readq(p->result);
@@ -39,6 +166,39 @@ static int cavium_rng_read(struct hwrng *rng, void *dat, size_t max, bool wait)
return max;
}
+static int cavium_map_pf_regs(struct cavium_rng *rng)
+{
+ struct pci_dev *pdev;
+
+ /* Health status is not supported on 83xx, skip mapping PF CSRs */
+ if (is_octeontx(rng->pdev)) {
+ rng->pf_regbase = NULL;
+ return 0;
+ }
+
+ pdev = pci_get_device(PCI_VENDOR_ID_CAVIUM,
+ PCI_DEVID_CAVIUM_RNG_PF, NULL);
+ if (!pdev) {
+ dev_err(&pdev->dev, "Cannot find RNG PF device\n");
+ return -EIO;
+ }
+
+ rng->pf_regbase = ioremap(pci_resource_start(pdev, 0),
+ pci_resource_len(pdev, 0));
+ if (!rng->pf_regbase) {
+ dev_err(&pdev->dev, "Failed to map PF CSR region\n");
+ pci_dev_put(pdev);
+ return -ENOMEM;
+ }
+
+ pci_dev_put(pdev);
+
+ /* Get co-processor clock rate */
+ rng->clock_rate = rng_get_coprocessor_clkrate();
+
+ return 0;
+}
+
/* Map Cavium RNG to an HWRNG object */
static int cavium_rng_probe_vf(struct pci_dev *pdev,
const struct pci_device_id *id)
@@ -50,6 +210,8 @@ static int cavium_rng_probe_vf(struct pci_dev *pdev,
if (!rng)
return -ENOMEM;
+ rng->pdev = pdev;
+
/* Map the RNG result */
rng->result = pcim_iomap(pdev, 0, 0);
if (!rng->result) {
@@ -67,6 +229,11 @@ static int cavium_rng_probe_vf(struct pci_dev *pdev,
pci_set_drvdata(pdev, rng);
+ /* Health status is available only at PF, hence map PF registers. */
+ ret = cavium_map_pf_regs(rng);
+ if (ret)
+ return ret;
+
ret = devm_hwrng_register(&pdev->dev, &rng->ops);
if (ret) {
dev_err(&pdev->dev, "Error registering device as HWRNG.\n");
@@ -76,10 +243,18 @@ static int cavium_rng_probe_vf(struct pci_dev *pdev,
return 0;
}
+/* Remove the VF */
+static void cavium_rng_remove_vf(struct pci_dev *pdev)
+{
+ struct cavium_rng *rng;
+
+ rng = pci_get_drvdata(pdev);
+ iounmap(rng->pf_regbase);
+}
static const struct pci_device_id cavium_rng_vf_id_table[] = {
- { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, 0xa033), 0, 0, 0},
- {0,},
+ { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVID_CAVIUM_RNG_VF) },
+ { 0, }
};
MODULE_DEVICE_TABLE(pci, cavium_rng_vf_id_table);
@@ -87,8 +262,9 @@ static struct pci_driver cavium_rng_vf_driver = {
.name = "cavium_rng_vf",
.id_table = cavium_rng_vf_id_table,
.probe = cavium_rng_probe_vf,
+ .remove = cavium_rng_remove_vf,
};
module_pci_driver(cavium_rng_vf_driver);
MODULE_AUTHOR("Omer Khaliq <okhaliq@caviumnetworks.com>");
-MODULE_LICENSE("GPL");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/char/hw_random/cavium-rng.c b/drivers/char/hw_random/cavium-rng.c
index 63d6e68c24d2..b96579222408 100644
--- a/drivers/char/hw_random/cavium-rng.c
+++ b/drivers/char/hw_random/cavium-rng.c
@@ -1,10 +1,7 @@
+// SPDX-License-Identifier: GPL-2.0
/*
- * Hardware Random Number Generator support for Cavium Inc.
- * Thunder processor family.
- *
- * This file is subject to the terms and conditions of the GNU General Public
- * License. See the file "COPYING" in the main directory of this archive
- * for more details.
+ * Hardware Random Number Generator support.
+ * Cavium Thunder, Marvell OcteonTx/Tx2 processor families.
*
* Copyright (C) 2016 Cavium, Inc.
*/
@@ -91,4 +88,4 @@ static struct pci_driver cavium_rng_pf_driver = {
module_pci_driver(cavium_rng_pf_driver);
MODULE_AUTHOR("Omer Khaliq <okhaliq@caviumnetworks.com>");
-MODULE_LICENSE("GPL");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/char/hw_random/cn10k-rng.c b/drivers/char/hw_random/cn10k-rng.c
new file mode 100644
index 000000000000..35001c63648b
--- /dev/null
+++ b/drivers/char/hw_random/cn10k-rng.c
@@ -0,0 +1,181 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Marvell CN10K RVU Hardware Random Number Generator.
+ *
+ * Copyright (C) 2021 Marvell.
+ *
+ */
+
+#include <linux/hw_random.h>
+#include <linux/io.h>
+#include <linux/module.h>
+#include <linux/pci.h>
+#include <linux/pci_ids.h>
+#include <linux/delay.h>
+
+#include <linux/arm-smccc.h>
+
+/* CSRs */
+#define RNM_CTL_STATUS 0x000
+#define RNM_ENTROPY_STATUS 0x008
+#define RNM_CONST 0x030
+#define RNM_EBG_ENT 0x048
+#define RNM_PF_EBG_HEALTH 0x050
+#define RNM_PF_RANDOM 0x400
+#define RNM_TRNG_RESULT 0x408
+
+struct cn10k_rng {
+ void __iomem *reg_base;
+ struct hwrng ops;
+ struct pci_dev *pdev;
+};
+
+#define PLAT_OCTEONTX_RESET_RNG_EBG_HEALTH_STATE 0xc2000b0f
+
+static int reset_rng_health_state(struct cn10k_rng *rng)
+{
+ struct arm_smccc_res res;
+
+ /* Send SMC service call to reset EBG health state */
+ arm_smccc_smc(PLAT_OCTEONTX_RESET_RNG_EBG_HEALTH_STATE, 0, 0, 0, 0, 0, 0, 0, &res);
+ if (res.a0 != 0UL)
+ return -EIO;
+
+ return 0;
+}
+
+static int check_rng_health(struct cn10k_rng *rng)
+{
+ u64 status;
+ int err;
+
+ /* Skip checking health */
+ if (!rng->reg_base)
+ return 0;
+
+ status = readq(rng->reg_base + RNM_PF_EBG_HEALTH);
+ if (status & BIT_ULL(20)) {
+ err = reset_rng_health_state(rng);
+ if (err) {
+ dev_err(&rng->pdev->dev, "HWRNG: Health test failed (status=%llx)\n",
+ status);
+ dev_err(&rng->pdev->dev, "HWRNG: error during reset\n");
+ }
+ }
+ return 0;
+}
+
+static void cn10k_read_trng(struct cn10k_rng *rng, u64 *value)
+{
+ u64 upper, lower;
+
+ *value = readq(rng->reg_base + RNM_PF_RANDOM);
+
+ /* HW can run out of entropy if large amount random data is read in
+ * quick succession. Zeros may not be real random data from HW.
+ */
+ if (!*value) {
+ upper = readq(rng->reg_base + RNM_PF_RANDOM);
+ lower = readq(rng->reg_base + RNM_PF_RANDOM);
+ while (!(upper & 0x00000000FFFFFFFFULL))
+ upper = readq(rng->reg_base + RNM_PF_RANDOM);
+ while (!(lower & 0xFFFFFFFF00000000ULL))
+ lower = readq(rng->reg_base + RNM_PF_RANDOM);
+
+ *value = (upper & 0xFFFFFFFF00000000) | (lower & 0xFFFFFFFF);
+ }
+}
+
+static int cn10k_rng_read(struct hwrng *hwrng, void *data,
+ size_t max, bool wait)
+{
+ struct cn10k_rng *rng = (struct cn10k_rng *)hwrng->priv;
+ unsigned int size;
+ int err = 0;
+ u64 value;
+
+ err = check_rng_health(rng);
+ if (err)
+ return err;
+
+ size = max;
+
+ while (size >= 8) {
+ cn10k_read_trng(rng, &value);
+
+ *((u64 *)data) = (u64)value;
+ size -= 8;
+ data += 8;
+ }
+
+ while (size > 0) {
+ cn10k_read_trng(rng, &value);
+
+ *((u8 *)data) = (u8)value;
+ size--;
+ data++;
+ }
+
+ return max - size;
+}
+
+static int cn10k_rng_probe(struct pci_dev *pdev, const struct pci_device_id *id)
+{
+ struct cn10k_rng *rng;
+ int err;
+
+ rng = devm_kzalloc(&pdev->dev, sizeof(*rng), GFP_KERNEL);
+ if (!rng)
+ return -ENOMEM;
+
+ rng->pdev = pdev;
+ pci_set_drvdata(pdev, rng);
+
+ rng->reg_base = pcim_iomap(pdev, 0, 0);
+ if (!rng->reg_base) {
+ dev_err(&pdev->dev, "Error while mapping CSRs, exiting\n");
+ return -ENOMEM;
+ }
+
+ rng->ops.name = devm_kasprintf(&pdev->dev, GFP_KERNEL,
+ "cn10k-rng-%s", dev_name(&pdev->dev));
+ if (!rng->ops.name)
+ return -ENOMEM;
+
+ rng->ops.read = cn10k_rng_read;
+ rng->ops.quality = 1000;
+ rng->ops.priv = (unsigned long)rng;
+
+ reset_rng_health_state(rng);
+
+ err = devm_hwrng_register(&pdev->dev, &rng->ops);
+ if (err) {
+ dev_err(&pdev->dev, "Could not register hwrng device.\n");
+ return err;
+ }
+
+ return 0;
+}
+
+static void cn10k_rng_remove(struct pci_dev *pdev)
+{
+ /* Nothing to do */
+}
+
+static const struct pci_device_id cn10k_rng_id_table[] = {
+ { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, 0xA098) }, /* RNG PF */
+ {0,},
+};
+
+MODULE_DEVICE_TABLE(pci, cn10k_rng_id_table);
+
+static struct pci_driver cn10k_rng_driver = {
+ .name = "cn10k_rng",
+ .id_table = cn10k_rng_id_table,
+ .probe = cn10k_rng_probe,
+ .remove = cn10k_rng_remove,
+};
+
+module_pci_driver(cn10k_rng_driver);
+MODULE_AUTHOR("Sunil Goutham <sgoutham@marvell.com>");
+MODULE_DESCRIPTION("Marvell CN10K HW RNG Driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-core.c b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-core.c
index 00194d1d9ae6..d8623c7e0d1d 100644
--- a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-core.c
+++ b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-core.c
@@ -106,6 +106,24 @@ static const struct ce_variant ce_a64_variant = {
.trng = CE_ID_NOTSUPP,
};
+static const struct ce_variant ce_d1_variant = {
+ .alg_cipher = { CE_ALG_AES, CE_ALG_DES, CE_ALG_3DES,
+ },
+ .alg_hash = { CE_ALG_MD5, CE_ALG_SHA1, CE_ALG_SHA224, CE_ALG_SHA256,
+ CE_ALG_SHA384, CE_ALG_SHA512
+ },
+ .op_mode = { CE_OP_ECB, CE_OP_CBC
+ },
+ .ce_clks = {
+ { "bus", 0, 200000000 },
+ { "mod", 300000000, 0 },
+ { "ram", 0, 400000000 },
+ },
+ .esr = ESR_D1,
+ .prng = CE_ALG_PRNG,
+ .trng = CE_ALG_TRNG,
+};
+
static const struct ce_variant ce_r40_variant = {
.alg_cipher = { CE_ALG_AES, CE_ALG_DES, CE_ALG_3DES,
},
@@ -192,6 +210,7 @@ int sun8i_ce_run_task(struct sun8i_ce_dev *ce, int flow, const char *name)
dev_err(ce->dev, "CE ERROR: keysram access error for AES\n");
break;
case ESR_A64:
+ case ESR_D1:
case ESR_H5:
case ESR_R40:
v >>= (flow * 4);
@@ -990,6 +1009,8 @@ static const struct of_device_id sun8i_ce_crypto_of_match_table[] = {
.data = &ce_h3_variant },
{ .compatible = "allwinner,sun8i-r40-crypto",
.data = &ce_r40_variant },
+ { .compatible = "allwinner,sun20i-d1-crypto",
+ .data = &ce_d1_variant },
{ .compatible = "allwinner,sun50i-a64-crypto",
.data = &ce_a64_variant },
{ .compatible = "allwinner,sun50i-h5-crypto",
diff --git a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce.h b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce.h
index cec781d5063c..624a5926f21f 100644
--- a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce.h
+++ b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce.h
@@ -94,6 +94,7 @@
#define ESR_R40 2
#define ESR_H5 3
#define ESR_H6 4
+#define ESR_D1 5
#define PRNG_DATA_SIZE (160 / 8)
#define PRNG_SEED_SIZE DIV_ROUND_UP(175, 8)
diff --git a/drivers/crypto/atmel-aes.c b/drivers/crypto/atmel-aes.c
index 9391ccc03382..fe0558403191 100644
--- a/drivers/crypto/atmel-aes.c
+++ b/drivers/crypto/atmel-aes.c
@@ -960,6 +960,7 @@ static int atmel_aes_handle_queue(struct atmel_aes_dev *dd,
ctx = crypto_tfm_ctx(areq->tfm);
dd->areq = areq;
+ dd->ctx = ctx;
start_async = (areq != new_areq);
dd->is_async = start_async;
@@ -1274,7 +1275,6 @@ static int atmel_aes_init_tfm(struct crypto_skcipher *tfm)
crypto_skcipher_set_reqsize(tfm, sizeof(struct atmel_aes_reqctx));
ctx->base.dd = dd;
- ctx->base.dd->ctx = &ctx->base;
ctx->base.start = atmel_aes_start;
return 0;
@@ -1291,7 +1291,6 @@ static int atmel_aes_ctr_init_tfm(struct crypto_skcipher *tfm)
crypto_skcipher_set_reqsize(tfm, sizeof(struct atmel_aes_reqctx));
ctx->base.dd = dd;
- ctx->base.dd->ctx = &ctx->base;
ctx->base.start = atmel_aes_ctr_start;
return 0;
@@ -1783,7 +1782,6 @@ static int atmel_aes_gcm_init(struct crypto_aead *tfm)
crypto_aead_set_reqsize(tfm, sizeof(struct atmel_aes_reqctx));
ctx->base.dd = dd;
- ctx->base.dd->ctx = &ctx->base;
ctx->base.start = atmel_aes_gcm_start;
return 0;
@@ -1927,7 +1925,6 @@ static int atmel_aes_xts_init_tfm(struct crypto_skcipher *tfm)
crypto_skcipher_set_reqsize(tfm, sizeof(struct atmel_aes_reqctx) +
crypto_skcipher_reqsize(ctx->fallback_tfm));
ctx->base.dd = dd;
- ctx->base.dd->ctx = &ctx->base;
ctx->base.start = atmel_aes_xts_start;
return 0;
@@ -2154,7 +2151,6 @@ static int atmel_aes_authenc_init_tfm(struct crypto_aead *tfm,
crypto_aead_set_reqsize(tfm, (sizeof(struct atmel_aes_authenc_reqctx) +
auth_reqsize));
ctx->base.dd = dd;
- ctx->base.dd->ctx = &ctx->base;
ctx->base.start = atmel_aes_authenc_start;
return 0;
diff --git a/drivers/crypto/caam/caamalg.c b/drivers/crypto/caam/caamalg.c
index 8697ae53b063..d3d8bb0a6990 100644
--- a/drivers/crypto/caam/caamalg.c
+++ b/drivers/crypto/caam/caamalg.c
@@ -1533,6 +1533,9 @@ static int aead_do_one_req(struct crypto_engine *engine, void *areq)
ret = caam_jr_enqueue(ctx->jrdev, desc, aead_crypt_done, req);
+ if (ret == -ENOSPC && engine->retry_support)
+ return ret;
+
if (ret != -EINPROGRESS) {
aead_unmap(ctx->jrdev, rctx->edesc, req);
kfree(rctx->edesc);
@@ -1762,6 +1765,9 @@ static int skcipher_do_one_req(struct crypto_engine *engine, void *areq)
ret = caam_jr_enqueue(ctx->jrdev, desc, skcipher_crypt_done, req);
+ if (ret == -ENOSPC && engine->retry_support)
+ return ret;
+
if (ret != -EINPROGRESS) {
skcipher_unmap(ctx->jrdev, rctx->edesc, req);
kfree(rctx->edesc);
diff --git a/drivers/crypto/caam/caamalg_qi2.c b/drivers/crypto/caam/caamalg_qi2.c
index 8b8ed77d8715..6753f0e6e55d 100644
--- a/drivers/crypto/caam/caamalg_qi2.c
+++ b/drivers/crypto/caam/caamalg_qi2.c
@@ -5470,7 +5470,7 @@ int dpaa2_caam_enqueue(struct device *dev, struct caam_request *req)
dpaa2_fd_set_len(&fd, dpaa2_fl_get_len(&req->fd_flt[1]));
dpaa2_fd_set_flc(&fd, req->flc_dma);
- ppriv = this_cpu_ptr(priv->ppriv);
+ ppriv = raw_cpu_ptr(priv->ppriv);
for (i = 0; i < (priv->dpseci_attr.num_tx_queues << 1); i++) {
err = dpaa2_io_service_enqueue_fq(ppriv->dpio, ppriv->req_fqid,
&fd);
diff --git a/drivers/crypto/caam/caamhash.c b/drivers/crypto/caam/caamhash.c
index e8a6d8bc43b5..36ef738e4a18 100644
--- a/drivers/crypto/caam/caamhash.c
+++ b/drivers/crypto/caam/caamhash.c
@@ -765,6 +765,9 @@ static int ahash_do_one_req(struct crypto_engine *engine, void *areq)
ret = caam_jr_enqueue(jrdev, desc, state->ahash_op_done, req);
+ if (ret == -ENOSPC && engine->retry_support)
+ return ret;
+
if (ret != -EINPROGRESS) {
ahash_unmap(jrdev, state->edesc, req, 0);
kfree(state->edesc);
diff --git a/drivers/crypto/caam/caampkc.c b/drivers/crypto/caam/caampkc.c
index bf6275ffc4aa..886727576710 100644
--- a/drivers/crypto/caam/caampkc.c
+++ b/drivers/crypto/caam/caampkc.c
@@ -380,6 +380,9 @@ static int akcipher_do_one_req(struct crypto_engine *engine, void *areq)
ret = caam_jr_enqueue(jrdev, desc, req_ctx->akcipher_op_done, req);
+ if (ret == -ENOSPC && engine->retry_support)
+ return ret;
+
if (ret != -EINPROGRESS) {
rsa_pub_unmap(jrdev, req_ctx->edesc, req);
rsa_io_unmap(jrdev, req_ctx->edesc, req);
diff --git a/drivers/crypto/cavium/cpt/cptvf_main.c b/drivers/crypto/cavium/cpt/cptvf_main.c
index 112b12a32542..c246920e6f54 100644
--- a/drivers/crypto/cavium/cpt/cptvf_main.c
+++ b/drivers/crypto/cavium/cpt/cptvf_main.c
@@ -104,17 +104,14 @@ static int alloc_pending_queues(struct pending_qinfo *pqinfo, u32 qlen,
u32 nr_queues)
{
u32 i;
- size_t size;
int ret;
struct pending_queue *queue = NULL;
pqinfo->nr_queues = nr_queues;
pqinfo->qlen = qlen;
- size = (qlen * sizeof(struct pending_entry));
-
for_each_pending_queue(pqinfo, queue, i) {
- queue->head = kzalloc((size), GFP_KERNEL);
+ queue->head = kcalloc(qlen, sizeof(*queue->head), GFP_KERNEL);
if (!queue->head) {
ret = -ENOMEM;
goto pending_qfail;
diff --git a/drivers/crypto/ccp/ccp-dev.c b/drivers/crypto/ccp/ccp-dev.c
index 9ce4b68e9c48..c531d13d971f 100644
--- a/drivers/crypto/ccp/ccp-dev.c
+++ b/drivers/crypto/ccp/ccp-dev.c
@@ -31,7 +31,7 @@
#define MAX_CCPS 32
/* Limit CCP use to a specifed number of queues per device */
-static unsigned int nqueues = 0;
+static unsigned int nqueues;
module_param(nqueues, uint, 0444);
MODULE_PARM_DESC(nqueues, "Number of queues per CCP (minimum 1; default: all available)");
diff --git a/drivers/crypto/ccp/sev-dev.c b/drivers/crypto/ccp/sev-dev.c
index e09925d86bf3..8fd774a10edc 100644
--- a/drivers/crypto/ccp/sev-dev.c
+++ b/drivers/crypto/ccp/sev-dev.c
@@ -22,6 +22,7 @@
#include <linux/firmware.h>
#include <linux/gfp.h>
#include <linux/cpufeature.h>
+#include <linux/fs.h>
#include <asm/smp.h>
@@ -43,6 +44,14 @@ static int psp_probe_timeout = 5;
module_param(psp_probe_timeout, int, 0644);
MODULE_PARM_DESC(psp_probe_timeout, " default timeout value, in seconds, during PSP device probe");
+static char *init_ex_path;
+module_param(init_ex_path, charp, 0444);
+MODULE_PARM_DESC(init_ex_path, " Path for INIT_EX data; if set try INIT_EX");
+
+static bool psp_init_on_probe = true;
+module_param(psp_init_on_probe, bool, 0444);
+MODULE_PARM_DESC(psp_init_on_probe, " if true, the PSP will be initialized on module init. Else the PSP will be initialized on the first command requiring it");
+
MODULE_FIRMWARE("amd/amd_sev_fam17h_model0xh.sbin"); /* 1st gen EPYC */
MODULE_FIRMWARE("amd/amd_sev_fam17h_model3xh.sbin"); /* 2nd gen EPYC */
MODULE_FIRMWARE("amd/amd_sev_fam19h_model0xh.sbin"); /* 3rd gen EPYC */
@@ -58,6 +67,14 @@ static int psp_timeout;
#define SEV_ES_TMR_SIZE (1024 * 1024)
static void *sev_es_tmr;
+/* INIT_EX NV Storage:
+ * The NV Storage is a 32Kb area and must be 4Kb page aligned. Use the page
+ * allocator to allocate the memory, which will return aligned memory for the
+ * specified allocation order.
+ */
+#define NV_LENGTH (32 * 1024)
+static void *sev_init_ex_buffer;
+
static inline bool sev_version_greater_or_equal(u8 maj, u8 min)
{
struct sev_device *sev = psp_master->sev_data;
@@ -107,6 +124,7 @@ static int sev_cmd_buffer_len(int cmd)
{
switch (cmd) {
case SEV_CMD_INIT: return sizeof(struct sev_data_init);
+ case SEV_CMD_INIT_EX: return sizeof(struct sev_data_init_ex);
case SEV_CMD_PLATFORM_STATUS: return sizeof(struct sev_user_data_status);
case SEV_CMD_PEK_CSR: return sizeof(struct sev_data_pek_csr);
case SEV_CMD_PEK_CERT_IMPORT: return sizeof(struct sev_data_pek_cert_import);
@@ -141,6 +159,112 @@ static int sev_cmd_buffer_len(int cmd)
return 0;
}
+static void *sev_fw_alloc(unsigned long len)
+{
+ struct page *page;
+
+ page = alloc_pages(GFP_KERNEL, get_order(len));
+ if (!page)
+ return NULL;
+
+ return page_address(page);
+}
+
+static int sev_read_init_ex_file(void)
+{
+ struct sev_device *sev = psp_master->sev_data;
+ struct file *fp;
+ ssize_t nread;
+
+ lockdep_assert_held(&sev_cmd_mutex);
+
+ if (!sev_init_ex_buffer)
+ return -EOPNOTSUPP;
+
+ fp = filp_open(init_ex_path, O_RDONLY, 0);
+ if (IS_ERR(fp)) {
+ int ret = PTR_ERR(fp);
+
+ dev_err(sev->dev,
+ "SEV: could not open %s for read, error %d\n",
+ init_ex_path, ret);
+ return ret;
+ }
+
+ nread = kernel_read(fp, sev_init_ex_buffer, NV_LENGTH, NULL);
+ if (nread != NV_LENGTH) {
+ dev_err(sev->dev,
+ "SEV: failed to read %u bytes to non volatile memory area, ret %ld\n",
+ NV_LENGTH, nread);
+ return -EIO;
+ }
+
+ dev_dbg(sev->dev, "SEV: read %ld bytes from NV file\n", nread);
+ filp_close(fp, NULL);
+
+ return 0;
+}
+
+static void sev_write_init_ex_file(void)
+{
+ struct sev_device *sev = psp_master->sev_data;
+ struct file *fp;
+ loff_t offset = 0;
+ ssize_t nwrite;
+
+ lockdep_assert_held(&sev_cmd_mutex);
+
+ if (!sev_init_ex_buffer)
+ return;
+
+ fp = filp_open(init_ex_path, O_CREAT | O_WRONLY, 0600);
+ if (IS_ERR(fp)) {
+ dev_err(sev->dev,
+ "SEV: could not open file for write, error %ld\n",
+ PTR_ERR(fp));
+ return;
+ }
+
+ nwrite = kernel_write(fp, sev_init_ex_buffer, NV_LENGTH, &offset);
+ vfs_fsync(fp, 0);
+ filp_close(fp, NULL);
+
+ if (nwrite != NV_LENGTH) {
+ dev_err(sev->dev,
+ "SEV: failed to write %u bytes to non volatile memory area, ret %ld\n",
+ NV_LENGTH, nwrite);
+ return;
+ }
+
+ dev_dbg(sev->dev, "SEV: write successful to NV file\n");
+}
+
+static void sev_write_init_ex_file_if_required(int cmd_id)
+{
+ lockdep_assert_held(&sev_cmd_mutex);
+
+ if (!sev_init_ex_buffer)
+ return;
+
+ /*
+ * Only a few platform commands modify the SPI/NV area, but none of the
+ * non-platform commands do. Only INIT(_EX), PLATFORM_RESET, PEK_GEN,
+ * PEK_CERT_IMPORT, and PDH_GEN do.
+ */
+ switch (cmd_id) {
+ case SEV_CMD_FACTORY_RESET:
+ case SEV_CMD_INIT_EX:
+ case SEV_CMD_PDH_GEN:
+ case SEV_CMD_PEK_CERT_IMPORT:
+ case SEV_CMD_PEK_GEN:
+ break;
+ default:
+ return;
+ }
+
+ sev_write_init_ex_file();
+}
+
static int __sev_do_cmd_locked(int cmd, void *data, int *psp_ret)
{
struct psp_device *psp = psp_master;
@@ -210,6 +334,8 @@ static int __sev_do_cmd_locked(int cmd, void *data, int *psp_ret)
dev_dbg(sev->dev, "sev command %#x failed (%#010x)\n",
cmd, reg & PSP_CMDRESP_ERR_MASK);
ret = -EIO;
+ } else {
+ sev_write_init_ex_file_if_required(cmd);
}
print_hex_dump_debug("(out): ", DUMP_PREFIX_OFFSET, 16, 2, data,
@@ -236,37 +362,85 @@ static int sev_do_cmd(int cmd, void *data, int *psp_ret)
return rc;
}
-static int __sev_platform_init_locked(int *error)
+static int __sev_init_locked(int *error)
{
- struct psp_device *psp = psp_master;
struct sev_data_init data;
- struct sev_device *sev;
- int rc = 0;
- if (!psp || !psp->sev_data)
- return -ENODEV;
+ memset(&data, 0, sizeof(data));
+ if (sev_es_tmr) {
+ /*
+ * Do not include the encryption mask on the physical
+ * address of the TMR (firmware should clear it anyway).
+ */
+ data.tmr_address = __pa(sev_es_tmr);
- sev = psp->sev_data;
+ data.flags |= SEV_INIT_FLAGS_SEV_ES;
+ data.tmr_len = SEV_ES_TMR_SIZE;
+ }
- if (sev->state == SEV_STATE_INIT)
- return 0;
+ return __sev_do_cmd_locked(SEV_CMD_INIT, &data, error);
+}
+
+static int __sev_init_ex_locked(int *error)
+{
+ struct sev_data_init_ex data;
+ int ret;
memset(&data, 0, sizeof(data));
- if (sev_es_tmr) {
- u64 tmr_pa;
+ data.length = sizeof(data);
+ data.nv_address = __psp_pa(sev_init_ex_buffer);
+ data.nv_len = NV_LENGTH;
+
+ ret = sev_read_init_ex_file();
+ if (ret)
+ return ret;
+ if (sev_es_tmr) {
/*
* Do not include the encryption mask on the physical
* address of the TMR (firmware should clear it anyway).
*/
- tmr_pa = __pa(sev_es_tmr);
+ data.tmr_address = __pa(sev_es_tmr);
data.flags |= SEV_INIT_FLAGS_SEV_ES;
- data.tmr_address = tmr_pa;
data.tmr_len = SEV_ES_TMR_SIZE;
}
- rc = __sev_do_cmd_locked(SEV_CMD_INIT, &data, error);
+ return __sev_do_cmd_locked(SEV_CMD_INIT_EX, &data, error);
+}
+
+static int __sev_platform_init_locked(int *error)
+{
+ struct psp_device *psp = psp_master;
+ struct sev_device *sev;
+ int rc, psp_ret;
+ int (*init_function)(int *error);
+
+ if (!psp || !psp->sev_data)
+ return -ENODEV;
+
+ sev = psp->sev_data;
+
+ if (sev->state == SEV_STATE_INIT)
+ return 0;
+
+ init_function = sev_init_ex_buffer ? __sev_init_ex_locked :
+ __sev_init_locked;
+ rc = init_function(&psp_ret);
+ if (rc && psp_ret == SEV_RET_SECURE_DATA_INVALID) {
+ /*
+ * Initialization command returned an integrity check failure
+ * status code, meaning that firmware load and validation of SEV
+ * related persistent data has failed. Retrying the
+ * initialization function should succeed by replacing the state
+ * with a reset state.
+ */
+ dev_dbg(sev->dev, "SEV: retrying INIT command");
+ rc = init_function(&psp_ret);
+ }
+ if (error)
+ *error = psp_ret;
+
if (rc)
return rc;
@@ -280,7 +454,10 @@ static int __sev_platform_init_locked(int *error)
dev_dbg(sev->dev, "SEV firmware initialized\n");
- return rc;
+ dev_info(sev->dev, "SEV API:%d.%d build:%d\n", sev->api_major,
+ sev->api_minor, sev->build);
+
+ return 0;
}
int sev_platform_init(int *error)
@@ -1034,6 +1211,12 @@ static void sev_firmware_shutdown(struct sev_device *sev)
get_order(SEV_ES_TMR_SIZE));
sev_es_tmr = NULL;
}
+
+ if (sev_init_ex_buffer) {
+ free_pages((unsigned long)sev_init_ex_buffer,
+ get_order(NV_LENGTH));
+ sev_init_ex_buffer = NULL;
+ }
}
void sev_dev_destroy(struct psp_device *psp)
@@ -1064,7 +1247,6 @@ EXPORT_SYMBOL_GPL(sev_issue_cmd_external_user);
void sev_pci_init(void)
{
struct sev_device *sev = psp_master->sev_data;
- struct page *tmr_page;
int error, rc;
if (!sev)
@@ -1079,37 +1261,32 @@ void sev_pci_init(void)
sev_update_firmware(sev->dev) == 0)
sev_get_api_version();
+ /* If an init_ex_path is provided rely on INIT_EX for PSP initialization
+ * instead of INIT.
+ */
+ if (init_ex_path) {
+ sev_init_ex_buffer = sev_fw_alloc(NV_LENGTH);
+ if (!sev_init_ex_buffer) {
+ dev_err(sev->dev,
+ "SEV: INIT_EX NV memory allocation failed\n");
+ goto err;
+ }
+ }
+
/* Obtain the TMR memory area for SEV-ES use */
- tmr_page = alloc_pages(GFP_KERNEL, get_order(SEV_ES_TMR_SIZE));
- if (tmr_page) {
- sev_es_tmr = page_address(tmr_page);
- } else {
- sev_es_tmr = NULL;
+ sev_es_tmr = sev_fw_alloc(SEV_ES_TMR_SIZE);
+ if (!sev_es_tmr)
dev_warn(sev->dev,
"SEV: TMR allocation failed, SEV-ES support unavailable\n");
- }
-
- /* Initialize the platform */
- rc = sev_platform_init(&error);
- if (rc && (error == SEV_RET_SECURE_DATA_INVALID)) {
- /*
- * INIT command returned an integrity check failure
- * status code, meaning that firmware load and
- * validation of SEV related persistent data has
- * failed and persistent state has been erased.
- * Retrying INIT command here should succeed.
- */
- dev_dbg(sev->dev, "SEV: retrying INIT command");
- rc = sev_platform_init(&error);
- }
- if (rc) {
- dev_err(sev->dev, "SEV: failed to INIT error %#x\n", error);
+ if (!psp_init_on_probe)
return;
- }
- dev_info(sev->dev, "SEV API:%d.%d build:%d\n", sev->api_major,
- sev->api_minor, sev->build);
+ /* Initialize the platform */
+ rc = sev_platform_init(&error);
+ if (rc)
+ dev_err(sev->dev, "SEV: failed to INIT error %#x, rc %d\n",
+ error, rc);
return;
diff --git a/drivers/crypto/ccree/cc_request_mgr.c b/drivers/crypto/ccree/cc_request_mgr.c
index 33fb27745d52..887162df50f9 100644
--- a/drivers/crypto/ccree/cc_request_mgr.c
+++ b/drivers/crypto/ccree/cc_request_mgr.c
@@ -101,7 +101,6 @@ void cc_req_mgr_fini(struct cc_drvdata *drvdata)
dev_dbg(dev, "max_used_sw_slots=%d\n", req_mgr_h->max_used_sw_slots);
#ifdef COMP_IN_WQ
- flush_workqueue(req_mgr_h->workq);
destroy_workqueue(req_mgr_h->workq);
#else
/* Kill tasklet */
diff --git a/drivers/crypto/hisilicon/hpre/hpre_crypto.c b/drivers/crypto/hisilicon/hpre/hpre_crypto.c
index a032c192ef1d..97d54c1465c2 100644
--- a/drivers/crypto/hisilicon/hpre/hpre_crypto.c
+++ b/drivers/crypto/hisilicon/hpre/hpre_crypto.c
@@ -1177,13 +1177,10 @@ static void hpre_rsa_exit_tfm(struct crypto_akcipher *tfm)
static void hpre_key_to_big_end(u8 *data, int len)
{
int i, j;
- u8 tmp;
for (i = 0; i < len / 2; i++) {
j = len - i - 1;
- tmp = data[j];
- data[j] = data[i];
- data[i] = tmp;
+ swap(data[j], data[i]);
}
}
@@ -1865,7 +1862,7 @@ static int hpre_curve25519_src_init(struct hpre_asym_request *hpre_req,
*/
if (memcmp(ptr, p, ctx->key_sz) == 0) {
dev_err(dev, "gx is p!\n");
- return -EINVAL;
+ goto err;
} else if (memcmp(ptr, p, ctx->key_sz) > 0) {
hpre_curve25519_src_modulo_p(ptr);
}
diff --git a/drivers/crypto/hisilicon/hpre/hpre_main.c b/drivers/crypto/hisilicon/hpre/hpre_main.c
index 65a641396c07..ebfab3e14499 100644
--- a/drivers/crypto/hisilicon/hpre/hpre_main.c
+++ b/drivers/crypto/hisilicon/hpre/hpre_main.c
@@ -103,7 +103,7 @@
#define HPRE_QM_PM_FLR BIT(11)
#define HPRE_QM_SRIOV_FLR BIT(12)
-#define HPRE_SHAPER_TYPE_RATE 128
+#define HPRE_SHAPER_TYPE_RATE 640
#define HPRE_VIA_MSI_DSM 1
#define HPRE_SQE_MASK_OFFSET 8
#define HPRE_SQE_MASK_LEN 24
diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c
index 52d6cca6262e..c5b84a5ea350 100644
--- a/drivers/crypto/hisilicon/qm.c
+++ b/drivers/crypto/hisilicon/qm.c
@@ -89,6 +89,10 @@
#define QM_AEQE_PHASE(aeqe) ((le32_to_cpu((aeqe)->dw0) >> 16) & 0x1)
#define QM_AEQE_TYPE_SHIFT 17
+#define QM_AEQE_CQN_MASK GENMASK(15, 0)
+#define QM_CQ_OVERFLOW 0
+#define QM_EQ_OVERFLOW 1
+#define QM_CQE_ERROR 2
#define QM_DOORBELL_CMD_SQ 0
#define QM_DOORBELL_CMD_CQ 1
@@ -122,6 +126,8 @@
#define QM_CQC_VFT 0x1
#define QM_VFT_CFG 0x100060
#define QM_VFT_CFG_OP_ENABLE 0x100054
+#define QM_PM_CTRL 0x100148
+#define QM_IDLE_DISABLE BIT(9)
#define QM_VFT_CFG_DATA_L 0x100064
#define QM_VFT_CFG_DATA_H 0x100068
@@ -501,10 +507,30 @@ static const char * const qp_s[] = {
"none", "init", "start", "stop", "close",
};
-static const u32 typical_qos_val[QM_QOS_TYPICAL_NUM] = {100, 250, 500, 1000,
- 10000, 25000, 50000, 100000};
-static const u32 typical_qos_cbs_s[QM_QOS_TYPICAL_NUM] = {9, 10, 11, 12, 16,
- 17, 18, 19};
+struct qm_typical_qos_table {
+ u32 start;
+ u32 end;
+ u32 val;
+};
+
+/* the qos step is 100 */
+static struct qm_typical_qos_table shaper_cir_s[] = {
+ {100, 100, 4},
+ {200, 200, 3},
+ {300, 500, 2},
+ {600, 1000, 1},
+ {1100, 100000, 0},
+};
+
+static struct qm_typical_qos_table shaper_cbs_s[] = {
+ {100, 200, 9},
+ {300, 500, 11},
+ {600, 1000, 12},
+ {1100, 10000, 16},
+ {10100, 25000, 17},
+ {25100, 50000, 18},
+ {50100, 100000, 19}
+};
static bool qm_avail_state(struct hisi_qm *qm, enum qm_state new)
{
@@ -585,6 +611,75 @@ static bool qm_qp_avail_state(struct hisi_qm *qm, struct hisi_qp *qp,
return avail;
}
+static u32 qm_get_hw_error_status(struct hisi_qm *qm)
+{
+ return readl(qm->io_base + QM_ABNORMAL_INT_STATUS);
+}
+
+static u32 qm_get_dev_err_status(struct hisi_qm *qm)
+{
+ return qm->err_ini->get_dev_hw_err_status(qm);
+}
+
+/* Check if the error causes the master ooo block */
+static int qm_check_dev_error(struct hisi_qm *qm)
+{
+ u32 val, dev_val;
+
+ if (qm->fun_type == QM_HW_VF)
+ return 0;
+
+ val = qm_get_hw_error_status(qm);
+ dev_val = qm_get_dev_err_status(qm);
+
+ if (qm->ver < QM_HW_V3)
+ return (val & QM_ECC_MBIT) ||
+ (dev_val & qm->err_info.ecc_2bits_mask);
+
+ return (val & readl(qm->io_base + QM_OOO_SHUTDOWN_SEL)) ||
+ (dev_val & (~qm->err_info.dev_ce_mask));
+}
+
+static int qm_wait_reset_finish(struct hisi_qm *qm)
+{
+ int delay = 0;
+
+ /* All reset requests need to be queued for processing */
+ while (test_and_set_bit(QM_RESETTING, &qm->misc_ctl)) {
+ msleep(++delay);
+ if (delay > QM_RESET_WAIT_TIMEOUT)
+ return -EBUSY;
+ }
+
+ return 0;
+}
+
+static int qm_reset_prepare_ready(struct hisi_qm *qm)
+{
+ struct pci_dev *pdev = qm->pdev;
+ struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(pdev));
+
+ /*
+ * PF and VF on host doesnot support resetting at the
+ * same time on Kunpeng920.
+ */
+ if (qm->ver < QM_HW_V3)
+ return qm_wait_reset_finish(pf_qm);
+
+ return qm_wait_reset_finish(qm);
+}
+
+static void qm_reset_bit_clear(struct hisi_qm *qm)
+{
+ struct pci_dev *pdev = qm->pdev;
+ struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(pdev));
+
+ if (qm->ver < QM_HW_V3)
+ clear_bit(QM_RESETTING, &pf_qm->misc_ctl);
+
+ clear_bit(QM_RESETTING, &qm->misc_ctl);
+}
+
static void qm_mb_pre_init(struct qm_mailbox *mailbox, u8 cmd,
u64 base, u16 queue, bool op)
{
@@ -707,6 +802,19 @@ static void qm_db(struct hisi_qm *qm, u16 qn, u8 cmd, u16 index, u8 priority)
qm->ops->qm_db(qm, qn, cmd, index, priority);
}
+static void qm_disable_clock_gate(struct hisi_qm *qm)
+{
+ u32 val;
+
+ /* if qm enables clock gating in Kunpeng930, qos will be inaccurate. */
+ if (qm->ver < QM_HW_V3)
+ return;
+
+ val = readl(qm->io_base + QM_PM_CTRL);
+ val |= QM_IDLE_DISABLE;
+ writel(val, qm->io_base + QM_PM_CTRL);
+}
+
static int qm_dev_mem_reset(struct hisi_qm *qm)
{
u32 val;
@@ -899,24 +1007,71 @@ static void qm_set_qp_disable(struct hisi_qp *qp, int offset)
mb();
}
-static irqreturn_t qm_aeq_irq(int irq, void *data)
+static void qm_disable_qp(struct hisi_qm *qm, u32 qp_id)
+{
+ struct hisi_qp *qp = &qm->qp_array[qp_id];
+
+ qm_set_qp_disable(qp, QM_RESET_STOP_TX_OFFSET);
+ hisi_qm_stop_qp(qp);
+ qm_set_qp_disable(qp, QM_RESET_STOP_RX_OFFSET);
+}
+
+static void qm_reset_function(struct hisi_qm *qm)
+{
+ struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(qm->pdev));
+ struct device *dev = &qm->pdev->dev;
+ int ret;
+
+ if (qm_check_dev_error(pf_qm))
+ return;
+
+ ret = qm_reset_prepare_ready(qm);
+ if (ret) {
+ dev_err(dev, "reset function not ready\n");
+ return;
+ }
+
+ ret = hisi_qm_stop(qm, QM_FLR);
+ if (ret) {
+ dev_err(dev, "failed to stop qm when reset function\n");
+ goto clear_bit;
+ }
+
+ ret = hisi_qm_start(qm);
+ if (ret)
+ dev_err(dev, "failed to start qm when reset function\n");
+
+clear_bit:
+ qm_reset_bit_clear(qm);
+}
+
+static irqreturn_t qm_aeq_thread(int irq, void *data)
{
struct hisi_qm *qm = data;
struct qm_aeqe *aeqe = qm->aeqe + qm->status.aeq_head;
- u32 type;
-
- atomic64_inc(&qm->debug.dfx.aeq_irq_cnt);
- if (!readl(qm->io_base + QM_VF_AEQ_INT_SOURCE))
- return IRQ_NONE;
+ u32 type, qp_id;
while (QM_AEQE_PHASE(aeqe) == qm->status.aeqc_phase) {
type = le32_to_cpu(aeqe->dw0) >> QM_AEQE_TYPE_SHIFT;
- if (type < ARRAY_SIZE(qm_fifo_overflow))
- dev_err(&qm->pdev->dev, "%s overflow\n",
- qm_fifo_overflow[type]);
- else
+ qp_id = le32_to_cpu(aeqe->dw0) & QM_AEQE_CQN_MASK;
+
+ switch (type) {
+ case QM_EQ_OVERFLOW:
+ dev_err(&qm->pdev->dev, "eq overflow, reset function\n");
+ qm_reset_function(qm);
+ return IRQ_HANDLED;
+ case QM_CQ_OVERFLOW:
+ dev_err(&qm->pdev->dev, "cq overflow, stop qp(%u)\n",
+ qp_id);
+ fallthrough;
+ case QM_CQE_ERROR:
+ qm_disable_qp(qm, qp_id);
+ break;
+ default:
dev_err(&qm->pdev->dev, "unknown error type %u\n",
type);
+ break;
+ }
if (qm->status.aeq_head == QM_Q_DEPTH - 1) {
qm->status.aeqc_phase = !qm->status.aeqc_phase;
@@ -926,13 +1081,24 @@ static irqreturn_t qm_aeq_irq(int irq, void *data)
aeqe++;
qm->status.aeq_head++;
}
-
- qm_db(qm, 0, QM_DOORBELL_CMD_AEQ, qm->status.aeq_head, 0);
}
+ qm_db(qm, 0, QM_DOORBELL_CMD_AEQ, qm->status.aeq_head, 0);
+
return IRQ_HANDLED;
}
+static irqreturn_t qm_aeq_irq(int irq, void *data)
+{
+ struct hisi_qm *qm = data;
+
+ atomic64_inc(&qm->debug.dfx.aeq_irq_cnt);
+ if (!readl(qm->io_base + QM_VF_AEQ_INT_SOURCE))
+ return IRQ_NONE;
+
+ return IRQ_WAKE_THREAD;
+}
+
static void qm_irq_unregister(struct hisi_qm *qm)
{
struct pci_dev *pdev = qm->pdev;
@@ -988,12 +1154,14 @@ static void qm_init_prefetch(struct hisi_qm *qm)
}
/*
+ * acc_shaper_para_calc() Get the IR value by the qos formula, the return value
+ * is the expected qos calculated.
* the formula:
* IR = X Mbps if ir = 1 means IR = 100 Mbps, if ir = 10000 means = 10Gbps
*
- * IR_b * (2 ^ IR_u) * 8
- * IR(Mbps) * 10 ^ -3 = -------------------------
- * Tick * (2 ^ IR_s)
+ * IR_b * (2 ^ IR_u) * 8000
+ * IR(Mbps) = -------------------------
+ * Tick * (2 ^ IR_s)
*/
static u32 acc_shaper_para_calc(u64 cir_b, u64 cir_u, u64 cir_s)
{
@@ -1003,17 +1171,28 @@ static u32 acc_shaper_para_calc(u64 cir_b, u64 cir_u, u64 cir_s)
static u32 acc_shaper_calc_cbs_s(u32 ir)
{
+ int table_size = ARRAY_SIZE(shaper_cbs_s);
int i;
- if (ir < typical_qos_val[0])
- return QM_SHAPER_MIN_CBS_S;
+ for (i = 0; i < table_size; i++) {
+ if (ir >= shaper_cbs_s[i].start && ir <= shaper_cbs_s[i].end)
+ return shaper_cbs_s[i].val;
+ }
+
+ return QM_SHAPER_MIN_CBS_S;
+}
+
+static u32 acc_shaper_calc_cir_s(u32 ir)
+{
+ int table_size = ARRAY_SIZE(shaper_cir_s);
+ int i;
- for (i = 1; i < QM_QOS_TYPICAL_NUM; i++) {
- if (ir >= typical_qos_val[i - 1] && ir < typical_qos_val[i])
- return typical_qos_cbs_s[i - 1];
+ for (i = 0; i < table_size; i++) {
+ if (ir >= shaper_cir_s[i].start && ir <= shaper_cir_s[i].end)
+ return shaper_cir_s[i].val;
}
- return typical_qos_cbs_s[QM_QOS_TYPICAL_NUM - 1];
+ return 0;
}
static int qm_get_shaper_para(u32 ir, struct qm_shaper_factor *factor)
@@ -1022,25 +1201,18 @@ static int qm_get_shaper_para(u32 ir, struct qm_shaper_factor *factor)
u32 error_rate;
factor->cbs_s = acc_shaper_calc_cbs_s(ir);
+ cir_s = acc_shaper_calc_cir_s(ir);
for (cir_b = QM_QOS_MIN_CIR_B; cir_b <= QM_QOS_MAX_CIR_B; cir_b++) {
for (cir_u = 0; cir_u <= QM_QOS_MAX_CIR_U; cir_u++) {
- for (cir_s = 0; cir_s <= QM_QOS_MAX_CIR_S; cir_s++) {
- /** the formula is changed to:
- * IR_b * (2 ^ IR_u) * DIVISOR_CLK
- * IR(Mbps) = -------------------------
- * 768 * (2 ^ IR_s)
- */
- ir_calc = acc_shaper_para_calc(cir_b, cir_u,
- cir_s);
- error_rate = QM_QOS_EXPAND_RATE * (u32)abs(ir_calc - ir) / ir;
- if (error_rate <= QM_QOS_MIN_ERROR_RATE) {
- factor->cir_b = cir_b;
- factor->cir_u = cir_u;
- factor->cir_s = cir_s;
-
- return 0;
- }
+ ir_calc = acc_shaper_para_calc(cir_b, cir_u, cir_s);
+
+ error_rate = QM_QOS_EXPAND_RATE * (u32)abs(ir_calc - ir) / ir;
+ if (error_rate <= QM_QOS_MIN_ERROR_RATE) {
+ factor->cir_b = cir_b;
+ factor->cir_u = cir_u;
+ factor->cir_s = cir_s;
+ return 0;
}
}
}
@@ -1126,10 +1298,10 @@ static int qm_set_vft_common(struct hisi_qm *qm, enum vft_type type,
static int qm_shaper_init_vft(struct hisi_qm *qm, u32 fun_num)
{
+ u32 qos = qm->factor[fun_num].func_qos;
int ret, i;
- qm->factor[fun_num].func_qos = QM_QOS_MAX_VAL;
- ret = qm_get_shaper_para(QM_QOS_MAX_VAL * QM_QOS_RATE, &qm->factor[fun_num]);
+ ret = qm_get_shaper_para(qos * QM_QOS_RATE, &qm->factor[fun_num]);
if (ret) {
dev_err(&qm->pdev->dev, "failed to calculate shaper parameter!\n");
return ret;
@@ -2082,35 +2254,6 @@ static enum acc_err_result qm_hw_error_handle_v2(struct hisi_qm *qm)
return ACC_ERR_RECOVERED;
}
-static u32 qm_get_hw_error_status(struct hisi_qm *qm)
-{
- return readl(qm->io_base + QM_ABNORMAL_INT_STATUS);
-}
-
-static u32 qm_get_dev_err_status(struct hisi_qm *qm)
-{
- return qm->err_ini->get_dev_hw_err_status(qm);
-}
-
-/* Check if the error causes the master ooo block */
-static int qm_check_dev_error(struct hisi_qm *qm)
-{
- u32 val, dev_val;
-
- if (qm->fun_type == QM_HW_VF)
- return 0;
-
- val = qm_get_hw_error_status(qm);
- dev_val = qm_get_dev_err_status(qm);
-
- if (qm->ver < QM_HW_V3)
- return (val & QM_ECC_MBIT) ||
- (dev_val & qm->err_info.ecc_2bits_mask);
-
- return (val & readl(qm->io_base + QM_OOO_SHUTDOWN_SEL)) ||
- (dev_val & (~qm->err_info.dev_ce_mask));
-}
-
static int qm_get_mb_cmd(struct hisi_qm *qm, u64 *msg, u16 fun_num)
{
struct qm_mailbox mailbox;
@@ -3399,6 +3542,7 @@ void hisi_qm_uninit(struct hisi_qm *qm)
dma_free_coherent(dev, qm->qdma.size,
qm->qdma.va, qm->qdma.dma);
}
+ up_write(&qm->qps_lock);
qm_irq_unregister(qm);
hisi_qm_pci_uninit(qm);
@@ -3406,8 +3550,6 @@ void hisi_qm_uninit(struct hisi_qm *qm)
uacce_remove(qm->uacce);
qm->uacce = NULL;
}
-
- up_write(&qm->qps_lock);
}
EXPORT_SYMBOL_GPL(hisi_qm_uninit);
@@ -3473,6 +3615,22 @@ static void qm_init_eq_aeq_status(struct hisi_qm *qm)
status->aeqc_phase = true;
}
+static void qm_enable_eq_aeq_interrupts(struct hisi_qm *qm)
+{
+ /* Clear eq/aeq interrupt source */
+ qm_db(qm, 0, QM_DOORBELL_CMD_AEQ, qm->status.aeq_head, 0);
+ qm_db(qm, 0, QM_DOORBELL_CMD_EQ, qm->status.eq_head, 0);
+
+ writel(0x0, qm->io_base + QM_VF_EQ_INT_MASK);
+ writel(0x0, qm->io_base + QM_VF_AEQ_INT_MASK);
+}
+
+static void qm_disable_eq_aeq_interrupts(struct hisi_qm *qm)
+{
+ writel(0x1, qm->io_base + QM_VF_EQ_INT_MASK);
+ writel(0x1, qm->io_base + QM_VF_AEQ_INT_MASK);
+}
+
static int qm_eq_ctx_cfg(struct hisi_qm *qm)
{
struct device *dev = &qm->pdev->dev;
@@ -3556,10 +3714,6 @@ static int __hisi_qm_start(struct hisi_qm *qm)
WARN_ON(!qm->qdma.va);
if (qm->fun_type == QM_HW_PF) {
- ret = qm_dev_mem_reset(qm);
- if (ret)
- return ret;
-
ret = hisi_qm_set_vft(qm, 0, qm->qp_base, qm->qp_num);
if (ret)
return ret;
@@ -3578,9 +3732,7 @@ static int __hisi_qm_start(struct hisi_qm *qm)
return ret;
qm_init_prefetch(qm);
-
- writel(0x0, qm->io_base + QM_VF_EQ_INT_MASK);
- writel(0x0, qm->io_base + QM_VF_AEQ_INT_MASK);
+ qm_enable_eq_aeq_interrupts(qm);
return 0;
}
@@ -3728,10 +3880,7 @@ int hisi_qm_stop(struct hisi_qm *qm, enum qm_stop_reason r)
hisi_qm_set_hw_reset(qm, QM_RESET_STOP_RX_OFFSET);
}
- /* Mask eq and aeq irq */
- writel(0x1, qm->io_base + QM_VF_EQ_INT_MASK);
- writel(0x1, qm->io_base + QM_VF_AEQ_INT_MASK);
-
+ qm_disable_eq_aeq_interrupts(qm);
if (qm->fun_type == QM_HW_PF) {
ret = hisi_qm_set_vft(qm, 0, 0, 0);
if (ret < 0) {
@@ -4231,66 +4380,69 @@ static ssize_t qm_qos_value_init(const char *buf, unsigned long *val)
return 0;
}
+static ssize_t qm_get_qos_value(struct hisi_qm *qm, const char *buf,
+ unsigned long *val,
+ unsigned int *fun_index)
+{
+ char tbuf_bdf[QM_DBG_READ_LEN] = {0};
+ char val_buf[QM_QOS_VAL_MAX_LEN] = {0};
+ u32 tmp1, device, function;
+ int ret, bus;
+
+ ret = sscanf(buf, "%s %s", tbuf_bdf, val_buf);
+ if (ret != QM_QOS_PARAM_NUM)
+ return -EINVAL;
+
+ ret = qm_qos_value_init(val_buf, val);
+ if (ret || *val == 0 || *val > QM_QOS_MAX_VAL) {
+ pci_err(qm->pdev, "input qos value is error, please set 1~1000!\n");
+ return -EINVAL;
+ }
+
+ ret = sscanf(tbuf_bdf, "%u:%x:%u.%u", &tmp1, &bus, &device, &function);
+ if (ret != QM_QOS_BDF_PARAM_NUM) {
+ pci_err(qm->pdev, "input pci bdf value is error!\n");
+ return -EINVAL;
+ }
+
+ *fun_index = PCI_DEVFN(device, function);
+
+ return 0;
+}
+
static ssize_t qm_algqos_write(struct file *filp, const char __user *buf,
size_t count, loff_t *pos)
{
struct hisi_qm *qm = filp->private_data;
char tbuf[QM_DBG_READ_LEN];
- int tmp1, bus, device, function;
- char tbuf_bdf[QM_DBG_READ_LEN] = {0};
- char val_buf[QM_QOS_VAL_MAX_LEN] = {0};
unsigned int fun_index;
- unsigned long val = 0;
+ unsigned long val;
int len, ret;
if (qm->fun_type == QM_HW_VF)
return -EINVAL;
- /* Mailbox and reset cannot be operated at the same time */
- if (test_and_set_bit(QM_RESETTING, &qm->misc_ctl)) {
- pci_err(qm->pdev, "dev resetting, write alg qos failed!\n");
- return -EAGAIN;
- }
-
- if (*pos != 0) {
- ret = 0;
- goto err_get_status;
- }
+ if (*pos != 0)
+ return 0;
- if (count >= QM_DBG_READ_LEN) {
- ret = -ENOSPC;
- goto err_get_status;
- }
+ if (count >= QM_DBG_READ_LEN)
+ return -ENOSPC;
len = simple_write_to_buffer(tbuf, QM_DBG_READ_LEN - 1, pos, buf, count);
- if (len < 0) {
- ret = len;
- goto err_get_status;
- }
+ if (len < 0)
+ return len;
tbuf[len] = '\0';
- ret = sscanf(tbuf, "%s %s", tbuf_bdf, val_buf);
- if (ret != QM_QOS_PARAM_NUM) {
- ret = -EINVAL;
- goto err_get_status;
- }
-
- ret = qm_qos_value_init(val_buf, &val);
- if (val == 0 || val > QM_QOS_MAX_VAL || ret) {
- pci_err(qm->pdev, "input qos value is error, please set 1~1000!\n");
- ret = -EINVAL;
- goto err_get_status;
- }
+ ret = qm_get_qos_value(qm, tbuf, &val, &fun_index);
+ if (ret)
+ return ret;
- ret = sscanf(tbuf_bdf, "%d:%x:%d.%d", &tmp1, &bus, &device, &function);
- if (ret != QM_QOS_BDF_PARAM_NUM) {
- pci_err(qm->pdev, "input pci bdf value is error!\n");
- ret = -EINVAL;
- goto err_get_status;
+ /* Mailbox and reset cannot be operated at the same time */
+ if (test_and_set_bit(QM_RESETTING, &qm->misc_ctl)) {
+ pci_err(qm->pdev, "dev resetting, write alg qos failed!\n");
+ return -EAGAIN;
}
- fun_index = device * 8 + function;
-
ret = qm_pm_get_sync(qm);
if (ret) {
ret = -EINVAL;
@@ -4304,6 +4456,8 @@ static ssize_t qm_algqos_write(struct file *filp, const char __user *buf,
goto err_put_sync;
}
+ pci_info(qm->pdev, "the qos value of function%u is set to %lu.\n",
+ fun_index, val);
ret = count;
err_put_sync:
@@ -4728,46 +4882,6 @@ static int qm_try_stop_vfs(struct hisi_qm *qm, u64 cmd,
return ret;
}
-static int qm_wait_reset_finish(struct hisi_qm *qm)
-{
- int delay = 0;
-
- /* All reset requests need to be queued for processing */
- while (test_and_set_bit(QM_RESETTING, &qm->misc_ctl)) {
- msleep(++delay);
- if (delay > QM_RESET_WAIT_TIMEOUT)
- return -EBUSY;
- }
-
- return 0;
-}
-
-static int qm_reset_prepare_ready(struct hisi_qm *qm)
-{
- struct pci_dev *pdev = qm->pdev;
- struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(pdev));
-
- /*
- * PF and VF on host doesnot support resetting at the
- * same time on Kunpeng920.
- */
- if (qm->ver < QM_HW_V3)
- return qm_wait_reset_finish(pf_qm);
-
- return qm_wait_reset_finish(qm);
-}
-
-static void qm_reset_bit_clear(struct hisi_qm *qm)
-{
- struct pci_dev *pdev = qm->pdev;
- struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(pdev));
-
- if (qm->ver < QM_HW_V3)
- clear_bit(QM_RESETTING, &pf_qm->misc_ctl);
-
- clear_bit(QM_RESETTING, &qm->misc_ctl);
-}
-
static int qm_controller_reset_prepare(struct hisi_qm *qm)
{
struct pci_dev *pdev = qm->pdev;
@@ -5053,6 +5167,12 @@ static int qm_controller_reset_done(struct hisi_qm *qm)
if (qm->err_ini->open_axi_master_ooo)
qm->err_ini->open_axi_master_ooo(qm);
+ ret = qm_dev_mem_reset(qm);
+ if (ret) {
+ pci_err(pdev, "failed to reset device memory\n");
+ return ret;
+ }
+
ret = qm_restart(qm);
if (ret) {
pci_err(pdev, "Failed to start QM!\n");
@@ -5267,8 +5387,10 @@ static int qm_irq_register(struct hisi_qm *qm)
return ret;
if (qm->ver > QM_HW_V1) {
- ret = request_irq(pci_irq_vector(pdev, QM_AEQ_EVENT_IRQ_VECTOR),
- qm_aeq_irq, 0, qm->dev_name, qm);
+ ret = request_threaded_irq(pci_irq_vector(pdev,
+ QM_AEQ_EVENT_IRQ_VECTOR),
+ qm_aeq_irq, qm_aeq_thread,
+ 0, qm->dev_name, qm);
if (ret)
goto err_aeq_irq;
@@ -5750,13 +5872,15 @@ err_init_qp_mem:
static int hisi_qm_memory_init(struct hisi_qm *qm)
{
struct device *dev = &qm->pdev->dev;
- int ret, total_vfs;
+ int ret, total_func, i;
size_t off = 0;
- total_vfs = pci_sriov_get_totalvfs(qm->pdev);
- qm->factor = kcalloc(total_vfs + 1, sizeof(struct qm_shaper_factor), GFP_KERNEL);
+ total_func = pci_sriov_get_totalvfs(qm->pdev) + 1;
+ qm->factor = kcalloc(total_func, sizeof(struct qm_shaper_factor), GFP_KERNEL);
if (!qm->factor)
return -ENOMEM;
+ for (i = 0; i < total_func; i++)
+ qm->factor[i].func_qos = QM_QOS_MAX_VAL;
#define QM_INIT_BUF(qm, type, num) do { \
(qm)->type = ((qm)->qdma.va + (off)); \
@@ -5825,6 +5949,15 @@ int hisi_qm_init(struct hisi_qm *qm)
goto err_irq_register;
}
+ if (qm->fun_type == QM_HW_PF) {
+ qm_disable_clock_gate(qm);
+ ret = qm_dev_mem_reset(qm);
+ if (ret) {
+ dev_err(dev, "failed to reset device memory\n");
+ goto err_irq_register;
+ }
+ }
+
if (qm->mode == UACCE_MODE_SVA) {
ret = qm_alloc_uacce(qm);
if (ret < 0)
@@ -5982,8 +6115,12 @@ static int qm_rebuild_for_resume(struct hisi_qm *qm)
qm_cmd_init(qm);
hisi_qm_dev_err_init(qm);
+ qm_disable_clock_gate(qm);
+ ret = qm_dev_mem_reset(qm);
+ if (ret)
+ pci_err(pdev, "failed to reset device memory\n");
- return 0;
+ return ret;
}
/**
@@ -6038,7 +6175,7 @@ int hisi_qm_resume(struct device *dev)
if (ret)
pci_err(pdev, "failed to start qm(%d)\n", ret);
- return 0;
+ return ret;
}
EXPORT_SYMBOL_GPL(hisi_qm_resume);
diff --git a/drivers/crypto/hisilicon/sec2/sec_main.c b/drivers/crypto/hisilicon/sec2/sec_main.c
index 90551bf38b52..26d3ab1d308b 100644
--- a/drivers/crypto/hisilicon/sec2/sec_main.c
+++ b/drivers/crypto/hisilicon/sec2/sec_main.c
@@ -105,7 +105,7 @@
#define SEC_SQE_MASK_OFFSET 64
#define SEC_SQE_MASK_LEN 48
-#define SEC_SHAPER_TYPE_RATE 128
+#define SEC_SHAPER_TYPE_RATE 400
struct sec_hw_error {
u32 int_msk;
diff --git a/drivers/crypto/hisilicon/zip/zip_main.c b/drivers/crypto/hisilicon/zip/zip_main.c
index 873971ef9aee..678f8b58ec42 100644
--- a/drivers/crypto/hisilicon/zip/zip_main.c
+++ b/drivers/crypto/hisilicon/zip/zip_main.c
@@ -103,8 +103,8 @@
#define HZIP_PREFETCH_ENABLE (~(BIT(26) | BIT(17) | BIT(0)))
#define HZIP_SVA_PREFETCH_DISABLE BIT(26)
#define HZIP_SVA_DISABLE_READY (BIT(26) | BIT(30))
-#define HZIP_SHAPER_RATE_COMPRESS 252
-#define HZIP_SHAPER_RATE_DECOMPRESS 229
+#define HZIP_SHAPER_RATE_COMPRESS 750
+#define HZIP_SHAPER_RATE_DECOMPRESS 140
#define HZIP_DELAY_1_US 1
#define HZIP_POLL_TIMEOUT_US 1000
@@ -364,15 +364,16 @@ static int hisi_zip_set_user_domain_and_cache(struct hisi_qm *qm)
/* user domain configurations */
writel(AXUSER_BASE, base + HZIP_BD_RUSER_32_63);
- writel(AXUSER_BASE, base + HZIP_SGL_RUSER_32_63);
writel(AXUSER_BASE, base + HZIP_BD_WUSER_32_63);
if (qm->use_sva && qm->ver == QM_HW_V2) {
writel(AXUSER_BASE | AXUSER_SSV, base + HZIP_DATA_RUSER_32_63);
writel(AXUSER_BASE | AXUSER_SSV, base + HZIP_DATA_WUSER_32_63);
+ writel(AXUSER_BASE | AXUSER_SSV, base + HZIP_SGL_RUSER_32_63);
} else {
writel(AXUSER_BASE, base + HZIP_DATA_RUSER_32_63);
writel(AXUSER_BASE, base + HZIP_DATA_WUSER_32_63);
+ writel(AXUSER_BASE, base + HZIP_SGL_RUSER_32_63);
}
/* let's open all compression/decompression cores */
@@ -829,7 +830,10 @@ static int hisi_zip_qm_init(struct hisi_qm *qm, struct pci_dev *pdev)
qm->pdev = pdev;
qm->ver = pdev->revision;
- qm->algs = "zlib\ngzip";
+ if (pdev->revision >= QM_HW_V3)
+ qm->algs = "zlib\ngzip\ndeflate\nlz77_zstd";
+ else
+ qm->algs = "zlib\ngzip";
qm->mode = uacce_mode;
qm->sqe_size = HZIP_SQE_SIZE;
qm->dev_name = hisi_zip_name;
diff --git a/drivers/crypto/keembay/keembay-ocs-ecc.c b/drivers/crypto/keembay/keembay-ocs-ecc.c
index 679e6ae295e0..5d0785d3f1b5 100644
--- a/drivers/crypto/keembay/keembay-ocs-ecc.c
+++ b/drivers/crypto/keembay/keembay-ocs-ecc.c
@@ -930,6 +930,7 @@ static int kmb_ocs_ecc_probe(struct platform_device *pdev)
ecc_dev->engine = crypto_engine_alloc_init(dev, 1);
if (!ecc_dev->engine) {
dev_err(dev, "Could not allocate crypto engine\n");
+ rc = -ENOMEM;
goto list_del;
}
diff --git a/drivers/crypto/marvell/octeontx/otx_cptvf_main.c b/drivers/crypto/marvell/octeontx/otx_cptvf_main.c
index c076d0b3ad5f..b681bd2dc6ad 100644
--- a/drivers/crypto/marvell/octeontx/otx_cptvf_main.c
+++ b/drivers/crypto/marvell/octeontx/otx_cptvf_main.c
@@ -94,15 +94,13 @@ static int alloc_pending_queues(struct otx_cpt_pending_qinfo *pqinfo, u32 qlen,
u32 num_queues)
{
struct otx_cpt_pending_queue *queue = NULL;
- size_t size;
int ret;
u32 i;
pqinfo->num_queues = num_queues;
- size = (qlen * sizeof(struct otx_cpt_pending_entry));
for_each_pending_queue(pqinfo, queue, i) {
- queue->head = kzalloc((size), GFP_KERNEL);
+ queue->head = kcalloc(qlen, sizeof(*queue->head), GFP_KERNEL);
if (!queue->head) {
ret = -ENOMEM;
goto pending_qfail;
diff --git a/drivers/crypto/marvell/octeontx2/Makefile b/drivers/crypto/marvell/octeontx2/Makefile
index c242d22008c3..965297e96954 100644
--- a/drivers/crypto/marvell/octeontx2/Makefile
+++ b/drivers/crypto/marvell/octeontx2/Makefile
@@ -3,7 +3,7 @@ obj-$(CONFIG_CRYPTO_DEV_OCTEONTX2_CPT) += rvu_cptpf.o rvu_cptvf.o
rvu_cptpf-objs := otx2_cptpf_main.o otx2_cptpf_mbox.o \
otx2_cpt_mbox_common.o otx2_cptpf_ucode.o otx2_cptlf.o \
- cn10k_cpt.o
+ cn10k_cpt.o otx2_cpt_devlink.o
rvu_cptvf-objs := otx2_cptvf_main.o otx2_cptvf_mbox.o otx2_cptlf.o \
otx2_cpt_mbox_common.o otx2_cptvf_reqmgr.o \
otx2_cptvf_algs.o cn10k_cpt.o
diff --git a/drivers/crypto/marvell/octeontx2/otx2_cpt_common.h b/drivers/crypto/marvell/octeontx2/otx2_cpt_common.h
index c5445b05f53c..fb56824cb0a6 100644
--- a/drivers/crypto/marvell/octeontx2/otx2_cpt_common.h
+++ b/drivers/crypto/marvell/octeontx2/otx2_cpt_common.h
@@ -10,6 +10,7 @@
#include <linux/module.h>
#include <linux/delay.h>
#include <linux/crypto.h>
+#include <net/devlink.h>
#include "otx2_cpt_hw_types.h"
#include "rvu.h"
#include "mbox.h"
diff --git a/drivers/crypto/marvell/octeontx2/otx2_cpt_devlink.c b/drivers/crypto/marvell/octeontx2/otx2_cpt_devlink.c
new file mode 100644
index 000000000000..bb02e0db3615
--- /dev/null
+++ b/drivers/crypto/marvell/octeontx2/otx2_cpt_devlink.c
@@ -0,0 +1,108 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/* Copyright (C) 2021 Marvell. */
+
+#include "otx2_cpt_devlink.h"
+
+static int otx2_cpt_dl_egrp_create(struct devlink *dl, u32 id,
+ struct devlink_param_gset_ctx *ctx)
+{
+ struct otx2_cpt_devlink *cpt_dl = devlink_priv(dl);
+ struct otx2_cptpf_dev *cptpf = cpt_dl->cptpf;
+
+ return otx2_cpt_dl_custom_egrp_create(cptpf, ctx);
+}
+
+static int otx2_cpt_dl_egrp_delete(struct devlink *dl, u32 id,
+ struct devlink_param_gset_ctx *ctx)
+{
+ struct otx2_cpt_devlink *cpt_dl = devlink_priv(dl);
+ struct otx2_cptpf_dev *cptpf = cpt_dl->cptpf;
+
+ return otx2_cpt_dl_custom_egrp_delete(cptpf, ctx);
+}
+
+static int otx2_cpt_dl_uc_info(struct devlink *dl, u32 id,
+ struct devlink_param_gset_ctx *ctx)
+{
+ struct otx2_cpt_devlink *cpt_dl = devlink_priv(dl);
+ struct otx2_cptpf_dev *cptpf = cpt_dl->cptpf;
+
+ otx2_cpt_print_uc_dbg_info(cptpf);
+
+ return 0;
+}
+
+enum otx2_cpt_dl_param_id {
+ OTX2_CPT_DEVLINK_PARAM_ID_BASE = DEVLINK_PARAM_GENERIC_ID_MAX,
+ OTX2_CPT_DEVLINK_PARAM_ID_EGRP_CREATE,
+ OTX2_CPT_DEVLINK_PARAM_ID_EGRP_DELETE,
+};
+
+static const struct devlink_param otx2_cpt_dl_params[] = {
+ DEVLINK_PARAM_DRIVER(OTX2_CPT_DEVLINK_PARAM_ID_EGRP_CREATE,
+ "egrp_create", DEVLINK_PARAM_TYPE_STRING,
+ BIT(DEVLINK_PARAM_CMODE_RUNTIME),
+ otx2_cpt_dl_uc_info, otx2_cpt_dl_egrp_create,
+ NULL),
+ DEVLINK_PARAM_DRIVER(OTX2_CPT_DEVLINK_PARAM_ID_EGRP_DELETE,
+ "egrp_delete", DEVLINK_PARAM_TYPE_STRING,
+ BIT(DEVLINK_PARAM_CMODE_RUNTIME),
+ otx2_cpt_dl_uc_info, otx2_cpt_dl_egrp_delete,
+ NULL),
+};
+
+static int otx2_cpt_devlink_info_get(struct devlink *devlink,
+ struct devlink_info_req *req,
+ struct netlink_ext_ack *extack)
+{
+ return devlink_info_driver_name_put(req, "rvu_cptpf");
+}
+
+static const struct devlink_ops otx2_cpt_devlink_ops = {
+ .info_get = otx2_cpt_devlink_info_get,
+};
+
+int otx2_cpt_register_dl(struct otx2_cptpf_dev *cptpf)
+{
+ struct device *dev = &cptpf->pdev->dev;
+ struct otx2_cpt_devlink *cpt_dl;
+ struct devlink *dl;
+ int ret;
+
+ dl = devlink_alloc(&otx2_cpt_devlink_ops,
+ sizeof(struct otx2_cpt_devlink), dev);
+ if (!dl) {
+ dev_warn(dev, "devlink_alloc failed\n");
+ return -ENOMEM;
+ }
+
+ cpt_dl = devlink_priv(dl);
+ cpt_dl->dl = dl;
+ cpt_dl->cptpf = cptpf;
+ cptpf->dl = dl;
+ ret = devlink_params_register(dl, otx2_cpt_dl_params,
+ ARRAY_SIZE(otx2_cpt_dl_params));
+ if (ret) {
+ dev_err(dev, "devlink params register failed with error %d",
+ ret);
+ devlink_free(dl);
+ return ret;
+ }
+
+ devlink_register(dl);
+
+ return 0;
+}
+
+void otx2_cpt_unregister_dl(struct otx2_cptpf_dev *cptpf)
+{
+ struct devlink *dl = cptpf->dl;
+
+ if (!dl)
+ return;
+
+ devlink_unregister(dl);
+ devlink_params_unregister(dl, otx2_cpt_dl_params,
+ ARRAY_SIZE(otx2_cpt_dl_params));
+ devlink_free(dl);
+}
diff --git a/drivers/crypto/marvell/octeontx2/otx2_cpt_devlink.h b/drivers/crypto/marvell/octeontx2/otx2_cpt_devlink.h
new file mode 100644
index 000000000000..8b7d88c5d519
--- /dev/null
+++ b/drivers/crypto/marvell/octeontx2/otx2_cpt_devlink.h
@@ -0,0 +1,20 @@
+/* SPDX-License-Identifier: GPL-2.0-only
+ * Copyright (C) 2021 Marvell.
+ */
+
+#ifndef __OTX2_CPT_DEVLINK_H
+#define __OTX2_CPT_DEVLINK_H
+
+#include "otx2_cpt_common.h"
+#include "otx2_cptpf.h"
+
+struct otx2_cpt_devlink {
+ struct devlink *dl;
+ struct otx2_cptpf_dev *cptpf;
+};
+
+/* Devlink APIs */
+int otx2_cpt_register_dl(struct otx2_cptpf_dev *cptpf);
+void otx2_cpt_unregister_dl(struct otx2_cptpf_dev *cptpf);
+
+#endif /* __OTX2_CPT_DEVLINK_H */
diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptpf.h b/drivers/crypto/marvell/octeontx2/otx2_cptpf.h
index 5ebba86c65d9..05b2d9c650e1 100644
--- a/drivers/crypto/marvell/octeontx2/otx2_cptpf.h
+++ b/drivers/crypto/marvell/octeontx2/otx2_cptpf.h
@@ -53,6 +53,9 @@ struct otx2_cptpf_dev {
u8 enabled_vfs; /* Number of enabled VFs */
u8 kvf_limits; /* Kernel crypto limits */
bool has_cpt1;
+
+ /* Devlink */
+ struct devlink *dl;
};
irqreturn_t otx2_cptpf_afpf_mbox_intr(int irq, void *arg);
diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptpf_main.c b/drivers/crypto/marvell/octeontx2/otx2_cptpf_main.c
index 146a55ac4b9b..1720a5bb7016 100644
--- a/drivers/crypto/marvell/octeontx2/otx2_cptpf_main.c
+++ b/drivers/crypto/marvell/octeontx2/otx2_cptpf_main.c
@@ -4,6 +4,7 @@
#include <linux/firmware.h>
#include "otx2_cpt_hw_types.h"
#include "otx2_cpt_common.h"
+#include "otx2_cpt_devlink.h"
#include "otx2_cptpf_ucode.h"
#include "otx2_cptpf.h"
#include "cn10k_cpt.h"
@@ -494,12 +495,11 @@ static ssize_t kvf_limits_store(struct device *dev,
{
struct otx2_cptpf_dev *cptpf = dev_get_drvdata(dev);
int lfs_num;
+ int ret;
- if (kstrtoint(buf, 0, &lfs_num)) {
- dev_err(dev, "lfs count %d must be in range [1 - %d]\n",
- lfs_num, num_online_cpus());
- return -EINVAL;
- }
+ ret = kstrtoint(buf, 0, &lfs_num);
+ if (ret)
+ return ret;
if (lfs_num < 1 || lfs_num > num_online_cpus()) {
dev_err(dev, "lfs count %d must be in range [1 - %d]\n",
lfs_num, num_online_cpus());
@@ -767,8 +767,15 @@ static int otx2_cptpf_probe(struct pci_dev *pdev,
err = sysfs_create_group(&dev->kobj, &cptpf_sysfs_group);
if (err)
goto cleanup_eng_grps;
+
+ err = otx2_cpt_register_dl(cptpf);
+ if (err)
+ goto sysfs_grp_del;
+
return 0;
+sysfs_grp_del:
+ sysfs_remove_group(&dev->kobj, &cptpf_sysfs_group);
cleanup_eng_grps:
otx2_cpt_cleanup_eng_grps(pdev, &cptpf->eng_grps);
unregister_intr:
@@ -788,6 +795,7 @@ static void otx2_cptpf_remove(struct pci_dev *pdev)
return;
cptpf_sriov_disable(pdev);
+ otx2_cpt_unregister_dl(cptpf);
/* Delete sysfs entry created for kernel VF limits */
sysfs_remove_group(&pdev->dev.kobj, &cptpf_sysfs_group);
/* Cleanup engine groups */
diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c b/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c
index dff34b3ec09e..4c8ebdf671ca 100644
--- a/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c
+++ b/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c
@@ -29,7 +29,8 @@ static struct otx2_cpt_bitmap get_cores_bmap(struct device *dev,
bool found = false;
int i;
- if (eng_grp->g->engs_num > OTX2_CPT_MAX_ENGINES) {
+ if (eng_grp->g->engs_num < 0 ||
+ eng_grp->g->engs_num > OTX2_CPT_MAX_ENGINES) {
dev_err(dev, "unsupported number of engines %d on octeontx2\n",
eng_grp->g->engs_num);
return bmap;
@@ -1110,18 +1111,19 @@ int otx2_cpt_create_eng_grps(struct otx2_cptpf_dev *cptpf,
struct otx2_cpt_engines engs[OTX2_CPT_MAX_ETYPES_PER_GRP] = { {0} };
struct pci_dev *pdev = cptpf->pdev;
struct fw_info_t fw_info;
- int ret;
+ int ret = 0;
+ mutex_lock(&eng_grps->lock);
/*
* We don't create engine groups if it was already
* made (when user enabled VFs for the first time)
*/
if (eng_grps->is_grps_created)
- return 0;
+ goto unlock;
ret = cpt_ucode_load_fw(pdev, &fw_info);
if (ret)
- return ret;
+ goto unlock;
/*
* Create engine group with SE engines for kernel
@@ -1186,7 +1188,7 @@ int otx2_cpt_create_eng_grps(struct otx2_cptpf_dev *cptpf,
cpt_ucode_release_fw(&fw_info);
if (is_dev_otx2(pdev))
- return 0;
+ goto unlock;
/*
* Configure engine group mask to allow context prefetching
* for the groups.
@@ -1201,12 +1203,15 @@ int otx2_cpt_create_eng_grps(struct otx2_cptpf_dev *cptpf,
*/
otx2_cpt_write_af_reg(&cptpf->afpf_mbox, pdev, CPT_AF_CTX_FLUSH_TIMER,
CTX_FLUSH_TIMER_CNT, BLKADDR_CPT0);
+ mutex_unlock(&eng_grps->lock);
return 0;
delete_eng_grp:
delete_engine_grps(pdev, eng_grps);
release_fw:
cpt_ucode_release_fw(&fw_info);
+unlock:
+ mutex_unlock(&eng_grps->lock);
return ret;
}
@@ -1286,6 +1291,7 @@ void otx2_cpt_cleanup_eng_grps(struct pci_dev *pdev,
struct otx2_cpt_eng_grp_info *grp;
int i, j;
+ mutex_lock(&eng_grps->lock);
delete_engine_grps(pdev, eng_grps);
/* Release memory */
for (i = 0; i < OTX2_CPT_MAX_ENGINE_GROUPS; i++) {
@@ -1295,6 +1301,7 @@ void otx2_cpt_cleanup_eng_grps(struct pci_dev *pdev,
grp->engs[j].bmap = NULL;
}
}
+ mutex_unlock(&eng_grps->lock);
}
int otx2_cpt_init_eng_grps(struct pci_dev *pdev,
@@ -1303,6 +1310,7 @@ int otx2_cpt_init_eng_grps(struct pci_dev *pdev,
struct otx2_cpt_eng_grp_info *grp;
int i, j, ret;
+ mutex_init(&eng_grps->lock);
eng_grps->obj = pci_get_drvdata(pdev);
eng_grps->avail.se_cnt = eng_grps->avail.max_se_cnt;
eng_grps->avail.ie_cnt = eng_grps->avail.max_ie_cnt;
@@ -1349,11 +1357,14 @@ static int create_eng_caps_discovery_grps(struct pci_dev *pdev,
struct fw_info_t fw_info;
int ret;
+ mutex_lock(&eng_grps->lock);
ret = cpt_ucode_load_fw(pdev, &fw_info);
- if (ret)
+ if (ret) {
+ mutex_unlock(&eng_grps->lock);
return ret;
+ }
- uc_info[0] = get_ucode(&fw_info, OTX2_CPT_SE_TYPES);
+ uc_info[0] = get_ucode(&fw_info, OTX2_CPT_AE_TYPES);
if (uc_info[0] == NULL) {
dev_err(&pdev->dev, "Unable to find firmware for AE\n");
ret = -EINVAL;
@@ -1396,12 +1407,14 @@ static int create_eng_caps_discovery_grps(struct pci_dev *pdev,
goto delete_eng_grp;
cpt_ucode_release_fw(&fw_info);
+ mutex_unlock(&eng_grps->lock);
return 0;
delete_eng_grp:
delete_engine_grps(pdev, eng_grps);
release_fw:
cpt_ucode_release_fw(&fw_info);
+ mutex_unlock(&eng_grps->lock);
return ret;
}
@@ -1501,3 +1514,291 @@ delete_grps:
return ret;
}
+
+int otx2_cpt_dl_custom_egrp_create(struct otx2_cptpf_dev *cptpf,
+ struct devlink_param_gset_ctx *ctx)
+{
+ struct otx2_cpt_engines engs[OTX2_CPT_MAX_ETYPES_PER_GRP] = { { 0 } };
+ struct otx2_cpt_uc_info_t *uc_info[OTX2_CPT_MAX_ETYPES_PER_GRP] = {};
+ struct otx2_cpt_eng_grps *eng_grps = &cptpf->eng_grps;
+ char *ucode_filename[OTX2_CPT_MAX_ETYPES_PER_GRP];
+ char tmp_buf[OTX2_CPT_NAME_LENGTH] = { 0 };
+ struct device *dev = &cptpf->pdev->dev;
+ char *start, *val, *err_msg, *tmp;
+ int grp_idx = 0, ret = -EINVAL;
+ bool has_se, has_ie, has_ae;
+ struct fw_info_t fw_info;
+ int ucode_idx = 0;
+
+ if (!eng_grps->is_grps_created) {
+ dev_err(dev, "Not allowed before creating the default groups\n");
+ return -EINVAL;
+ }
+ err_msg = "Invalid engine group format";
+ strscpy(tmp_buf, ctx->val.vstr, strlen(ctx->val.vstr) + 1);
+ start = tmp_buf;
+
+ has_se = has_ie = has_ae = false;
+
+ for (;;) {
+ val = strsep(&start, ";");
+ if (!val)
+ break;
+ val = strim(val);
+ if (!*val)
+ continue;
+
+ if (!strncasecmp(val, "se", 2) && strchr(val, ':')) {
+ if (has_se || ucode_idx)
+ goto err_print;
+ tmp = strim(strsep(&val, ":"));
+ if (!val)
+ goto err_print;
+ if (strlen(tmp) != 2)
+ goto err_print;
+ if (kstrtoint(strim(val), 10, &engs[grp_idx].count))
+ goto err_print;
+ engs[grp_idx++].type = OTX2_CPT_SE_TYPES;
+ has_se = true;
+ } else if (!strncasecmp(val, "ae", 2) && strchr(val, ':')) {
+ if (has_ae || ucode_idx)
+ goto err_print;
+ tmp = strim(strsep(&val, ":"));
+ if (!val)
+ goto err_print;
+ if (strlen(tmp) != 2)
+ goto err_print;
+ if (kstrtoint(strim(val), 10, &engs[grp_idx].count))
+ goto err_print;
+ engs[grp_idx++].type = OTX2_CPT_AE_TYPES;
+ has_ae = true;
+ } else if (!strncasecmp(val, "ie", 2) && strchr(val, ':')) {
+ if (has_ie || ucode_idx)
+ goto err_print;
+ tmp = strim(strsep(&val, ":"));
+ if (!val)
+ goto err_print;
+ if (strlen(tmp) != 2)
+ goto err_print;
+ if (kstrtoint(strim(val), 10, &engs[grp_idx].count))
+ goto err_print;
+ engs[grp_idx++].type = OTX2_CPT_IE_TYPES;
+ has_ie = true;
+ } else {
+ if (ucode_idx > 1)
+ goto err_print;
+ if (!strlen(val))
+ goto err_print;
+ if (strnstr(val, " ", strlen(val)))
+ goto err_print;
+ ucode_filename[ucode_idx++] = val;
+ }
+ }
+
+ /* Validate input parameters */
+ if (!(grp_idx && ucode_idx))
+ goto err_print;
+
+ if (ucode_idx > 1 && grp_idx < 2)
+ goto err_print;
+
+ if (grp_idx > OTX2_CPT_MAX_ETYPES_PER_GRP) {
+ err_msg = "Error max 2 engine types can be attached";
+ goto err_print;
+ }
+
+ if (grp_idx > 1) {
+ if ((engs[0].type + engs[1].type) !=
+ (OTX2_CPT_SE_TYPES + OTX2_CPT_IE_TYPES)) {
+ err_msg = "Only combination of SE+IE engines is allowed";
+ goto err_print;
+ }
+ /* Keep SE engines at zero index */
+ if (engs[1].type == OTX2_CPT_SE_TYPES)
+ swap(engs[0], engs[1]);
+ }
+ mutex_lock(&eng_grps->lock);
+
+ if (cptpf->enabled_vfs) {
+ dev_err(dev, "Disable VFs before modifying engine groups\n");
+ ret = -EACCES;
+ goto err_unlock;
+ }
+ INIT_LIST_HEAD(&fw_info.ucodes);
+ ret = load_fw(dev, &fw_info, ucode_filename[0]);
+ if (ret) {
+ dev_err(dev, "Unable to load firmware %s\n", ucode_filename[0]);
+ goto err_unlock;
+ }
+ if (ucode_idx > 1) {
+ ret = load_fw(dev, &fw_info, ucode_filename[1]);
+ if (ret) {
+ dev_err(dev, "Unable to load firmware %s\n",
+ ucode_filename[1]);
+ goto release_fw;
+ }
+ }
+ uc_info[0] = get_ucode(&fw_info, engs[0].type);
+ if (uc_info[0] == NULL) {
+ dev_err(dev, "Unable to find firmware for %s\n",
+ get_eng_type_str(engs[0].type));
+ ret = -EINVAL;
+ goto release_fw;
+ }
+ if (ucode_idx > 1) {
+ uc_info[1] = get_ucode(&fw_info, engs[1].type);
+ if (uc_info[1] == NULL) {
+ dev_err(dev, "Unable to find firmware for %s\n",
+ get_eng_type_str(engs[1].type));
+ ret = -EINVAL;
+ goto release_fw;
+ }
+ }
+ ret = create_engine_group(dev, eng_grps, engs, grp_idx,
+ (void **)uc_info, 1);
+
+release_fw:
+ cpt_ucode_release_fw(&fw_info);
+err_unlock:
+ mutex_unlock(&eng_grps->lock);
+ return ret;
+err_print:
+ dev_err(dev, "%s\n", err_msg);
+ return ret;
+}
+
+int otx2_cpt_dl_custom_egrp_delete(struct otx2_cptpf_dev *cptpf,
+ struct devlink_param_gset_ctx *ctx)
+{
+ struct otx2_cpt_eng_grps *eng_grps = &cptpf->eng_grps;
+ struct device *dev = &cptpf->pdev->dev;
+ char *tmp, *err_msg;
+ int egrp;
+ int ret;
+
+ err_msg = "Invalid input string format(ex: egrp:0)";
+ if (strncasecmp(ctx->val.vstr, "egrp", 4))
+ goto err_print;
+ tmp = ctx->val.vstr;
+ strsep(&tmp, ":");
+ if (!tmp)
+ goto err_print;
+ if (kstrtoint(tmp, 10, &egrp))
+ goto err_print;
+
+ if (egrp < 0 || egrp >= OTX2_CPT_MAX_ENGINE_GROUPS) {
+ dev_err(dev, "Invalid engine group %d", egrp);
+ return -EINVAL;
+ }
+ if (!eng_grps->grp[egrp].is_enabled) {
+ dev_err(dev, "Error engine_group%d is not configured", egrp);
+ return -EINVAL;
+ }
+ mutex_lock(&eng_grps->lock);
+ ret = delete_engine_group(dev, &eng_grps->grp[egrp]);
+ mutex_unlock(&eng_grps->lock);
+
+ return ret;
+
+err_print:
+ dev_err(dev, "%s\n", err_msg);
+ return -EINVAL;
+}
+
+static void get_engs_info(struct otx2_cpt_eng_grp_info *eng_grp, char *buf,
+ int size, int idx)
+{
+ struct otx2_cpt_engs_rsvd *mirrored_engs = NULL;
+ struct otx2_cpt_engs_rsvd *engs;
+ int len, i;
+
+ buf[0] = '\0';
+ for (i = 0; i < OTX2_CPT_MAX_ETYPES_PER_GRP; i++) {
+ engs = &eng_grp->engs[i];
+ if (!engs->type)
+ continue;
+ if (idx != -1 && idx != i)
+ continue;
+
+ if (eng_grp->mirror.is_ena)
+ mirrored_engs = find_engines_by_type(
+ &eng_grp->g->grp[eng_grp->mirror.idx],
+ engs->type);
+ if (i > 0 && idx == -1) {
+ len = strlen(buf);
+ scnprintf(buf + len, size - len, ", ");
+ }
+
+ len = strlen(buf);
+ scnprintf(buf + len, size - len, "%d %s ",
+ mirrored_engs ? engs->count + mirrored_engs->count :
+ engs->count,
+ get_eng_type_str(engs->type));
+ if (mirrored_engs) {
+ len = strlen(buf);
+ scnprintf(buf + len, size - len,
+ "(%d shared with engine_group%d) ",
+ engs->count <= 0 ?
+ engs->count + mirrored_engs->count :
+ mirrored_engs->count,
+ eng_grp->mirror.idx);
+ }
+ }
+}
+
+void otx2_cpt_print_uc_dbg_info(struct otx2_cptpf_dev *cptpf)
+{
+ struct otx2_cpt_eng_grps *eng_grps = &cptpf->eng_grps;
+ struct otx2_cpt_eng_grp_info *mirrored_grp;
+ char engs_info[2 * OTX2_CPT_NAME_LENGTH];
+ struct otx2_cpt_eng_grp_info *grp;
+ struct otx2_cpt_engs_rsvd *engs;
+ u32 mask[4];
+ int i, j;
+
+ pr_debug("Engine groups global info");
+ pr_debug("max SE %d, max IE %d, max AE %d", eng_grps->avail.max_se_cnt,
+ eng_grps->avail.max_ie_cnt, eng_grps->avail.max_ae_cnt);
+ pr_debug("free SE %d", eng_grps->avail.se_cnt);
+ pr_debug("free IE %d", eng_grps->avail.ie_cnt);
+ pr_debug("free AE %d", eng_grps->avail.ae_cnt);
+
+ for (i = 0; i < OTX2_CPT_MAX_ENGINE_GROUPS; i++) {
+ grp = &eng_grps->grp[i];
+ pr_debug("engine_group%d, state %s", i,
+ grp->is_enabled ? "enabled" : "disabled");
+ if (grp->is_enabled) {
+ mirrored_grp = &eng_grps->grp[grp->mirror.idx];
+ pr_debug("Ucode0 filename %s, version %s",
+ grp->mirror.is_ena ?
+ mirrored_grp->ucode[0].filename :
+ grp->ucode[0].filename,
+ grp->mirror.is_ena ?
+ mirrored_grp->ucode[0].ver_str :
+ grp->ucode[0].ver_str);
+ if (is_2nd_ucode_used(grp))
+ pr_debug("Ucode1 filename %s, version %s",
+ grp->ucode[1].filename,
+ grp->ucode[1].ver_str);
+ }
+
+ for (j = 0; j < OTX2_CPT_MAX_ETYPES_PER_GRP; j++) {
+ engs = &grp->engs[j];
+ if (engs->type) {
+ get_engs_info(grp, engs_info,
+ 2 * OTX2_CPT_NAME_LENGTH, j);
+ pr_debug("Slot%d: %s", j, engs_info);
+ bitmap_to_arr32(mask, engs->bmap,
+ eng_grps->engs_num);
+ if (is_dev_otx2(cptpf->pdev))
+ pr_debug("Mask: %8.8x %8.8x %8.8x %8.8x",
+ mask[3], mask[2], mask[1],
+ mask[0]);
+ else
+ pr_debug("Mask: %8.8x %8.8x %8.8x %8.8x %8.8x",
+ mask[4], mask[3], mask[2], mask[1],
+ mask[0]);
+ }
+ }
+ }
+}
diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.h b/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.h
index fe019ab730b2..8f4d4e5f531a 100644
--- a/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.h
+++ b/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.h
@@ -143,6 +143,7 @@ struct otx2_cpt_eng_grp_info {
};
struct otx2_cpt_eng_grps {
+ struct mutex lock;
struct otx2_cpt_eng_grp_info grp[OTX2_CPT_MAX_ENGINE_GROUPS];
struct otx2_cpt_engs_available avail;
void *obj; /* device specific data */
@@ -160,5 +161,9 @@ int otx2_cpt_create_eng_grps(struct otx2_cptpf_dev *cptpf,
int otx2_cpt_disable_all_cores(struct otx2_cptpf_dev *cptpf);
int otx2_cpt_get_eng_grp(struct otx2_cpt_eng_grps *eng_grps, int eng_type);
int otx2_cpt_discover_eng_capabilities(struct otx2_cptpf_dev *cptpf);
-
+int otx2_cpt_dl_custom_egrp_create(struct otx2_cptpf_dev *cptpf,
+ struct devlink_param_gset_ctx *ctx);
+int otx2_cpt_dl_custom_egrp_delete(struct otx2_cptpf_dev *cptpf,
+ struct devlink_param_gset_ctx *ctx);
+void otx2_cpt_print_uc_dbg_info(struct otx2_cptpf_dev *cptpf);
#endif /* __OTX2_CPTPF_UCODE_H */
diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptvf_algs.c b/drivers/crypto/marvell/octeontx2/otx2_cptvf_algs.c
index 877a948469bd..2748a3327e39 100644
--- a/drivers/crypto/marvell/octeontx2/otx2_cptvf_algs.c
+++ b/drivers/crypto/marvell/octeontx2/otx2_cptvf_algs.c
@@ -1682,11 +1682,8 @@ static void swap_func(void *lptr, void *rptr, int size)
{
struct cpt_device_desc *ldesc = lptr;
struct cpt_device_desc *rdesc = rptr;
- struct cpt_device_desc desc;
- desc = *ldesc;
- *ldesc = *rdesc;
- *rdesc = desc;
+ swap(*ldesc, *rdesc);
}
int otx2_cpt_crypto_init(struct pci_dev *pdev, struct module *mod,
diff --git a/drivers/crypto/omap-aes.c b/drivers/crypto/omap-aes.c
index 9b968ac4ee7b..a196bb8b1701 100644
--- a/drivers/crypto/omap-aes.c
+++ b/drivers/crypto/omap-aes.c
@@ -1302,7 +1302,7 @@ static int omap_aes_suspend(struct device *dev)
static int omap_aes_resume(struct device *dev)
{
- pm_runtime_resume_and_get(dev);
+ pm_runtime_get_sync(dev);
return 0;
}
#endif
diff --git a/drivers/crypto/omap-des.c b/drivers/crypto/omap-des.c
index be77656864e3..538aff80869f 100644
--- a/drivers/crypto/omap-des.c
+++ b/drivers/crypto/omap-des.c
@@ -735,7 +735,7 @@ static struct skcipher_alg algs_ecb_cbc[] = {
{
.base.cra_name = "ecb(des)",
.base.cra_driver_name = "ecb-des-omap",
- .base.cra_priority = 100,
+ .base.cra_priority = 300,
.base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY |
CRYPTO_ALG_ASYNC,
.base.cra_blocksize = DES_BLOCK_SIZE,
@@ -752,7 +752,7 @@ static struct skcipher_alg algs_ecb_cbc[] = {
{
.base.cra_name = "cbc(des)",
.base.cra_driver_name = "cbc-des-omap",
- .base.cra_priority = 100,
+ .base.cra_priority = 300,
.base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY |
CRYPTO_ALG_ASYNC,
.base.cra_blocksize = DES_BLOCK_SIZE,
@@ -770,7 +770,7 @@ static struct skcipher_alg algs_ecb_cbc[] = {
{
.base.cra_name = "ecb(des3_ede)",
.base.cra_driver_name = "ecb-des3-omap",
- .base.cra_priority = 100,
+ .base.cra_priority = 300,
.base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY |
CRYPTO_ALG_ASYNC,
.base.cra_blocksize = DES3_EDE_BLOCK_SIZE,
@@ -787,7 +787,7 @@ static struct skcipher_alg algs_ecb_cbc[] = {
{
.base.cra_name = "cbc(des3_ede)",
.base.cra_driver_name = "cbc-des3-omap",
- .base.cra_priority = 100,
+ .base.cra_priority = 300,
.base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY |
CRYPTO_ALG_ASYNC,
.base.cra_blocksize = DES3_EDE_BLOCK_SIZE,
diff --git a/drivers/crypto/qat/Kconfig b/drivers/crypto/qat/Kconfig
index 77783feb62b2..4b90c0f22b03 100644
--- a/drivers/crypto/qat/Kconfig
+++ b/drivers/crypto/qat/Kconfig
@@ -13,6 +13,7 @@ config CRYPTO_DEV_QAT
select CRYPTO_SHA512
select CRYPTO_LIB_AES
select FW_LOADER
+ select CRC8
config CRYPTO_DEV_QAT_DH895xCC
tristate "Support for Intel(R) DH895xCC"
diff --git a/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c b/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c
index fd29861526d6..6d10edc40aca 100644
--- a/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c
+++ b/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c
@@ -1,10 +1,11 @@
// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
-/* Copyright(c) 2020 Intel Corporation */
+/* Copyright(c) 2020 - 2021 Intel Corporation */
#include <linux/iopoll.h>
#include <adf_accel_devices.h>
+#include <adf_cfg.h>
#include <adf_common_drv.h>
-#include <adf_pf2vf_msg.h>
#include <adf_gen4_hw_data.h>
+#include <adf_gen4_pfvf.h>
#include "adf_4xxx_hw_data.h"
#include "icp_qat_hw.h"
@@ -13,12 +14,18 @@ struct adf_fw_config {
char *obj_name;
};
-static struct adf_fw_config adf_4xxx_fw_config[] = {
+static struct adf_fw_config adf_4xxx_fw_cy_config[] = {
{0xF0, ADF_4XXX_SYM_OBJ},
{0xF, ADF_4XXX_ASYM_OBJ},
{0x100, ADF_4XXX_ADMIN_OBJ},
};
+static struct adf_fw_config adf_4xxx_fw_dc_config[] = {
+ {0xF0, ADF_4XXX_DC_OBJ},
+ {0xF, ADF_4XXX_DC_OBJ},
+ {0x100, ADF_4XXX_ADMIN_OBJ},
+};
+
/* Worker thread to service arbiter mappings */
static const u32 thrd_to_arb_map[ADF_4XXX_MAX_ACCELENGINES] = {
0x5555555, 0x5555555, 0x5555555, 0x5555555,
@@ -32,6 +39,39 @@ static struct adf_hw_device_class adf_4xxx_class = {
.instances = 0,
};
+enum dev_services {
+ SVC_CY = 0,
+ SVC_DC,
+};
+
+static const char *const dev_cfg_services[] = {
+ [SVC_CY] = ADF_CFG_CY,
+ [SVC_DC] = ADF_CFG_DC,
+};
+
+static int get_service_enabled(struct adf_accel_dev *accel_dev)
+{
+ char services[ADF_CFG_MAX_VAL_LEN_IN_BYTES] = {0};
+ u32 ret;
+
+ ret = adf_cfg_get_param_value(accel_dev, ADF_GENERAL_SEC,
+ ADF_SERVICES_ENABLED, services);
+ if (ret) {
+ dev_err(&GET_DEV(accel_dev),
+ ADF_SERVICES_ENABLED " param not found\n");
+ return ret;
+ }
+
+ ret = match_string(dev_cfg_services, ARRAY_SIZE(dev_cfg_services),
+ services);
+ if (ret < 0)
+ dev_err(&GET_DEV(accel_dev),
+ "Invalid value of " ADF_SERVICES_ENABLED " param: %s\n",
+ services);
+
+ return ret;
+}
+
static u32 get_accel_mask(struct adf_hw_device_data *self)
{
return ADF_4XXX_ACCELERATORS_MASK;
@@ -96,23 +136,67 @@ static void set_msix_default_rttable(struct adf_accel_dev *accel_dev)
static u32 get_accel_cap(struct adf_accel_dev *accel_dev)
{
struct pci_dev *pdev = accel_dev->accel_pci_dev.pci_dev;
+ u32 capabilities_cy, capabilities_dc;
u32 fusectl1;
- u32 capabilities = ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC |
- ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC |
- ICP_ACCEL_CAPABILITIES_AUTHENTICATION |
- ICP_ACCEL_CAPABILITIES_AES_V2;
/* Read accelerator capabilities mask */
pci_read_config_dword(pdev, ADF_4XXX_FUSECTL1_OFFSET, &fusectl1);
- if (fusectl1 & ICP_ACCEL_4XXX_MASK_CIPHER_SLICE)
- capabilities &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC;
- if (fusectl1 & ICP_ACCEL_4XXX_MASK_AUTH_SLICE)
- capabilities &= ~ICP_ACCEL_CAPABILITIES_AUTHENTICATION;
- if (fusectl1 & ICP_ACCEL_4XXX_MASK_PKE_SLICE)
- capabilities &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC;
+ capabilities_cy = ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC |
+ ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC |
+ ICP_ACCEL_CAPABILITIES_CIPHER |
+ ICP_ACCEL_CAPABILITIES_AUTHENTICATION |
+ ICP_ACCEL_CAPABILITIES_SHA3 |
+ ICP_ACCEL_CAPABILITIES_SHA3_EXT |
+ ICP_ACCEL_CAPABILITIES_HKDF |
+ ICP_ACCEL_CAPABILITIES_ECEDMONT |
+ ICP_ACCEL_CAPABILITIES_CHACHA_POLY |
+ ICP_ACCEL_CAPABILITIES_AESGCM_SPC |
+ ICP_ACCEL_CAPABILITIES_AES_V2;
+
+ /* A set bit in fusectl1 means the feature is OFF in this SKU */
+ if (fusectl1 & ICP_ACCEL_4XXX_MASK_CIPHER_SLICE) {
+ capabilities_cy &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC;
+ capabilities_cy &= ~ICP_ACCEL_CAPABILITIES_HKDF;
+ capabilities_cy &= ~ICP_ACCEL_CAPABILITIES_CIPHER;
+ }
+ if (fusectl1 & ICP_ACCEL_4XXX_MASK_UCS_SLICE) {
+ capabilities_cy &= ~ICP_ACCEL_CAPABILITIES_CHACHA_POLY;
+ capabilities_cy &= ~ICP_ACCEL_CAPABILITIES_AESGCM_SPC;
+ capabilities_cy &= ~ICP_ACCEL_CAPABILITIES_AES_V2;
+ capabilities_cy &= ~ICP_ACCEL_CAPABILITIES_CIPHER;
+ }
+ if (fusectl1 & ICP_ACCEL_4XXX_MASK_AUTH_SLICE) {
+ capabilities_cy &= ~ICP_ACCEL_CAPABILITIES_AUTHENTICATION;
+ capabilities_cy &= ~ICP_ACCEL_CAPABILITIES_SHA3;
+ capabilities_cy &= ~ICP_ACCEL_CAPABILITIES_SHA3_EXT;
+ capabilities_cy &= ~ICP_ACCEL_CAPABILITIES_CIPHER;
+ }
+ if (fusectl1 & ICP_ACCEL_4XXX_MASK_PKE_SLICE) {
+ capabilities_cy &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC;
+ capabilities_cy &= ~ICP_ACCEL_CAPABILITIES_ECEDMONT;
+ }
+
+ capabilities_dc = ICP_ACCEL_CAPABILITIES_COMPRESSION |
+ ICP_ACCEL_CAPABILITIES_LZ4_COMPRESSION |
+ ICP_ACCEL_CAPABILITIES_LZ4S_COMPRESSION |
+ ICP_ACCEL_CAPABILITIES_CNV_INTEGRITY64;
+
+ if (fusectl1 & ICP_ACCEL_4XXX_MASK_COMPRESS_SLICE) {
+ capabilities_dc &= ~ICP_ACCEL_CAPABILITIES_COMPRESSION;
+ capabilities_dc &= ~ICP_ACCEL_CAPABILITIES_LZ4_COMPRESSION;
+ capabilities_dc &= ~ICP_ACCEL_CAPABILITIES_LZ4S_COMPRESSION;
+ capabilities_dc &= ~ICP_ACCEL_CAPABILITIES_CNV_INTEGRITY64;
+ }
+
+ switch (get_service_enabled(accel_dev)) {
+ case SVC_CY:
+ return capabilities_cy;
+ case SVC_DC:
+ return capabilities_dc;
+ }
- return capabilities;
+ return 0;
}
static enum dev_sku_info get_sku(struct adf_hw_device_data *self)
@@ -191,29 +275,36 @@ static int adf_init_device(struct adf_accel_dev *accel_dev)
return ret;
}
-static int pfvf_comms_disabled(struct adf_accel_dev *accel_dev)
-{
- return 0;
-}
-
static u32 uof_get_num_objs(void)
{
- return ARRAY_SIZE(adf_4xxx_fw_config);
-}
+ BUILD_BUG_ON_MSG(ARRAY_SIZE(adf_4xxx_fw_cy_config) !=
+ ARRAY_SIZE(adf_4xxx_fw_dc_config),
+ "Size mismatch between adf_4xxx_fw_*_config arrays");
-static char *uof_get_name(u32 obj_num)
-{
- return adf_4xxx_fw_config[obj_num].obj_name;
+ return ARRAY_SIZE(adf_4xxx_fw_cy_config);
}
-static u32 uof_get_ae_mask(u32 obj_num)
+static char *uof_get_name(struct adf_accel_dev *accel_dev, u32 obj_num)
{
- return adf_4xxx_fw_config[obj_num].ae_mask;
+ switch (get_service_enabled(accel_dev)) {
+ case SVC_CY:
+ return adf_4xxx_fw_cy_config[obj_num].obj_name;
+ case SVC_DC:
+ return adf_4xxx_fw_dc_config[obj_num].obj_name;
+ }
+
+ return NULL;
}
-static u32 get_vf2pf_sources(void __iomem *pmisc_addr)
+static u32 uof_get_ae_mask(struct adf_accel_dev *accel_dev, u32 obj_num)
{
- /* For the moment do not report vf2pf sources */
+ switch (get_service_enabled(accel_dev)) {
+ case SVC_CY:
+ return adf_4xxx_fw_cy_config[obj_num].ae_mask;
+ case SVC_DC:
+ return adf_4xxx_fw_dc_config[obj_num].ae_mask;
+ }
+
return 0;
}
@@ -222,12 +313,14 @@ void adf_init_hw_data_4xxx(struct adf_hw_device_data *hw_data)
hw_data->dev_class = &adf_4xxx_class;
hw_data->instance_id = adf_4xxx_class.instances++;
hw_data->num_banks = ADF_4XXX_ETR_MAX_BANKS;
+ hw_data->num_banks_per_vf = ADF_4XXX_NUM_BANKS_PER_VF;
hw_data->num_rings_per_bank = ADF_4XXX_NUM_RINGS_PER_BANK;
hw_data->num_accel = ADF_4XXX_MAX_ACCELERATORS;
hw_data->num_engines = ADF_4XXX_MAX_ACCELENGINES;
hw_data->num_logical_accel = 1;
hw_data->tx_rx_gap = ADF_4XXX_RX_RINGS_OFFSET;
hw_data->tx_rings_mask = ADF_4XXX_TX_RINGS_MASK;
+ hw_data->ring_to_svc_map = ADF_GEN4_DEFAULT_RING_TO_SRV_MAP;
hw_data->alloc_irq = adf_isr_resource_alloc;
hw_data->free_irq = adf_isr_resource_free;
hw_data->enable_error_correction = adf_enable_error_correction;
@@ -259,12 +352,11 @@ void adf_init_hw_data_4xxx(struct adf_hw_device_data *hw_data)
hw_data->uof_get_ae_mask = uof_get_ae_mask;
hw_data->set_msix_rttable = set_msix_default_rttable;
hw_data->set_ssm_wdtimer = adf_gen4_set_ssm_wdtimer;
- hw_data->enable_pfvf_comms = pfvf_comms_disabled;
- hw_data->get_vf2pf_sources = get_vf2pf_sources;
hw_data->disable_iov = adf_disable_sriov;
- hw_data->min_iov_compat_ver = ADF_PFVF_COMPAT_THIS_VERSION;
+ hw_data->ring_pair_reset = adf_gen4_ring_pair_reset;
adf_gen4_init_hw_csr_ops(&hw_data->csr_ops);
+ adf_gen4_init_pf_pfvf_ops(&hw_data->pfvf_ops);
}
void adf_clean_hw_data_4xxx(struct adf_hw_device_data *hw_data)
diff --git a/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.h b/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.h
index 924bac6feb37..12e4fb9b40ce 100644
--- a/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.h
+++ b/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.h
@@ -37,6 +37,7 @@
/* Bank and ring configuration */
#define ADF_4XXX_NUM_RINGS_PER_BANK 2
+#define ADF_4XXX_NUM_BANKS_PER_VF 4
/* Error source registers */
#define ADF_4XXX_ERRSOU0 (0x41A200)
@@ -76,6 +77,7 @@
#define ADF_4XXX_FW "qat_4xxx.bin"
#define ADF_4XXX_MMP "qat_4xxx_mmp.bin"
#define ADF_4XXX_SYM_OBJ "qat_4xxx_sym.bin"
+#define ADF_4XXX_DC_OBJ "qat_4xxx_dc.bin"
#define ADF_4XXX_ASYM_OBJ "qat_4xxx_asym.bin"
#define ADF_4XXX_ADMIN_OBJ "qat_4xxx_admin.bin"
diff --git a/drivers/crypto/qat/qat_4xxx/adf_drv.c b/drivers/crypto/qat/qat_4xxx/adf_drv.c
index 71ef065914b2..a6c78b9c730b 100644
--- a/drivers/crypto/qat/qat_4xxx/adf_drv.c
+++ b/drivers/crypto/qat/qat_4xxx/adf_drv.c
@@ -29,6 +29,29 @@ static void adf_cleanup_accel(struct adf_accel_dev *accel_dev)
adf_devmgr_rm_dev(accel_dev, NULL);
}
+static int adf_cfg_dev_init(struct adf_accel_dev *accel_dev)
+{
+ const char *config;
+ int ret;
+
+ config = accel_dev->accel_id % 2 ? ADF_CFG_DC : ADF_CFG_CY;
+
+ ret = adf_cfg_section_add(accel_dev, ADF_GENERAL_SEC);
+ if (ret)
+ return ret;
+
+ /* Default configuration is crypto only for even devices
+ * and compression for odd devices
+ */
+ ret = adf_cfg_add_key_value_param(accel_dev, ADF_GENERAL_SEC,
+ ADF_SERVICES_ENABLED, config,
+ ADF_STR);
+ if (ret)
+ return ret;
+
+ return 0;
+}
+
static int adf_crypto_dev_config(struct adf_accel_dev *accel_dev)
{
char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES];
@@ -227,8 +250,18 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
goto out_err;
}
+ ret = adf_cfg_dev_init(accel_dev);
+ if (ret) {
+ dev_err(&pdev->dev, "Failed to initialize configuration.\n");
+ goto out_err;
+ }
+
/* Get accelerator capabilities mask */
hw_data->accel_capabilities_mask = hw_data->get_accel_cap(accel_dev);
+ if (!hw_data->accel_capabilities_mask) {
+ dev_err(&pdev->dev, "Failed to get capabilities mask.\n");
+ goto out_err;
+ }
/* Find and map all the device's BARS */
bar_mask = pci_select_bars(pdev, IORESOURCE_MEM) & ADF_4XXX_BAR_MASK;
diff --git a/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c b/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c
index 1fa690219d92..b941fe3713ff 100644
--- a/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c
+++ b/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c
@@ -1,9 +1,9 @@
// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
-/* Copyright(c) 2014 - 2020 Intel Corporation */
+/* Copyright(c) 2014 - 2021 Intel Corporation */
#include <adf_accel_devices.h>
#include <adf_common_drv.h>
-#include <adf_pf2vf_msg.h>
#include <adf_gen2_hw_data.h>
+#include <adf_gen2_pfvf.h>
#include "adf_c3xxx_hw_data.h"
#include "icp_qat_hw.h"
@@ -109,6 +109,7 @@ void adf_init_hw_data_c3xxx(struct adf_hw_device_data *hw_data)
hw_data->num_engines = ADF_C3XXX_MAX_ACCELENGINES;
hw_data->tx_rx_gap = ADF_GEN2_RX_RINGS_OFFSET;
hw_data->tx_rings_mask = ADF_GEN2_TX_RINGS_MASK;
+ hw_data->ring_to_svc_map = ADF_GEN2_DEFAULT_RING_TO_SRV_MAP;
hw_data->alloc_irq = adf_isr_resource_alloc;
hw_data->free_irq = adf_isr_resource_free;
hw_data->enable_error_correction = adf_gen2_enable_error_correction;
@@ -135,14 +136,9 @@ void adf_init_hw_data_c3xxx(struct adf_hw_device_data *hw_data)
hw_data->enable_ints = adf_enable_ints;
hw_data->reset_device = adf_reset_flr;
hw_data->set_ssm_wdtimer = adf_gen2_set_ssm_wdtimer;
- hw_data->get_pf2vf_offset = adf_gen2_get_pf2vf_offset;
- hw_data->get_vf2pf_sources = adf_gen2_get_vf2pf_sources;
- hw_data->enable_vf2pf_interrupts = adf_gen2_enable_vf2pf_interrupts;
- hw_data->disable_vf2pf_interrupts = adf_gen2_disable_vf2pf_interrupts;
- hw_data->enable_pfvf_comms = adf_enable_pf2vf_comms;
hw_data->disable_iov = adf_disable_sriov;
- hw_data->min_iov_compat_ver = ADF_PFVF_COMPAT_THIS_VERSION;
+ adf_gen2_init_pf_pfvf_ops(&hw_data->pfvf_ops);
adf_gen2_init_hw_csr_ops(&hw_data->csr_ops);
}
diff --git a/drivers/crypto/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c b/drivers/crypto/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c
index 3e69b520e82f..a9fbe57b32ae 100644
--- a/drivers/crypto/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c
+++ b/drivers/crypto/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c
@@ -1,9 +1,10 @@
// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
-/* Copyright(c) 2015 - 2020 Intel Corporation */
+/* Copyright(c) 2015 - 2021 Intel Corporation */
#include <adf_accel_devices.h>
-#include <adf_pf2vf_msg.h>
#include <adf_common_drv.h>
#include <adf_gen2_hw_data.h>
+#include <adf_gen2_pfvf.h>
+#include <adf_pfvf_vf_msg.h>
#include "adf_c3xxxvf_hw_data.h"
static struct adf_hw_device_class c3xxxiov_class = {
@@ -47,11 +48,6 @@ static enum dev_sku_info get_sku(struct adf_hw_device_data *self)
return DEV_SKU_VF;
}
-static u32 get_pf2vf_offset(u32 i)
-{
- return ADF_C3XXXIOV_PF2VF_OFFSET;
-}
-
static int adf_vf_int_noop(struct adf_accel_dev *accel_dev)
{
return 0;
@@ -71,6 +67,7 @@ void adf_init_hw_data_c3xxxiov(struct adf_hw_device_data *hw_data)
hw_data->num_engines = ADF_C3XXXIOV_MAX_ACCELENGINES;
hw_data->tx_rx_gap = ADF_C3XXXIOV_RX_RINGS_OFFSET;
hw_data->tx_rings_mask = ADF_C3XXXIOV_TX_RINGS_MASK;
+ hw_data->ring_to_svc_map = ADF_GEN2_DEFAULT_RING_TO_SRV_MAP;
hw_data->alloc_irq = adf_vf_isr_resource_alloc;
hw_data->free_irq = adf_vf_isr_resource_free;
hw_data->enable_error_correction = adf_vf_void_noop;
@@ -86,13 +83,11 @@ void adf_init_hw_data_c3xxxiov(struct adf_hw_device_data *hw_data)
hw_data->get_num_aes = get_num_aes;
hw_data->get_etr_bar_id = get_etr_bar_id;
hw_data->get_misc_bar_id = get_misc_bar_id;
- hw_data->get_pf2vf_offset = get_pf2vf_offset;
hw_data->get_sku = get_sku;
hw_data->enable_ints = adf_vf_void_noop;
- hw_data->enable_pfvf_comms = adf_enable_vf2pf_comms;
- hw_data->min_iov_compat_ver = ADF_PFVF_COMPAT_THIS_VERSION;
hw_data->dev_class->instances++;
adf_devmgr_update_class_index(hw_data);
+ adf_gen2_init_vf_pfvf_ops(&hw_data->pfvf_ops);
adf_gen2_init_hw_csr_ops(&hw_data->csr_ops);
}
diff --git a/drivers/crypto/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.h b/drivers/crypto/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.h
index f5de4ce66014..6b4bf181d15b 100644
--- a/drivers/crypto/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.h
+++ b/drivers/crypto/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.h
@@ -12,7 +12,6 @@
#define ADF_C3XXXIOV_TX_RINGS_MASK 0xFF
#define ADF_C3XXXIOV_ETR_BAR 0
#define ADF_C3XXXIOV_ETR_MAX_BANKS 1
-#define ADF_C3XXXIOV_PF2VF_OFFSET 0x200
void adf_init_hw_data_c3xxxiov(struct adf_hw_device_data *hw_data);
void adf_clean_hw_data_c3xxxiov(struct adf_hw_device_data *hw_data);
diff --git a/drivers/crypto/qat/qat_c3xxxvf/adf_drv.c b/drivers/crypto/qat/qat_c3xxxvf/adf_drv.c
index 1df1b868978d..fa18d8009f53 100644
--- a/drivers/crypto/qat/qat_c3xxxvf/adf_drv.c
+++ b/drivers/crypto/qat/qat_c3xxxvf/adf_drv.c
@@ -171,11 +171,7 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
}
pci_set_master(pdev);
/* Completion for VF2PF request/response message exchange */
- init_completion(&accel_dev->vf.iov_msg_completion);
-
- ret = qat_crypto_dev_config(accel_dev);
- if (ret)
- goto out_err_free_reg;
+ init_completion(&accel_dev->vf.msg_received);
ret = adf_dev_init(accel_dev);
if (ret)
diff --git a/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c b/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c
index 0613db077689..b1eac2f81faa 100644
--- a/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c
+++ b/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c
@@ -1,9 +1,9 @@
// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
-/* Copyright(c) 2014 - 2020 Intel Corporation */
+/* Copyright(c) 2014 - 2021 Intel Corporation */
#include <adf_accel_devices.h>
#include <adf_common_drv.h>
-#include <adf_pf2vf_msg.h>
#include <adf_gen2_hw_data.h>
+#include <adf_gen2_pfvf.h>
#include "adf_c62x_hw_data.h"
#include "icp_qat_hw.h"
@@ -111,6 +111,7 @@ void adf_init_hw_data_c62x(struct adf_hw_device_data *hw_data)
hw_data->num_engines = ADF_C62X_MAX_ACCELENGINES;
hw_data->tx_rx_gap = ADF_GEN2_RX_RINGS_OFFSET;
hw_data->tx_rings_mask = ADF_GEN2_TX_RINGS_MASK;
+ hw_data->ring_to_svc_map = ADF_GEN2_DEFAULT_RING_TO_SRV_MAP;
hw_data->alloc_irq = adf_isr_resource_alloc;
hw_data->free_irq = adf_isr_resource_free;
hw_data->enable_error_correction = adf_gen2_enable_error_correction;
@@ -137,14 +138,9 @@ void adf_init_hw_data_c62x(struct adf_hw_device_data *hw_data)
hw_data->enable_ints = adf_enable_ints;
hw_data->reset_device = adf_reset_flr;
hw_data->set_ssm_wdtimer = adf_gen2_set_ssm_wdtimer;
- hw_data->get_pf2vf_offset = adf_gen2_get_pf2vf_offset;
- hw_data->get_vf2pf_sources = adf_gen2_get_vf2pf_sources;
- hw_data->enable_vf2pf_interrupts = adf_gen2_enable_vf2pf_interrupts;
- hw_data->disable_vf2pf_interrupts = adf_gen2_disable_vf2pf_interrupts;
- hw_data->enable_pfvf_comms = adf_enable_pf2vf_comms;
hw_data->disable_iov = adf_disable_sriov;
- hw_data->min_iov_compat_ver = ADF_PFVF_COMPAT_THIS_VERSION;
+ adf_gen2_init_pf_pfvf_ops(&hw_data->pfvf_ops);
adf_gen2_init_hw_csr_ops(&hw_data->csr_ops);
}
diff --git a/drivers/crypto/qat/qat_c62xvf/adf_c62xvf_hw_data.c b/drivers/crypto/qat/qat_c62xvf/adf_c62xvf_hw_data.c
index 3bee3e467363..0282038fca54 100644
--- a/drivers/crypto/qat/qat_c62xvf/adf_c62xvf_hw_data.c
+++ b/drivers/crypto/qat/qat_c62xvf/adf_c62xvf_hw_data.c
@@ -1,9 +1,10 @@
// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
-/* Copyright(c) 2015 - 2020 Intel Corporation */
+/* Copyright(c) 2015 - 2021 Intel Corporation */
#include <adf_accel_devices.h>
-#include <adf_pf2vf_msg.h>
#include <adf_common_drv.h>
#include <adf_gen2_hw_data.h>
+#include <adf_gen2_pfvf.h>
+#include <adf_pfvf_vf_msg.h>
#include "adf_c62xvf_hw_data.h"
static struct adf_hw_device_class c62xiov_class = {
@@ -47,11 +48,6 @@ static enum dev_sku_info get_sku(struct adf_hw_device_data *self)
return DEV_SKU_VF;
}
-static u32 get_pf2vf_offset(u32 i)
-{
- return ADF_C62XIOV_PF2VF_OFFSET;
-}
-
static int adf_vf_int_noop(struct adf_accel_dev *accel_dev)
{
return 0;
@@ -71,6 +67,7 @@ void adf_init_hw_data_c62xiov(struct adf_hw_device_data *hw_data)
hw_data->num_engines = ADF_C62XIOV_MAX_ACCELENGINES;
hw_data->tx_rx_gap = ADF_C62XIOV_RX_RINGS_OFFSET;
hw_data->tx_rings_mask = ADF_C62XIOV_TX_RINGS_MASK;
+ hw_data->ring_to_svc_map = ADF_GEN2_DEFAULT_RING_TO_SRV_MAP;
hw_data->alloc_irq = adf_vf_isr_resource_alloc;
hw_data->free_irq = adf_vf_isr_resource_free;
hw_data->enable_error_correction = adf_vf_void_noop;
@@ -86,13 +83,11 @@ void adf_init_hw_data_c62xiov(struct adf_hw_device_data *hw_data)
hw_data->get_num_aes = get_num_aes;
hw_data->get_etr_bar_id = get_etr_bar_id;
hw_data->get_misc_bar_id = get_misc_bar_id;
- hw_data->get_pf2vf_offset = get_pf2vf_offset;
hw_data->get_sku = get_sku;
hw_data->enable_ints = adf_vf_void_noop;
- hw_data->enable_pfvf_comms = adf_enable_vf2pf_comms;
- hw_data->min_iov_compat_ver = ADF_PFVF_COMPAT_THIS_VERSION;
hw_data->dev_class->instances++;
adf_devmgr_update_class_index(hw_data);
+ adf_gen2_init_vf_pfvf_ops(&hw_data->pfvf_ops);
adf_gen2_init_hw_csr_ops(&hw_data->csr_ops);
}
diff --git a/drivers/crypto/qat/qat_c62xvf/adf_c62xvf_hw_data.h b/drivers/crypto/qat/qat_c62xvf/adf_c62xvf_hw_data.h
index 794778c48678..a1a62c003ebf 100644
--- a/drivers/crypto/qat/qat_c62xvf/adf_c62xvf_hw_data.h
+++ b/drivers/crypto/qat/qat_c62xvf/adf_c62xvf_hw_data.h
@@ -12,7 +12,6 @@
#define ADF_C62XIOV_TX_RINGS_MASK 0xFF
#define ADF_C62XIOV_ETR_BAR 0
#define ADF_C62XIOV_ETR_MAX_BANKS 1
-#define ADF_C62XIOV_PF2VF_OFFSET 0x200
void adf_init_hw_data_c62xiov(struct adf_hw_device_data *hw_data);
void adf_clean_hw_data_c62xiov(struct adf_hw_device_data *hw_data);
diff --git a/drivers/crypto/qat/qat_c62xvf/adf_drv.c b/drivers/crypto/qat/qat_c62xvf/adf_drv.c
index 8103bd81d617..686ec752d0e9 100644
--- a/drivers/crypto/qat/qat_c62xvf/adf_drv.c
+++ b/drivers/crypto/qat/qat_c62xvf/adf_drv.c
@@ -171,11 +171,7 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
}
pci_set_master(pdev);
/* Completion for VF2PF request/response message exchange */
- init_completion(&accel_dev->vf.iov_msg_completion);
-
- ret = qat_crypto_dev_config(accel_dev);
- if (ret)
- goto out_err_free_reg;
+ init_completion(&accel_dev->vf.msg_received);
ret = adf_dev_init(accel_dev);
if (ret)
diff --git a/drivers/crypto/qat/qat_common/Makefile b/drivers/crypto/qat/qat_common/Makefile
index 9c57abdf56b7..7e191a42a5c7 100644
--- a/drivers/crypto/qat/qat_common/Makefile
+++ b/drivers/crypto/qat/qat_common/Makefile
@@ -19,5 +19,7 @@ intel_qat-objs := adf_cfg.o \
qat_hal.o
intel_qat-$(CONFIG_DEBUG_FS) += adf_transport_debug.o
-intel_qat-$(CONFIG_PCI_IOV) += adf_sriov.o adf_pf2vf_msg.o \
- adf_vf2pf_msg.o adf_vf_isr.o
+intel_qat-$(CONFIG_PCI_IOV) += adf_sriov.o adf_vf_isr.o adf_pfvf_utils.o \
+ adf_pfvf_pf_msg.o adf_pfvf_pf_proto.o \
+ adf_pfvf_vf_msg.o adf_pfvf_vf_proto.o \
+ adf_gen2_pfvf.o adf_gen4_pfvf.o
diff --git a/drivers/crypto/qat/qat_common/adf_accel_devices.h b/drivers/crypto/qat/qat_common/adf_accel_devices.h
index 57d9ca08e611..2d4cd7c7cf33 100644
--- a/drivers/crypto/qat/qat_common/adf_accel_devices.h
+++ b/drivers/crypto/qat/qat_common/adf_accel_devices.h
@@ -8,6 +8,7 @@
#include <linux/io.h>
#include <linux/ratelimit.h>
#include "adf_cfg_common.h"
+#include "adf_pfvf_msg.h"
#define ADF_DH895XCC_DEVICE_NAME "dh895xcc"
#define ADF_DH895XCCVF_DEVICE_NAME "dh895xccvf"
@@ -147,6 +148,19 @@ struct adf_accel_dev;
struct adf_etr_data;
struct adf_etr_ring_data;
+struct adf_pfvf_ops {
+ int (*enable_comms)(struct adf_accel_dev *accel_dev);
+ u32 (*get_pf2vf_offset)(u32 i);
+ u32 (*get_vf2pf_offset)(u32 i);
+ u32 (*get_vf2pf_sources)(void __iomem *pmisc_addr);
+ void (*enable_vf2pf_interrupts)(void __iomem *pmisc_addr, u32 vf_mask);
+ void (*disable_vf2pf_interrupts)(void __iomem *pmisc_addr, u32 vf_mask);
+ int (*send_msg)(struct adf_accel_dev *accel_dev, struct pfvf_message msg,
+ u32 pfvf_offset, struct mutex *csr_lock);
+ struct pfvf_message (*recv_msg)(struct adf_accel_dev *accel_dev,
+ u32 pfvf_offset, u8 compat_ver);
+};
+
struct adf_hw_device_data {
struct adf_hw_device_class *dev_class;
u32 (*get_accel_mask)(struct adf_hw_device_data *self);
@@ -157,7 +171,6 @@ struct adf_hw_device_data {
u32 (*get_etr_bar_id)(struct adf_hw_device_data *self);
u32 (*get_num_aes)(struct adf_hw_device_data *self);
u32 (*get_num_accels)(struct adf_hw_device_data *self);
- u32 (*get_pf2vf_offset)(u32 i);
void (*get_arb_info)(struct arb_info *arb_csrs_info);
void (*get_admin_info)(struct admin_info *admin_csrs_info);
enum dev_sku_info (*get_sku)(struct adf_hw_device_data *self);
@@ -176,35 +189,34 @@ struct adf_hw_device_data {
bool enable);
void (*enable_ints)(struct adf_accel_dev *accel_dev);
void (*set_ssm_wdtimer)(struct adf_accel_dev *accel_dev);
- int (*enable_pfvf_comms)(struct adf_accel_dev *accel_dev);
- u32 (*get_vf2pf_sources)(void __iomem *pmisc_addr);
- void (*enable_vf2pf_interrupts)(void __iomem *pmisc_bar_addr,
- u32 vf_mask);
- void (*disable_vf2pf_interrupts)(void __iomem *pmisc_bar_addr,
- u32 vf_mask);
+ int (*ring_pair_reset)(struct adf_accel_dev *accel_dev, u32 bank_nr);
void (*reset_device)(struct adf_accel_dev *accel_dev);
void (*set_msix_rttable)(struct adf_accel_dev *accel_dev);
- char *(*uof_get_name)(u32 obj_num);
+ char *(*uof_get_name)(struct adf_accel_dev *accel_dev, u32 obj_num);
u32 (*uof_get_num_objs)(void);
- u32 (*uof_get_ae_mask)(u32 obj_num);
+ u32 (*uof_get_ae_mask)(struct adf_accel_dev *accel_dev, u32 obj_num);
+ struct adf_pfvf_ops pfvf_ops;
struct adf_hw_csr_ops csr_ops;
const char *fw_name;
const char *fw_mmp_name;
u32 fuses;
u32 straps;
u32 accel_capabilities_mask;
+ u32 extended_dc_capabilities;
+ u32 clock_frequency;
u32 instance_id;
u16 accel_mask;
u32 ae_mask;
u32 admin_ae_mask;
u16 tx_rings_mask;
+ u16 ring_to_svc_map;
u8 tx_rx_gap;
u8 num_banks;
+ u16 num_banks_per_vf;
u8 num_rings_per_bank;
u8 num_accel;
u8 num_logical_accel;
u8 num_engines;
- u8 min_iov_compat_ver;
};
/* CSR write macro */
@@ -214,14 +226,22 @@ struct adf_hw_device_data {
/* CSR read macro */
#define ADF_CSR_RD(csr_base, csr_offset) __raw_readl(csr_base + csr_offset)
+#define ADF_CFG_NUM_SERVICES 4
+#define ADF_SRV_TYPE_BIT_LEN 3
+#define ADF_SRV_TYPE_MASK 0x7
+
#define GET_DEV(accel_dev) ((accel_dev)->accel_pci_dev.pci_dev->dev)
#define GET_BARS(accel_dev) ((accel_dev)->accel_pci_dev.pci_bars)
#define GET_HW_DATA(accel_dev) (accel_dev->hw_device)
#define GET_MAX_BANKS(accel_dev) (GET_HW_DATA(accel_dev)->num_banks)
#define GET_NUM_RINGS_PER_BANK(accel_dev) \
GET_HW_DATA(accel_dev)->num_rings_per_bank
+#define GET_SRV_TYPE(accel_dev, idx) \
+ (((GET_HW_DATA(accel_dev)->ring_to_svc_map) >> (ADF_SRV_TYPE_BIT_LEN * (idx))) \
+ & ADF_SRV_TYPE_MASK)
#define GET_MAX_ACCELENGINES(accel_dev) (GET_HW_DATA(accel_dev)->num_engines)
#define GET_CSR_OPS(accel_dev) (&(accel_dev)->hw_device->csr_ops)
+#define GET_PFVF_OPS(accel_dev) (&(accel_dev)->hw_device->pfvf_ops)
#define accel_to_pci_dev(accel_ptr) accel_ptr->accel_pci_dev.pci_dev
struct adf_admin_comms;
@@ -238,6 +258,7 @@ struct adf_accel_vf_info {
struct ratelimit_state vf2pf_ratelimit;
u32 vf_nr;
bool init;
+ u8 vf_compat_ver;
};
struct adf_accel_dev {
@@ -265,9 +286,9 @@ struct adf_accel_dev {
char irq_name[ADF_MAX_MSIX_VECTOR_NAME];
struct tasklet_struct pf2vf_bh_tasklet;
struct mutex vf2pf_lock; /* protect CSR access */
- struct completion iov_msg_completion;
- u8 compatible;
- u8 pf_version;
+ struct completion msg_received;
+ struct pfvf_message response; /* temp field holding pf2vf response */
+ u8 pf_compat_ver;
} vf;
};
bool is_vf;
diff --git a/drivers/crypto/qat/qat_common/adf_accel_engine.c b/drivers/crypto/qat/qat_common/adf_accel_engine.c
index ca4eae8cdd0b..4ce2b666929e 100644
--- a/drivers/crypto/qat/qat_common/adf_accel_engine.c
+++ b/drivers/crypto/qat/qat_common/adf_accel_engine.c
@@ -22,8 +22,12 @@ static int adf_ae_fw_load_images(struct adf_accel_dev *accel_dev, void *fw_addr,
num_objs = hw_device->uof_get_num_objs();
for (i = 0; i < num_objs; i++) {
- obj_name = hw_device->uof_get_name(i);
- ae_mask = hw_device->uof_get_ae_mask(i);
+ obj_name = hw_device->uof_get_name(accel_dev, i);
+ ae_mask = hw_device->uof_get_ae_mask(accel_dev, i);
+ if (!obj_name || !ae_mask) {
+ dev_err(&GET_DEV(accel_dev), "Invalid UOF image\n");
+ goto out_err;
+ }
if (qat_uclo_set_cfg_ae_mask(loader, ae_mask)) {
dev_err(&GET_DEV(accel_dev),
diff --git a/drivers/crypto/qat/qat_common/adf_admin.c b/drivers/crypto/qat/qat_common/adf_admin.c
index 43680e178242..498eb6f690e3 100644
--- a/drivers/crypto/qat/qat_common/adf_admin.c
+++ b/drivers/crypto/qat/qat_common/adf_admin.c
@@ -194,6 +194,35 @@ static int adf_set_fw_constants(struct adf_accel_dev *accel_dev)
return adf_send_admin(accel_dev, &req, &resp, ae_mask);
}
+static int adf_get_dc_capabilities(struct adf_accel_dev *accel_dev,
+ u32 *capabilities)
+{
+ struct adf_hw_device_data *hw_device = accel_dev->hw_device;
+ struct icp_qat_fw_init_admin_resp resp;
+ struct icp_qat_fw_init_admin_req req;
+ unsigned long ae_mask;
+ unsigned long ae;
+ int ret;
+
+ /* Target only service accelerator engines */
+ ae_mask = hw_device->ae_mask & ~hw_device->admin_ae_mask;
+
+ memset(&req, 0, sizeof(req));
+ memset(&resp, 0, sizeof(resp));
+ req.cmd_id = ICP_QAT_FW_COMP_CAPABILITY_GET;
+
+ *capabilities = 0;
+ for_each_set_bit(ae, &ae_mask, GET_MAX_ACCELENGINES(accel_dev)) {
+ ret = adf_send_admin(accel_dev, &req, &resp, 1ULL << ae);
+ if (ret)
+ return ret;
+
+ *capabilities |= resp.extended_features;
+ }
+
+ return 0;
+}
+
/**
* adf_send_admin_init() - Function sends init message to FW
* @accel_dev: Pointer to acceleration device.
@@ -204,8 +233,16 @@ static int adf_set_fw_constants(struct adf_accel_dev *accel_dev)
*/
int adf_send_admin_init(struct adf_accel_dev *accel_dev)
{
+ u32 dc_capabilities = 0;
int ret;
+ ret = adf_get_dc_capabilities(accel_dev, &dc_capabilities);
+ if (ret) {
+ dev_err(&GET_DEV(accel_dev), "Cannot get dc capabilities\n");
+ return ret;
+ }
+ accel_dev->hw_device->extended_dc_capabilities = dc_capabilities;
+
ret = adf_set_fw_constants(accel_dev);
if (ret)
return ret;
@@ -218,9 +255,7 @@ int adf_init_admin_comms(struct adf_accel_dev *accel_dev)
{
struct adf_admin_comms *admin;
struct adf_hw_device_data *hw_data = accel_dev->hw_device;
- struct adf_bar *pmisc =
- &GET_BARS(accel_dev)[hw_data->get_misc_bar_id(hw_data)];
- void __iomem *csr = pmisc->virt_addr;
+ void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev);
struct admin_info admin_csrs_info;
u32 mailbox_offset, adminmsg_u, adminmsg_l;
void __iomem *mailbox;
@@ -254,13 +289,13 @@ int adf_init_admin_comms(struct adf_accel_dev *accel_dev)
hw_data->get_admin_info(&admin_csrs_info);
mailbox_offset = admin_csrs_info.mailbox_offset;
- mailbox = csr + mailbox_offset;
+ mailbox = pmisc_addr + mailbox_offset;
adminmsg_u = admin_csrs_info.admin_msg_ur;
adminmsg_l = admin_csrs_info.admin_msg_lr;
reg_val = (u64)admin->phy_addr;
- ADF_CSR_WR(csr, adminmsg_u, upper_32_bits(reg_val));
- ADF_CSR_WR(csr, adminmsg_l, lower_32_bits(reg_val));
+ ADF_CSR_WR(pmisc_addr, adminmsg_u, upper_32_bits(reg_val));
+ ADF_CSR_WR(pmisc_addr, adminmsg_l, lower_32_bits(reg_val));
mutex_init(&admin->lock);
admin->mailbox_addr = mailbox;
diff --git a/drivers/crypto/qat/qat_common/adf_cfg.c b/drivers/crypto/qat/qat_common/adf_cfg.c
index 575b6f002303..b5b208cbe5a1 100644
--- a/drivers/crypto/qat/qat_common/adf_cfg.c
+++ b/drivers/crypto/qat/qat_common/adf_cfg.c
@@ -297,3 +297,4 @@ int adf_cfg_get_param_value(struct adf_accel_dev *accel_dev,
up_read(&cfg->lock);
return ret;
}
+EXPORT_SYMBOL_GPL(adf_cfg_get_param_value);
diff --git a/drivers/crypto/qat/qat_common/adf_cfg_common.h b/drivers/crypto/qat/qat_common/adf_cfg_common.h
index 4fabb70b1f18..6e5de1dab97b 100644
--- a/drivers/crypto/qat/qat_common/adf_cfg_common.h
+++ b/drivers/crypto/qat/qat_common/adf_cfg_common.h
@@ -19,6 +19,19 @@
#define ADF_MAX_DEVICES (32 * 32)
#define ADF_DEVS_ARRAY_SIZE BITS_TO_LONGS(ADF_MAX_DEVICES)
+#define ADF_CFG_SERV_RING_PAIR_0_SHIFT 0
+#define ADF_CFG_SERV_RING_PAIR_1_SHIFT 3
+#define ADF_CFG_SERV_RING_PAIR_2_SHIFT 6
+#define ADF_CFG_SERV_RING_PAIR_3_SHIFT 9
+enum adf_cfg_service_type {
+ UNUSED = 0,
+ CRYPTO,
+ COMP,
+ SYM,
+ ASYM,
+ USED
+};
+
enum adf_cfg_val_type {
ADF_DEC,
ADF_HEX,
diff --git a/drivers/crypto/qat/qat_common/adf_cfg_strings.h b/drivers/crypto/qat/qat_common/adf_cfg_strings.h
index 09651e1f937a..655248dbf962 100644
--- a/drivers/crypto/qat/qat_common/adf_cfg_strings.h
+++ b/drivers/crypto/qat/qat_common/adf_cfg_strings.h
@@ -22,6 +22,9 @@
#define ADF_RING_ASYM_BANK_NUM "BankAsymNumber"
#define ADF_CY "Cy"
#define ADF_DC "Dc"
+#define ADF_CFG_DC "dc"
+#define ADF_CFG_CY "sym;asym"
+#define ADF_SERVICES_ENABLED "ServicesEnabled"
#define ADF_ETRMGR_COALESCING_ENABLED "InterruptCoalescingEnabled"
#define ADF_ETRMGR_COALESCING_ENABLED_FORMAT \
ADF_ETRMGR_BANK "%d" ADF_ETRMGR_COALESCING_ENABLED
diff --git a/drivers/crypto/qat/qat_common/adf_common_drv.h b/drivers/crypto/qat/qat_common/adf_common_drv.h
index de94b76a6d2c..76f4f96ec5eb 100644
--- a/drivers/crypto/qat/qat_common/adf_common_drv.h
+++ b/drivers/crypto/qat/qat_common/adf_common_drv.h
@@ -1,5 +1,5 @@
/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */
-/* Copyright(c) 2014 - 2020 Intel Corporation */
+/* Copyright(c) 2014 - 2021 Intel Corporation */
#ifndef ADF_DRV_H
#define ADF_DRV_H
@@ -62,9 +62,6 @@ int adf_dev_start(struct adf_accel_dev *accel_dev);
void adf_dev_stop(struct adf_accel_dev *accel_dev);
void adf_dev_shutdown(struct adf_accel_dev *accel_dev);
-void adf_pf2vf_notify_restarting(struct adf_accel_dev *accel_dev);
-int adf_enable_vf2pf_comms(struct adf_accel_dev *accel_dev);
-void adf_vf2pf_req_hndl(struct adf_accel_vf_info *vf_info);
void adf_devmgr_update_class_index(struct adf_hw_device_data *hw_data);
void adf_clean_vf_map(bool);
@@ -117,6 +114,7 @@ void adf_cleanup_etr_data(struct adf_accel_dev *accel_dev);
int qat_crypto_register(void);
int qat_crypto_unregister(void);
int qat_crypto_dev_config(struct adf_accel_dev *accel_dev);
+int qat_crypto_vf_dev_config(struct adf_accel_dev *accel_dev);
struct qat_crypto_instance *qat_crypto_get_instance_node(int node);
void qat_crypto_put_instance(struct qat_crypto_instance *inst);
void qat_alg_callback(void *resp);
@@ -131,6 +129,8 @@ void adf_isr_resource_free(struct adf_accel_dev *accel_dev);
int adf_vf_isr_resource_alloc(struct adf_accel_dev *accel_dev);
void adf_vf_isr_resource_free(struct adf_accel_dev *accel_dev);
+int adf_pfvf_comms_disabled(struct adf_accel_dev *accel_dev);
+
int qat_hal_init(struct adf_accel_dev *accel_dev);
void qat_hal_deinit(struct icp_qat_fw_loader_handle *handle);
int qat_hal_start(struct icp_qat_fw_loader_handle *handle);
@@ -193,17 +193,14 @@ int adf_sriov_configure(struct pci_dev *pdev, int numvfs);
void adf_disable_sriov(struct adf_accel_dev *accel_dev);
void adf_disable_vf2pf_interrupts(struct adf_accel_dev *accel_dev,
u32 vf_mask);
-void adf_disable_vf2pf_interrupts_irq(struct adf_accel_dev *accel_dev,
- u32 vf_mask);
void adf_enable_vf2pf_interrupts(struct adf_accel_dev *accel_dev,
u32 vf_mask);
-int adf_enable_pf2vf_comms(struct adf_accel_dev *accel_dev);
+bool adf_recv_and_handle_pf2vf_msg(struct adf_accel_dev *accel_dev);
+bool adf_recv_and_handle_vf2pf_msg(struct adf_accel_dev *accel_dev, u32 vf_nr);
+int adf_pf2vf_handle_pf_restarting(struct adf_accel_dev *accel_dev);
void adf_enable_pf2vf_interrupts(struct adf_accel_dev *accel_dev);
void adf_disable_pf2vf_interrupts(struct adf_accel_dev *accel_dev);
void adf_schedule_vf2pf_handler(struct adf_accel_vf_info *vf_info);
-int adf_send_vf2pf_msg(struct adf_accel_dev *accel_dev, u32 msg);
-int adf_vf2pf_notify_init(struct adf_accel_dev *accel_dev);
-void adf_vf2pf_notify_shutdown(struct adf_accel_dev *accel_dev);
int adf_init_pf_wq(void);
void adf_exit_pf_wq(void);
int adf_init_vf_wq(void);
@@ -212,11 +209,6 @@ void adf_flush_vf_wq(struct adf_accel_dev *accel_dev);
#else
#define adf_sriov_configure NULL
-static inline int adf_enable_pf2vf_comms(struct adf_accel_dev *accel_dev)
-{
- return 0;
-}
-
static inline void adf_disable_sriov(struct adf_accel_dev *accel_dev)
{
}
@@ -229,15 +221,6 @@ static inline void adf_disable_pf2vf_interrupts(struct adf_accel_dev *accel_dev)
{
}
-static inline int adf_vf2pf_notify_init(struct adf_accel_dev *accel_dev)
-{
- return 0;
-}
-
-static inline void adf_vf2pf_notify_shutdown(struct adf_accel_dev *accel_dev)
-{
-}
-
static inline int adf_init_pf_wq(void)
{
return 0;
@@ -261,4 +244,15 @@ static inline void adf_flush_vf_wq(struct adf_accel_dev *accel_dev)
}
#endif
+
+static inline void __iomem *adf_get_pmisc_base(struct adf_accel_dev *accel_dev)
+{
+ struct adf_hw_device_data *hw_data = accel_dev->hw_device;
+ struct adf_bar *pmisc;
+
+ pmisc = &GET_BARS(accel_dev)[hw_data->get_misc_bar_id(hw_data)];
+
+ return pmisc->virt_addr;
+}
+
#endif
diff --git a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c
index 262bdc05dab4..57035b7dd4b2 100644
--- a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c
+++ b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c
@@ -1,57 +1,10 @@
// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
/* Copyright(c) 2020 Intel Corporation */
+#include "adf_common_drv.h"
#include "adf_gen2_hw_data.h"
#include "icp_qat_hw.h"
#include <linux/pci.h>
-#define ADF_GEN2_PF2VF_OFFSET(i) (0x3A000 + 0x280 + ((i) * 0x04))
-
-u32 adf_gen2_get_pf2vf_offset(u32 i)
-{
- return ADF_GEN2_PF2VF_OFFSET(i);
-}
-EXPORT_SYMBOL_GPL(adf_gen2_get_pf2vf_offset);
-
-u32 adf_gen2_get_vf2pf_sources(void __iomem *pmisc_addr)
-{
- u32 errsou3, errmsk3, vf_int_mask;
-
- /* Get the interrupt sources triggered by VFs */
- errsou3 = ADF_CSR_RD(pmisc_addr, ADF_GEN2_ERRSOU3);
- vf_int_mask = ADF_GEN2_ERR_REG_VF2PF(errsou3);
-
- /* To avoid adding duplicate entries to work queue, clear
- * vf_int_mask_sets bits that are already masked in ERRMSK register.
- */
- errmsk3 = ADF_CSR_RD(pmisc_addr, ADF_GEN2_ERRMSK3);
- vf_int_mask &= ~ADF_GEN2_ERR_REG_VF2PF(errmsk3);
-
- return vf_int_mask;
-}
-EXPORT_SYMBOL_GPL(adf_gen2_get_vf2pf_sources);
-
-void adf_gen2_enable_vf2pf_interrupts(void __iomem *pmisc_addr, u32 vf_mask)
-{
- /* Enable VF2PF Messaging Ints - VFs 0 through 15 per vf_mask[15:0] */
- if (vf_mask & 0xFFFF) {
- u32 val = ADF_CSR_RD(pmisc_addr, ADF_GEN2_ERRMSK3)
- & ~ADF_GEN2_ERR_MSK_VF2PF(vf_mask);
- ADF_CSR_WR(pmisc_addr, ADF_GEN2_ERRMSK3, val);
- }
-}
-EXPORT_SYMBOL_GPL(adf_gen2_enable_vf2pf_interrupts);
-
-void adf_gen2_disable_vf2pf_interrupts(void __iomem *pmisc_addr, u32 vf_mask)
-{
- /* Disable VF2PF interrupts for VFs 0 through 15 per vf_mask[15:0] */
- if (vf_mask & 0xFFFF) {
- u32 val = ADF_CSR_RD(pmisc_addr, ADF_GEN2_ERRMSK3)
- | ADF_GEN2_ERR_MSK_VF2PF(vf_mask);
- ADF_CSR_WR(pmisc_addr, ADF_GEN2_ERRMSK3, val);
- }
-}
-EXPORT_SYMBOL_GPL(adf_gen2_disable_vf2pf_interrupts);
-
u32 adf_gen2_get_num_accels(struct adf_hw_device_data *self)
{
if (!self || !self->accel_mask)
@@ -73,31 +26,29 @@ EXPORT_SYMBOL_GPL(adf_gen2_get_num_aes);
void adf_gen2_enable_error_correction(struct adf_accel_dev *accel_dev)
{
struct adf_hw_device_data *hw_data = accel_dev->hw_device;
- struct adf_bar *misc_bar = &GET_BARS(accel_dev)
- [hw_data->get_misc_bar_id(hw_data)];
+ void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev);
unsigned long accel_mask = hw_data->accel_mask;
unsigned long ae_mask = hw_data->ae_mask;
- void __iomem *csr = misc_bar->virt_addr;
unsigned int val, i;
/* Enable Accel Engine error detection & correction */
for_each_set_bit(i, &ae_mask, hw_data->num_engines) {
- val = ADF_CSR_RD(csr, ADF_GEN2_AE_CTX_ENABLES(i));
+ val = ADF_CSR_RD(pmisc_addr, ADF_GEN2_AE_CTX_ENABLES(i));
val |= ADF_GEN2_ENABLE_AE_ECC_ERR;
- ADF_CSR_WR(csr, ADF_GEN2_AE_CTX_ENABLES(i), val);
- val = ADF_CSR_RD(csr, ADF_GEN2_AE_MISC_CONTROL(i));
+ ADF_CSR_WR(pmisc_addr, ADF_GEN2_AE_CTX_ENABLES(i), val);
+ val = ADF_CSR_RD(pmisc_addr, ADF_GEN2_AE_MISC_CONTROL(i));
val |= ADF_GEN2_ENABLE_AE_ECC_PARITY_CORR;
- ADF_CSR_WR(csr, ADF_GEN2_AE_MISC_CONTROL(i), val);
+ ADF_CSR_WR(pmisc_addr, ADF_GEN2_AE_MISC_CONTROL(i), val);
}
/* Enable shared memory error detection & correction */
for_each_set_bit(i, &accel_mask, hw_data->num_accel) {
- val = ADF_CSR_RD(csr, ADF_GEN2_UERRSSMSH(i));
+ val = ADF_CSR_RD(pmisc_addr, ADF_GEN2_UERRSSMSH(i));
val |= ADF_GEN2_ERRSSMSH_EN;
- ADF_CSR_WR(csr, ADF_GEN2_UERRSSMSH(i), val);
- val = ADF_CSR_RD(csr, ADF_GEN2_CERRSSMSH(i));
+ ADF_CSR_WR(pmisc_addr, ADF_GEN2_UERRSSMSH(i), val);
+ val = ADF_CSR_RD(pmisc_addr, ADF_GEN2_CERRSSMSH(i));
val |= ADF_GEN2_ERRSSMSH_EN;
- ADF_CSR_WR(csr, ADF_GEN2_CERRSSMSH(i), val);
+ ADF_CSR_WR(pmisc_addr, ADF_GEN2_CERRSSMSH(i), val);
}
}
EXPORT_SYMBOL_GPL(adf_gen2_enable_error_correction);
@@ -105,15 +56,9 @@ EXPORT_SYMBOL_GPL(adf_gen2_enable_error_correction);
void adf_gen2_cfg_iov_thds(struct adf_accel_dev *accel_dev, bool enable,
int num_a_regs, int num_b_regs)
{
- struct adf_hw_device_data *hw_data = accel_dev->hw_device;
- void __iomem *pmisc_addr;
- struct adf_bar *pmisc;
- int pmisc_id, i;
+ void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev);
u32 reg;
-
- pmisc_id = hw_data->get_misc_bar_id(hw_data);
- pmisc = &GET_BARS(accel_dev)[pmisc_id];
- pmisc_addr = pmisc->virt_addr;
+ int i;
/* Set/Unset Valid bit in AE Thread to PCIe Function Mapping Group A */
for (i = 0; i < num_a_regs; i++) {
@@ -259,21 +204,33 @@ u32 adf_gen2_get_accel_cap(struct adf_accel_dev *accel_dev)
u32 legfuses;
u32 capabilities = ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC |
ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC |
- ICP_ACCEL_CAPABILITIES_AUTHENTICATION;
+ ICP_ACCEL_CAPABILITIES_AUTHENTICATION |
+ ICP_ACCEL_CAPABILITIES_CIPHER |
+ ICP_ACCEL_CAPABILITIES_COMPRESSION;
/* Read accelerator capabilities mask */
pci_read_config_dword(pdev, ADF_DEVICE_LEGFUSE_OFFSET, &legfuses);
- if (legfuses & ICP_ACCEL_MASK_CIPHER_SLICE)
+ /* A set bit in legfuses means the feature is OFF in this SKU */
+ if (legfuses & ICP_ACCEL_MASK_CIPHER_SLICE) {
capabilities &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC;
+ capabilities &= ~ICP_ACCEL_CAPABILITIES_CIPHER;
+ }
if (legfuses & ICP_ACCEL_MASK_PKE_SLICE)
capabilities &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC;
- if (legfuses & ICP_ACCEL_MASK_AUTH_SLICE)
+ if (legfuses & ICP_ACCEL_MASK_AUTH_SLICE) {
capabilities &= ~ICP_ACCEL_CAPABILITIES_AUTHENTICATION;
+ capabilities &= ~ICP_ACCEL_CAPABILITIES_CIPHER;
+ }
+ if (legfuses & ICP_ACCEL_MASK_COMPRESS_SLICE)
+ capabilities &= ~ICP_ACCEL_CAPABILITIES_COMPRESSION;
if ((straps | fuses) & ADF_POWERGATE_PKE)
capabilities &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC;
+ if ((straps | fuses) & ADF_POWERGATE_DC)
+ capabilities &= ~ICP_ACCEL_CAPABILITIES_COMPRESSION;
+
return capabilities;
}
EXPORT_SYMBOL_GPL(adf_gen2_get_accel_cap);
@@ -281,18 +238,12 @@ EXPORT_SYMBOL_GPL(adf_gen2_get_accel_cap);
void adf_gen2_set_ssm_wdtimer(struct adf_accel_dev *accel_dev)
{
struct adf_hw_device_data *hw_data = accel_dev->hw_device;
+ void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev);
u32 timer_val_pke = ADF_SSM_WDT_PKE_DEFAULT_VALUE;
u32 timer_val = ADF_SSM_WDT_DEFAULT_VALUE;
unsigned long accel_mask = hw_data->accel_mask;
- void __iomem *pmisc_addr;
- struct adf_bar *pmisc;
- int pmisc_id;
u32 i = 0;
- pmisc_id = hw_data->get_misc_bar_id(hw_data);
- pmisc = &GET_BARS(accel_dev)[pmisc_id];
- pmisc_addr = pmisc->virt_addr;
-
/* Configures WDT timers */
for_each_set_bit(i, &accel_mask, hw_data->num_accel) {
/* Enable WDT for sym and dc */
diff --git a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h
index c169d704097d..f2e0451b11c0 100644
--- a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h
+++ b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h
@@ -4,6 +4,7 @@
#define ADF_GEN2_HW_DATA_H_
#include "adf_accel_devices.h"
+#include "adf_cfg_common.h"
/* Transport access */
#define ADF_BANK_INT_SRC_SEL_MASK_0 0x4444444CUL
@@ -113,8 +114,16 @@ do { \
(ADF_ARB_REG_SLOT * (index)), value)
/* Power gating */
+#define ADF_POWERGATE_DC BIT(23)
#define ADF_POWERGATE_PKE BIT(24)
+/* Default ring mapping */
+#define ADF_GEN2_DEFAULT_RING_TO_SRV_MAP \
+ (CRYPTO << ADF_CFG_SERV_RING_PAIR_0_SHIFT | \
+ CRYPTO << ADF_CFG_SERV_RING_PAIR_1_SHIFT | \
+ UNUSED << ADF_CFG_SERV_RING_PAIR_2_SHIFT | \
+ COMP << ADF_CFG_SERV_RING_PAIR_3_SHIFT)
+
/* WDT timers
*
* Timeout is in cycles. Clock speed may vary across products but this
@@ -136,19 +145,6 @@ do { \
#define ADF_GEN2_CERRSSMSH(i) ((i) * 0x4000 + 0x10)
#define ADF_GEN2_ERRSSMSH_EN BIT(3)
- /* VF2PF interrupts */
-#define ADF_GEN2_ERRSOU3 (0x3A000 + 0x0C)
-#define ADF_GEN2_ERRSOU5 (0x3A000 + 0xD8)
-#define ADF_GEN2_ERRMSK3 (0x3A000 + 0x1C)
-#define ADF_GEN2_ERRMSK5 (0x3A000 + 0xDC)
-#define ADF_GEN2_ERR_REG_VF2PF(vf_src) (((vf_src) & 0x01FFFE00) >> 9)
-#define ADF_GEN2_ERR_MSK_VF2PF(vf_mask) (((vf_mask) & 0xFFFF) << 9)
-
-u32 adf_gen2_get_pf2vf_offset(u32 i);
-u32 adf_gen2_get_vf2pf_sources(void __iomem *pmisc_bar);
-void adf_gen2_enable_vf2pf_interrupts(void __iomem *pmisc_addr, u32 vf_mask);
-void adf_gen2_disable_vf2pf_interrupts(void __iomem *pmisc_addr, u32 vf_mask);
-
u32 adf_gen2_get_num_accels(struct adf_hw_device_data *self);
u32 adf_gen2_get_num_aes(struct adf_hw_device_data *self);
void adf_gen2_enable_error_correction(struct adf_accel_dev *accel_dev);
diff --git a/drivers/crypto/qat/qat_common/adf_gen2_pfvf.c b/drivers/crypto/qat/qat_common/adf_gen2_pfvf.c
new file mode 100644
index 000000000000..1a9072aac2ca
--- /dev/null
+++ b/drivers/crypto/qat/qat_common/adf_gen2_pfvf.c
@@ -0,0 +1,381 @@
+// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
+/* Copyright(c) 2021 Intel Corporation */
+#include <linux/delay.h>
+#include <linux/iopoll.h>
+#include <linux/mutex.h>
+#include <linux/types.h>
+#include "adf_accel_devices.h"
+#include "adf_common_drv.h"
+#include "adf_gen2_pfvf.h"
+#include "adf_pfvf_msg.h"
+#include "adf_pfvf_pf_proto.h"
+#include "adf_pfvf_vf_proto.h"
+#include "adf_pfvf_utils.h"
+
+ /* VF2PF interrupts */
+#define ADF_GEN2_ERR_REG_VF2PF(vf_src) (((vf_src) & 0x01FFFE00) >> 9)
+#define ADF_GEN2_ERR_MSK_VF2PF(vf_mask) (((vf_mask) & 0xFFFF) << 9)
+
+#define ADF_GEN2_PF_PF2VF_OFFSET(i) (0x3A000 + 0x280 + ((i) * 0x04))
+#define ADF_GEN2_VF_PF2VF_OFFSET 0x200
+
+#define ADF_GEN2_CSR_IN_USE 0x6AC2
+#define ADF_GEN2_CSR_IN_USE_MASK 0xFFFE
+
+enum gen2_csr_pos {
+ ADF_GEN2_CSR_PF2VF_OFFSET = 0,
+ ADF_GEN2_CSR_VF2PF_OFFSET = 16,
+};
+
+#define ADF_PFVF_GEN2_MSGTYPE_SHIFT 2
+#define ADF_PFVF_GEN2_MSGTYPE_MASK 0x0F
+#define ADF_PFVF_GEN2_MSGDATA_SHIFT 6
+#define ADF_PFVF_GEN2_MSGDATA_MASK 0x3FF
+
+static const struct pfvf_csr_format csr_gen2_fmt = {
+ { ADF_PFVF_GEN2_MSGTYPE_SHIFT, ADF_PFVF_GEN2_MSGTYPE_MASK },
+ { ADF_PFVF_GEN2_MSGDATA_SHIFT, ADF_PFVF_GEN2_MSGDATA_MASK },
+};
+
+#define ADF_PFVF_MSG_RETRY_DELAY 5
+#define ADF_PFVF_MSG_MAX_RETRIES 3
+
+static u32 adf_gen2_pf_get_pfvf_offset(u32 i)
+{
+ return ADF_GEN2_PF_PF2VF_OFFSET(i);
+}
+
+static u32 adf_gen2_vf_get_pfvf_offset(u32 i)
+{
+ return ADF_GEN2_VF_PF2VF_OFFSET;
+}
+
+static u32 adf_gen2_get_vf2pf_sources(void __iomem *pmisc_addr)
+{
+ u32 errsou3, errmsk3, vf_int_mask;
+
+ /* Get the interrupt sources triggered by VFs */
+ errsou3 = ADF_CSR_RD(pmisc_addr, ADF_GEN2_ERRSOU3);
+ vf_int_mask = ADF_GEN2_ERR_REG_VF2PF(errsou3);
+
+ /* To avoid adding duplicate entries to work queue, clear
+ * vf_int_mask_sets bits that are already masked in ERRMSK register.
+ */
+ errmsk3 = ADF_CSR_RD(pmisc_addr, ADF_GEN2_ERRMSK3);
+ vf_int_mask &= ~ADF_GEN2_ERR_REG_VF2PF(errmsk3);
+
+ return vf_int_mask;
+}
+
+static void adf_gen2_enable_vf2pf_interrupts(void __iomem *pmisc_addr,
+ u32 vf_mask)
+{
+ /* Enable VF2PF Messaging Ints - VFs 0 through 15 per vf_mask[15:0] */
+ if (vf_mask & 0xFFFF) {
+ u32 val = ADF_CSR_RD(pmisc_addr, ADF_GEN2_ERRMSK3)
+ & ~ADF_GEN2_ERR_MSK_VF2PF(vf_mask);
+ ADF_CSR_WR(pmisc_addr, ADF_GEN2_ERRMSK3, val);
+ }
+}
+
+static void adf_gen2_disable_vf2pf_interrupts(void __iomem *pmisc_addr,
+ u32 vf_mask)
+{
+ /* Disable VF2PF interrupts for VFs 0 through 15 per vf_mask[15:0] */
+ if (vf_mask & 0xFFFF) {
+ u32 val = ADF_CSR_RD(pmisc_addr, ADF_GEN2_ERRMSK3)
+ | ADF_GEN2_ERR_MSK_VF2PF(vf_mask);
+ ADF_CSR_WR(pmisc_addr, ADF_GEN2_ERRMSK3, val);
+ }
+}
+
+static u32 gen2_csr_get_int_bit(enum gen2_csr_pos offset)
+{
+ return ADF_PFVF_INT << offset;
+}
+
+static u32 gen2_csr_msg_to_position(u32 csr_msg, enum gen2_csr_pos offset)
+{
+ return (csr_msg & 0xFFFF) << offset;
+}
+
+static u32 gen2_csr_msg_from_position(u32 csr_val, enum gen2_csr_pos offset)
+{
+ return (csr_val >> offset) & 0xFFFF;
+}
+
+static bool gen2_csr_is_in_use(u32 msg, enum gen2_csr_pos offset)
+{
+ return ((msg >> offset) & ADF_GEN2_CSR_IN_USE_MASK) == ADF_GEN2_CSR_IN_USE;
+}
+
+static void gen2_csr_clear_in_use(u32 *msg, enum gen2_csr_pos offset)
+{
+ *msg &= ~(ADF_GEN2_CSR_IN_USE_MASK << offset);
+}
+
+static void gen2_csr_set_in_use(u32 *msg, enum gen2_csr_pos offset)
+{
+ *msg |= (ADF_GEN2_CSR_IN_USE << offset);
+}
+
+static bool is_legacy_user_pfvf_message(u32 msg)
+{
+ return !(msg & ADF_PFVF_MSGORIGIN_SYSTEM);
+}
+
+static bool is_pf2vf_notification(u8 msg_type)
+{
+ switch (msg_type) {
+ case ADF_PF2VF_MSGTYPE_RESTARTING:
+ return true;
+ default:
+ return false;
+ }
+}
+
+static bool is_vf2pf_notification(u8 msg_type)
+{
+ switch (msg_type) {
+ case ADF_VF2PF_MSGTYPE_INIT:
+ case ADF_VF2PF_MSGTYPE_SHUTDOWN:
+ return true;
+ default:
+ return false;
+ }
+}
+
+struct pfvf_gen2_params {
+ u32 pfvf_offset;
+ struct mutex *csr_lock; /* lock preventing concurrent access of CSR */
+ enum gen2_csr_pos local_offset;
+ enum gen2_csr_pos remote_offset;
+ bool (*is_notification_message)(u8 msg_type);
+ u8 compat_ver;
+};
+
+static int adf_gen2_pfvf_send(struct adf_accel_dev *accel_dev,
+ struct pfvf_message msg,
+ struct pfvf_gen2_params *params)
+{
+ void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev);
+ enum gen2_csr_pos remote_offset = params->remote_offset;
+ enum gen2_csr_pos local_offset = params->local_offset;
+ unsigned int retries = ADF_PFVF_MSG_MAX_RETRIES;
+ struct mutex *lock = params->csr_lock;
+ u32 pfvf_offset = params->pfvf_offset;
+ u32 int_bit;
+ u32 csr_val;
+ u32 csr_msg;
+ int ret;
+
+ /* Gen2 messages, both PF->VF and VF->PF, are all 16 bits long. This
+ * allows us to build and read messages as if they where all 0 based.
+ * However, send and receive are in a single shared 32 bits register,
+ * so we need to shift and/or mask the message half before decoding
+ * it and after encoding it. Which one to shift depends on the
+ * direction.
+ */
+
+ int_bit = gen2_csr_get_int_bit(local_offset);
+
+ csr_msg = adf_pfvf_csr_msg_of(accel_dev, msg, &csr_gen2_fmt);
+ if (unlikely(!csr_msg))
+ return -EINVAL;
+
+ /* Prepare for CSR format, shifting the wire message in place and
+ * setting the in use pattern
+ */
+ csr_msg = gen2_csr_msg_to_position(csr_msg, local_offset);
+ gen2_csr_set_in_use(&csr_msg, remote_offset);
+
+ mutex_lock(lock);
+
+start:
+ /* Check if the PFVF CSR is in use by remote function */
+ csr_val = ADF_CSR_RD(pmisc_addr, pfvf_offset);
+ if (gen2_csr_is_in_use(csr_val, local_offset)) {
+ dev_dbg(&GET_DEV(accel_dev),
+ "PFVF CSR in use by remote function\n");
+ goto retry;
+ }
+
+ /* Attempt to get ownership of the PFVF CSR */
+ ADF_CSR_WR(pmisc_addr, pfvf_offset, csr_msg | int_bit);
+
+ /* Wait for confirmation from remote func it received the message */
+ ret = read_poll_timeout(ADF_CSR_RD, csr_val, !(csr_val & int_bit),
+ ADF_PFVF_MSG_ACK_DELAY_US,
+ ADF_PFVF_MSG_ACK_MAX_DELAY_US,
+ true, pmisc_addr, pfvf_offset);
+ if (unlikely(ret < 0)) {
+ dev_dbg(&GET_DEV(accel_dev), "ACK not received from remote\n");
+ csr_val &= ~int_bit;
+ }
+
+ /* For fire-and-forget notifications, the receiver does not clear
+ * the in-use pattern. This is used to detect collisions.
+ */
+ if (params->is_notification_message(msg.type) && csr_val != csr_msg) {
+ /* Collision must have overwritten the message */
+ dev_err(&GET_DEV(accel_dev),
+ "Collision on notification - PFVF CSR overwritten by remote function\n");
+ goto retry;
+ }
+
+ /* If the far side did not clear the in-use pattern it is either
+ * 1) Notification - message left intact to detect collision
+ * 2) Older protocol (compatibility version < 3) on the far side
+ * where the sender is responsible for clearing the in-use
+ * pattern after the received has acknowledged receipt.
+ * In either case, clear the in-use pattern now.
+ */
+ if (gen2_csr_is_in_use(csr_val, remote_offset)) {
+ gen2_csr_clear_in_use(&csr_val, remote_offset);
+ ADF_CSR_WR(pmisc_addr, pfvf_offset, csr_val);
+ }
+
+out:
+ mutex_unlock(lock);
+ return ret;
+
+retry:
+ if (--retries) {
+ msleep(ADF_PFVF_MSG_RETRY_DELAY);
+ goto start;
+ } else {
+ ret = -EBUSY;
+ goto out;
+ }
+}
+
+static struct pfvf_message adf_gen2_pfvf_recv(struct adf_accel_dev *accel_dev,
+ struct pfvf_gen2_params *params)
+{
+ void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev);
+ enum gen2_csr_pos remote_offset = params->remote_offset;
+ enum gen2_csr_pos local_offset = params->local_offset;
+ u32 pfvf_offset = params->pfvf_offset;
+ struct pfvf_message msg = { 0 };
+ u32 int_bit;
+ u32 csr_val;
+ u16 csr_msg;
+
+ int_bit = gen2_csr_get_int_bit(local_offset);
+
+ /* Read message */
+ csr_val = ADF_CSR_RD(pmisc_addr, pfvf_offset);
+ if (!(csr_val & int_bit)) {
+ dev_info(&GET_DEV(accel_dev),
+ "Spurious PFVF interrupt, msg 0x%.8x. Ignored\n", csr_val);
+ return msg;
+ }
+
+ /* Extract the message from the CSR */
+ csr_msg = gen2_csr_msg_from_position(csr_val, local_offset);
+
+ /* Ignore legacy non-system (non-kernel) messages */
+ if (unlikely(is_legacy_user_pfvf_message(csr_msg))) {
+ dev_dbg(&GET_DEV(accel_dev),
+ "Ignored non-system message (0x%.8x);\n", csr_val);
+ /* Because this must be a legacy message, the far side
+ * must clear the in-use pattern, so don't do it.
+ */
+ return msg;
+ }
+
+ /* Return the pfvf_message format */
+ msg = adf_pfvf_message_of(accel_dev, csr_msg, &csr_gen2_fmt);
+
+ /* The in-use pattern is not cleared for notifications (so that
+ * it can be used for collision detection) or older implementations
+ */
+ if (params->compat_ver >= ADF_PFVF_COMPAT_FAST_ACK &&
+ !params->is_notification_message(msg.type))
+ gen2_csr_clear_in_use(&csr_val, remote_offset);
+
+ /* To ACK, clear the INT bit */
+ csr_val &= ~int_bit;
+ ADF_CSR_WR(pmisc_addr, pfvf_offset, csr_val);
+
+ return msg;
+}
+
+static int adf_gen2_pf2vf_send(struct adf_accel_dev *accel_dev, struct pfvf_message msg,
+ u32 pfvf_offset, struct mutex *csr_lock)
+{
+ struct pfvf_gen2_params params = {
+ .csr_lock = csr_lock,
+ .pfvf_offset = pfvf_offset,
+ .local_offset = ADF_GEN2_CSR_PF2VF_OFFSET,
+ .remote_offset = ADF_GEN2_CSR_VF2PF_OFFSET,
+ .is_notification_message = is_pf2vf_notification,
+ };
+
+ return adf_gen2_pfvf_send(accel_dev, msg, &params);
+}
+
+static int adf_gen2_vf2pf_send(struct adf_accel_dev *accel_dev, struct pfvf_message msg,
+ u32 pfvf_offset, struct mutex *csr_lock)
+{
+ struct pfvf_gen2_params params = {
+ .csr_lock = csr_lock,
+ .pfvf_offset = pfvf_offset,
+ .local_offset = ADF_GEN2_CSR_VF2PF_OFFSET,
+ .remote_offset = ADF_GEN2_CSR_PF2VF_OFFSET,
+ .is_notification_message = is_vf2pf_notification,
+ };
+
+ return adf_gen2_pfvf_send(accel_dev, msg, &params);
+}
+
+static struct pfvf_message adf_gen2_pf2vf_recv(struct adf_accel_dev *accel_dev,
+ u32 pfvf_offset, u8 compat_ver)
+{
+ struct pfvf_gen2_params params = {
+ .pfvf_offset = pfvf_offset,
+ .local_offset = ADF_GEN2_CSR_PF2VF_OFFSET,
+ .remote_offset = ADF_GEN2_CSR_VF2PF_OFFSET,
+ .is_notification_message = is_pf2vf_notification,
+ .compat_ver = compat_ver,
+ };
+
+ return adf_gen2_pfvf_recv(accel_dev, &params);
+}
+
+static struct pfvf_message adf_gen2_vf2pf_recv(struct adf_accel_dev *accel_dev,
+ u32 pfvf_offset, u8 compat_ver)
+{
+ struct pfvf_gen2_params params = {
+ .pfvf_offset = pfvf_offset,
+ .local_offset = ADF_GEN2_CSR_VF2PF_OFFSET,
+ .remote_offset = ADF_GEN2_CSR_PF2VF_OFFSET,
+ .is_notification_message = is_vf2pf_notification,
+ .compat_ver = compat_ver,
+ };
+
+ return adf_gen2_pfvf_recv(accel_dev, &params);
+}
+
+void adf_gen2_init_pf_pfvf_ops(struct adf_pfvf_ops *pfvf_ops)
+{
+ pfvf_ops->enable_comms = adf_enable_pf2vf_comms;
+ pfvf_ops->get_pf2vf_offset = adf_gen2_pf_get_pfvf_offset;
+ pfvf_ops->get_vf2pf_offset = adf_gen2_pf_get_pfvf_offset;
+ pfvf_ops->get_vf2pf_sources = adf_gen2_get_vf2pf_sources;
+ pfvf_ops->enable_vf2pf_interrupts = adf_gen2_enable_vf2pf_interrupts;
+ pfvf_ops->disable_vf2pf_interrupts = adf_gen2_disable_vf2pf_interrupts;
+ pfvf_ops->send_msg = adf_gen2_pf2vf_send;
+ pfvf_ops->recv_msg = adf_gen2_vf2pf_recv;
+}
+EXPORT_SYMBOL_GPL(adf_gen2_init_pf_pfvf_ops);
+
+void adf_gen2_init_vf_pfvf_ops(struct adf_pfvf_ops *pfvf_ops)
+{
+ pfvf_ops->enable_comms = adf_enable_vf2pf_comms;
+ pfvf_ops->get_pf2vf_offset = adf_gen2_vf_get_pfvf_offset;
+ pfvf_ops->get_vf2pf_offset = adf_gen2_vf_get_pfvf_offset;
+ pfvf_ops->send_msg = adf_gen2_vf2pf_send;
+ pfvf_ops->recv_msg = adf_gen2_pf2vf_recv;
+}
+EXPORT_SYMBOL_GPL(adf_gen2_init_vf_pfvf_ops);
diff --git a/drivers/crypto/qat/qat_common/adf_gen2_pfvf.h b/drivers/crypto/qat/qat_common/adf_gen2_pfvf.h
new file mode 100644
index 000000000000..a716545a764c
--- /dev/null
+++ b/drivers/crypto/qat/qat_common/adf_gen2_pfvf.h
@@ -0,0 +1,29 @@
+/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */
+/* Copyright(c) 2021 Intel Corporation */
+#ifndef ADF_GEN2_PFVF_H
+#define ADF_GEN2_PFVF_H
+
+#include <linux/types.h>
+#include "adf_accel_devices.h"
+
+#define ADF_GEN2_ERRSOU3 (0x3A000 + 0x0C)
+#define ADF_GEN2_ERRSOU5 (0x3A000 + 0xD8)
+#define ADF_GEN2_ERRMSK3 (0x3A000 + 0x1C)
+#define ADF_GEN2_ERRMSK5 (0x3A000 + 0xDC)
+
+#if defined(CONFIG_PCI_IOV)
+void adf_gen2_init_pf_pfvf_ops(struct adf_pfvf_ops *pfvf_ops);
+void adf_gen2_init_vf_pfvf_ops(struct adf_pfvf_ops *pfvf_ops);
+#else
+static inline void adf_gen2_init_pf_pfvf_ops(struct adf_pfvf_ops *pfvf_ops)
+{
+ pfvf_ops->enable_comms = adf_pfvf_comms_disabled;
+}
+
+static inline void adf_gen2_init_vf_pfvf_ops(struct adf_pfvf_ops *pfvf_ops)
+{
+ pfvf_ops->enable_comms = adf_pfvf_comms_disabled;
+}
+#endif
+
+#endif /* ADF_GEN2_PFVF_H */
diff --git a/drivers/crypto/qat/qat_common/adf_gen4_hw_data.c b/drivers/crypto/qat/qat_common/adf_gen4_hw_data.c
index 000528327b29..3148a62938fd 100644
--- a/drivers/crypto/qat/qat_common/adf_gen4_hw_data.c
+++ b/drivers/crypto/qat/qat_common/adf_gen4_hw_data.c
@@ -1,6 +1,8 @@
// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
/* Copyright(c) 2020 Intel Corporation */
+#include <linux/iopoll.h>
#include "adf_accel_devices.h"
+#include "adf_common_drv.h"
#include "adf_gen4_hw_data.h"
static u64 build_csr_ring_base_addr(dma_addr_t addr, u32 size)
@@ -109,20 +111,13 @@ static inline void adf_gen4_unpack_ssm_wdtimer(u64 value, u32 *upper,
void adf_gen4_set_ssm_wdtimer(struct adf_accel_dev *accel_dev)
{
- struct adf_hw_device_data *hw_data = accel_dev->hw_device;
+ void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev);
u64 timer_val_pke = ADF_SSM_WDT_PKE_DEFAULT_VALUE;
u64 timer_val = ADF_SSM_WDT_DEFAULT_VALUE;
u32 ssm_wdt_pke_high = 0;
u32 ssm_wdt_pke_low = 0;
u32 ssm_wdt_high = 0;
u32 ssm_wdt_low = 0;
- void __iomem *pmisc_addr;
- struct adf_bar *pmisc;
- int pmisc_id;
-
- pmisc_id = hw_data->get_misc_bar_id(hw_data);
- pmisc = &GET_BARS(accel_dev)[pmisc_id];
- pmisc_addr = pmisc->virt_addr;
/* Convert 64bit WDT timer value into 32bit values for
* mmio write to 32bit CSRs.
@@ -139,3 +134,61 @@ void adf_gen4_set_ssm_wdtimer(struct adf_accel_dev *accel_dev)
ADF_CSR_WR(pmisc_addr, ADF_SSMWDTPKEH_OFFSET, ssm_wdt_pke_high);
}
EXPORT_SYMBOL_GPL(adf_gen4_set_ssm_wdtimer);
+
+int adf_pfvf_comms_disabled(struct adf_accel_dev *accel_dev)
+{
+ return 0;
+}
+EXPORT_SYMBOL_GPL(adf_pfvf_comms_disabled);
+
+static int reset_ring_pair(void __iomem *csr, u32 bank_number)
+{
+ u32 status;
+ int ret;
+
+ /* Write rpresetctl register BIT(0) as 1
+ * Since rpresetctl registers have no RW fields, no need to preserve
+ * values for other bits. Just write directly.
+ */
+ ADF_CSR_WR(csr, ADF_WQM_CSR_RPRESETCTL(bank_number),
+ ADF_WQM_CSR_RPRESETCTL_RESET);
+
+ /* Read rpresetsts register and wait for rp reset to complete */
+ ret = read_poll_timeout(ADF_CSR_RD, status,
+ status & ADF_WQM_CSR_RPRESETSTS_STATUS,
+ ADF_RPRESET_POLL_DELAY_US,
+ ADF_RPRESET_POLL_TIMEOUT_US, true,
+ csr, ADF_WQM_CSR_RPRESETSTS(bank_number));
+ if (!ret) {
+ /* When rp reset is done, clear rpresetsts */
+ ADF_CSR_WR(csr, ADF_WQM_CSR_RPRESETSTS(bank_number),
+ ADF_WQM_CSR_RPRESETSTS_STATUS);
+ }
+
+ return ret;
+}
+
+int adf_gen4_ring_pair_reset(struct adf_accel_dev *accel_dev, u32 bank_number)
+{
+ struct adf_hw_device_data *hw_data = accel_dev->hw_device;
+ u32 etr_bar_id = hw_data->get_etr_bar_id(hw_data);
+ void __iomem *csr;
+ int ret;
+
+ if (bank_number >= hw_data->num_banks)
+ return -EINVAL;
+
+ dev_dbg(&GET_DEV(accel_dev),
+ "ring pair reset for bank:%d\n", bank_number);
+
+ csr = (&GET_BARS(accel_dev)[etr_bar_id])->virt_addr;
+ ret = reset_ring_pair(csr, bank_number);
+ if (ret)
+ dev_err(&GET_DEV(accel_dev),
+ "ring pair reset failed (timeout)\n");
+ else
+ dev_dbg(&GET_DEV(accel_dev), "ring pair reset successful\n");
+
+ return ret;
+}
+EXPORT_SYMBOL_GPL(adf_gen4_ring_pair_reset);
diff --git a/drivers/crypto/qat/qat_common/adf_gen4_hw_data.h b/drivers/crypto/qat/qat_common/adf_gen4_hw_data.h
index b8fca1ff7aab..f0f71ca44ca3 100644
--- a/drivers/crypto/qat/qat_common/adf_gen4_hw_data.h
+++ b/drivers/crypto/qat/qat_common/adf_gen4_hw_data.h
@@ -4,6 +4,7 @@
#define ADF_GEN4_HW_CSR_DATA_H_
#include "adf_accel_devices.h"
+#include "adf_cfg_common.h"
/* Transport access */
#define ADF_BANK_INT_SRC_SEL_MASK 0x44UL
@@ -94,6 +95,13 @@ do { \
ADF_RING_BUNDLE_SIZE * (bank) + \
ADF_RING_CSR_RING_SRV_ARB_EN, (value))
+/* Default ring mapping */
+#define ADF_GEN4_DEFAULT_RING_TO_SRV_MAP \
+ (ASYM << ADF_CFG_SERV_RING_PAIR_0_SHIFT | \
+ SYM << ADF_CFG_SERV_RING_PAIR_1_SHIFT | \
+ ASYM << ADF_CFG_SERV_RING_PAIR_2_SHIFT | \
+ SYM << ADF_CFG_SERV_RING_PAIR_3_SHIFT)
+
/* WDT timers
*
* Timeout is in cycles. Clock speed may vary across products but this
@@ -106,6 +114,15 @@ do { \
#define ADF_SSMWDTPKEL_OFFSET 0x58
#define ADF_SSMWDTPKEH_OFFSET 0x60
+/* Ring reset */
+#define ADF_RPRESET_POLL_TIMEOUT_US (5 * USEC_PER_SEC)
+#define ADF_RPRESET_POLL_DELAY_US 20
+#define ADF_WQM_CSR_RPRESETCTL_RESET BIT(0)
+#define ADF_WQM_CSR_RPRESETCTL(bank) (0x6000 + ((bank) << 3))
+#define ADF_WQM_CSR_RPRESETSTS_STATUS BIT(0)
+#define ADF_WQM_CSR_RPRESETSTS(bank) (ADF_WQM_CSR_RPRESETCTL(bank) + 4)
+
void adf_gen4_set_ssm_wdtimer(struct adf_accel_dev *accel_dev);
void adf_gen4_init_hw_csr_ops(struct adf_hw_csr_ops *csr_ops);
+int adf_gen4_ring_pair_reset(struct adf_accel_dev *accel_dev, u32 bank_number);
#endif
diff --git a/drivers/crypto/qat/qat_common/adf_gen4_pfvf.c b/drivers/crypto/qat/qat_common/adf_gen4_pfvf.c
new file mode 100644
index 000000000000..8efbedf63bc8
--- /dev/null
+++ b/drivers/crypto/qat/qat_common/adf_gen4_pfvf.c
@@ -0,0 +1,148 @@
+// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
+/* Copyright(c) 2021 Intel Corporation */
+#include <linux/iopoll.h>
+#include <linux/mutex.h>
+#include <linux/types.h>
+#include "adf_accel_devices.h"
+#include "adf_common_drv.h"
+#include "adf_gen4_pfvf.h"
+#include "adf_pfvf_pf_proto.h"
+#include "adf_pfvf_utils.h"
+
+#define ADF_4XXX_MAX_NUM_VFS 16
+
+#define ADF_4XXX_PF2VM_OFFSET(i) (0x40B010 + ((i) * 0x20))
+#define ADF_4XXX_VM2PF_OFFSET(i) (0x40B014 + ((i) * 0x20))
+
+/* VF2PF interrupt source registers */
+#define ADF_4XXX_VM2PF_SOU(i) (0x41A180 + ((i) * 4))
+#define ADF_4XXX_VM2PF_MSK(i) (0x41A1C0 + ((i) * 4))
+#define ADF_4XXX_VM2PF_INT_EN_MSK BIT(0)
+
+#define ADF_PFVF_GEN4_MSGTYPE_SHIFT 2
+#define ADF_PFVF_GEN4_MSGTYPE_MASK 0x3F
+#define ADF_PFVF_GEN4_MSGDATA_SHIFT 8
+#define ADF_PFVF_GEN4_MSGDATA_MASK 0xFFFFFF
+
+static const struct pfvf_csr_format csr_gen4_fmt = {
+ { ADF_PFVF_GEN4_MSGTYPE_SHIFT, ADF_PFVF_GEN4_MSGTYPE_MASK },
+ { ADF_PFVF_GEN4_MSGDATA_SHIFT, ADF_PFVF_GEN4_MSGDATA_MASK },
+};
+
+static u32 adf_gen4_pf_get_pf2vf_offset(u32 i)
+{
+ return ADF_4XXX_PF2VM_OFFSET(i);
+}
+
+static u32 adf_gen4_pf_get_vf2pf_offset(u32 i)
+{
+ return ADF_4XXX_VM2PF_OFFSET(i);
+}
+
+static u32 adf_gen4_get_vf2pf_sources(void __iomem *pmisc_addr)
+{
+ int i;
+ u32 sou, mask;
+ int num_csrs = ADF_4XXX_MAX_NUM_VFS;
+ u32 vf_mask = 0;
+
+ for (i = 0; i < num_csrs; i++) {
+ sou = ADF_CSR_RD(pmisc_addr, ADF_4XXX_VM2PF_SOU(i));
+ mask = ADF_CSR_RD(pmisc_addr, ADF_4XXX_VM2PF_MSK(i));
+ sou &= ~mask;
+ vf_mask |= sou << i;
+ }
+
+ return vf_mask;
+}
+
+static void adf_gen4_enable_vf2pf_interrupts(void __iomem *pmisc_addr,
+ u32 vf_mask)
+{
+ int num_csrs = ADF_4XXX_MAX_NUM_VFS;
+ unsigned long mask = vf_mask;
+ unsigned int val;
+ int i;
+
+ for_each_set_bit(i, &mask, num_csrs) {
+ unsigned int offset = ADF_4XXX_VM2PF_MSK(i);
+
+ val = ADF_CSR_RD(pmisc_addr, offset) & ~ADF_4XXX_VM2PF_INT_EN_MSK;
+ ADF_CSR_WR(pmisc_addr, offset, val);
+ }
+}
+
+static void adf_gen4_disable_vf2pf_interrupts(void __iomem *pmisc_addr,
+ u32 vf_mask)
+{
+ int num_csrs = ADF_4XXX_MAX_NUM_VFS;
+ unsigned long mask = vf_mask;
+ unsigned int val;
+ int i;
+
+ for_each_set_bit(i, &mask, num_csrs) {
+ unsigned int offset = ADF_4XXX_VM2PF_MSK(i);
+
+ val = ADF_CSR_RD(pmisc_addr, offset) | ADF_4XXX_VM2PF_INT_EN_MSK;
+ ADF_CSR_WR(pmisc_addr, offset, val);
+ }
+}
+
+static int adf_gen4_pfvf_send(struct adf_accel_dev *accel_dev,
+ struct pfvf_message msg, u32 pfvf_offset,
+ struct mutex *csr_lock)
+{
+ void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev);
+ u32 csr_val;
+ int ret;
+
+ csr_val = adf_pfvf_csr_msg_of(accel_dev, msg, &csr_gen4_fmt);
+ if (unlikely(!csr_val))
+ return -EINVAL;
+
+ mutex_lock(csr_lock);
+
+ ADF_CSR_WR(pmisc_addr, pfvf_offset, csr_val | ADF_PFVF_INT);
+
+ /* Wait for confirmation from remote that it received the message */
+ ret = read_poll_timeout(ADF_CSR_RD, csr_val, !(csr_val & ADF_PFVF_INT),
+ ADF_PFVF_MSG_ACK_DELAY_US,
+ ADF_PFVF_MSG_ACK_MAX_DELAY_US,
+ true, pmisc_addr, pfvf_offset);
+ if (ret < 0)
+ dev_dbg(&GET_DEV(accel_dev), "ACK not received from remote\n");
+
+ mutex_unlock(csr_lock);
+ return ret;
+}
+
+static struct pfvf_message adf_gen4_pfvf_recv(struct adf_accel_dev *accel_dev,
+ u32 pfvf_offset, u8 compat_ver)
+{
+ void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev);
+ u32 csr_val;
+
+ /* Read message from the CSR */
+ csr_val = ADF_CSR_RD(pmisc_addr, pfvf_offset);
+
+ /* We can now acknowledge the message reception by clearing the
+ * interrupt bit
+ */
+ ADF_CSR_WR(pmisc_addr, pfvf_offset, csr_val & ~ADF_PFVF_INT);
+
+ /* Return the pfvf_message format */
+ return adf_pfvf_message_of(accel_dev, csr_val, &csr_gen4_fmt);
+}
+
+void adf_gen4_init_pf_pfvf_ops(struct adf_pfvf_ops *pfvf_ops)
+{
+ pfvf_ops->enable_comms = adf_enable_pf2vf_comms;
+ pfvf_ops->get_pf2vf_offset = adf_gen4_pf_get_pf2vf_offset;
+ pfvf_ops->get_vf2pf_offset = adf_gen4_pf_get_vf2pf_offset;
+ pfvf_ops->get_vf2pf_sources = adf_gen4_get_vf2pf_sources;
+ pfvf_ops->enable_vf2pf_interrupts = adf_gen4_enable_vf2pf_interrupts;
+ pfvf_ops->disable_vf2pf_interrupts = adf_gen4_disable_vf2pf_interrupts;
+ pfvf_ops->send_msg = adf_gen4_pfvf_send;
+ pfvf_ops->recv_msg = adf_gen4_pfvf_recv;
+}
+EXPORT_SYMBOL_GPL(adf_gen4_init_pf_pfvf_ops);
diff --git a/drivers/crypto/qat/qat_common/adf_gen4_pfvf.h b/drivers/crypto/qat/qat_common/adf_gen4_pfvf.h
new file mode 100644
index 000000000000..17d1b774d4a8
--- /dev/null
+++ b/drivers/crypto/qat/qat_common/adf_gen4_pfvf.h
@@ -0,0 +1,17 @@
+/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */
+/* Copyright(c) 2021 Intel Corporation */
+#ifndef ADF_GEN4_PFVF_H
+#define ADF_GEN4_PFVF_H
+
+#include "adf_accel_devices.h"
+
+#ifdef CONFIG_PCI_IOV
+void adf_gen4_init_pf_pfvf_ops(struct adf_pfvf_ops *pfvf_ops);
+#else
+static inline void adf_gen4_init_pf_pfvf_ops(struct adf_pfvf_ops *pfvf_ops)
+{
+ pfvf_ops->enable_comms = adf_pfvf_comms_disabled;
+}
+#endif
+
+#endif /* ADF_GEN4_PFVF_H */
diff --git a/drivers/crypto/qat/qat_common/adf_init.c b/drivers/crypto/qat/qat_common/adf_init.c
index e3749e5817d9..2edc63c6b6ca 100644
--- a/drivers/crypto/qat/qat_common/adf_init.c
+++ b/drivers/crypto/qat/qat_common/adf_init.c
@@ -69,7 +69,8 @@ int adf_dev_init(struct adf_accel_dev *accel_dev)
return -EFAULT;
}
- if (!test_bit(ADF_STATUS_CONFIGURED, &accel_dev->status)) {
+ if (!test_bit(ADF_STATUS_CONFIGURED, &accel_dev->status) &&
+ !accel_dev->is_vf) {
dev_err(&GET_DEV(accel_dev), "Device not configured\n");
return -EFAULT;
}
@@ -117,10 +118,16 @@ int adf_dev_init(struct adf_accel_dev *accel_dev)
hw_data->enable_ints(accel_dev);
hw_data->enable_error_correction(accel_dev);
- ret = hw_data->enable_pfvf_comms(accel_dev);
+ ret = hw_data->pfvf_ops.enable_comms(accel_dev);
if (ret)
return ret;
+ if (!test_bit(ADF_STATUS_CONFIGURED, &accel_dev->status) &&
+ accel_dev->is_vf) {
+ if (qat_crypto_vf_dev_config(accel_dev))
+ return -EFAULT;
+ }
+
/*
* Subservice initialisation is divided into two stages: init and start.
* This is to facilitate any ordering dependencies between services
diff --git a/drivers/crypto/qat/qat_common/adf_isr.c b/drivers/crypto/qat/qat_common/adf_isr.c
index 40593c9449a2..4ca482aa69f7 100644
--- a/drivers/crypto/qat/qat_common/adf_isr.c
+++ b/drivers/crypto/qat/qat_common/adf_isr.c
@@ -54,52 +54,83 @@ static irqreturn_t adf_msix_isr_bundle(int irq, void *bank_ptr)
return IRQ_HANDLED;
}
-static irqreturn_t adf_msix_isr_ae(int irq, void *dev_ptr)
+#ifdef CONFIG_PCI_IOV
+void adf_enable_vf2pf_interrupts(struct adf_accel_dev *accel_dev, u32 vf_mask)
{
- struct adf_accel_dev *accel_dev = dev_ptr;
+ void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev);
+ unsigned long flags;
-#ifdef CONFIG_PCI_IOV
- /* If SR-IOV is enabled (vf_info is non-NULL), check for VF->PF ints */
- if (accel_dev->pf.vf_info) {
- struct adf_hw_device_data *hw_data = accel_dev->hw_device;
- struct adf_bar *pmisc =
- &GET_BARS(accel_dev)[hw_data->get_misc_bar_id(hw_data)];
- void __iomem *pmisc_addr = pmisc->virt_addr;
- unsigned long vf_mask;
-
- /* Get the interrupt sources triggered by VFs */
- vf_mask = hw_data->get_vf2pf_sources(pmisc_addr);
-
- if (vf_mask) {
- struct adf_accel_vf_info *vf_info;
- bool irq_handled = false;
- int i;
-
- /* Disable VF2PF interrupts for VFs with pending ints */
- adf_disable_vf2pf_interrupts_irq(accel_dev, vf_mask);
-
- /*
- * Handle VF2PF interrupt unless the VF is malicious and
- * is attempting to flood the host OS with VF2PF interrupts.
- */
- for_each_set_bit(i, &vf_mask, ADF_MAX_NUM_VFS) {
- vf_info = accel_dev->pf.vf_info + i;
-
- if (!__ratelimit(&vf_info->vf2pf_ratelimit)) {
- dev_info(&GET_DEV(accel_dev),
- "Too many ints from VF%d\n",
- vf_info->vf_nr + 1);
- continue;
- }
-
- adf_schedule_vf2pf_handler(vf_info);
- irq_handled = true;
+ spin_lock_irqsave(&accel_dev->pf.vf2pf_ints_lock, flags);
+ GET_PFVF_OPS(accel_dev)->enable_vf2pf_interrupts(pmisc_addr, vf_mask);
+ spin_unlock_irqrestore(&accel_dev->pf.vf2pf_ints_lock, flags);
+}
+
+void adf_disable_vf2pf_interrupts(struct adf_accel_dev *accel_dev, u32 vf_mask)
+{
+ void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev);
+ unsigned long flags;
+
+ spin_lock_irqsave(&accel_dev->pf.vf2pf_ints_lock, flags);
+ GET_PFVF_OPS(accel_dev)->disable_vf2pf_interrupts(pmisc_addr, vf_mask);
+ spin_unlock_irqrestore(&accel_dev->pf.vf2pf_ints_lock, flags);
+}
+
+static void adf_disable_vf2pf_interrupts_irq(struct adf_accel_dev *accel_dev,
+ u32 vf_mask)
+{
+ void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev);
+
+ spin_lock(&accel_dev->pf.vf2pf_ints_lock);
+ GET_PFVF_OPS(accel_dev)->disable_vf2pf_interrupts(pmisc_addr, vf_mask);
+ spin_unlock(&accel_dev->pf.vf2pf_ints_lock);
+}
+
+static bool adf_handle_vf2pf_int(struct adf_accel_dev *accel_dev)
+{
+ void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev);
+ bool irq_handled = false;
+ unsigned long vf_mask;
+
+ /* Get the interrupt sources triggered by VFs */
+ vf_mask = GET_PFVF_OPS(accel_dev)->get_vf2pf_sources(pmisc_addr);
+
+ if (vf_mask) {
+ struct adf_accel_vf_info *vf_info;
+ int i;
+
+ /* Disable VF2PF interrupts for VFs with pending ints */
+ adf_disable_vf2pf_interrupts_irq(accel_dev, vf_mask);
+
+ /*
+ * Handle VF2PF interrupt unless the VF is malicious and
+ * is attempting to flood the host OS with VF2PF interrupts.
+ */
+ for_each_set_bit(i, &vf_mask, ADF_MAX_NUM_VFS) {
+ vf_info = accel_dev->pf.vf_info + i;
+
+ if (!__ratelimit(&vf_info->vf2pf_ratelimit)) {
+ dev_info(&GET_DEV(accel_dev),
+ "Too many ints from VF%d\n",
+ vf_info->vf_nr);
+ continue;
}
- if (irq_handled)
- return IRQ_HANDLED;
+ adf_schedule_vf2pf_handler(vf_info);
+ irq_handled = true;
}
}
+ return irq_handled;
+}
+#endif /* CONFIG_PCI_IOV */
+
+static irqreturn_t adf_msix_isr_ae(int irq, void *dev_ptr)
+{
+ struct adf_accel_dev *accel_dev = dev_ptr;
+
+#ifdef CONFIG_PCI_IOV
+ /* If SR-IOV is enabled (vf_info is non-NULL), check for VF->PF ints */
+ if (accel_dev->pf.vf_info && adf_handle_vf2pf_int(accel_dev))
+ return IRQ_HANDLED;
#endif /* CONFIG_PCI_IOV */
dev_dbg(&GET_DEV(accel_dev), "qat_dev%d spurious AE interrupt\n",
diff --git a/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c b/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c
deleted file mode 100644
index 59860bdaedb6..000000000000
--- a/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c
+++ /dev/null
@@ -1,416 +0,0 @@
-// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
-/* Copyright(c) 2015 - 2020 Intel Corporation */
-#include <linux/delay.h>
-#include "adf_accel_devices.h"
-#include "adf_common_drv.h"
-#include "adf_pf2vf_msg.h"
-
-#define ADF_PFVF_MSG_COLLISION_DETECT_DELAY 10
-#define ADF_PFVF_MSG_ACK_DELAY 2
-#define ADF_PFVF_MSG_ACK_MAX_RETRY 100
-#define ADF_PFVF_MSG_RETRY_DELAY 5
-#define ADF_PFVF_MSG_MAX_RETRIES 3
-#define ADF_PFVF_MSG_RESP_TIMEOUT (ADF_PFVF_MSG_ACK_DELAY * \
- ADF_PFVF_MSG_ACK_MAX_RETRY + \
- ADF_PFVF_MSG_COLLISION_DETECT_DELAY)
-
-void adf_enable_vf2pf_interrupts(struct adf_accel_dev *accel_dev, u32 vf_mask)
-{
- struct adf_hw_device_data *hw_data = accel_dev->hw_device;
- u32 misc_bar_id = hw_data->get_misc_bar_id(hw_data);
- struct adf_bar *pmisc = &GET_BARS(accel_dev)[misc_bar_id];
- void __iomem *pmisc_addr = pmisc->virt_addr;
- unsigned long flags;
-
- spin_lock_irqsave(&accel_dev->pf.vf2pf_ints_lock, flags);
- hw_data->enable_vf2pf_interrupts(pmisc_addr, vf_mask);
- spin_unlock_irqrestore(&accel_dev->pf.vf2pf_ints_lock, flags);
-}
-
-void adf_disable_vf2pf_interrupts(struct adf_accel_dev *accel_dev, u32 vf_mask)
-{
- struct adf_hw_device_data *hw_data = accel_dev->hw_device;
- u32 misc_bar_id = hw_data->get_misc_bar_id(hw_data);
- struct adf_bar *pmisc = &GET_BARS(accel_dev)[misc_bar_id];
- void __iomem *pmisc_addr = pmisc->virt_addr;
- unsigned long flags;
-
- spin_lock_irqsave(&accel_dev->pf.vf2pf_ints_lock, flags);
- hw_data->disable_vf2pf_interrupts(pmisc_addr, vf_mask);
- spin_unlock_irqrestore(&accel_dev->pf.vf2pf_ints_lock, flags);
-}
-
-void adf_disable_vf2pf_interrupts_irq(struct adf_accel_dev *accel_dev,
- u32 vf_mask)
-{
- struct adf_hw_device_data *hw_data = accel_dev->hw_device;
- u32 misc_bar_id = hw_data->get_misc_bar_id(hw_data);
- struct adf_bar *pmisc = &GET_BARS(accel_dev)[misc_bar_id];
- void __iomem *pmisc_addr = pmisc->virt_addr;
-
- spin_lock(&accel_dev->pf.vf2pf_ints_lock);
- hw_data->disable_vf2pf_interrupts(pmisc_addr, vf_mask);
- spin_unlock(&accel_dev->pf.vf2pf_ints_lock);
-}
-
-static int __adf_iov_putmsg(struct adf_accel_dev *accel_dev, u32 msg, u8 vf_nr)
-{
- struct adf_accel_pci *pci_info = &accel_dev->accel_pci_dev;
- struct adf_hw_device_data *hw_data = accel_dev->hw_device;
- void __iomem *pmisc_bar_addr =
- pci_info->pci_bars[hw_data->get_misc_bar_id(hw_data)].virt_addr;
- u32 val, pf2vf_offset, count = 0;
- u32 local_in_use_mask, local_in_use_pattern;
- u32 remote_in_use_mask, remote_in_use_pattern;
- struct mutex *lock; /* lock preventing concurrent acces of CSR */
- u32 int_bit;
- int ret = 0;
-
- if (accel_dev->is_vf) {
- pf2vf_offset = hw_data->get_pf2vf_offset(0);
- lock = &accel_dev->vf.vf2pf_lock;
- local_in_use_mask = ADF_VF2PF_IN_USE_BY_VF_MASK;
- local_in_use_pattern = ADF_VF2PF_IN_USE_BY_VF;
- remote_in_use_mask = ADF_PF2VF_IN_USE_BY_PF_MASK;
- remote_in_use_pattern = ADF_PF2VF_IN_USE_BY_PF;
- int_bit = ADF_VF2PF_INT;
- } else {
- pf2vf_offset = hw_data->get_pf2vf_offset(vf_nr);
- lock = &accel_dev->pf.vf_info[vf_nr].pf2vf_lock;
- local_in_use_mask = ADF_PF2VF_IN_USE_BY_PF_MASK;
- local_in_use_pattern = ADF_PF2VF_IN_USE_BY_PF;
- remote_in_use_mask = ADF_VF2PF_IN_USE_BY_VF_MASK;
- remote_in_use_pattern = ADF_VF2PF_IN_USE_BY_VF;
- int_bit = ADF_PF2VF_INT;
- }
-
- mutex_lock(lock);
-
- /* Check if the PFVF CSR is in use by remote function */
- val = ADF_CSR_RD(pmisc_bar_addr, pf2vf_offset);
- if ((val & remote_in_use_mask) == remote_in_use_pattern) {
- dev_dbg(&GET_DEV(accel_dev),
- "PFVF CSR in use by remote function\n");
- ret = -EBUSY;
- goto out;
- }
-
- msg &= ~local_in_use_mask;
- msg |= local_in_use_pattern;
-
- /* Attempt to get ownership of the PFVF CSR */
- ADF_CSR_WR(pmisc_bar_addr, pf2vf_offset, msg | int_bit);
-
- /* Wait for confirmation from remote func it received the message */
- do {
- msleep(ADF_PFVF_MSG_ACK_DELAY);
- val = ADF_CSR_RD(pmisc_bar_addr, pf2vf_offset);
- } while ((val & int_bit) && (count++ < ADF_PFVF_MSG_ACK_MAX_RETRY));
-
- if (val != msg) {
- dev_dbg(&GET_DEV(accel_dev),
- "Collision - PFVF CSR overwritten by remote function\n");
- ret = -EIO;
- goto out;
- }
-
- if (val & int_bit) {
- dev_dbg(&GET_DEV(accel_dev), "ACK not received from remote\n");
- val &= ~int_bit;
- ret = -EIO;
- }
-
- /* Finished with the PFVF CSR; relinquish it and leave msg in CSR */
- ADF_CSR_WR(pmisc_bar_addr, pf2vf_offset, val & ~local_in_use_mask);
-out:
- mutex_unlock(lock);
- return ret;
-}
-
-/**
- * adf_iov_putmsg() - send PFVF message
- * @accel_dev: Pointer to acceleration device.
- * @msg: Message to send
- * @vf_nr: VF number to which the message will be sent if on PF, ignored
- * otherwise
- *
- * Function sends a message through the PFVF channel
- *
- * Return: 0 on success, error code otherwise.
- */
-static int adf_iov_putmsg(struct adf_accel_dev *accel_dev, u32 msg, u8 vf_nr)
-{
- u32 count = 0;
- int ret;
-
- do {
- ret = __adf_iov_putmsg(accel_dev, msg, vf_nr);
- if (ret)
- msleep(ADF_PFVF_MSG_RETRY_DELAY);
- } while (ret && (count++ < ADF_PFVF_MSG_MAX_RETRIES));
-
- return ret;
-}
-
-/**
- * adf_send_pf2vf_msg() - send PF to VF message
- * @accel_dev: Pointer to acceleration device
- * @vf_nr: VF number to which the message will be sent
- * @msg: Message to send
- *
- * This function allows the PF to send a message to a specific VF.
- *
- * Return: 0 on success, error code otherwise.
- */
-static int adf_send_pf2vf_msg(struct adf_accel_dev *accel_dev, u8 vf_nr, u32 msg)
-{
- return adf_iov_putmsg(accel_dev, msg, vf_nr);
-}
-
-/**
- * adf_send_vf2pf_msg() - send VF to PF message
- * @accel_dev: Pointer to acceleration device
- * @msg: Message to send
- *
- * This function allows the VF to send a message to the PF.
- *
- * Return: 0 on success, error code otherwise.
- */
-int adf_send_vf2pf_msg(struct adf_accel_dev *accel_dev, u32 msg)
-{
- return adf_iov_putmsg(accel_dev, msg, 0);
-}
-
-/**
- * adf_send_vf2pf_req() - send VF2PF request message
- * @accel_dev: Pointer to acceleration device.
- * @msg: Request message to send
- *
- * This function sends a message that requires a response from the VF to the PF
- * and waits for a reply.
- *
- * Return: 0 on success, error code otherwise.
- */
-static int adf_send_vf2pf_req(struct adf_accel_dev *accel_dev, u32 msg)
-{
- unsigned long timeout = msecs_to_jiffies(ADF_PFVF_MSG_RESP_TIMEOUT);
- int ret;
-
- reinit_completion(&accel_dev->vf.iov_msg_completion);
-
- /* Send request from VF to PF */
- ret = adf_send_vf2pf_msg(accel_dev, msg);
- if (ret) {
- dev_err(&GET_DEV(accel_dev),
- "Failed to send request msg to PF\n");
- return ret;
- }
-
- /* Wait for response */
- if (!wait_for_completion_timeout(&accel_dev->vf.iov_msg_completion,
- timeout)) {
- dev_err(&GET_DEV(accel_dev),
- "PFVF request/response message timeout expired\n");
- return -EIO;
- }
-
- return 0;
-}
-
-void adf_vf2pf_req_hndl(struct adf_accel_vf_info *vf_info)
-{
- struct adf_accel_dev *accel_dev = vf_info->accel_dev;
- struct adf_hw_device_data *hw_data = accel_dev->hw_device;
- int bar_id = hw_data->get_misc_bar_id(hw_data);
- struct adf_bar *pmisc = &GET_BARS(accel_dev)[bar_id];
- void __iomem *pmisc_addr = pmisc->virt_addr;
- u32 msg, resp = 0, vf_nr = vf_info->vf_nr;
-
- /* Read message from the VF */
- msg = ADF_CSR_RD(pmisc_addr, hw_data->get_pf2vf_offset(vf_nr));
- if (!(msg & ADF_VF2PF_INT)) {
- dev_info(&GET_DEV(accel_dev),
- "Spurious VF2PF interrupt, msg %X. Ignored\n", msg);
- goto out;
- }
-
- /* To ACK, clear the VF2PFINT bit */
- msg &= ~ADF_VF2PF_INT;
- ADF_CSR_WR(pmisc_addr, hw_data->get_pf2vf_offset(vf_nr), msg);
-
- if (!(msg & ADF_VF2PF_MSGORIGIN_SYSTEM))
- /* Ignore legacy non-system (non-kernel) VF2PF messages */
- goto err;
-
- switch ((msg & ADF_VF2PF_MSGTYPE_MASK) >> ADF_VF2PF_MSGTYPE_SHIFT) {
- case ADF_VF2PF_MSGTYPE_COMPAT_VER_REQ:
- {
- u8 vf_compat_ver = msg >> ADF_VF2PF_COMPAT_VER_REQ_SHIFT;
-
- resp = (ADF_PF2VF_MSGORIGIN_SYSTEM |
- (ADF_PF2VF_MSGTYPE_VERSION_RESP <<
- ADF_PF2VF_MSGTYPE_SHIFT) |
- (ADF_PFVF_COMPAT_THIS_VERSION <<
- ADF_PF2VF_VERSION_RESP_VERS_SHIFT));
-
- dev_dbg(&GET_DEV(accel_dev),
- "Compatibility Version Request from VF%d vers=%u\n",
- vf_nr + 1, vf_compat_ver);
-
- if (vf_compat_ver < hw_data->min_iov_compat_ver) {
- dev_err(&GET_DEV(accel_dev),
- "VF (vers %d) incompatible with PF (vers %d)\n",
- vf_compat_ver, ADF_PFVF_COMPAT_THIS_VERSION);
- resp |= ADF_PF2VF_VF_INCOMPATIBLE <<
- ADF_PF2VF_VERSION_RESP_RESULT_SHIFT;
- } else if (vf_compat_ver > ADF_PFVF_COMPAT_THIS_VERSION) {
- dev_err(&GET_DEV(accel_dev),
- "VF (vers %d) compat with PF (vers %d) unkn.\n",
- vf_compat_ver, ADF_PFVF_COMPAT_THIS_VERSION);
- resp |= ADF_PF2VF_VF_COMPAT_UNKNOWN <<
- ADF_PF2VF_VERSION_RESP_RESULT_SHIFT;
- } else {
- dev_dbg(&GET_DEV(accel_dev),
- "VF (vers %d) compatible with PF (vers %d)\n",
- vf_compat_ver, ADF_PFVF_COMPAT_THIS_VERSION);
- resp |= ADF_PF2VF_VF_COMPATIBLE <<
- ADF_PF2VF_VERSION_RESP_RESULT_SHIFT;
- }
- }
- break;
- case ADF_VF2PF_MSGTYPE_VERSION_REQ:
- dev_dbg(&GET_DEV(accel_dev),
- "Legacy VersionRequest received from VF%d 0x%x\n",
- vf_nr + 1, msg);
- resp = (ADF_PF2VF_MSGORIGIN_SYSTEM |
- (ADF_PF2VF_MSGTYPE_VERSION_RESP <<
- ADF_PF2VF_MSGTYPE_SHIFT) |
- (ADF_PFVF_COMPAT_THIS_VERSION <<
- ADF_PF2VF_VERSION_RESP_VERS_SHIFT));
- resp |= ADF_PF2VF_VF_COMPATIBLE <<
- ADF_PF2VF_VERSION_RESP_RESULT_SHIFT;
- /* Set legacy major and minor version num */
- resp |= 1 << ADF_PF2VF_MAJORVERSION_SHIFT |
- 1 << ADF_PF2VF_MINORVERSION_SHIFT;
- break;
- case ADF_VF2PF_MSGTYPE_INIT:
- {
- dev_dbg(&GET_DEV(accel_dev),
- "Init message received from VF%d 0x%x\n",
- vf_nr + 1, msg);
- vf_info->init = true;
- }
- break;
- case ADF_VF2PF_MSGTYPE_SHUTDOWN:
- {
- dev_dbg(&GET_DEV(accel_dev),
- "Shutdown message received from VF%d 0x%x\n",
- vf_nr + 1, msg);
- vf_info->init = false;
- }
- break;
- default:
- goto err;
- }
-
- if (resp && adf_send_pf2vf_msg(accel_dev, vf_nr, resp))
- dev_err(&GET_DEV(accel_dev), "Failed to send response to VF\n");
-
-out:
- /* re-enable interrupt on PF from this VF */
- adf_enable_vf2pf_interrupts(accel_dev, (1 << vf_nr));
-
- return;
-err:
- dev_dbg(&GET_DEV(accel_dev), "Unknown message from VF%d (0x%x);\n",
- vf_nr + 1, msg);
-}
-
-void adf_pf2vf_notify_restarting(struct adf_accel_dev *accel_dev)
-{
- struct adf_accel_vf_info *vf;
- u32 msg = (ADF_PF2VF_MSGORIGIN_SYSTEM |
- (ADF_PF2VF_MSGTYPE_RESTARTING << ADF_PF2VF_MSGTYPE_SHIFT));
- int i, num_vfs = pci_num_vf(accel_to_pci_dev(accel_dev));
-
- for (i = 0, vf = accel_dev->pf.vf_info; i < num_vfs; i++, vf++) {
- if (vf->init && adf_send_pf2vf_msg(accel_dev, i, msg))
- dev_err(&GET_DEV(accel_dev),
- "Failed to send restarting msg to VF%d\n", i);
- }
-}
-
-static int adf_vf2pf_request_version(struct adf_accel_dev *accel_dev)
-{
- struct adf_hw_device_data *hw_data = accel_dev->hw_device;
- u32 msg = 0;
- int ret;
-
- msg = ADF_VF2PF_MSGORIGIN_SYSTEM;
- msg |= ADF_VF2PF_MSGTYPE_COMPAT_VER_REQ << ADF_VF2PF_MSGTYPE_SHIFT;
- msg |= ADF_PFVF_COMPAT_THIS_VERSION << ADF_VF2PF_COMPAT_VER_REQ_SHIFT;
- BUILD_BUG_ON(ADF_PFVF_COMPAT_THIS_VERSION > 255);
-
- ret = adf_send_vf2pf_req(accel_dev, msg);
- if (ret) {
- dev_err(&GET_DEV(accel_dev),
- "Failed to send Compatibility Version Request.\n");
- return ret;
- }
-
- /* Response from PF received, check compatibility */
- switch (accel_dev->vf.compatible) {
- case ADF_PF2VF_VF_COMPATIBLE:
- break;
- case ADF_PF2VF_VF_COMPAT_UNKNOWN:
- /* VF is newer than PF and decides whether it is compatible */
- if (accel_dev->vf.pf_version >= hw_data->min_iov_compat_ver) {
- accel_dev->vf.compatible = ADF_PF2VF_VF_COMPATIBLE;
- break;
- }
- fallthrough;
- case ADF_PF2VF_VF_INCOMPATIBLE:
- dev_err(&GET_DEV(accel_dev),
- "PF (vers %d) and VF (vers %d) are not compatible\n",
- accel_dev->vf.pf_version,
- ADF_PFVF_COMPAT_THIS_VERSION);
- return -EINVAL;
- default:
- dev_err(&GET_DEV(accel_dev),
- "Invalid response from PF; assume not compatible\n");
- return -EINVAL;
- }
- return ret;
-}
-
-/**
- * adf_enable_vf2pf_comms() - Function enables communication from vf to pf
- *
- * @accel_dev: Pointer to acceleration device virtual function.
- *
- * Return: 0 on success, error code otherwise.
- */
-int adf_enable_vf2pf_comms(struct adf_accel_dev *accel_dev)
-{
- adf_enable_pf2vf_interrupts(accel_dev);
- return adf_vf2pf_request_version(accel_dev);
-}
-EXPORT_SYMBOL_GPL(adf_enable_vf2pf_comms);
-
-/**
- * adf_enable_pf2vf_comms() - Function enables communication from pf to vf
- *
- * @accel_dev: Pointer to acceleration device virtual function.
- *
- * This function carries out the necessary steps to setup and start the PFVF
- * communication channel, if any.
- *
- * Return: 0 on success, error code otherwise.
- */
-int adf_enable_pf2vf_comms(struct adf_accel_dev *accel_dev)
-{
- spin_lock_init(&accel_dev->pf.vf2pf_ints_lock);
-
- return 0;
-}
-EXPORT_SYMBOL_GPL(adf_enable_pf2vf_comms);
diff --git a/drivers/crypto/qat/qat_common/adf_pf2vf_msg.h b/drivers/crypto/qat/qat_common/adf_pf2vf_msg.h
deleted file mode 100644
index a7d8f8367345..000000000000
--- a/drivers/crypto/qat/qat_common/adf_pf2vf_msg.h
+++ /dev/null
@@ -1,93 +0,0 @@
-/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */
-/* Copyright(c) 2015 - 2020 Intel Corporation */
-#ifndef ADF_PF2VF_MSG_H
-#define ADF_PF2VF_MSG_H
-
-/*
- * PF<->VF Messaging
- * The PF has an array of 32-bit PF2VF registers, one for each VF. The
- * PF can access all these registers; each VF can access only the one
- * register associated with that particular VF.
- *
- * The register functionally is split into two parts:
- * The bottom half is for PF->VF messages. In particular when the first
- * bit of this register (bit 0) gets set an interrupt will be triggered
- * in the respective VF.
- * The top half is for VF->PF messages. In particular when the first bit
- * of this half of register (bit 16) gets set an interrupt will be triggered
- * in the PF.
- *
- * The remaining bits within this register are available to encode messages.
- * and implement a collision control mechanism to prevent concurrent use of
- * the PF2VF register by both the PF and VF.
- *
- * 31 30 29 28 27 26 25 24 23 22 21 20 19 18 17 16
- * _______________________________________________
- * | | | | | | | | | | | | | | | | |
- * +-----------------------------------------------+
- * \___________________________/ \_________/ ^ ^
- * ^ ^ | |
- * | | | VF2PF Int
- * | | Message Origin
- * | Message Type
- * Message-specific Data/Reserved
- *
- * 15 14 13 12 11 10 9 8 7 6 5 4 3 2 1 0
- * _______________________________________________
- * | | | | | | | | | | | | | | | | |
- * +-----------------------------------------------+
- * \___________________________/ \_________/ ^ ^
- * ^ ^ | |
- * | | | PF2VF Int
- * | | Message Origin
- * | Message Type
- * Message-specific Data/Reserved
- *
- * Message Origin (Should always be 1)
- * A legacy out-of-tree QAT driver allowed for a set of messages not supported
- * by this driver; these had a Msg Origin of 0 and are ignored by this driver.
- *
- * When a PF or VF attempts to send a message in the lower or upper 16 bits,
- * respectively, the other 16 bits are written to first with a defined
- * IN_USE_BY pattern as part of a collision control scheme (see adf_iov_putmsg).
- */
-
-#define ADF_PFVF_COMPAT_THIS_VERSION 0x1 /* PF<->VF compat */
-
-/* PF->VF messages */
-#define ADF_PF2VF_INT BIT(0)
-#define ADF_PF2VF_MSGORIGIN_SYSTEM BIT(1)
-#define ADF_PF2VF_MSGTYPE_MASK 0x0000003C
-#define ADF_PF2VF_MSGTYPE_SHIFT 2
-#define ADF_PF2VF_MSGTYPE_RESTARTING 0x01
-#define ADF_PF2VF_MSGTYPE_VERSION_RESP 0x02
-#define ADF_PF2VF_IN_USE_BY_PF 0x6AC20000
-#define ADF_PF2VF_IN_USE_BY_PF_MASK 0xFFFE0000
-
-/* PF->VF Version Response */
-#define ADF_PF2VF_VERSION_RESP_VERS_MASK 0x00003FC0
-#define ADF_PF2VF_VERSION_RESP_VERS_SHIFT 6
-#define ADF_PF2VF_VERSION_RESP_RESULT_MASK 0x0000C000
-#define ADF_PF2VF_VERSION_RESP_RESULT_SHIFT 14
-#define ADF_PF2VF_MINORVERSION_SHIFT 6
-#define ADF_PF2VF_MAJORVERSION_SHIFT 10
-#define ADF_PF2VF_VF_COMPATIBLE 1
-#define ADF_PF2VF_VF_INCOMPATIBLE 2
-#define ADF_PF2VF_VF_COMPAT_UNKNOWN 3
-
-/* VF->PF messages */
-#define ADF_VF2PF_IN_USE_BY_VF 0x00006AC2
-#define ADF_VF2PF_IN_USE_BY_VF_MASK 0x0000FFFE
-#define ADF_VF2PF_INT BIT(16)
-#define ADF_VF2PF_MSGORIGIN_SYSTEM BIT(17)
-#define ADF_VF2PF_MSGTYPE_MASK 0x003C0000
-#define ADF_VF2PF_MSGTYPE_SHIFT 18
-#define ADF_VF2PF_MSGTYPE_INIT 0x3
-#define ADF_VF2PF_MSGTYPE_SHUTDOWN 0x4
-#define ADF_VF2PF_MSGTYPE_VERSION_REQ 0x5
-#define ADF_VF2PF_MSGTYPE_COMPAT_VER_REQ 0x6
-
-/* VF->PF Compatible Version Request */
-#define ADF_VF2PF_COMPAT_VER_REQ_SHIFT 22
-
-#endif /* ADF_IOV_MSG_H */
diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_msg.h b/drivers/crypto/qat/qat_common/adf_pfvf_msg.h
new file mode 100644
index 000000000000..9c37a2661392
--- /dev/null
+++ b/drivers/crypto/qat/qat_common/adf_pfvf_msg.h
@@ -0,0 +1,259 @@
+/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */
+/* Copyright(c) 2015 - 2021 Intel Corporation */
+#ifndef ADF_PFVF_MSG_H
+#define ADF_PFVF_MSG_H
+
+#include <linux/bits.h>
+
+/*
+ * PF<->VF Gen2 Messaging format
+ *
+ * The PF has an array of 32-bit PF2VF registers, one for each VF. The
+ * PF can access all these registers; each VF can access only the one
+ * register associated with that particular VF.
+ *
+ * The register functionally is split into two parts:
+ * The bottom half is for PF->VF messages. In particular when the first
+ * bit of this register (bit 0) gets set an interrupt will be triggered
+ * in the respective VF.
+ * The top half is for VF->PF messages. In particular when the first bit
+ * of this half of register (bit 16) gets set an interrupt will be triggered
+ * in the PF.
+ *
+ * The remaining bits within this register are available to encode messages.
+ * and implement a collision control mechanism to prevent concurrent use of
+ * the PF2VF register by both the PF and VF.
+ *
+ * 31 30 29 28 27 26 25 24 23 22 21 20 19 18 17 16
+ * _______________________________________________
+ * | | | | | | | | | | | | | | | | |
+ * +-----------------------------------------------+
+ * \___________________________/ \_________/ ^ ^
+ * ^ ^ | |
+ * | | | VF2PF Int
+ * | | Message Origin
+ * | Message Type
+ * Message-specific Data/Reserved
+ *
+ * 15 14 13 12 11 10 9 8 7 6 5 4 3 2 1 0
+ * _______________________________________________
+ * | | | | | | | | | | | | | | | | |
+ * +-----------------------------------------------+
+ * \___________________________/ \_________/ ^ ^
+ * ^ ^ | |
+ * | | | PF2VF Int
+ * | | Message Origin
+ * | Message Type
+ * Message-specific Data/Reserved
+ *
+ * Message Origin (Should always be 1)
+ * A legacy out-of-tree QAT driver allowed for a set of messages not supported
+ * by this driver; these had a Msg Origin of 0 and are ignored by this driver.
+ *
+ * When a PF or VF attempts to send a message in the lower or upper 16 bits,
+ * respectively, the other 16 bits are written to first with a defined
+ * IN_USE_BY pattern as part of a collision control scheme (see function
+ * adf_gen2_pfvf_send() in adf_pf2vf_msg.c).
+ *
+ *
+ * PF<->VF Gen4 Messaging format
+ *
+ * Similarly to the gen2 messaging format, 32-bit long registers are used for
+ * communication between PF and VFs. However, each VF and PF share a pair of
+ * 32-bits register to avoid collisions: one for PV to VF messages and one
+ * for VF to PF messages.
+ *
+ * Both the Interrupt bit and the Message Origin bit retain the same position
+ * and meaning, although non-system messages are now deprecated and not
+ * expected.
+ *
+ * 31 30 9 8 7 6 5 4 3 2 1 0
+ * _______________________________________________
+ * | | | . . . | | | | | | | | | | |
+ * +-----------------------------------------------+
+ * \_____________________/ \_______________/ ^ ^
+ * ^ ^ | |
+ * | | | PF/VF Int
+ * | | Message Origin
+ * | Message Type
+ * Message-specific Data/Reserved
+ *
+ * For both formats, the message reception is acknowledged by lowering the
+ * interrupt bit on the register where the message was sent.
+ */
+
+/* PFVF message common bits */
+#define ADF_PFVF_INT BIT(0)
+#define ADF_PFVF_MSGORIGIN_SYSTEM BIT(1)
+
+/* Different generations have different CSR layouts, use this struct
+ * to abstract these differences away
+ */
+struct pfvf_message {
+ u8 type;
+ u32 data;
+};
+
+/* PF->VF messages */
+enum pf2vf_msgtype {
+ ADF_PF2VF_MSGTYPE_RESTARTING = 0x01,
+ ADF_PF2VF_MSGTYPE_VERSION_RESP = 0x02,
+ ADF_PF2VF_MSGTYPE_BLKMSG_RESP = 0x03,
+/* Values from 0x10 are Gen4 specific, message type is only 4 bits in Gen2 devices. */
+ ADF_PF2VF_MSGTYPE_RP_RESET_RESP = 0x10,
+};
+
+/* VF->PF messages */
+enum vf2pf_msgtype {
+ ADF_VF2PF_MSGTYPE_INIT = 0x03,
+ ADF_VF2PF_MSGTYPE_SHUTDOWN = 0x04,
+ ADF_VF2PF_MSGTYPE_VERSION_REQ = 0x05,
+ ADF_VF2PF_MSGTYPE_COMPAT_VER_REQ = 0x06,
+ ADF_VF2PF_MSGTYPE_LARGE_BLOCK_REQ = 0x07,
+ ADF_VF2PF_MSGTYPE_MEDIUM_BLOCK_REQ = 0x08,
+ ADF_VF2PF_MSGTYPE_SMALL_BLOCK_REQ = 0x09,
+/* Values from 0x10 are Gen4 specific, message type is only 4 bits in Gen2 devices. */
+ ADF_VF2PF_MSGTYPE_RP_RESET = 0x10,
+};
+
+/* VF/PF compatibility version. */
+enum pfvf_compatibility_version {
+ /* Support for extended capabilities */
+ ADF_PFVF_COMPAT_CAPABILITIES = 0x02,
+ /* In-use pattern cleared by receiver */
+ ADF_PFVF_COMPAT_FAST_ACK = 0x03,
+ /* Ring to service mapping support for non-standard mappings */
+ ADF_PFVF_COMPAT_RING_TO_SVC_MAP = 0x04,
+ /* Reference to the latest version */
+ ADF_PFVF_COMPAT_THIS_VERSION = 0x04,
+};
+
+/* PF->VF Version Response */
+#define ADF_PF2VF_VERSION_RESP_VERS_MASK GENMASK(7, 0)
+#define ADF_PF2VF_VERSION_RESP_RESULT_MASK GENMASK(9, 8)
+
+enum pf2vf_compat_response {
+ ADF_PF2VF_VF_COMPATIBLE = 0x01,
+ ADF_PF2VF_VF_INCOMPATIBLE = 0x02,
+ ADF_PF2VF_VF_COMPAT_UNKNOWN = 0x03,
+};
+
+enum ring_reset_result {
+ RPRESET_SUCCESS = 0x00,
+ RPRESET_NOT_SUPPORTED = 0x01,
+ RPRESET_INVAL_BANK = 0x02,
+ RPRESET_TIMEOUT = 0x03,
+};
+
+#define ADF_VF2PF_RNG_RESET_RP_MASK GENMASK(1, 0)
+#define ADF_VF2PF_RNG_RESET_RSVD_MASK GENMASK(25, 2)
+
+/* PF->VF Block Responses */
+#define ADF_PF2VF_BLKMSG_RESP_TYPE_MASK GENMASK(1, 0)
+#define ADF_PF2VF_BLKMSG_RESP_DATA_MASK GENMASK(9, 2)
+
+enum pf2vf_blkmsg_resp_type {
+ ADF_PF2VF_BLKMSG_RESP_TYPE_DATA = 0x00,
+ ADF_PF2VF_BLKMSG_RESP_TYPE_CRC = 0x01,
+ ADF_PF2VF_BLKMSG_RESP_TYPE_ERROR = 0x02,
+};
+
+/* PF->VF Block Error Code */
+enum pf2vf_blkmsg_error {
+ ADF_PF2VF_INVALID_BLOCK_TYPE = 0x00,
+ ADF_PF2VF_INVALID_BYTE_NUM_REQ = 0x01,
+ ADF_PF2VF_PAYLOAD_TRUNCATED = 0x02,
+ ADF_PF2VF_UNSPECIFIED_ERROR = 0x03,
+};
+
+/* VF->PF Block Requests */
+#define ADF_VF2PF_LARGE_BLOCK_TYPE_MASK GENMASK(1, 0)
+#define ADF_VF2PF_LARGE_BLOCK_BYTE_MASK GENMASK(8, 2)
+#define ADF_VF2PF_MEDIUM_BLOCK_TYPE_MASK GENMASK(2, 0)
+#define ADF_VF2PF_MEDIUM_BLOCK_BYTE_MASK GENMASK(8, 3)
+#define ADF_VF2PF_SMALL_BLOCK_TYPE_MASK GENMASK(3, 0)
+#define ADF_VF2PF_SMALL_BLOCK_BYTE_MASK GENMASK(8, 4)
+#define ADF_VF2PF_BLOCK_CRC_REQ_MASK BIT(9)
+
+/* PF->VF Block Request Types
+ * 0..15 - 32 byte message
+ * 16..23 - 64 byte message
+ * 24..27 - 128 byte message
+ */
+enum vf2pf_blkmsg_req_type {
+ ADF_VF2PF_BLKMSG_REQ_CAP_SUMMARY = 0x02,
+ ADF_VF2PF_BLKMSG_REQ_RING_SVC_MAP = 0x03,
+};
+
+#define ADF_VF2PF_SMALL_BLOCK_TYPE_MAX \
+ (FIELD_MAX(ADF_VF2PF_SMALL_BLOCK_TYPE_MASK))
+
+#define ADF_VF2PF_MEDIUM_BLOCK_TYPE_MAX \
+ (FIELD_MAX(ADF_VF2PF_MEDIUM_BLOCK_TYPE_MASK) + \
+ ADF_VF2PF_SMALL_BLOCK_TYPE_MAX + 1)
+
+#define ADF_VF2PF_LARGE_BLOCK_TYPE_MAX \
+ (FIELD_MAX(ADF_VF2PF_LARGE_BLOCK_TYPE_MASK) + \
+ ADF_VF2PF_MEDIUM_BLOCK_TYPE_MAX)
+
+#define ADF_VF2PF_SMALL_BLOCK_BYTE_MAX \
+ FIELD_MAX(ADF_VF2PF_SMALL_BLOCK_BYTE_MASK)
+
+#define ADF_VF2PF_MEDIUM_BLOCK_BYTE_MAX \
+ FIELD_MAX(ADF_VF2PF_MEDIUM_BLOCK_BYTE_MASK)
+
+#define ADF_VF2PF_LARGE_BLOCK_BYTE_MAX \
+ FIELD_MAX(ADF_VF2PF_LARGE_BLOCK_BYTE_MASK)
+
+struct pfvf_blkmsg_header {
+ u8 version;
+ u8 payload_size;
+} __packed;
+
+#define ADF_PFVF_BLKMSG_HEADER_SIZE (sizeof(struct pfvf_blkmsg_header))
+#define ADF_PFVF_BLKMSG_PAYLOAD_SIZE(blkmsg) (sizeof(blkmsg) - \
+ ADF_PFVF_BLKMSG_HEADER_SIZE)
+#define ADF_PFVF_BLKMSG_MSG_SIZE(blkmsg) (ADF_PFVF_BLKMSG_HEADER_SIZE + \
+ (blkmsg)->hdr.payload_size)
+#define ADF_PFVF_BLKMSG_MSG_MAX_SIZE 128
+
+/* PF->VF Block message header bytes */
+#define ADF_PFVF_BLKMSG_VER_BYTE 0
+#define ADF_PFVF_BLKMSG_LEN_BYTE 1
+
+/* PF/VF Capabilities message values */
+enum blkmsg_capabilities_versions {
+ ADF_PFVF_CAPABILITIES_V1_VERSION = 0x01,
+ ADF_PFVF_CAPABILITIES_V2_VERSION = 0x02,
+ ADF_PFVF_CAPABILITIES_V3_VERSION = 0x03,
+};
+
+struct capabilities_v1 {
+ struct pfvf_blkmsg_header hdr;
+ u32 ext_dc_caps;
+} __packed;
+
+struct capabilities_v2 {
+ struct pfvf_blkmsg_header hdr;
+ u32 ext_dc_caps;
+ u32 capabilities;
+} __packed;
+
+struct capabilities_v3 {
+ struct pfvf_blkmsg_header hdr;
+ u32 ext_dc_caps;
+ u32 capabilities;
+ u32 frequency;
+} __packed;
+
+/* PF/VF Ring to service mapping values */
+enum blkmsg_ring_to_svc_versions {
+ ADF_PFVF_RING_TO_SVC_VERSION = 0x01,
+};
+
+struct ring_to_svc_map_v1 {
+ struct pfvf_blkmsg_header hdr;
+ u16 map;
+} __packed;
+
+#endif /* ADF_PFVF_MSG_H */
diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_pf_msg.c b/drivers/crypto/qat/qat_common/adf_pfvf_pf_msg.c
new file mode 100644
index 000000000000..14c069f0d71a
--- /dev/null
+++ b/drivers/crypto/qat/qat_common/adf_pfvf_pf_msg.c
@@ -0,0 +1,52 @@
+// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
+/* Copyright(c) 2015 - 2021 Intel Corporation */
+#include <linux/pci.h>
+#include "adf_accel_devices.h"
+#include "adf_pfvf_msg.h"
+#include "adf_pfvf_pf_msg.h"
+#include "adf_pfvf_pf_proto.h"
+
+void adf_pf2vf_notify_restarting(struct adf_accel_dev *accel_dev)
+{
+ struct adf_accel_vf_info *vf;
+ struct pfvf_message msg = { .type = ADF_PF2VF_MSGTYPE_RESTARTING };
+ int i, num_vfs = pci_num_vf(accel_to_pci_dev(accel_dev));
+
+ for (i = 0, vf = accel_dev->pf.vf_info; i < num_vfs; i++, vf++) {
+ if (vf->init && adf_send_pf2vf_msg(accel_dev, i, msg))
+ dev_err(&GET_DEV(accel_dev),
+ "Failed to send restarting msg to VF%d\n", i);
+ }
+}
+
+int adf_pf_capabilities_msg_provider(struct adf_accel_dev *accel_dev,
+ u8 *buffer, u8 compat)
+{
+ struct adf_hw_device_data *hw_data = accel_dev->hw_device;
+ struct capabilities_v2 caps_msg;
+
+ caps_msg.ext_dc_caps = hw_data->extended_dc_capabilities;
+ caps_msg.capabilities = hw_data->accel_capabilities_mask;
+
+ caps_msg.hdr.version = ADF_PFVF_CAPABILITIES_V2_VERSION;
+ caps_msg.hdr.payload_size =
+ ADF_PFVF_BLKMSG_PAYLOAD_SIZE(struct capabilities_v2);
+
+ memcpy(buffer, &caps_msg, sizeof(caps_msg));
+
+ return 0;
+}
+
+int adf_pf_ring_to_svc_msg_provider(struct adf_accel_dev *accel_dev,
+ u8 *buffer, u8 compat)
+{
+ struct ring_to_svc_map_v1 rts_map_msg;
+
+ rts_map_msg.map = accel_dev->hw_device->ring_to_svc_map;
+ rts_map_msg.hdr.version = ADF_PFVF_RING_TO_SVC_VERSION;
+ rts_map_msg.hdr.payload_size = ADF_PFVF_BLKMSG_PAYLOAD_SIZE(rts_map_msg);
+
+ memcpy(buffer, &rts_map_msg, sizeof(rts_map_msg));
+
+ return 0;
+}
diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_pf_msg.h b/drivers/crypto/qat/qat_common/adf_pfvf_pf_msg.h
new file mode 100644
index 000000000000..e8982d1ac896
--- /dev/null
+++ b/drivers/crypto/qat/qat_common/adf_pfvf_pf_msg.h
@@ -0,0 +1,18 @@
+/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */
+/* Copyright(c) 2021 Intel Corporation */
+#ifndef ADF_PFVF_PF_MSG_H
+#define ADF_PFVF_PF_MSG_H
+
+#include "adf_accel_devices.h"
+
+void adf_pf2vf_notify_restarting(struct adf_accel_dev *accel_dev);
+
+typedef int (*adf_pf2vf_blkmsg_provider)(struct adf_accel_dev *accel_dev,
+ u8 *buffer, u8 compat);
+
+int adf_pf_capabilities_msg_provider(struct adf_accel_dev *accel_dev,
+ u8 *buffer, u8 comapt);
+int adf_pf_ring_to_svc_msg_provider(struct adf_accel_dev *accel_dev,
+ u8 *buffer, u8 comapt);
+
+#endif /* ADF_PFVF_PF_MSG_H */
diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.c b/drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.c
new file mode 100644
index 000000000000..588352de1ef0
--- /dev/null
+++ b/drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.c
@@ -0,0 +1,346 @@
+// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
+/* Copyright(c) 2015 - 2021 Intel Corporation */
+#include <linux/bitfield.h>
+#include <linux/spinlock.h>
+#include <linux/types.h>
+#include "adf_accel_devices.h"
+#include "adf_common_drv.h"
+#include "adf_pfvf_msg.h"
+#include "adf_pfvf_pf_msg.h"
+#include "adf_pfvf_pf_proto.h"
+#include "adf_pfvf_utils.h"
+
+typedef u8 (*pf2vf_blkmsg_data_getter_fn)(u8 const *blkmsg, u8 byte);
+
+static const adf_pf2vf_blkmsg_provider pf2vf_blkmsg_providers[] = {
+ NULL, /* no message type defined for value 0 */
+ NULL, /* no message type defined for value 1 */
+ adf_pf_capabilities_msg_provider, /* ADF_VF2PF_BLKMSG_REQ_CAP_SUMMARY */
+ adf_pf_ring_to_svc_msg_provider, /* ADF_VF2PF_BLKMSG_REQ_RING_SVC_MAP */
+};
+
+/**
+ * adf_send_pf2vf_msg() - send PF to VF message
+ * @accel_dev: Pointer to acceleration device
+ * @vf_nr: VF number to which the message will be sent
+ * @msg: Message to send
+ *
+ * This function allows the PF to send a message to a specific VF.
+ *
+ * Return: 0 on success, error code otherwise.
+ */
+int adf_send_pf2vf_msg(struct adf_accel_dev *accel_dev, u8 vf_nr, struct pfvf_message msg)
+{
+ struct adf_pfvf_ops *pfvf_ops = GET_PFVF_OPS(accel_dev);
+ u32 pfvf_offset = pfvf_ops->get_pf2vf_offset(vf_nr);
+
+ return pfvf_ops->send_msg(accel_dev, msg, pfvf_offset,
+ &accel_dev->pf.vf_info[vf_nr].pf2vf_lock);
+}
+
+/**
+ * adf_recv_vf2pf_msg() - receive a VF to PF message
+ * @accel_dev: Pointer to acceleration device
+ * @vf_nr: Number of the VF from where the message will be received
+ *
+ * This function allows the PF to receive a message from a specific VF.
+ *
+ * Return: a valid message on success, zero otherwise.
+ */
+static struct pfvf_message adf_recv_vf2pf_msg(struct adf_accel_dev *accel_dev, u8 vf_nr)
+{
+ struct adf_accel_vf_info *vf_info = &accel_dev->pf.vf_info[vf_nr];
+ struct adf_pfvf_ops *pfvf_ops = GET_PFVF_OPS(accel_dev);
+ u32 pfvf_offset = pfvf_ops->get_vf2pf_offset(vf_nr);
+
+ return pfvf_ops->recv_msg(accel_dev, pfvf_offset, vf_info->vf_compat_ver);
+}
+
+static adf_pf2vf_blkmsg_provider get_blkmsg_response_provider(u8 type)
+{
+ if (type >= ARRAY_SIZE(pf2vf_blkmsg_providers))
+ return NULL;
+
+ return pf2vf_blkmsg_providers[type];
+}
+
+/* Byte pf2vf_blkmsg_data_getter_fn callback */
+static u8 adf_pf2vf_blkmsg_get_byte(u8 const *blkmsg, u8 index)
+{
+ return blkmsg[index];
+}
+
+/* CRC pf2vf_blkmsg_data_getter_fn callback */
+static u8 adf_pf2vf_blkmsg_get_crc(u8 const *blkmsg, u8 count)
+{
+ /* count is 0-based, turn it into a length */
+ return adf_pfvf_calc_blkmsg_crc(blkmsg, count + 1);
+}
+
+static int adf_pf2vf_blkmsg_get_data(struct adf_accel_vf_info *vf_info,
+ u8 type, u8 byte, u8 max_size, u8 *data,
+ pf2vf_blkmsg_data_getter_fn data_getter)
+{
+ u8 blkmsg[ADF_PFVF_BLKMSG_MSG_MAX_SIZE] = { 0 };
+ struct adf_accel_dev *accel_dev = vf_info->accel_dev;
+ adf_pf2vf_blkmsg_provider provider;
+ u8 msg_size;
+
+ provider = get_blkmsg_response_provider(type);
+
+ if (unlikely(!provider)) {
+ pr_err("QAT: No registered provider for message %d\n", type);
+ *data = ADF_PF2VF_INVALID_BLOCK_TYPE;
+ return -EINVAL;
+ }
+
+ if (unlikely((*provider)(accel_dev, blkmsg, vf_info->vf_compat_ver))) {
+ pr_err("QAT: unknown error from provider for message %d\n", type);
+ *data = ADF_PF2VF_UNSPECIFIED_ERROR;
+ return -EINVAL;
+ }
+
+ msg_size = ADF_PFVF_BLKMSG_HEADER_SIZE + blkmsg[ADF_PFVF_BLKMSG_LEN_BYTE];
+
+ if (unlikely(msg_size >= max_size)) {
+ pr_err("QAT: Invalid size %d provided for message type %d\n",
+ msg_size, type);
+ *data = ADF_PF2VF_PAYLOAD_TRUNCATED;
+ return -EINVAL;
+ }
+
+ if (unlikely(byte >= msg_size)) {
+ pr_err("QAT: Out-of-bound byte number %d (msg size %d)\n",
+ byte, msg_size);
+ *data = ADF_PF2VF_INVALID_BYTE_NUM_REQ;
+ return -EINVAL;
+ }
+
+ *data = data_getter(blkmsg, byte);
+ return 0;
+}
+
+static struct pfvf_message handle_blkmsg_req(struct adf_accel_vf_info *vf_info,
+ struct pfvf_message req)
+{
+ u8 resp_type = ADF_PF2VF_BLKMSG_RESP_TYPE_ERROR;
+ struct pfvf_message resp = { 0 };
+ u8 resp_data = 0;
+ u8 blk_type;
+ u8 blk_byte;
+ u8 byte_max;
+
+ switch (req.type) {
+ case ADF_VF2PF_MSGTYPE_LARGE_BLOCK_REQ:
+ blk_type = FIELD_GET(ADF_VF2PF_LARGE_BLOCK_TYPE_MASK, req.data)
+ + ADF_VF2PF_MEDIUM_BLOCK_TYPE_MAX + 1;
+ blk_byte = FIELD_GET(ADF_VF2PF_LARGE_BLOCK_BYTE_MASK, req.data);
+ byte_max = ADF_VF2PF_LARGE_BLOCK_BYTE_MAX;
+ break;
+ case ADF_VF2PF_MSGTYPE_MEDIUM_BLOCK_REQ:
+ blk_type = FIELD_GET(ADF_VF2PF_MEDIUM_BLOCK_TYPE_MASK, req.data)
+ + ADF_VF2PF_SMALL_BLOCK_TYPE_MAX + 1;
+ blk_byte = FIELD_GET(ADF_VF2PF_MEDIUM_BLOCK_BYTE_MASK, req.data);
+ byte_max = ADF_VF2PF_MEDIUM_BLOCK_BYTE_MAX;
+ break;
+ case ADF_VF2PF_MSGTYPE_SMALL_BLOCK_REQ:
+ blk_type = FIELD_GET(ADF_VF2PF_SMALL_BLOCK_TYPE_MASK, req.data);
+ blk_byte = FIELD_GET(ADF_VF2PF_SMALL_BLOCK_BYTE_MASK, req.data);
+ byte_max = ADF_VF2PF_SMALL_BLOCK_BYTE_MAX;
+ break;
+ }
+
+ /* Is this a request for CRC or data? */
+ if (FIELD_GET(ADF_VF2PF_BLOCK_CRC_REQ_MASK, req.data)) {
+ dev_dbg(&GET_DEV(vf_info->accel_dev),
+ "BlockMsg of type %d for CRC over %d bytes received from VF%d\n",
+ blk_type, blk_byte, vf_info->vf_nr);
+
+ if (!adf_pf2vf_blkmsg_get_data(vf_info, blk_type, blk_byte,
+ byte_max, &resp_data,
+ adf_pf2vf_blkmsg_get_crc))
+ resp_type = ADF_PF2VF_BLKMSG_RESP_TYPE_CRC;
+ } else {
+ dev_dbg(&GET_DEV(vf_info->accel_dev),
+ "BlockMsg of type %d for data byte %d received from VF%d\n",
+ blk_type, blk_byte, vf_info->vf_nr);
+
+ if (!adf_pf2vf_blkmsg_get_data(vf_info, blk_type, blk_byte,
+ byte_max, &resp_data,
+ adf_pf2vf_blkmsg_get_byte))
+ resp_type = ADF_PF2VF_BLKMSG_RESP_TYPE_DATA;
+ }
+
+ resp.type = ADF_PF2VF_MSGTYPE_BLKMSG_RESP;
+ resp.data = FIELD_PREP(ADF_PF2VF_BLKMSG_RESP_TYPE_MASK, resp_type) |
+ FIELD_PREP(ADF_PF2VF_BLKMSG_RESP_DATA_MASK, resp_data);
+
+ return resp;
+}
+
+static struct pfvf_message handle_rp_reset_req(struct adf_accel_dev *accel_dev, u8 vf_nr,
+ struct pfvf_message req)
+{
+ struct adf_hw_device_data *hw_data = accel_dev->hw_device;
+ struct pfvf_message resp = {
+ .type = ADF_PF2VF_MSGTYPE_RP_RESET_RESP,
+ .data = RPRESET_SUCCESS
+ };
+ u32 bank_number;
+ u32 rsvd_field;
+
+ bank_number = FIELD_GET(ADF_VF2PF_RNG_RESET_RP_MASK, req.data);
+ rsvd_field = FIELD_GET(ADF_VF2PF_RNG_RESET_RSVD_MASK, req.data);
+
+ dev_dbg(&GET_DEV(accel_dev),
+ "Ring Pair Reset Message received from VF%d for bank 0x%x\n",
+ vf_nr, bank_number);
+
+ if (!hw_data->ring_pair_reset || rsvd_field) {
+ dev_dbg(&GET_DEV(accel_dev),
+ "Ring Pair Reset for VF%d is not supported\n", vf_nr);
+ resp.data = RPRESET_NOT_SUPPORTED;
+ goto out;
+ }
+
+ if (bank_number >= hw_data->num_banks_per_vf) {
+ dev_err(&GET_DEV(accel_dev),
+ "Invalid bank number (0x%x) from VF%d for Ring Reset\n",
+ bank_number, vf_nr);
+ resp.data = RPRESET_INVAL_BANK;
+ goto out;
+ }
+
+ /* Convert the VF provided value to PF bank number */
+ bank_number = vf_nr * hw_data->num_banks_per_vf + bank_number;
+ if (hw_data->ring_pair_reset(accel_dev, bank_number)) {
+ dev_dbg(&GET_DEV(accel_dev),
+ "Ring pair reset for VF%d failure\n", vf_nr);
+ resp.data = RPRESET_TIMEOUT;
+ goto out;
+ }
+
+ dev_dbg(&GET_DEV(accel_dev),
+ "Ring pair reset for VF%d successfully\n", vf_nr);
+
+out:
+ return resp;
+}
+
+static int adf_handle_vf2pf_msg(struct adf_accel_dev *accel_dev, u8 vf_nr,
+ struct pfvf_message msg, struct pfvf_message *resp)
+{
+ struct adf_accel_vf_info *vf_info = &accel_dev->pf.vf_info[vf_nr];
+
+ switch (msg.type) {
+ case ADF_VF2PF_MSGTYPE_COMPAT_VER_REQ:
+ {
+ u8 vf_compat_ver = msg.data;
+ u8 compat;
+
+ dev_dbg(&GET_DEV(accel_dev),
+ "VersionRequest received from VF%d (vers %d) to PF (vers %d)\n",
+ vf_nr, vf_compat_ver, ADF_PFVF_COMPAT_THIS_VERSION);
+
+ if (vf_compat_ver <= ADF_PFVF_COMPAT_THIS_VERSION)
+ compat = ADF_PF2VF_VF_COMPATIBLE;
+ else
+ compat = ADF_PF2VF_VF_COMPAT_UNKNOWN;
+
+ vf_info->vf_compat_ver = vf_compat_ver;
+
+ resp->type = ADF_PF2VF_MSGTYPE_VERSION_RESP;
+ resp->data = FIELD_PREP(ADF_PF2VF_VERSION_RESP_VERS_MASK,
+ ADF_PFVF_COMPAT_THIS_VERSION) |
+ FIELD_PREP(ADF_PF2VF_VERSION_RESP_RESULT_MASK, compat);
+ }
+ break;
+ case ADF_VF2PF_MSGTYPE_VERSION_REQ:
+ {
+ u8 compat;
+
+ dev_dbg(&GET_DEV(accel_dev),
+ "Legacy VersionRequest received from VF%d to PF (vers 1.1)\n",
+ vf_nr);
+
+ /* legacy driver, VF compat_ver is 0 */
+ vf_info->vf_compat_ver = 0;
+
+ /* PF always newer than legacy VF */
+ compat = ADF_PF2VF_VF_COMPATIBLE;
+
+ /* Set legacy major and minor version to the latest, 1.1 */
+ resp->type = ADF_PF2VF_MSGTYPE_VERSION_RESP;
+ resp->data = FIELD_PREP(ADF_PF2VF_VERSION_RESP_VERS_MASK, 0x11) |
+ FIELD_PREP(ADF_PF2VF_VERSION_RESP_RESULT_MASK, compat);
+ }
+ break;
+ case ADF_VF2PF_MSGTYPE_INIT:
+ {
+ dev_dbg(&GET_DEV(accel_dev),
+ "Init message received from VF%d\n", vf_nr);
+ vf_info->init = true;
+ }
+ break;
+ case ADF_VF2PF_MSGTYPE_SHUTDOWN:
+ {
+ dev_dbg(&GET_DEV(accel_dev),
+ "Shutdown message received from VF%d\n", vf_nr);
+ vf_info->init = false;
+ }
+ break;
+ case ADF_VF2PF_MSGTYPE_LARGE_BLOCK_REQ:
+ case ADF_VF2PF_MSGTYPE_MEDIUM_BLOCK_REQ:
+ case ADF_VF2PF_MSGTYPE_SMALL_BLOCK_REQ:
+ *resp = handle_blkmsg_req(vf_info, msg);
+ break;
+ case ADF_VF2PF_MSGTYPE_RP_RESET:
+ *resp = handle_rp_reset_req(accel_dev, vf_nr, msg);
+ break;
+ default:
+ dev_dbg(&GET_DEV(accel_dev),
+ "Unknown message from VF%d (type 0x%.4x, data: 0x%.4x)\n",
+ vf_nr, msg.type, msg.data);
+ return -ENOMSG;
+ }
+
+ return 0;
+}
+
+bool adf_recv_and_handle_vf2pf_msg(struct adf_accel_dev *accel_dev, u32 vf_nr)
+{
+ struct pfvf_message req;
+ struct pfvf_message resp = {0};
+
+ req = adf_recv_vf2pf_msg(accel_dev, vf_nr);
+ if (!req.type) /* Legacy or no message */
+ return true;
+
+ if (adf_handle_vf2pf_msg(accel_dev, vf_nr, req, &resp))
+ return false;
+
+ if (resp.type && adf_send_pf2vf_msg(accel_dev, vf_nr, resp))
+ dev_err(&GET_DEV(accel_dev),
+ "Failed to send response to VF%d\n", vf_nr);
+
+ return true;
+}
+
+/**
+ * adf_enable_pf2vf_comms() - Function enables communication from pf to vf
+ *
+ * @accel_dev: Pointer to acceleration device virtual function.
+ *
+ * This function carries out the necessary steps to setup and start the PFVF
+ * communication channel, if any.
+ *
+ * Return: 0 on success, error code otherwise.
+ */
+int adf_enable_pf2vf_comms(struct adf_accel_dev *accel_dev)
+{
+ adf_pfvf_crc_init();
+ spin_lock_init(&accel_dev->pf.vf2pf_ints_lock);
+
+ return 0;
+}
+EXPORT_SYMBOL_GPL(adf_enable_pf2vf_comms);
diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.h b/drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.h
new file mode 100644
index 000000000000..165d266d023d
--- /dev/null
+++ b/drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.h
@@ -0,0 +1,13 @@
+/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */
+/* Copyright(c) 2021 Intel Corporation */
+#ifndef ADF_PFVF_PF_PROTO_H
+#define ADF_PFVF_PF_PROTO_H
+
+#include <linux/types.h>
+#include "adf_accel_devices.h"
+
+int adf_send_pf2vf_msg(struct adf_accel_dev *accel_dev, u8 vf_nr, struct pfvf_message msg);
+
+int adf_enable_pf2vf_comms(struct adf_accel_dev *accel_dev);
+
+#endif /* ADF_PFVF_PF_PROTO_H */
diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_utils.c b/drivers/crypto/qat/qat_common/adf_pfvf_utils.c
new file mode 100644
index 000000000000..c5f6d77d4bb8
--- /dev/null
+++ b/drivers/crypto/qat/qat_common/adf_pfvf_utils.c
@@ -0,0 +1,65 @@
+// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
+/* Copyright(c) 2021 Intel Corporation */
+#include <linux/crc8.h>
+#include <linux/pci.h>
+#include <linux/types.h>
+#include "adf_accel_devices.h"
+#include "adf_pfvf_msg.h"
+#include "adf_pfvf_utils.h"
+
+/* CRC Calculation */
+DECLARE_CRC8_TABLE(pfvf_crc8_table);
+#define ADF_PFVF_CRC8_POLYNOMIAL 0x97
+
+void adf_pfvf_crc_init(void)
+{
+ crc8_populate_msb(pfvf_crc8_table, ADF_PFVF_CRC8_POLYNOMIAL);
+}
+
+u8 adf_pfvf_calc_blkmsg_crc(u8 const *buf, u8 buf_len)
+{
+ return crc8(pfvf_crc8_table, buf, buf_len, CRC8_INIT_VALUE);
+}
+
+static bool set_value_on_csr_msg(struct adf_accel_dev *accel_dev, u32 *csr_msg,
+ u32 value, const struct pfvf_field_format *fmt)
+{
+ if (unlikely((value & fmt->mask) != value)) {
+ dev_err(&GET_DEV(accel_dev),
+ "PFVF message value 0x%X out of range, %u max allowed\n",
+ value, fmt->mask);
+ return false;
+ }
+
+ *csr_msg |= value << fmt->offset;
+
+ return true;
+}
+
+u32 adf_pfvf_csr_msg_of(struct adf_accel_dev *accel_dev,
+ struct pfvf_message msg,
+ const struct pfvf_csr_format *fmt)
+{
+ u32 csr_msg = 0;
+
+ if (!set_value_on_csr_msg(accel_dev, &csr_msg, msg.type, &fmt->type) ||
+ !set_value_on_csr_msg(accel_dev, &csr_msg, msg.data, &fmt->data))
+ return 0;
+
+ return csr_msg | ADF_PFVF_MSGORIGIN_SYSTEM;
+}
+
+struct pfvf_message adf_pfvf_message_of(struct adf_accel_dev *accel_dev, u32 csr_msg,
+ const struct pfvf_csr_format *fmt)
+{
+ struct pfvf_message msg = { 0 };
+
+ msg.type = (csr_msg >> fmt->type.offset) & fmt->type.mask;
+ msg.data = (csr_msg >> fmt->data.offset) & fmt->data.mask;
+
+ if (unlikely(!msg.type))
+ dev_err(&GET_DEV(accel_dev),
+ "Invalid PFVF msg with no type received\n");
+
+ return msg;
+}
diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_utils.h b/drivers/crypto/qat/qat_common/adf_pfvf_utils.h
new file mode 100644
index 000000000000..2be048e2287b
--- /dev/null
+++ b/drivers/crypto/qat/qat_common/adf_pfvf_utils.h
@@ -0,0 +1,31 @@
+/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */
+/* Copyright(c) 2021 Intel Corporation */
+#ifndef ADF_PFVF_UTILS_H
+#define ADF_PFVF_UTILS_H
+
+#include <linux/types.h>
+#include "adf_pfvf_msg.h"
+
+/* How long to wait for far side to acknowledge receipt */
+#define ADF_PFVF_MSG_ACK_DELAY_US 4
+#define ADF_PFVF_MSG_ACK_MAX_DELAY_US (1 * USEC_PER_SEC)
+
+u8 adf_pfvf_calc_blkmsg_crc(u8 const *buf, u8 buf_len);
+void adf_pfvf_crc_init(void);
+
+struct pfvf_field_format {
+ u8 offset;
+ u32 mask;
+};
+
+struct pfvf_csr_format {
+ struct pfvf_field_format type;
+ struct pfvf_field_format data;
+};
+
+u32 adf_pfvf_csr_msg_of(struct adf_accel_dev *accel_dev, struct pfvf_message msg,
+ const struct pfvf_csr_format *fmt);
+struct pfvf_message adf_pfvf_message_of(struct adf_accel_dev *accel_dev, u32 raw_msg,
+ const struct pfvf_csr_format *fmt);
+
+#endif /* ADF_PFVF_UTILS_H */
diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_vf_msg.c b/drivers/crypto/qat/qat_common/adf_pfvf_vf_msg.c
new file mode 100644
index 000000000000..14b222691c9c
--- /dev/null
+++ b/drivers/crypto/qat/qat_common/adf_pfvf_vf_msg.c
@@ -0,0 +1,167 @@
+// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
+/* Copyright(c) 2015 - 2021 Intel Corporation */
+#include <linux/bitfield.h>
+#include "adf_accel_devices.h"
+#include "adf_common_drv.h"
+#include "adf_pfvf_msg.h"
+#include "adf_pfvf_vf_msg.h"
+#include "adf_pfvf_vf_proto.h"
+
+/**
+ * adf_vf2pf_notify_init() - send init msg to PF
+ * @accel_dev: Pointer to acceleration VF device.
+ *
+ * Function sends an init message from the VF to a PF
+ *
+ * Return: 0 on success, error code otherwise.
+ */
+int adf_vf2pf_notify_init(struct adf_accel_dev *accel_dev)
+{
+ struct pfvf_message msg = { .type = ADF_VF2PF_MSGTYPE_INIT };
+
+ if (adf_send_vf2pf_msg(accel_dev, msg)) {
+ dev_err(&GET_DEV(accel_dev),
+ "Failed to send Init event to PF\n");
+ return -EFAULT;
+ }
+ set_bit(ADF_STATUS_PF_RUNNING, &accel_dev->status);
+ return 0;
+}
+EXPORT_SYMBOL_GPL(adf_vf2pf_notify_init);
+
+/**
+ * adf_vf2pf_notify_shutdown() - send shutdown msg to PF
+ * @accel_dev: Pointer to acceleration VF device.
+ *
+ * Function sends a shutdown message from the VF to a PF
+ *
+ * Return: void
+ */
+void adf_vf2pf_notify_shutdown(struct adf_accel_dev *accel_dev)
+{
+ struct pfvf_message msg = { .type = ADF_VF2PF_MSGTYPE_SHUTDOWN };
+
+ if (test_bit(ADF_STATUS_PF_RUNNING, &accel_dev->status))
+ if (adf_send_vf2pf_msg(accel_dev, msg))
+ dev_err(&GET_DEV(accel_dev),
+ "Failed to send Shutdown event to PF\n");
+}
+EXPORT_SYMBOL_GPL(adf_vf2pf_notify_shutdown);
+
+int adf_vf2pf_request_version(struct adf_accel_dev *accel_dev)
+{
+ u8 pf_version;
+ int compat;
+ int ret;
+ struct pfvf_message resp;
+ struct pfvf_message msg = {
+ .type = ADF_VF2PF_MSGTYPE_COMPAT_VER_REQ,
+ .data = ADF_PFVF_COMPAT_THIS_VERSION,
+ };
+
+ BUILD_BUG_ON(ADF_PFVF_COMPAT_THIS_VERSION > 255);
+
+ ret = adf_send_vf2pf_req(accel_dev, msg, &resp);
+ if (ret) {
+ dev_err(&GET_DEV(accel_dev),
+ "Failed to send Compatibility Version Request.\n");
+ return ret;
+ }
+
+ pf_version = FIELD_GET(ADF_PF2VF_VERSION_RESP_VERS_MASK, resp.data);
+ compat = FIELD_GET(ADF_PF2VF_VERSION_RESP_RESULT_MASK, resp.data);
+
+ /* Response from PF received, check compatibility */
+ switch (compat) {
+ case ADF_PF2VF_VF_COMPATIBLE:
+ break;
+ case ADF_PF2VF_VF_COMPAT_UNKNOWN:
+ /* VF is newer than PF - compatible for now */
+ break;
+ case ADF_PF2VF_VF_INCOMPATIBLE:
+ dev_err(&GET_DEV(accel_dev),
+ "PF (vers %d) and VF (vers %d) are not compatible\n",
+ pf_version, ADF_PFVF_COMPAT_THIS_VERSION);
+ return -EINVAL;
+ default:
+ dev_err(&GET_DEV(accel_dev),
+ "Invalid response from PF; assume not compatible\n");
+ return -EINVAL;
+ }
+
+ accel_dev->vf.pf_compat_ver = pf_version;
+ return 0;
+}
+
+int adf_vf2pf_get_capabilities(struct adf_accel_dev *accel_dev)
+{
+ struct adf_hw_device_data *hw_data = accel_dev->hw_device;
+ struct capabilities_v3 cap_msg = { { 0 }, };
+ unsigned int len = sizeof(cap_msg);
+
+ if (accel_dev->vf.pf_compat_ver < ADF_PFVF_COMPAT_CAPABILITIES)
+ /* The PF is too old to support the extended capabilities */
+ return 0;
+
+ if (adf_send_vf2pf_blkmsg_req(accel_dev, ADF_VF2PF_BLKMSG_REQ_CAP_SUMMARY,
+ (u8 *)&cap_msg, &len)) {
+ dev_err(&GET_DEV(accel_dev),
+ "QAT: Failed to get block message response\n");
+ return -EFAULT;
+ }
+
+ switch (cap_msg.hdr.version) {
+ default:
+ /* Newer version received, handle only the know parts */
+ fallthrough;
+ case ADF_PFVF_CAPABILITIES_V3_VERSION:
+ if (likely(len >= sizeof(struct capabilities_v3)))
+ hw_data->clock_frequency = cap_msg.frequency;
+ else
+ dev_info(&GET_DEV(accel_dev), "Could not get frequency");
+ fallthrough;
+ case ADF_PFVF_CAPABILITIES_V2_VERSION:
+ if (likely(len >= sizeof(struct capabilities_v2)))
+ hw_data->accel_capabilities_mask = cap_msg.capabilities;
+ else
+ dev_info(&GET_DEV(accel_dev), "Could not get capabilities");
+ fallthrough;
+ case ADF_PFVF_CAPABILITIES_V1_VERSION:
+ if (likely(len >= sizeof(struct capabilities_v1))) {
+ hw_data->extended_dc_capabilities = cap_msg.ext_dc_caps;
+ } else {
+ dev_err(&GET_DEV(accel_dev),
+ "Capabilities message truncated to %d bytes\n", len);
+ return -EFAULT;
+ }
+ }
+
+ return 0;
+}
+
+int adf_vf2pf_get_ring_to_svc(struct adf_accel_dev *accel_dev)
+{
+ struct ring_to_svc_map_v1 rts_map_msg = { { 0 }, };
+ unsigned int len = sizeof(rts_map_msg);
+
+ if (accel_dev->vf.pf_compat_ver < ADF_PFVF_COMPAT_RING_TO_SVC_MAP)
+ /* Use already set default mappings */
+ return 0;
+
+ if (adf_send_vf2pf_blkmsg_req(accel_dev, ADF_VF2PF_BLKMSG_REQ_RING_SVC_MAP,
+ (u8 *)&rts_map_msg, &len)) {
+ dev_err(&GET_DEV(accel_dev),
+ "QAT: Failed to get block message response\n");
+ return -EFAULT;
+ }
+
+ if (unlikely(len < sizeof(struct ring_to_svc_map_v1))) {
+ dev_err(&GET_DEV(accel_dev),
+ "RING_TO_SVC message truncated to %d bytes\n", len);
+ return -EFAULT;
+ }
+
+ /* Only v1 at present */
+ accel_dev->hw_device->ring_to_svc_map = rts_map_msg.map;
+ return 0;
+}
diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_vf_msg.h b/drivers/crypto/qat/qat_common/adf_pfvf_vf_msg.h
new file mode 100644
index 000000000000..71bc0e3f1d93
--- /dev/null
+++ b/drivers/crypto/qat/qat_common/adf_pfvf_vf_msg.h
@@ -0,0 +1,23 @@
+/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */
+/* Copyright(c) 2021 Intel Corporation */
+#ifndef ADF_PFVF_VF_MSG_H
+#define ADF_PFVF_VF_MSG_H
+
+#if defined(CONFIG_PCI_IOV)
+int adf_vf2pf_notify_init(struct adf_accel_dev *accel_dev);
+void adf_vf2pf_notify_shutdown(struct adf_accel_dev *accel_dev);
+int adf_vf2pf_request_version(struct adf_accel_dev *accel_dev);
+int adf_vf2pf_get_capabilities(struct adf_accel_dev *accel_dev);
+int adf_vf2pf_get_ring_to_svc(struct adf_accel_dev *accel_dev);
+#else
+static inline int adf_vf2pf_notify_init(struct adf_accel_dev *accel_dev)
+{
+ return 0;
+}
+
+static inline void adf_vf2pf_notify_shutdown(struct adf_accel_dev *accel_dev)
+{
+}
+#endif
+
+#endif /* ADF_PFVF_VF_MSG_H */
diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_vf_proto.c b/drivers/crypto/qat/qat_common/adf_pfvf_vf_proto.c
new file mode 100644
index 000000000000..1015155b6374
--- /dev/null
+++ b/drivers/crypto/qat/qat_common/adf_pfvf_vf_proto.c
@@ -0,0 +1,368 @@
+// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
+/* Copyright(c) 2015 - 2021 Intel Corporation */
+#include <linux/bitfield.h>
+#include <linux/completion.h>
+#include <linux/minmax.h>
+#include <linux/types.h>
+#include "adf_accel_devices.h"
+#include "adf_common_drv.h"
+#include "adf_pfvf_msg.h"
+#include "adf_pfvf_utils.h"
+#include "adf_pfvf_vf_msg.h"
+#include "adf_pfvf_vf_proto.h"
+
+#define ADF_PFVF_MSG_COLLISION_DETECT_DELAY 10
+#define ADF_PFVF_MSG_ACK_DELAY 2
+#define ADF_PFVF_MSG_ACK_MAX_RETRY 100
+
+/* How often to retry if there is no response */
+#define ADF_PFVF_MSG_RESP_RETRIES 5
+#define ADF_PFVF_MSG_RESP_TIMEOUT (ADF_PFVF_MSG_ACK_DELAY * \
+ ADF_PFVF_MSG_ACK_MAX_RETRY + \
+ ADF_PFVF_MSG_COLLISION_DETECT_DELAY)
+
+/**
+ * adf_send_vf2pf_msg() - send VF to PF message
+ * @accel_dev: Pointer to acceleration device
+ * @msg: Message to send
+ *
+ * This function allows the VF to send a message to the PF.
+ *
+ * Return: 0 on success, error code otherwise.
+ */
+int adf_send_vf2pf_msg(struct adf_accel_dev *accel_dev, struct pfvf_message msg)
+{
+ struct adf_pfvf_ops *pfvf_ops = GET_PFVF_OPS(accel_dev);
+ u32 pfvf_offset = pfvf_ops->get_vf2pf_offset(0);
+
+ return pfvf_ops->send_msg(accel_dev, msg, pfvf_offset,
+ &accel_dev->vf.vf2pf_lock);
+}
+
+/**
+ * adf_recv_pf2vf_msg() - receive a PF to VF message
+ * @accel_dev: Pointer to acceleration device
+ *
+ * This function allows the VF to receive a message from the PF.
+ *
+ * Return: a valid message on success, zero otherwise.
+ */
+static struct pfvf_message adf_recv_pf2vf_msg(struct adf_accel_dev *accel_dev)
+{
+ struct adf_pfvf_ops *pfvf_ops = GET_PFVF_OPS(accel_dev);
+ u32 pfvf_offset = pfvf_ops->get_pf2vf_offset(0);
+
+ return pfvf_ops->recv_msg(accel_dev, pfvf_offset, accel_dev->vf.pf_compat_ver);
+}
+
+/**
+ * adf_send_vf2pf_req() - send VF2PF request message
+ * @accel_dev: Pointer to acceleration device.
+ * @msg: Request message to send
+ * @resp: Returned PF response
+ *
+ * This function sends a message that requires a response from the VF to the PF
+ * and waits for a reply.
+ *
+ * Return: 0 on success, error code otherwise.
+ */
+int adf_send_vf2pf_req(struct adf_accel_dev *accel_dev, struct pfvf_message msg,
+ struct pfvf_message *resp)
+{
+ unsigned long timeout = msecs_to_jiffies(ADF_PFVF_MSG_RESP_TIMEOUT);
+ unsigned int retries = ADF_PFVF_MSG_RESP_RETRIES;
+ int ret;
+
+ reinit_completion(&accel_dev->vf.msg_received);
+
+ /* Send request from VF to PF */
+ do {
+ ret = adf_send_vf2pf_msg(accel_dev, msg);
+ if (ret) {
+ dev_err(&GET_DEV(accel_dev),
+ "Failed to send request msg to PF\n");
+ return ret;
+ }
+
+ /* Wait for response, if it times out retry */
+ ret = wait_for_completion_timeout(&accel_dev->vf.msg_received,
+ timeout);
+ if (ret) {
+ if (likely(resp))
+ *resp = accel_dev->vf.response;
+
+ /* Once copied, set to an invalid value */
+ accel_dev->vf.response.type = 0;
+
+ return 0;
+ }
+
+ dev_err(&GET_DEV(accel_dev), "PFVF response message timeout\n");
+ } while (--retries);
+
+ return -EIO;
+}
+
+static int adf_vf2pf_blkmsg_data_req(struct adf_accel_dev *accel_dev, bool crc,
+ u8 *type, u8 *data)
+{
+ struct pfvf_message req = { 0 };
+ struct pfvf_message resp = { 0 };
+ u8 blk_type;
+ u8 blk_byte;
+ u8 msg_type;
+ u8 max_data;
+ int err;
+
+ /* Convert the block type to {small, medium, large} size category */
+ if (*type <= ADF_VF2PF_SMALL_BLOCK_TYPE_MAX) {
+ msg_type = ADF_VF2PF_MSGTYPE_SMALL_BLOCK_REQ;
+ blk_type = FIELD_PREP(ADF_VF2PF_SMALL_BLOCK_TYPE_MASK, *type);
+ blk_byte = FIELD_PREP(ADF_VF2PF_SMALL_BLOCK_BYTE_MASK, *data);
+ max_data = ADF_VF2PF_SMALL_BLOCK_BYTE_MAX;
+ } else if (*type <= ADF_VF2PF_MEDIUM_BLOCK_TYPE_MAX) {
+ msg_type = ADF_VF2PF_MSGTYPE_MEDIUM_BLOCK_REQ;
+ blk_type = FIELD_PREP(ADF_VF2PF_MEDIUM_BLOCK_TYPE_MASK,
+ *type - ADF_VF2PF_SMALL_BLOCK_TYPE_MAX);
+ blk_byte = FIELD_PREP(ADF_VF2PF_MEDIUM_BLOCK_BYTE_MASK, *data);
+ max_data = ADF_VF2PF_MEDIUM_BLOCK_BYTE_MAX;
+ } else if (*type <= ADF_VF2PF_LARGE_BLOCK_TYPE_MAX) {
+ msg_type = ADF_VF2PF_MSGTYPE_LARGE_BLOCK_REQ;
+ blk_type = FIELD_PREP(ADF_VF2PF_LARGE_BLOCK_TYPE_MASK,
+ *type - ADF_VF2PF_MEDIUM_BLOCK_TYPE_MAX);
+ blk_byte = FIELD_PREP(ADF_VF2PF_LARGE_BLOCK_BYTE_MASK, *data);
+ max_data = ADF_VF2PF_LARGE_BLOCK_BYTE_MAX;
+ } else {
+ dev_err(&GET_DEV(accel_dev), "Invalid message type %u\n", *type);
+ return -EINVAL;
+ }
+
+ /* Sanity check */
+ if (*data > max_data) {
+ dev_err(&GET_DEV(accel_dev),
+ "Invalid byte %s %u for message type %u\n",
+ crc ? "count" : "index", *data, *type);
+ return -EINVAL;
+ }
+
+ /* Build the block message */
+ req.type = msg_type;
+ req.data = blk_type | blk_byte | FIELD_PREP(ADF_VF2PF_BLOCK_CRC_REQ_MASK, crc);
+
+ err = adf_send_vf2pf_req(accel_dev, req, &resp);
+ if (err)
+ return err;
+
+ *type = FIELD_GET(ADF_PF2VF_BLKMSG_RESP_TYPE_MASK, resp.data);
+ *data = FIELD_GET(ADF_PF2VF_BLKMSG_RESP_DATA_MASK, resp.data);
+
+ return 0;
+}
+
+static int adf_vf2pf_blkmsg_get_byte(struct adf_accel_dev *accel_dev, u8 type,
+ u8 index, u8 *data)
+{
+ int ret;
+
+ ret = adf_vf2pf_blkmsg_data_req(accel_dev, false, &type, &index);
+ if (ret < 0)
+ return ret;
+
+ if (unlikely(type != ADF_PF2VF_BLKMSG_RESP_TYPE_DATA)) {
+ dev_err(&GET_DEV(accel_dev),
+ "Unexpected BLKMSG response type %u, byte 0x%x\n",
+ type, index);
+ return -EFAULT;
+ }
+
+ *data = index;
+ return 0;
+}
+
+static int adf_vf2pf_blkmsg_get_crc(struct adf_accel_dev *accel_dev, u8 type,
+ u8 bytes, u8 *crc)
+{
+ int ret;
+
+ /* The count of bytes refers to a length, however shift it to a 0-based
+ * count to avoid overflows. Thus, a request for 0 bytes is technically
+ * valid.
+ */
+ --bytes;
+
+ ret = adf_vf2pf_blkmsg_data_req(accel_dev, true, &type, &bytes);
+ if (ret < 0)
+ return ret;
+
+ if (unlikely(type != ADF_PF2VF_BLKMSG_RESP_TYPE_CRC)) {
+ dev_err(&GET_DEV(accel_dev),
+ "Unexpected CRC BLKMSG response type %u, crc 0x%x\n",
+ type, bytes);
+ return -EFAULT;
+ }
+
+ *crc = bytes;
+ return 0;
+}
+
+/**
+ * adf_send_vf2pf_blkmsg_req() - retrieve block message
+ * @accel_dev: Pointer to acceleration VF device.
+ * @type: The block message type, see adf_pfvf_msg.h for allowed values
+ * @buffer: input buffer where to place the received data
+ * @buffer_len: buffer length as input, the amount of written bytes on output
+ *
+ * Request a message of type 'type' over the block message transport.
+ * This function will send the required amount block message requests and
+ * return the overall content back to the caller through the provided buffer.
+ * The buffer should be large enough to contain the requested message type,
+ * otherwise the response will be truncated.
+ *
+ * Return: 0 on success, error code otherwise.
+ */
+int adf_send_vf2pf_blkmsg_req(struct adf_accel_dev *accel_dev, u8 type,
+ u8 *buffer, unsigned int *buffer_len)
+{
+ unsigned int index;
+ unsigned int msg_len;
+ int ret;
+ u8 remote_crc;
+ u8 local_crc;
+
+ if (unlikely(type > ADF_VF2PF_LARGE_BLOCK_TYPE_MAX)) {
+ dev_err(&GET_DEV(accel_dev), "Invalid block message type %d\n",
+ type);
+ return -EINVAL;
+ }
+
+ if (unlikely(*buffer_len < ADF_PFVF_BLKMSG_HEADER_SIZE)) {
+ dev_err(&GET_DEV(accel_dev),
+ "Buffer size too small for a block message\n");
+ return -EINVAL;
+ }
+
+ ret = adf_vf2pf_blkmsg_get_byte(accel_dev, type,
+ ADF_PFVF_BLKMSG_VER_BYTE,
+ &buffer[ADF_PFVF_BLKMSG_VER_BYTE]);
+ if (unlikely(ret))
+ return ret;
+
+ if (unlikely(!buffer[ADF_PFVF_BLKMSG_VER_BYTE])) {
+ dev_err(&GET_DEV(accel_dev),
+ "Invalid version 0 received for block request %u", type);
+ return -EFAULT;
+ }
+
+ ret = adf_vf2pf_blkmsg_get_byte(accel_dev, type,
+ ADF_PFVF_BLKMSG_LEN_BYTE,
+ &buffer[ADF_PFVF_BLKMSG_LEN_BYTE]);
+ if (unlikely(ret))
+ return ret;
+
+ if (unlikely(!buffer[ADF_PFVF_BLKMSG_LEN_BYTE])) {
+ dev_err(&GET_DEV(accel_dev),
+ "Invalid size 0 received for block request %u", type);
+ return -EFAULT;
+ }
+
+ /* We need to pick the minimum since there is no way to request a
+ * specific version. As a consequence any scenario is possible:
+ * - PF has a newer (longer) version which doesn't fit in the buffer
+ * - VF expects a newer (longer) version, so we must not ask for
+ * bytes in excess
+ * - PF and VF share the same version, no problem
+ */
+ msg_len = ADF_PFVF_BLKMSG_HEADER_SIZE + buffer[ADF_PFVF_BLKMSG_LEN_BYTE];
+ msg_len = min(*buffer_len, msg_len);
+
+ /* Get the payload */
+ for (index = ADF_PFVF_BLKMSG_HEADER_SIZE; index < msg_len; index++) {
+ ret = adf_vf2pf_blkmsg_get_byte(accel_dev, type, index,
+ &buffer[index]);
+ if (unlikely(ret))
+ return ret;
+ }
+
+ ret = adf_vf2pf_blkmsg_get_crc(accel_dev, type, msg_len, &remote_crc);
+ if (unlikely(ret))
+ return ret;
+
+ local_crc = adf_pfvf_calc_blkmsg_crc(buffer, msg_len);
+ if (unlikely(local_crc != remote_crc)) {
+ dev_err(&GET_DEV(accel_dev),
+ "CRC error on msg type %d. Local %02X, remote %02X\n",
+ type, local_crc, remote_crc);
+ return -EIO;
+ }
+
+ *buffer_len = msg_len;
+ return 0;
+}
+
+static bool adf_handle_pf2vf_msg(struct adf_accel_dev *accel_dev,
+ struct pfvf_message msg)
+{
+ switch (msg.type) {
+ case ADF_PF2VF_MSGTYPE_RESTARTING:
+ dev_dbg(&GET_DEV(accel_dev), "Restarting message received from PF\n");
+
+ adf_pf2vf_handle_pf_restarting(accel_dev);
+ return false;
+ case ADF_PF2VF_MSGTYPE_VERSION_RESP:
+ case ADF_PF2VF_MSGTYPE_BLKMSG_RESP:
+ case ADF_PF2VF_MSGTYPE_RP_RESET_RESP:
+ dev_dbg(&GET_DEV(accel_dev),
+ "Response Message received from PF (type 0x%.4x, data 0x%.4x)\n",
+ msg.type, msg.data);
+ accel_dev->vf.response = msg;
+ complete(&accel_dev->vf.msg_received);
+ return true;
+ default:
+ dev_err(&GET_DEV(accel_dev),
+ "Unknown message from PF (type 0x%.4x, data: 0x%.4x)\n",
+ msg.type, msg.data);
+ }
+
+ return false;
+}
+
+bool adf_recv_and_handle_pf2vf_msg(struct adf_accel_dev *accel_dev)
+{
+ struct pfvf_message msg;
+
+ msg = adf_recv_pf2vf_msg(accel_dev);
+ if (msg.type) /* Invalid or no message */
+ return adf_handle_pf2vf_msg(accel_dev, msg);
+
+ /* No replies for PF->VF messages at present */
+
+ return true;
+}
+
+/**
+ * adf_enable_vf2pf_comms() - Function enables communication from vf to pf
+ *
+ * @accel_dev: Pointer to acceleration device virtual function.
+ *
+ * Return: 0 on success, error code otherwise.
+ */
+int adf_enable_vf2pf_comms(struct adf_accel_dev *accel_dev)
+{
+ int ret;
+
+ adf_pfvf_crc_init();
+ adf_enable_pf2vf_interrupts(accel_dev);
+
+ ret = adf_vf2pf_request_version(accel_dev);
+ if (ret)
+ return ret;
+
+ ret = adf_vf2pf_get_capabilities(accel_dev);
+ if (ret)
+ return ret;
+
+ ret = adf_vf2pf_get_ring_to_svc(accel_dev);
+
+ return ret;
+}
+EXPORT_SYMBOL_GPL(adf_enable_vf2pf_comms);
diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_vf_proto.h b/drivers/crypto/qat/qat_common/adf_pfvf_vf_proto.h
new file mode 100644
index 000000000000..f6ee9b38c0e1
--- /dev/null
+++ b/drivers/crypto/qat/qat_common/adf_pfvf_vf_proto.h
@@ -0,0 +1,17 @@
+/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */
+/* Copyright(c) 2021 Intel Corporation */
+#ifndef ADF_PFVF_VF_PROTO_H
+#define ADF_PFVF_VF_PROTO_H
+
+#include <linux/types.h>
+#include "adf_accel_devices.h"
+
+int adf_send_vf2pf_msg(struct adf_accel_dev *accel_dev, struct pfvf_message msg);
+int adf_send_vf2pf_req(struct adf_accel_dev *accel_dev, struct pfvf_message msg,
+ struct pfvf_message *resp);
+int adf_send_vf2pf_blkmsg_req(struct adf_accel_dev *accel_dev, u8 type,
+ u8 *buffer, unsigned int *buffer_len);
+
+int adf_enable_vf2pf_comms(struct adf_accel_dev *accel_dev);
+
+#endif /* ADF_PFVF_VF_PROTO_H */
diff --git a/drivers/crypto/qat/qat_common/adf_sriov.c b/drivers/crypto/qat/qat_common/adf_sriov.c
index 90ec057f9183..b960bca1f9d2 100644
--- a/drivers/crypto/qat/qat_common/adf_sriov.c
+++ b/drivers/crypto/qat/qat_common/adf_sriov.c
@@ -1,12 +1,15 @@
// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
-/* Copyright(c) 2015 - 2020 Intel Corporation */
+/* Copyright(c) 2015 - 2021 Intel Corporation */
#include <linux/workqueue.h>
#include <linux/pci.h>
#include <linux/device.h>
#include <linux/iommu.h>
#include "adf_common_drv.h"
#include "adf_cfg.h"
-#include "adf_pf2vf_msg.h"
+#include "adf_pfvf_pf_msg.h"
+
+#define ADF_VF2PF_RATELIMIT_INTERVAL 8
+#define ADF_VF2PF_RATELIMIT_BURST 130
static struct workqueue_struct *pf2vf_resp_wq;
@@ -19,8 +22,16 @@ static void adf_iov_send_resp(struct work_struct *work)
{
struct adf_pf2vf_resp *pf2vf_resp =
container_of(work, struct adf_pf2vf_resp, pf2vf_resp_work);
+ struct adf_accel_vf_info *vf_info = pf2vf_resp->vf_info;
+ struct adf_accel_dev *accel_dev = vf_info->accel_dev;
+ u32 vf_nr = vf_info->vf_nr;
+ bool ret;
+
+ ret = adf_recv_and_handle_vf2pf_msg(accel_dev, vf_nr);
+ if (ret)
+ /* re-enable interrupt on PF from this VF */
+ adf_enable_vf2pf_interrupts(accel_dev, 1 << vf_nr);
- adf_vf2pf_req_hndl(pf2vf_resp->vf_info);
kfree(pf2vf_resp);
}
@@ -50,11 +61,12 @@ static int adf_enable_sriov(struct adf_accel_dev *accel_dev)
/* This ptr will be populated when VFs will be created */
vf_info->accel_dev = accel_dev;
vf_info->vf_nr = i;
+ vf_info->vf_compat_ver = 0;
mutex_init(&vf_info->pf2vf_lock);
ratelimit_state_init(&vf_info->vf2pf_ratelimit,
- DEFAULT_RATELIMIT_INTERVAL,
- DEFAULT_RATELIMIT_BURST);
+ ADF_VF2PF_RATELIMIT_INTERVAL,
+ ADF_VF2PF_RATELIMIT_BURST);
}
/* Set Valid bits in AE Thread to PCIe Function Mapping */
@@ -62,7 +74,7 @@ static int adf_enable_sriov(struct adf_accel_dev *accel_dev)
hw_data->configure_iov_threads(accel_dev, true);
/* Enable VF to PF interrupts for all VFs */
- if (hw_data->get_pf2vf_offset)
+ if (hw_data->pfvf_ops.get_pf2vf_offset)
adf_enable_vf2pf_interrupts(accel_dev, BIT_ULL(totalvfs) - 1);
/*
@@ -92,13 +104,13 @@ void adf_disable_sriov(struct adf_accel_dev *accel_dev)
if (!accel_dev->pf.vf_info)
return;
- if (hw_data->get_pf2vf_offset)
+ if (hw_data->pfvf_ops.get_pf2vf_offset)
adf_pf2vf_notify_restarting(accel_dev);
pci_disable_sriov(accel_to_pci_dev(accel_dev));
/* Disable VF to PF interrupts */
- if (hw_data->get_pf2vf_offset)
+ if (hw_data->pfvf_ops.get_pf2vf_offset)
adf_disable_vf2pf_interrupts(accel_dev, GENMASK(31, 0));
/* Clear Valid bits in AE Thread to PCIe Function Mapping */
@@ -114,6 +126,32 @@ void adf_disable_sriov(struct adf_accel_dev *accel_dev)
}
EXPORT_SYMBOL_GPL(adf_disable_sriov);
+static int adf_sriov_prepare_restart(struct adf_accel_dev *accel_dev)
+{
+ char services[ADF_CFG_MAX_VAL_LEN_IN_BYTES] = {0};
+ int ret;
+
+ ret = adf_cfg_get_param_value(accel_dev, ADF_GENERAL_SEC,
+ ADF_SERVICES_ENABLED, services);
+
+ adf_dev_stop(accel_dev);
+ adf_dev_shutdown(accel_dev);
+
+ if (!ret) {
+ ret = adf_cfg_section_add(accel_dev, ADF_GENERAL_SEC);
+ if (ret)
+ return ret;
+
+ ret = adf_cfg_add_key_value_param(accel_dev, ADF_GENERAL_SEC,
+ ADF_SERVICES_ENABLED,
+ services, ADF_STR);
+ if (ret)
+ return ret;
+ }
+
+ return 0;
+}
+
/**
* adf_sriov_configure() - Enable SRIOV for the device
* @pdev: Pointer to PCI device.
@@ -153,8 +191,9 @@ int adf_sriov_configure(struct pci_dev *pdev, int numvfs)
return -EBUSY;
}
- adf_dev_stop(accel_dev);
- adf_dev_shutdown(accel_dev);
+ ret = adf_sriov_prepare_restart(accel_dev);
+ if (ret)
+ return ret;
}
if (adf_cfg_section_add(accel_dev, ADF_KERNEL_SEC))
diff --git a/drivers/crypto/qat/qat_common/adf_vf2pf_msg.c b/drivers/crypto/qat/qat_common/adf_vf2pf_msg.c
deleted file mode 100644
index 8d11bb24cea0..000000000000
--- a/drivers/crypto/qat/qat_common/adf_vf2pf_msg.c
+++ /dev/null
@@ -1,48 +0,0 @@
-// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
-/* Copyright(c) 2015 - 2020 Intel Corporation */
-#include "adf_accel_devices.h"
-#include "adf_common_drv.h"
-#include "adf_pf2vf_msg.h"
-
-/**
- * adf_vf2pf_notify_init() - send init msg to PF
- * @accel_dev: Pointer to acceleration VF device.
- *
- * Function sends an init message from the VF to a PF
- *
- * Return: 0 on success, error code otherwise.
- */
-int adf_vf2pf_notify_init(struct adf_accel_dev *accel_dev)
-{
- u32 msg = (ADF_VF2PF_MSGORIGIN_SYSTEM |
- (ADF_VF2PF_MSGTYPE_INIT << ADF_VF2PF_MSGTYPE_SHIFT));
-
- if (adf_send_vf2pf_msg(accel_dev, msg)) {
- dev_err(&GET_DEV(accel_dev),
- "Failed to send Init event to PF\n");
- return -EFAULT;
- }
- set_bit(ADF_STATUS_PF_RUNNING, &accel_dev->status);
- return 0;
-}
-EXPORT_SYMBOL_GPL(adf_vf2pf_notify_init);
-
-/**
- * adf_vf2pf_notify_shutdown() - send shutdown msg to PF
- * @accel_dev: Pointer to acceleration VF device.
- *
- * Function sends a shutdown message from the VF to a PF
- *
- * Return: void
- */
-void adf_vf2pf_notify_shutdown(struct adf_accel_dev *accel_dev)
-{
- u32 msg = (ADF_VF2PF_MSGORIGIN_SYSTEM |
- (ADF_VF2PF_MSGTYPE_SHUTDOWN << ADF_VF2PF_MSGTYPE_SHIFT));
-
- if (test_bit(ADF_STATUS_PF_RUNNING, &accel_dev->status))
- if (adf_send_vf2pf_msg(accel_dev, msg))
- dev_err(&GET_DEV(accel_dev),
- "Failed to send Shutdown event to PF\n");
-}
-EXPORT_SYMBOL_GPL(adf_vf2pf_notify_shutdown);
diff --git a/drivers/crypto/qat/qat_common/adf_vf_isr.c b/drivers/crypto/qat/qat_common/adf_vf_isr.c
index db5e7abbe5f3..86c3bd0c9c2b 100644
--- a/drivers/crypto/qat/qat_common/adf_vf_isr.c
+++ b/drivers/crypto/qat/qat_common/adf_vf_isr.c
@@ -15,7 +15,6 @@
#include "adf_cfg_common.h"
#include "adf_transport_access_macros.h"
#include "adf_transport_internal.h"
-#include "adf_pf2vf_msg.h"
#define ADF_VINTSOU_OFFSET 0x204
#define ADF_VINTMSK_OFFSET 0x208
@@ -31,22 +30,16 @@ struct adf_vf_stop_data {
void adf_enable_pf2vf_interrupts(struct adf_accel_dev *accel_dev)
{
- struct adf_accel_pci *pci_info = &accel_dev->accel_pci_dev;
- struct adf_hw_device_data *hw_data = accel_dev->hw_device;
- void __iomem *pmisc_bar_addr =
- pci_info->pci_bars[hw_data->get_misc_bar_id(hw_data)].virt_addr;
+ void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev);
- ADF_CSR_WR(pmisc_bar_addr, ADF_VINTMSK_OFFSET, 0x0);
+ ADF_CSR_WR(pmisc_addr, ADF_VINTMSK_OFFSET, 0x0);
}
void adf_disable_pf2vf_interrupts(struct adf_accel_dev *accel_dev)
{
- struct adf_accel_pci *pci_info = &accel_dev->accel_pci_dev;
- struct adf_hw_device_data *hw_data = accel_dev->hw_device;
- void __iomem *pmisc_bar_addr =
- pci_info->pci_bars[hw_data->get_misc_bar_id(hw_data)].virt_addr;
+ void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev);
- ADF_CSR_WR(pmisc_bar_addr, ADF_VINTMSK_OFFSET, 0x2);
+ ADF_CSR_WR(pmisc_addr, ADF_VINTMSK_OFFSET, 0x2);
}
EXPORT_SYMBOL_GPL(adf_disable_pf2vf_interrupts);
@@ -85,78 +78,37 @@ static void adf_dev_stop_async(struct work_struct *work)
kfree(stop_data);
}
-static void adf_pf2vf_bh_handler(void *data)
+int adf_pf2vf_handle_pf_restarting(struct adf_accel_dev *accel_dev)
{
- struct adf_accel_dev *accel_dev = data;
- struct adf_hw_device_data *hw_data = accel_dev->hw_device;
- struct adf_bar *pmisc =
- &GET_BARS(accel_dev)[hw_data->get_misc_bar_id(hw_data)];
- void __iomem *pmisc_bar_addr = pmisc->virt_addr;
- u32 msg;
-
- /* Read the message from PF */
- msg = ADF_CSR_RD(pmisc_bar_addr, hw_data->get_pf2vf_offset(0));
- if (!(msg & ADF_PF2VF_INT)) {
- dev_info(&GET_DEV(accel_dev),
- "Spurious PF2VF interrupt, msg %X. Ignored\n", msg);
- goto out;
- }
+ struct adf_vf_stop_data *stop_data;
- if (!(msg & ADF_PF2VF_MSGORIGIN_SYSTEM))
- /* Ignore legacy non-system (non-kernel) PF2VF messages */
- goto err;
-
- switch ((msg & ADF_PF2VF_MSGTYPE_MASK) >> ADF_PF2VF_MSGTYPE_SHIFT) {
- case ADF_PF2VF_MSGTYPE_RESTARTING: {
- struct adf_vf_stop_data *stop_data;
-
- dev_dbg(&GET_DEV(accel_dev),
- "Restarting msg received from PF 0x%x\n", msg);
-
- clear_bit(ADF_STATUS_PF_RUNNING, &accel_dev->status);
-
- stop_data = kzalloc(sizeof(*stop_data), GFP_ATOMIC);
- if (!stop_data) {
- dev_err(&GET_DEV(accel_dev),
- "Couldn't schedule stop for vf_%d\n",
- accel_dev->accel_id);
- return;
- }
- stop_data->accel_dev = accel_dev;
- INIT_WORK(&stop_data->work, adf_dev_stop_async);
- queue_work(adf_vf_stop_wq, &stop_data->work);
- /* To ack, clear the PF2VFINT bit */
- msg &= ~ADF_PF2VF_INT;
- ADF_CSR_WR(pmisc_bar_addr, hw_data->get_pf2vf_offset(0), msg);
- return;
- }
- case ADF_PF2VF_MSGTYPE_VERSION_RESP:
- dev_dbg(&GET_DEV(accel_dev),
- "Version resp received from PF 0x%x\n", msg);
- accel_dev->vf.pf_version =
- (msg & ADF_PF2VF_VERSION_RESP_VERS_MASK) >>
- ADF_PF2VF_VERSION_RESP_VERS_SHIFT;
- accel_dev->vf.compatible =
- (msg & ADF_PF2VF_VERSION_RESP_RESULT_MASK) >>
- ADF_PF2VF_VERSION_RESP_RESULT_SHIFT;
- complete(&accel_dev->vf.iov_msg_completion);
- break;
- default:
- goto err;
+ clear_bit(ADF_STATUS_PF_RUNNING, &accel_dev->status);
+ stop_data = kzalloc(sizeof(*stop_data), GFP_ATOMIC);
+ if (!stop_data) {
+ dev_err(&GET_DEV(accel_dev),
+ "Couldn't schedule stop for vf_%d\n",
+ accel_dev->accel_id);
+ return -ENOMEM;
}
+ stop_data->accel_dev = accel_dev;
+ INIT_WORK(&stop_data->work, adf_dev_stop_async);
+ queue_work(adf_vf_stop_wq, &stop_data->work);
- /* To ack, clear the PF2VFINT bit */
- msg &= ~ADF_PF2VF_INT;
- ADF_CSR_WR(pmisc_bar_addr, hw_data->get_pf2vf_offset(0), msg);
+ return 0;
+}
+
+static void adf_pf2vf_bh_handler(void *data)
+{
+ struct adf_accel_dev *accel_dev = data;
+ bool ret;
+
+ ret = adf_recv_and_handle_pf2vf_msg(accel_dev);
+ if (ret)
+ /* Re-enable PF2VF interrupts */
+ adf_enable_pf2vf_interrupts(accel_dev);
-out:
- /* Re-enable PF2VF interrupts */
- adf_enable_pf2vf_interrupts(accel_dev);
return;
-err:
- dev_err(&GET_DEV(accel_dev),
- "Unknown message from PF (0x%x); leaving PF2VF ints disabled\n",
- msg);
+
}
static int adf_setup_pf2vf_bh(struct adf_accel_dev *accel_dev)
diff --git a/drivers/crypto/qat/qat_common/icp_qat_fw_init_admin.h b/drivers/crypto/qat/qat_common/icp_qat_fw_init_admin.h
index f05ad17fbdd6..afe59a7684ac 100644
--- a/drivers/crypto/qat/qat_common/icp_qat_fw_init_admin.h
+++ b/drivers/crypto/qat/qat_common/icp_qat_fw_init_admin.h
@@ -14,7 +14,8 @@ enum icp_qat_fw_init_admin_cmd_id {
ICP_QAT_FW_COUNTERS_GET = 5,
ICP_QAT_FW_LOOPBACK = 6,
ICP_QAT_FW_HEARTBEAT_SYNC = 7,
- ICP_QAT_FW_HEARTBEAT_GET = 8
+ ICP_QAT_FW_HEARTBEAT_GET = 8,
+ ICP_QAT_FW_COMP_CAPABILITY_GET = 9,
};
enum icp_qat_fw_init_admin_resp_status {
@@ -52,6 +53,7 @@ struct icp_qat_fw_init_admin_resp {
__u16 version_minor_num;
__u16 version_major_num;
};
+ __u32 extended_features;
};
__u64 opaque_data;
union {
diff --git a/drivers/crypto/qat/qat_common/icp_qat_hw.h b/drivers/crypto/qat/qat_common/icp_qat_hw.h
index e39e8a2d51a7..433304cad2ed 100644
--- a/drivers/crypto/qat/qat_common/icp_qat_hw.h
+++ b/drivers/crypto/qat/qat_common/icp_qat_hw.h
@@ -91,7 +91,18 @@ enum icp_qat_capabilities_mask {
ICP_ACCEL_CAPABILITIES_RAND = BIT(7),
ICP_ACCEL_CAPABILITIES_ZUC = BIT(8),
ICP_ACCEL_CAPABILITIES_SHA3 = BIT(9),
- /* Bits 10-25 are currently reserved */
+ /* Bits 10-11 are currently reserved */
+ ICP_ACCEL_CAPABILITIES_HKDF = BIT(12),
+ ICP_ACCEL_CAPABILITIES_ECEDMONT = BIT(13),
+ /* Bit 14 is currently reserved */
+ ICP_ACCEL_CAPABILITIES_SHA3_EXT = BIT(15),
+ ICP_ACCEL_CAPABILITIES_AESGCM_SPC = BIT(16),
+ ICP_ACCEL_CAPABILITIES_CHACHA_POLY = BIT(17),
+ /* Bits 18-21 are currently reserved */
+ ICP_ACCEL_CAPABILITIES_CNV_INTEGRITY = BIT(22),
+ ICP_ACCEL_CAPABILITIES_CNV_INTEGRITY64 = BIT(23),
+ ICP_ACCEL_CAPABILITIES_LZ4_COMPRESSION = BIT(24),
+ ICP_ACCEL_CAPABILITIES_LZ4S_COMPRESSION = BIT(25),
ICP_ACCEL_CAPABILITIES_AES_V2 = BIT(26)
};
diff --git a/drivers/crypto/qat/qat_common/qat_crypto.c b/drivers/crypto/qat/qat_common/qat_crypto.c
index ece6776fbd53..7234c4940fae 100644
--- a/drivers/crypto/qat/qat_common/qat_crypto.c
+++ b/drivers/crypto/qat/qat_common/qat_crypto.c
@@ -8,6 +8,7 @@
#include "adf_transport_access_macros.h"
#include "adf_cfg.h"
#include "adf_cfg_strings.h"
+#include "adf_gen2_hw_data.h"
#include "qat_crypto.h"
#include "icp_qat_fw.h"
@@ -105,6 +106,30 @@ struct qat_crypto_instance *qat_crypto_get_instance_node(int node)
}
/**
+ * qat_crypto_vf_dev_config()
+ * create dev config required to create crypto inst.
+ *
+ * @accel_dev: Pointer to acceleration device.
+ *
+ * Function creates device configuration required to create
+ * asym, sym or, crypto instances
+ *
+ * Return: 0 on success, error code otherwise.
+ */
+int qat_crypto_vf_dev_config(struct adf_accel_dev *accel_dev)
+{
+ u16 ring_to_svc_map = GET_HW_DATA(accel_dev)->ring_to_svc_map;
+
+ if (ring_to_svc_map != ADF_GEN2_DEFAULT_RING_TO_SRV_MAP) {
+ dev_err(&GET_DEV(accel_dev),
+ "Unsupported ring/service mapping present on PF");
+ return -EFAULT;
+ }
+
+ return qat_crypto_dev_config(accel_dev);
+}
+
+/**
* qat_crypto_dev_config() - create dev config required to create crypto inst.
*
* @accel_dev: Pointer to acceleration device.
diff --git a/drivers/crypto/qat/qat_common/qat_hal.c b/drivers/crypto/qat/qat_common/qat_hal.c
index 12ca6b8764aa..4bfd8f3566f7 100644
--- a/drivers/crypto/qat/qat_common/qat_hal.c
+++ b/drivers/crypto/qat/qat_common/qat_hal.c
@@ -684,8 +684,7 @@ static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle,
{
struct adf_accel_pci *pci_info = &accel_dev->accel_pci_dev;
struct adf_hw_device_data *hw_data = accel_dev->hw_device;
- struct adf_bar *misc_bar =
- &pci_info->pci_bars[hw_data->get_misc_bar_id(hw_data)];
+ void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev);
unsigned int max_en_ae_id = 0;
struct adf_bar *sram_bar;
unsigned int csr_val = 0;
@@ -715,18 +714,12 @@ static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle,
handle->chip_info->fcu_loaded_ae_csr = FCU_AE_LOADED_4XXX;
handle->chip_info->fcu_loaded_ae_pos = 0;
- handle->hal_cap_g_ctl_csr_addr_v =
- (void __iomem *)((uintptr_t)misc_bar->virt_addr +
- ICP_QAT_CAP_OFFSET_4XXX);
- handle->hal_cap_ae_xfer_csr_addr_v =
- (void __iomem *)((uintptr_t)misc_bar->virt_addr +
- ICP_QAT_AE_OFFSET_4XXX);
- handle->hal_ep_csr_addr_v =
- (void __iomem *)((uintptr_t)misc_bar->virt_addr +
- ICP_QAT_EP_OFFSET_4XXX);
+ handle->hal_cap_g_ctl_csr_addr_v = pmisc_addr + ICP_QAT_CAP_OFFSET_4XXX;
+ handle->hal_cap_ae_xfer_csr_addr_v = pmisc_addr + ICP_QAT_AE_OFFSET_4XXX;
+ handle->hal_ep_csr_addr_v = pmisc_addr + ICP_QAT_EP_OFFSET_4XXX;
handle->hal_cap_ae_local_csr_addr_v =
(void __iomem *)((uintptr_t)handle->hal_cap_ae_xfer_csr_addr_v
- + LOCAL_TO_XFER_REG_OFFSET);
+ + LOCAL_TO_XFER_REG_OFFSET);
break;
case PCI_DEVICE_ID_INTEL_QAT_C62X:
case PCI_DEVICE_ID_INTEL_QAT_C3XXX:
@@ -749,15 +742,9 @@ static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle,
handle->chip_info->fcu_dram_addr_lo = FCU_DRAM_ADDR_LO;
handle->chip_info->fcu_loaded_ae_csr = FCU_STATUS;
handle->chip_info->fcu_loaded_ae_pos = FCU_LOADED_AE_POS;
- handle->hal_cap_g_ctl_csr_addr_v =
- (void __iomem *)((uintptr_t)misc_bar->virt_addr +
- ICP_QAT_CAP_OFFSET);
- handle->hal_cap_ae_xfer_csr_addr_v =
- (void __iomem *)((uintptr_t)misc_bar->virt_addr +
- ICP_QAT_AE_OFFSET);
- handle->hal_ep_csr_addr_v =
- (void __iomem *)((uintptr_t)misc_bar->virt_addr +
- ICP_QAT_EP_OFFSET);
+ handle->hal_cap_g_ctl_csr_addr_v = pmisc_addr + ICP_QAT_CAP_OFFSET;
+ handle->hal_cap_ae_xfer_csr_addr_v = pmisc_addr + ICP_QAT_AE_OFFSET;
+ handle->hal_ep_csr_addr_v = pmisc_addr + ICP_QAT_EP_OFFSET;
handle->hal_cap_ae_local_csr_addr_v =
(void __iomem *)((uintptr_t)handle->hal_cap_ae_xfer_csr_addr_v
+ LOCAL_TO_XFER_REG_OFFSET);
@@ -782,15 +769,9 @@ static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle,
handle->chip_info->fcu_dram_addr_lo = 0;
handle->chip_info->fcu_loaded_ae_csr = 0;
handle->chip_info->fcu_loaded_ae_pos = 0;
- handle->hal_cap_g_ctl_csr_addr_v =
- (void __iomem *)((uintptr_t)misc_bar->virt_addr +
- ICP_QAT_CAP_OFFSET);
- handle->hal_cap_ae_xfer_csr_addr_v =
- (void __iomem *)((uintptr_t)misc_bar->virt_addr +
- ICP_QAT_AE_OFFSET);
- handle->hal_ep_csr_addr_v =
- (void __iomem *)((uintptr_t)misc_bar->virt_addr +
- ICP_QAT_EP_OFFSET);
+ handle->hal_cap_g_ctl_csr_addr_v = pmisc_addr + ICP_QAT_CAP_OFFSET;
+ handle->hal_cap_ae_xfer_csr_addr_v = pmisc_addr + ICP_QAT_AE_OFFSET;
+ handle->hal_ep_csr_addr_v = pmisc_addr + ICP_QAT_EP_OFFSET;
handle->hal_cap_ae_local_csr_addr_v =
(void __iomem *)((uintptr_t)handle->hal_cap_ae_xfer_csr_addr_v
+ LOCAL_TO_XFER_REG_OFFSET);
diff --git a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c
index 8e2e1554dcf6..09599fe4d2f3 100644
--- a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c
+++ b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c
@@ -1,9 +1,9 @@
// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
-/* Copyright(c) 2014 - 2020 Intel Corporation */
+/* Copyright(c) 2014 - 2021 Intel Corporation */
#include <adf_accel_devices.h>
-#include <adf_pf2vf_msg.h>
#include <adf_common_drv.h>
#include <adf_gen2_hw_data.h>
+#include <adf_gen2_pfvf.h>
#include "adf_dh895xcc_hw_data.h"
#include "icp_qat_hw.h"
@@ -69,6 +69,8 @@ static u32 get_accel_cap(struct adf_accel_dev *accel_dev)
capabilities &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC;
if (legfuses & ICP_ACCEL_MASK_AUTH_SLICE)
capabilities &= ~ICP_ACCEL_CAPABILITIES_AUTHENTICATION;
+ if (legfuses & ICP_ACCEL_MASK_COMPRESS_SLICE)
+ capabilities &= ~ICP_ACCEL_CAPABILITIES_COMPRESSION;
return capabilities;
}
@@ -114,14 +116,19 @@ static void adf_enable_ints(struct adf_accel_dev *accel_dev)
static u32 get_vf2pf_sources(void __iomem *pmisc_bar)
{
- u32 errsou5, errmsk5, vf_int_mask;
+ u32 errsou3, errmsk3, errsou5, errmsk5, vf_int_mask;
- vf_int_mask = adf_gen2_get_vf2pf_sources(pmisc_bar);
+ /* Get the interrupt sources triggered by VFs */
+ errsou3 = ADF_CSR_RD(pmisc_bar, ADF_GEN2_ERRSOU3);
+ vf_int_mask = ADF_DH895XCC_ERR_REG_VF2PF_L(errsou3);
- /* Get the interrupt sources triggered by VFs, but to avoid duplicates
- * in the work queue, clear vf_int_mask_sets bits that are already
- * masked in ERRMSK register.
+ /* To avoid adding duplicate entries to work queue, clear
+ * vf_int_mask_sets bits that are already masked in ERRMSK register.
*/
+ errmsk3 = ADF_CSR_RD(pmisc_bar, ADF_GEN2_ERRMSK3);
+ vf_int_mask &= ~ADF_DH895XCC_ERR_REG_VF2PF_L(errmsk3);
+
+ /* Do the same for ERRSOU5 */
errsou5 = ADF_CSR_RD(pmisc_bar, ADF_GEN2_ERRSOU5);
errmsk5 = ADF_CSR_RD(pmisc_bar, ADF_GEN2_ERRMSK5);
vf_int_mask |= ADF_DH895XCC_ERR_REG_VF2PF_U(errsou5);
@@ -133,7 +140,11 @@ static u32 get_vf2pf_sources(void __iomem *pmisc_bar)
static void enable_vf2pf_interrupts(void __iomem *pmisc_addr, u32 vf_mask)
{
/* Enable VF2PF Messaging Ints - VFs 0 through 15 per vf_mask[15:0] */
- adf_gen2_enable_vf2pf_interrupts(pmisc_addr, vf_mask);
+ if (vf_mask & 0xFFFF) {
+ u32 val = ADF_CSR_RD(pmisc_addr, ADF_GEN2_ERRMSK3)
+ & ~ADF_DH895XCC_ERR_MSK_VF2PF_L(vf_mask);
+ ADF_CSR_WR(pmisc_addr, ADF_GEN2_ERRMSK3, val);
+ }
/* Enable VF2PF Messaging Ints - VFs 16 through 31 per vf_mask[31:16] */
if (vf_mask >> 16) {
@@ -147,7 +158,11 @@ static void enable_vf2pf_interrupts(void __iomem *pmisc_addr, u32 vf_mask)
static void disable_vf2pf_interrupts(void __iomem *pmisc_addr, u32 vf_mask)
{
/* Disable VF2PF interrupts for VFs 0 through 15 per vf_mask[15:0] */
- adf_gen2_disable_vf2pf_interrupts(pmisc_addr, vf_mask);
+ if (vf_mask & 0xFFFF) {
+ u32 val = ADF_CSR_RD(pmisc_addr, ADF_GEN2_ERRMSK3)
+ | ADF_DH895XCC_ERR_MSK_VF2PF_L(vf_mask);
+ ADF_CSR_WR(pmisc_addr, ADF_GEN2_ERRMSK3, val);
+ }
/* Disable VF2PF interrupts for VFs 16 through 31 per vf_mask[31:16] */
if (vf_mask >> 16) {
@@ -176,6 +191,7 @@ void adf_init_hw_data_dh895xcc(struct adf_hw_device_data *hw_data)
hw_data->num_engines = ADF_DH895XCC_MAX_ACCELENGINES;
hw_data->tx_rx_gap = ADF_GEN2_RX_RINGS_OFFSET;
hw_data->tx_rings_mask = ADF_GEN2_TX_RINGS_MASK;
+ hw_data->ring_to_svc_map = ADF_GEN2_DEFAULT_RING_TO_SRV_MAP;
hw_data->alloc_irq = adf_isr_resource_alloc;
hw_data->free_irq = adf_isr_resource_free;
hw_data->enable_error_correction = adf_gen2_enable_error_correction;
@@ -201,14 +217,12 @@ void adf_init_hw_data_dh895xcc(struct adf_hw_device_data *hw_data)
hw_data->get_arb_mapping = adf_get_arbiter_mapping;
hw_data->enable_ints = adf_enable_ints;
hw_data->reset_device = adf_reset_sbr;
- hw_data->get_pf2vf_offset = adf_gen2_get_pf2vf_offset;
- hw_data->get_vf2pf_sources = get_vf2pf_sources;
- hw_data->enable_vf2pf_interrupts = enable_vf2pf_interrupts;
- hw_data->disable_vf2pf_interrupts = disable_vf2pf_interrupts;
- hw_data->enable_pfvf_comms = adf_enable_pf2vf_comms;
hw_data->disable_iov = adf_disable_sriov;
- hw_data->min_iov_compat_ver = ADF_PFVF_COMPAT_THIS_VERSION;
+ adf_gen2_init_pf_pfvf_ops(&hw_data->pfvf_ops);
+ hw_data->pfvf_ops.get_vf2pf_sources = get_vf2pf_sources;
+ hw_data->pfvf_ops.enable_vf2pf_interrupts = enable_vf2pf_interrupts;
+ hw_data->pfvf_ops.disable_vf2pf_interrupts = disable_vf2pf_interrupts;
adf_gen2_init_hw_csr_ops(&hw_data->csr_ops);
}
diff --git a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.h b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.h
index 0af34dd8708a..aa17272a1507 100644
--- a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.h
+++ b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.h
@@ -25,6 +25,8 @@
#define ADF_DH895XCC_SMIA1_MASK 0x1
/* Masks for VF2PF interrupts */
+#define ADF_DH895XCC_ERR_REG_VF2PF_L(vf_src) (((vf_src) & 0x01FFFE00) >> 9)
+#define ADF_DH895XCC_ERR_MSK_VF2PF_L(vf_mask) (((vf_mask) & 0xFFFF) << 9)
#define ADF_DH895XCC_ERR_REG_VF2PF_U(vf_src) (((vf_src) & 0x0000FFFF) << 16)
#define ADF_DH895XCC_ERR_MSK_VF2PF_U(vf_mask) ((vf_mask) >> 16)
diff --git a/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c b/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c
index 7c6ed6bc8abf..31c14d7e1c11 100644
--- a/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c
+++ b/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c
@@ -1,9 +1,10 @@
// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
-/* Copyright(c) 2015 - 2020 Intel Corporation */
+/* Copyright(c) 2015 - 2021 Intel Corporation */
#include <adf_accel_devices.h>
-#include <adf_pf2vf_msg.h>
#include <adf_common_drv.h>
#include <adf_gen2_hw_data.h>
+#include <adf_gen2_pfvf.h>
+#include <adf_pfvf_vf_msg.h>
#include "adf_dh895xccvf_hw_data.h"
static struct adf_hw_device_class dh895xcciov_class = {
@@ -47,11 +48,6 @@ static enum dev_sku_info get_sku(struct adf_hw_device_data *self)
return DEV_SKU_VF;
}
-static u32 get_pf2vf_offset(u32 i)
-{
- return ADF_DH895XCCIOV_PF2VF_OFFSET;
-}
-
static int adf_vf_int_noop(struct adf_accel_dev *accel_dev)
{
return 0;
@@ -71,6 +67,7 @@ void adf_init_hw_data_dh895xcciov(struct adf_hw_device_data *hw_data)
hw_data->num_engines = ADF_DH895XCCIOV_MAX_ACCELENGINES;
hw_data->tx_rx_gap = ADF_DH895XCCIOV_RX_RINGS_OFFSET;
hw_data->tx_rings_mask = ADF_DH895XCCIOV_TX_RINGS_MASK;
+ hw_data->ring_to_svc_map = ADF_GEN2_DEFAULT_RING_TO_SRV_MAP;
hw_data->alloc_irq = adf_vf_isr_resource_alloc;
hw_data->free_irq = adf_vf_isr_resource_free;
hw_data->enable_error_correction = adf_vf_void_noop;
@@ -86,13 +83,11 @@ void adf_init_hw_data_dh895xcciov(struct adf_hw_device_data *hw_data)
hw_data->get_num_aes = get_num_aes;
hw_data->get_etr_bar_id = get_etr_bar_id;
hw_data->get_misc_bar_id = get_misc_bar_id;
- hw_data->get_pf2vf_offset = get_pf2vf_offset;
hw_data->get_sku = get_sku;
hw_data->enable_ints = adf_vf_void_noop;
- hw_data->enable_pfvf_comms = adf_enable_vf2pf_comms;
- hw_data->min_iov_compat_ver = ADF_PFVF_COMPAT_THIS_VERSION;
hw_data->dev_class->instances++;
adf_devmgr_update_class_index(hw_data);
+ adf_gen2_init_vf_pfvf_ops(&hw_data->pfvf_ops);
adf_gen2_init_hw_csr_ops(&hw_data->csr_ops);
}
diff --git a/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.h b/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.h
index 306ebb71a408..6973fa967bc8 100644
--- a/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.h
+++ b/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.h
@@ -12,7 +12,6 @@
#define ADF_DH895XCCIOV_TX_RINGS_MASK 0xFF
#define ADF_DH895XCCIOV_ETR_BAR 0
#define ADF_DH895XCCIOV_ETR_MAX_BANKS 1
-#define ADF_DH895XCCIOV_PF2VF_OFFSET 0x200
void adf_init_hw_data_dh895xcciov(struct adf_hw_device_data *hw_data);
void adf_clean_hw_data_dh895xcciov(struct adf_hw_device_data *hw_data);
diff --git a/drivers/crypto/qat/qat_dh895xccvf/adf_drv.c b/drivers/crypto/qat/qat_dh895xccvf/adf_drv.c
index 99d90f3ea2b7..18756b2e1c91 100644
--- a/drivers/crypto/qat/qat_dh895xccvf/adf_drv.c
+++ b/drivers/crypto/qat/qat_dh895xccvf/adf_drv.c
@@ -171,11 +171,7 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
}
pci_set_master(pdev);
/* Completion for VF2PF request/response message exchange */
- init_completion(&accel_dev->vf.iov_msg_completion);
-
- ret = qat_crypto_dev_config(accel_dev);
- if (ret)
- goto out_err_free_reg;
+ init_completion(&accel_dev->vf.msg_received);
ret = adf_dev_init(accel_dev);
if (ret)
diff --git a/drivers/crypto/qce/aead.c b/drivers/crypto/qce/aead.c
index 290e2446a2f3..97a530171f07 100644
--- a/drivers/crypto/qce/aead.c
+++ b/drivers/crypto/qce/aead.c
@@ -802,8 +802,8 @@ static int qce_aead_register_one(const struct qce_aead_def *def, struct qce_devi
ret = crypto_register_aead(alg);
if (ret) {
- kfree(tmpl);
dev_err(qce->dev, "%s registration failed\n", alg->base.cra_name);
+ kfree(tmpl);
return ret;
}
diff --git a/drivers/crypto/qce/sha.c b/drivers/crypto/qce/sha.c
index 8e6fcf2c21cc..59159f5e64e5 100644
--- a/drivers/crypto/qce/sha.c
+++ b/drivers/crypto/qce/sha.c
@@ -498,8 +498,8 @@ static int qce_ahash_register_one(const struct qce_ahash_def *def,
ret = crypto_register_ahash(alg);
if (ret) {
- kfree(tmpl);
dev_err(qce->dev, "%s registration failed\n", base->cra_name);
+ kfree(tmpl);
return ret;
}
diff --git a/drivers/crypto/qce/skcipher.c b/drivers/crypto/qce/skcipher.c
index 8ff10928f581..3d27cd5210ef 100644
--- a/drivers/crypto/qce/skcipher.c
+++ b/drivers/crypto/qce/skcipher.c
@@ -484,8 +484,8 @@ static int qce_skcipher_register_one(const struct qce_skcipher_def *def,
ret = crypto_register_skcipher(alg);
if (ret) {
- kfree(tmpl);
dev_err(qce->dev, "%s registration failed\n", alg->base.cra_name);
+ kfree(tmpl);
return ret;
}
diff --git a/drivers/crypto/sa2ul.c b/drivers/crypto/sa2ul.c
index bcbc38dc6ae8..51b58e57153f 100644
--- a/drivers/crypto/sa2ul.c
+++ b/drivers/crypto/sa2ul.c
@@ -8,6 +8,7 @@
* Vitaly Andrianov
* Tero Kristo
*/
+#include <linux/bitfield.h>
#include <linux/clk.h>
#include <linux/dma-mapping.h>
#include <linux/dmaengine.h>
@@ -646,8 +647,8 @@ static inline void sa_update_cmdl(struct sa_req *req, u32 *cmdl,
cmdl[upd_info->enc_offset.index] &=
~SA_CMDL_SOP_BYPASS_LEN_MASK;
cmdl[upd_info->enc_offset.index] |=
- ((u32)req->enc_offset <<
- __ffs(SA_CMDL_SOP_BYPASS_LEN_MASK));
+ FIELD_PREP(SA_CMDL_SOP_BYPASS_LEN_MASK,
+ req->enc_offset);
if (likely(upd_info->flags & SA_CMDL_UPD_ENC_IV)) {
__be32 *data = (__be32 *)&cmdl[upd_info->enc_iv.index];
@@ -666,8 +667,8 @@ static inline void sa_update_cmdl(struct sa_req *req, u32 *cmdl,
cmdl[upd_info->auth_offset.index] &=
~SA_CMDL_SOP_BYPASS_LEN_MASK;
cmdl[upd_info->auth_offset.index] |=
- ((u32)req->auth_offset <<
- __ffs(SA_CMDL_SOP_BYPASS_LEN_MASK));
+ FIELD_PREP(SA_CMDL_SOP_BYPASS_LEN_MASK,
+ req->auth_offset);
if (upd_info->flags & SA_CMDL_UPD_AUTH_IV) {
sa_copy_iv((void *)&cmdl[upd_info->auth_iv.index],
req->auth_iv,
@@ -689,16 +690,16 @@ void sa_set_swinfo(u8 eng_id, u16 sc_id, dma_addr_t sc_phys,
u8 hash_size, u32 *swinfo)
{
swinfo[0] = sc_id;
- swinfo[0] |= (flags << __ffs(SA_SW0_FLAGS_MASK));
+ swinfo[0] |= FIELD_PREP(SA_SW0_FLAGS_MASK, flags);
if (likely(cmdl_present))
- swinfo[0] |= ((cmdl_offset | SA_SW0_CMDL_PRESENT) <<
- __ffs(SA_SW0_CMDL_INFO_MASK));
- swinfo[0] |= (eng_id << __ffs(SA_SW0_ENG_ID_MASK));
+ swinfo[0] |= FIELD_PREP(SA_SW0_CMDL_INFO_MASK,
+ cmdl_offset | SA_SW0_CMDL_PRESENT);
+ swinfo[0] |= FIELD_PREP(SA_SW0_ENG_ID_MASK, eng_id);
swinfo[0] |= SA_SW0_DEST_INFO_PRESENT;
swinfo[1] = (u32)(sc_phys & 0xFFFFFFFFULL);
swinfo[2] = (u32)((sc_phys & 0xFFFFFFFF00000000ULL) >> 32);
- swinfo[2] |= (hash_size << __ffs(SA_SW2_EGRESS_LENGTH));
+ swinfo[2] |= FIELD_PREP(SA_SW2_EGRESS_LENGTH, hash_size);
}
/* Dump the security context */
diff --git a/drivers/crypto/stm32/stm32-crc32.c b/drivers/crypto/stm32/stm32-crc32.c
index 75867c0b0017..be1bf39a317d 100644
--- a/drivers/crypto/stm32/stm32-crc32.c
+++ b/drivers/crypto/stm32/stm32-crc32.c
@@ -279,7 +279,7 @@ static struct shash_alg algs[] = {
.digestsize = CHKSUM_DIGEST_SIZE,
.base = {
.cra_name = "crc32",
- .cra_driver_name = DRIVER_NAME,
+ .cra_driver_name = "stm32-crc32-crc32",
.cra_priority = 200,
.cra_flags = CRYPTO_ALG_OPTIONAL_KEY,
.cra_blocksize = CHKSUM_BLOCK_SIZE,
@@ -301,7 +301,7 @@ static struct shash_alg algs[] = {
.digestsize = CHKSUM_DIGEST_SIZE,
.base = {
.cra_name = "crc32c",
- .cra_driver_name = DRIVER_NAME,
+ .cra_driver_name = "stm32-crc32-crc32c",
.cra_priority = 200,
.cra_flags = CRYPTO_ALG_OPTIONAL_KEY,
.cra_blocksize = CHKSUM_BLOCK_SIZE,
diff --git a/drivers/crypto/stm32/stm32-cryp.c b/drivers/crypto/stm32/stm32-cryp.c
index 7389a0536ff0..59ef541123ae 100644
--- a/drivers/crypto/stm32/stm32-cryp.c
+++ b/drivers/crypto/stm32/stm32-cryp.c
@@ -37,7 +37,6 @@
/* Mode mask = bits [15..0] */
#define FLG_MODE_MASK GENMASK(15, 0)
/* Bit [31..16] status */
-#define FLG_CCM_PADDED_WA BIT(16)
/* Registers */
#define CRYP_CR 0x00000000
@@ -105,8 +104,6 @@
/* Misc */
#define AES_BLOCK_32 (AES_BLOCK_SIZE / sizeof(u32))
#define GCM_CTR_INIT 2
-#define _walked_in (cryp->in_walk.offset - cryp->in_sg->offset)
-#define _walked_out (cryp->out_walk.offset - cryp->out_sg->offset)
#define CRYP_AUTOSUSPEND_DELAY 50
struct stm32_cryp_caps {
@@ -144,26 +141,16 @@ struct stm32_cryp {
size_t authsize;
size_t hw_blocksize;
- size_t total_in;
- size_t total_in_save;
- size_t total_out;
- size_t total_out_save;
+ size_t payload_in;
+ size_t header_in;
+ size_t payload_out;
- struct scatterlist *in_sg;
struct scatterlist *out_sg;
- struct scatterlist *out_sg_save;
-
- struct scatterlist in_sgl;
- struct scatterlist out_sgl;
- bool sgs_copied;
-
- int in_sg_len;
- int out_sg_len;
struct scatter_walk in_walk;
struct scatter_walk out_walk;
- u32 last_ctr[4];
+ __be32 last_ctr[4];
u32 gcm_ctr;
};
@@ -245,6 +232,11 @@ static inline int stm32_cryp_wait_busy(struct stm32_cryp *cryp)
!(status & SR_BUSY), 10, 100000);
}
+static inline void stm32_cryp_enable(struct stm32_cryp *cryp)
+{
+ writel_relaxed(readl_relaxed(cryp->regs + CRYP_CR) | CR_CRYPEN, cryp->regs + CRYP_CR);
+}
+
static inline int stm32_cryp_wait_enable(struct stm32_cryp *cryp)
{
u32 status;
@@ -262,6 +254,7 @@ static inline int stm32_cryp_wait_output(struct stm32_cryp *cryp)
}
static int stm32_cryp_read_auth_tag(struct stm32_cryp *cryp);
+static void stm32_cryp_finish_req(struct stm32_cryp *cryp, int err);
static struct stm32_cryp *stm32_cryp_find_dev(struct stm32_cryp_ctx *ctx)
{
@@ -283,103 +276,6 @@ static struct stm32_cryp *stm32_cryp_find_dev(struct stm32_cryp_ctx *ctx)
return cryp;
}
-static int stm32_cryp_check_aligned(struct scatterlist *sg, size_t total,
- size_t align)
-{
- int len = 0;
-
- if (!total)
- return 0;
-
- if (!IS_ALIGNED(total, align))
- return -EINVAL;
-
- while (sg) {
- if (!IS_ALIGNED(sg->offset, sizeof(u32)))
- return -EINVAL;
-
- if (!IS_ALIGNED(sg->length, align))
- return -EINVAL;
-
- len += sg->length;
- sg = sg_next(sg);
- }
-
- if (len != total)
- return -EINVAL;
-
- return 0;
-}
-
-static int stm32_cryp_check_io_aligned(struct stm32_cryp *cryp)
-{
- int ret;
-
- ret = stm32_cryp_check_aligned(cryp->in_sg, cryp->total_in,
- cryp->hw_blocksize);
- if (ret)
- return ret;
-
- ret = stm32_cryp_check_aligned(cryp->out_sg, cryp->total_out,
- cryp->hw_blocksize);
-
- return ret;
-}
-
-static void sg_copy_buf(void *buf, struct scatterlist *sg,
- unsigned int start, unsigned int nbytes, int out)
-{
- struct scatter_walk walk;
-
- if (!nbytes)
- return;
-
- scatterwalk_start(&walk, sg);
- scatterwalk_advance(&walk, start);
- scatterwalk_copychunks(buf, &walk, nbytes, out);
- scatterwalk_done(&walk, out, 0);
-}
-
-static int stm32_cryp_copy_sgs(struct stm32_cryp *cryp)
-{
- void *buf_in, *buf_out;
- int pages, total_in, total_out;
-
- if (!stm32_cryp_check_io_aligned(cryp)) {
- cryp->sgs_copied = 0;
- return 0;
- }
-
- total_in = ALIGN(cryp->total_in, cryp->hw_blocksize);
- pages = total_in ? get_order(total_in) : 1;
- buf_in = (void *)__get_free_pages(GFP_ATOMIC, pages);
-
- total_out = ALIGN(cryp->total_out, cryp->hw_blocksize);
- pages = total_out ? get_order(total_out) : 1;
- buf_out = (void *)__get_free_pages(GFP_ATOMIC, pages);
-
- if (!buf_in || !buf_out) {
- dev_err(cryp->dev, "Can't allocate pages when unaligned\n");
- cryp->sgs_copied = 0;
- return -EFAULT;
- }
-
- sg_copy_buf(buf_in, cryp->in_sg, 0, cryp->total_in, 0);
-
- sg_init_one(&cryp->in_sgl, buf_in, total_in);
- cryp->in_sg = &cryp->in_sgl;
- cryp->in_sg_len = 1;
-
- sg_init_one(&cryp->out_sgl, buf_out, total_out);
- cryp->out_sg_save = cryp->out_sg;
- cryp->out_sg = &cryp->out_sgl;
- cryp->out_sg_len = 1;
-
- cryp->sgs_copied = 1;
-
- return 0;
-}
-
static void stm32_cryp_hw_write_iv(struct stm32_cryp *cryp, __be32 *iv)
{
if (!iv)
@@ -481,16 +377,99 @@ static int stm32_cryp_gcm_init(struct stm32_cryp *cryp, u32 cfg)
/* Wait for end of processing */
ret = stm32_cryp_wait_enable(cryp);
- if (ret)
+ if (ret) {
dev_err(cryp->dev, "Timeout (gcm init)\n");
+ return ret;
+ }
- return ret;
+ /* Prepare next phase */
+ if (cryp->areq->assoclen) {
+ cfg |= CR_PH_HEADER;
+ stm32_cryp_write(cryp, CRYP_CR, cfg);
+ } else if (stm32_cryp_get_input_text_len(cryp)) {
+ cfg |= CR_PH_PAYLOAD;
+ stm32_cryp_write(cryp, CRYP_CR, cfg);
+ }
+
+ return 0;
+}
+
+static void stm32_crypt_gcmccm_end_header(struct stm32_cryp *cryp)
+{
+ u32 cfg;
+ int err;
+
+ /* Check if whole header written */
+ if (!cryp->header_in) {
+ /* Wait for completion */
+ err = stm32_cryp_wait_busy(cryp);
+ if (err) {
+ dev_err(cryp->dev, "Timeout (gcm/ccm header)\n");
+ stm32_cryp_write(cryp, CRYP_IMSCR, 0);
+ stm32_cryp_finish_req(cryp, err);
+ return;
+ }
+
+ if (stm32_cryp_get_input_text_len(cryp)) {
+ /* Phase 3 : payload */
+ cfg = stm32_cryp_read(cryp, CRYP_CR);
+ cfg &= ~CR_CRYPEN;
+ stm32_cryp_write(cryp, CRYP_CR, cfg);
+
+ cfg &= ~CR_PH_MASK;
+ cfg |= CR_PH_PAYLOAD | CR_CRYPEN;
+ stm32_cryp_write(cryp, CRYP_CR, cfg);
+ } else {
+ /*
+ * Phase 4 : tag.
+ * Nothing to read, nothing to write, caller have to
+ * end request
+ */
+ }
+ }
+}
+
+static void stm32_cryp_write_ccm_first_header(struct stm32_cryp *cryp)
+{
+ unsigned int i;
+ size_t written;
+ size_t len;
+ u32 alen = cryp->areq->assoclen;
+ u32 block[AES_BLOCK_32] = {0};
+ u8 *b8 = (u8 *)block;
+
+ if (alen <= 65280) {
+ /* Write first u32 of B1 */
+ b8[0] = (alen >> 8) & 0xFF;
+ b8[1] = alen & 0xFF;
+ len = 2;
+ } else {
+ /* Build the two first u32 of B1 */
+ b8[0] = 0xFF;
+ b8[1] = 0xFE;
+ b8[2] = (alen & 0xFF000000) >> 24;
+ b8[3] = (alen & 0x00FF0000) >> 16;
+ b8[4] = (alen & 0x0000FF00) >> 8;
+ b8[5] = alen & 0x000000FF;
+ len = 6;
+ }
+
+ written = min_t(size_t, AES_BLOCK_SIZE - len, alen);
+
+ scatterwalk_copychunks((char *)block + len, &cryp->in_walk, written, 0);
+ for (i = 0; i < AES_BLOCK_32; i++)
+ stm32_cryp_write(cryp, CRYP_DIN, block[i]);
+
+ cryp->header_in -= written;
+
+ stm32_crypt_gcmccm_end_header(cryp);
}
static int stm32_cryp_ccm_init(struct stm32_cryp *cryp, u32 cfg)
{
int ret;
- u8 iv[AES_BLOCK_SIZE], b0[AES_BLOCK_SIZE];
+ u32 iv_32[AES_BLOCK_32], b0_32[AES_BLOCK_32];
+ u8 *iv = (u8 *)iv_32, *b0 = (u8 *)b0_32;
__be32 *bd;
u32 *d;
unsigned int i, textlen;
@@ -531,10 +510,24 @@ static int stm32_cryp_ccm_init(struct stm32_cryp *cryp, u32 cfg)
/* Wait for end of processing */
ret = stm32_cryp_wait_enable(cryp);
- if (ret)
+ if (ret) {
dev_err(cryp->dev, "Timeout (ccm init)\n");
+ return ret;
+ }
- return ret;
+ /* Prepare next phase */
+ if (cryp->areq->assoclen) {
+ cfg |= CR_PH_HEADER | CR_CRYPEN;
+ stm32_cryp_write(cryp, CRYP_CR, cfg);
+
+ /* Write first (special) block (may move to next phase [payload]) */
+ stm32_cryp_write_ccm_first_header(cryp);
+ } else if (stm32_cryp_get_input_text_len(cryp)) {
+ cfg |= CR_PH_PAYLOAD;
+ stm32_cryp_write(cryp, CRYP_CR, cfg);
+ }
+
+ return 0;
}
static int stm32_cryp_hw_init(struct stm32_cryp *cryp)
@@ -542,14 +535,11 @@ static int stm32_cryp_hw_init(struct stm32_cryp *cryp)
int ret;
u32 cfg, hw_mode;
- pm_runtime_resume_and_get(cryp->dev);
+ pm_runtime_get_sync(cryp->dev);
/* Disable interrupt */
stm32_cryp_write(cryp, CRYP_IMSCR, 0);
- /* Set key */
- stm32_cryp_hw_write_key(cryp);
-
/* Set configuration */
cfg = CR_DATA8 | CR_FFLUSH;
@@ -575,23 +565,36 @@ static int stm32_cryp_hw_init(struct stm32_cryp *cryp)
/* AES ECB/CBC decrypt: run key preparation first */
if (is_decrypt(cryp) &&
((hw_mode == CR_AES_ECB) || (hw_mode == CR_AES_CBC))) {
- stm32_cryp_write(cryp, CRYP_CR, cfg | CR_AES_KP | CR_CRYPEN);
+ /* Configure in key preparation mode */
+ stm32_cryp_write(cryp, CRYP_CR, cfg | CR_AES_KP);
+ /* Set key only after full configuration done */
+ stm32_cryp_hw_write_key(cryp);
+
+ /* Start prepare key */
+ stm32_cryp_enable(cryp);
/* Wait for end of processing */
ret = stm32_cryp_wait_busy(cryp);
if (ret) {
dev_err(cryp->dev, "Timeout (key preparation)\n");
return ret;
}
- }
- cfg |= hw_mode;
+ cfg |= hw_mode | CR_DEC_NOT_ENC;
- if (is_decrypt(cryp))
- cfg |= CR_DEC_NOT_ENC;
+ /* Apply updated config (Decrypt + algo) and flush */
+ stm32_cryp_write(cryp, CRYP_CR, cfg);
+ } else {
+ cfg |= hw_mode;
+ if (is_decrypt(cryp))
+ cfg |= CR_DEC_NOT_ENC;
- /* Apply config and flush (valid when CRYPEN = 0) */
- stm32_cryp_write(cryp, CRYP_CR, cfg);
+ /* Apply config and flush */
+ stm32_cryp_write(cryp, CRYP_CR, cfg);
+
+ /* Set key only after configuration done */
+ stm32_cryp_hw_write_key(cryp);
+ }
switch (hw_mode) {
case CR_AES_GCM:
@@ -605,16 +608,6 @@ static int stm32_cryp_hw_init(struct stm32_cryp *cryp)
if (ret)
return ret;
- /* Phase 2 : header (authenticated data) */
- if (cryp->areq->assoclen) {
- cfg |= CR_PH_HEADER;
- } else if (stm32_cryp_get_input_text_len(cryp)) {
- cfg |= CR_PH_PAYLOAD;
- stm32_cryp_write(cryp, CRYP_CR, cfg);
- } else {
- cfg |= CR_PH_INIT;
- }
-
break;
case CR_DES_CBC:
@@ -629,11 +622,7 @@ static int stm32_cryp_hw_init(struct stm32_cryp *cryp)
}
/* Enable now */
- cfg |= CR_CRYPEN;
-
- stm32_cryp_write(cryp, CRYP_CR, cfg);
-
- cryp->flags &= ~FLG_CCM_PADDED_WA;
+ stm32_cryp_enable(cryp);
return 0;
}
@@ -644,28 +633,9 @@ static void stm32_cryp_finish_req(struct stm32_cryp *cryp, int err)
/* Phase 4 : output tag */
err = stm32_cryp_read_auth_tag(cryp);
- if (!err && (!(is_gcm(cryp) || is_ccm(cryp))))
+ if (!err && (!(is_gcm(cryp) || is_ccm(cryp) || is_ecb(cryp))))
stm32_cryp_get_iv(cryp);
- if (cryp->sgs_copied) {
- void *buf_in, *buf_out;
- int pages, len;
-
- buf_in = sg_virt(&cryp->in_sgl);
- buf_out = sg_virt(&cryp->out_sgl);
-
- sg_copy_buf(buf_out, cryp->out_sg_save, 0,
- cryp->total_out_save, 1);
-
- len = ALIGN(cryp->total_in_save, cryp->hw_blocksize);
- pages = len ? get_order(len) : 1;
- free_pages((unsigned long)buf_in, pages);
-
- len = ALIGN(cryp->total_out_save, cryp->hw_blocksize);
- pages = len ? get_order(len) : 1;
- free_pages((unsigned long)buf_out, pages);
- }
-
pm_runtime_mark_last_busy(cryp->dev);
pm_runtime_put_autosuspend(cryp->dev);
@@ -674,8 +644,6 @@ static void stm32_cryp_finish_req(struct stm32_cryp *cryp, int err)
else
crypto_finalize_skcipher_request(cryp->engine, cryp->req,
err);
-
- memset(cryp->ctx->key, 0, cryp->ctx->keylen);
}
static int stm32_cryp_cpu_start(struct stm32_cryp *cryp)
@@ -801,7 +769,20 @@ static int stm32_cryp_aes_aead_setkey(struct crypto_aead *tfm, const u8 *key,
static int stm32_cryp_aes_gcm_setauthsize(struct crypto_aead *tfm,
unsigned int authsize)
{
- return authsize == AES_BLOCK_SIZE ? 0 : -EINVAL;
+ switch (authsize) {
+ case 4:
+ case 8:
+ case 12:
+ case 13:
+ case 14:
+ case 15:
+ case 16:
+ break;
+ default:
+ return -EINVAL;
+ }
+
+ return 0;
}
static int stm32_cryp_aes_ccm_setauthsize(struct crypto_aead *tfm,
@@ -825,31 +806,61 @@ static int stm32_cryp_aes_ccm_setauthsize(struct crypto_aead *tfm,
static int stm32_cryp_aes_ecb_encrypt(struct skcipher_request *req)
{
+ if (req->cryptlen % AES_BLOCK_SIZE)
+ return -EINVAL;
+
+ if (req->cryptlen == 0)
+ return 0;
+
return stm32_cryp_crypt(req, FLG_AES | FLG_ECB | FLG_ENCRYPT);
}
static int stm32_cryp_aes_ecb_decrypt(struct skcipher_request *req)
{
+ if (req->cryptlen % AES_BLOCK_SIZE)
+ return -EINVAL;
+
+ if (req->cryptlen == 0)
+ return 0;
+
return stm32_cryp_crypt(req, FLG_AES | FLG_ECB);
}
static int stm32_cryp_aes_cbc_encrypt(struct skcipher_request *req)
{
+ if (req->cryptlen % AES_BLOCK_SIZE)
+ return -EINVAL;
+
+ if (req->cryptlen == 0)
+ return 0;
+
return stm32_cryp_crypt(req, FLG_AES | FLG_CBC | FLG_ENCRYPT);
}
static int stm32_cryp_aes_cbc_decrypt(struct skcipher_request *req)
{
+ if (req->cryptlen % AES_BLOCK_SIZE)
+ return -EINVAL;
+
+ if (req->cryptlen == 0)
+ return 0;
+
return stm32_cryp_crypt(req, FLG_AES | FLG_CBC);
}
static int stm32_cryp_aes_ctr_encrypt(struct skcipher_request *req)
{
+ if (req->cryptlen == 0)
+ return 0;
+
return stm32_cryp_crypt(req, FLG_AES | FLG_CTR | FLG_ENCRYPT);
}
static int stm32_cryp_aes_ctr_decrypt(struct skcipher_request *req)
{
+ if (req->cryptlen == 0)
+ return 0;
+
return stm32_cryp_crypt(req, FLG_AES | FLG_CTR);
}
@@ -863,53 +874,122 @@ static int stm32_cryp_aes_gcm_decrypt(struct aead_request *req)
return stm32_cryp_aead_crypt(req, FLG_AES | FLG_GCM);
}
+static inline int crypto_ccm_check_iv(const u8 *iv)
+{
+ /* 2 <= L <= 8, so 1 <= L' <= 7. */
+ if (iv[0] < 1 || iv[0] > 7)
+ return -EINVAL;
+
+ return 0;
+}
+
static int stm32_cryp_aes_ccm_encrypt(struct aead_request *req)
{
+ int err;
+
+ err = crypto_ccm_check_iv(req->iv);
+ if (err)
+ return err;
+
return stm32_cryp_aead_crypt(req, FLG_AES | FLG_CCM | FLG_ENCRYPT);
}
static int stm32_cryp_aes_ccm_decrypt(struct aead_request *req)
{
+ int err;
+
+ err = crypto_ccm_check_iv(req->iv);
+ if (err)
+ return err;
+
return stm32_cryp_aead_crypt(req, FLG_AES | FLG_CCM);
}
static int stm32_cryp_des_ecb_encrypt(struct skcipher_request *req)
{
+ if (req->cryptlen % DES_BLOCK_SIZE)
+ return -EINVAL;
+
+ if (req->cryptlen == 0)
+ return 0;
+
return stm32_cryp_crypt(req, FLG_DES | FLG_ECB | FLG_ENCRYPT);
}
static int stm32_cryp_des_ecb_decrypt(struct skcipher_request *req)
{
+ if (req->cryptlen % DES_BLOCK_SIZE)
+ return -EINVAL;
+
+ if (req->cryptlen == 0)
+ return 0;
+
return stm32_cryp_crypt(req, FLG_DES | FLG_ECB);
}
static int stm32_cryp_des_cbc_encrypt(struct skcipher_request *req)
{
+ if (req->cryptlen % DES_BLOCK_SIZE)
+ return -EINVAL;
+
+ if (req->cryptlen == 0)
+ return 0;
+
return stm32_cryp_crypt(req, FLG_DES | FLG_CBC | FLG_ENCRYPT);
}
static int stm32_cryp_des_cbc_decrypt(struct skcipher_request *req)
{
+ if (req->cryptlen % DES_BLOCK_SIZE)
+ return -EINVAL;
+
+ if (req->cryptlen == 0)
+ return 0;
+
return stm32_cryp_crypt(req, FLG_DES | FLG_CBC);
}
static int stm32_cryp_tdes_ecb_encrypt(struct skcipher_request *req)
{
+ if (req->cryptlen % DES_BLOCK_SIZE)
+ return -EINVAL;
+
+ if (req->cryptlen == 0)
+ return 0;
+
return stm32_cryp_crypt(req, FLG_TDES | FLG_ECB | FLG_ENCRYPT);
}
static int stm32_cryp_tdes_ecb_decrypt(struct skcipher_request *req)
{
+ if (req->cryptlen % DES_BLOCK_SIZE)
+ return -EINVAL;
+
+ if (req->cryptlen == 0)
+ return 0;
+
return stm32_cryp_crypt(req, FLG_TDES | FLG_ECB);
}
static int stm32_cryp_tdes_cbc_encrypt(struct skcipher_request *req)
{
+ if (req->cryptlen % DES_BLOCK_SIZE)
+ return -EINVAL;
+
+ if (req->cryptlen == 0)
+ return 0;
+
return stm32_cryp_crypt(req, FLG_TDES | FLG_CBC | FLG_ENCRYPT);
}
static int stm32_cryp_tdes_cbc_decrypt(struct skcipher_request *req)
{
+ if (req->cryptlen % DES_BLOCK_SIZE)
+ return -EINVAL;
+
+ if (req->cryptlen == 0)
+ return 0;
+
return stm32_cryp_crypt(req, FLG_TDES | FLG_CBC);
}
@@ -919,6 +999,7 @@ static int stm32_cryp_prepare_req(struct skcipher_request *req,
struct stm32_cryp_ctx *ctx;
struct stm32_cryp *cryp;
struct stm32_cryp_reqctx *rctx;
+ struct scatterlist *in_sg;
int ret;
if (!req && !areq)
@@ -944,76 +1025,55 @@ static int stm32_cryp_prepare_req(struct skcipher_request *req,
if (req) {
cryp->req = req;
cryp->areq = NULL;
- cryp->total_in = req->cryptlen;
- cryp->total_out = cryp->total_in;
+ cryp->header_in = 0;
+ cryp->payload_in = req->cryptlen;
+ cryp->payload_out = req->cryptlen;
+ cryp->authsize = 0;
} else {
/*
* Length of input and output data:
* Encryption case:
- * INPUT = AssocData || PlainText
+ * INPUT = AssocData || PlainText
* <- assoclen -> <- cryptlen ->
- * <------- total_in ----------->
*
- * OUTPUT = AssocData || CipherText || AuthTag
- * <- assoclen -> <- cryptlen -> <- authsize ->
- * <---------------- total_out ----------------->
+ * OUTPUT = AssocData || CipherText || AuthTag
+ * <- assoclen -> <-- cryptlen --> <- authsize ->
*
* Decryption case:
- * INPUT = AssocData || CipherText || AuthTag
- * <- assoclen -> <--------- cryptlen --------->
- * <- authsize ->
- * <---------------- total_in ------------------>
+ * INPUT = AssocData || CipherTex || AuthTag
+ * <- assoclen ---> <---------- cryptlen ---------->
*
- * OUTPUT = AssocData || PlainText
- * <- assoclen -> <- crypten - authsize ->
- * <---------- total_out ----------------->
+ * OUTPUT = AssocData || PlainText
+ * <- assoclen -> <- cryptlen - authsize ->
*/
cryp->areq = areq;
cryp->req = NULL;
cryp->authsize = crypto_aead_authsize(crypto_aead_reqtfm(areq));
- cryp->total_in = areq->assoclen + areq->cryptlen;
- if (is_encrypt(cryp))
- /* Append auth tag to output */
- cryp->total_out = cryp->total_in + cryp->authsize;
- else
- /* No auth tag in output */
- cryp->total_out = cryp->total_in - cryp->authsize;
+ if (is_encrypt(cryp)) {
+ cryp->payload_in = areq->cryptlen;
+ cryp->header_in = areq->assoclen;
+ cryp->payload_out = areq->cryptlen;
+ } else {
+ cryp->payload_in = areq->cryptlen - cryp->authsize;
+ cryp->header_in = areq->assoclen;
+ cryp->payload_out = cryp->payload_in;
+ }
}
- cryp->total_in_save = cryp->total_in;
- cryp->total_out_save = cryp->total_out;
+ in_sg = req ? req->src : areq->src;
+ scatterwalk_start(&cryp->in_walk, in_sg);
- cryp->in_sg = req ? req->src : areq->src;
cryp->out_sg = req ? req->dst : areq->dst;
- cryp->out_sg_save = cryp->out_sg;
-
- cryp->in_sg_len = sg_nents_for_len(cryp->in_sg, cryp->total_in);
- if (cryp->in_sg_len < 0) {
- dev_err(cryp->dev, "Cannot get in_sg_len\n");
- ret = cryp->in_sg_len;
- return ret;
- }
-
- cryp->out_sg_len = sg_nents_for_len(cryp->out_sg, cryp->total_out);
- if (cryp->out_sg_len < 0) {
- dev_err(cryp->dev, "Cannot get out_sg_len\n");
- ret = cryp->out_sg_len;
- return ret;
- }
-
- ret = stm32_cryp_copy_sgs(cryp);
- if (ret)
- return ret;
-
- scatterwalk_start(&cryp->in_walk, cryp->in_sg);
scatterwalk_start(&cryp->out_walk, cryp->out_sg);
if (is_gcm(cryp) || is_ccm(cryp)) {
/* In output, jump after assoc data */
- scatterwalk_advance(&cryp->out_walk, cryp->areq->assoclen);
- cryp->total_out -= cryp->areq->assoclen;
+ scatterwalk_copychunks(NULL, &cryp->out_walk, cryp->areq->assoclen, 2);
}
+ if (is_ctr(cryp))
+ memset(cryp->last_ctr, 0, sizeof(cryp->last_ctr));
+
ret = stm32_cryp_hw_init(cryp);
return ret;
}
@@ -1061,8 +1121,7 @@ static int stm32_cryp_aead_one_req(struct crypto_engine *engine, void *areq)
if (!cryp)
return -ENODEV;
- if (unlikely(!cryp->areq->assoclen &&
- !stm32_cryp_get_input_text_len(cryp))) {
+ if (unlikely(!cryp->payload_in && !cryp->header_in)) {
/* No input data to process: get tag and finish */
stm32_cryp_finish_req(cryp, 0);
return 0;
@@ -1071,43 +1130,10 @@ static int stm32_cryp_aead_one_req(struct crypto_engine *engine, void *areq)
return stm32_cryp_cpu_start(cryp);
}
-static u32 *stm32_cryp_next_out(struct stm32_cryp *cryp, u32 *dst,
- unsigned int n)
-{
- scatterwalk_advance(&cryp->out_walk, n);
-
- if (unlikely(cryp->out_sg->length == _walked_out)) {
- cryp->out_sg = sg_next(cryp->out_sg);
- if (cryp->out_sg) {
- scatterwalk_start(&cryp->out_walk, cryp->out_sg);
- return (sg_virt(cryp->out_sg) + _walked_out);
- }
- }
-
- return (u32 *)((u8 *)dst + n);
-}
-
-static u32 *stm32_cryp_next_in(struct stm32_cryp *cryp, u32 *src,
- unsigned int n)
-{
- scatterwalk_advance(&cryp->in_walk, n);
-
- if (unlikely(cryp->in_sg->length == _walked_in)) {
- cryp->in_sg = sg_next(cryp->in_sg);
- if (cryp->in_sg) {
- scatterwalk_start(&cryp->in_walk, cryp->in_sg);
- return (sg_virt(cryp->in_sg) + _walked_in);
- }
- }
-
- return (u32 *)((u8 *)src + n);
-}
-
static int stm32_cryp_read_auth_tag(struct stm32_cryp *cryp)
{
- u32 cfg, size_bit, *dst, d32;
- u8 *d8;
- unsigned int i, j;
+ u32 cfg, size_bit;
+ unsigned int i;
int ret = 0;
/* Update Config */
@@ -1130,7 +1156,7 @@ static int stm32_cryp_read_auth_tag(struct stm32_cryp *cryp)
stm32_cryp_write(cryp, CRYP_DIN, size_bit);
size_bit = is_encrypt(cryp) ? cryp->areq->cryptlen :
- cryp->areq->cryptlen - AES_BLOCK_SIZE;
+ cryp->areq->cryptlen - cryp->authsize;
size_bit *= 8;
if (cryp->caps->swap_final)
size_bit = (__force u32)cpu_to_be32(size_bit);
@@ -1139,11 +1165,9 @@ static int stm32_cryp_read_auth_tag(struct stm32_cryp *cryp)
stm32_cryp_write(cryp, CRYP_DIN, size_bit);
} else {
/* CCM: write CTR0 */
- u8 iv[AES_BLOCK_SIZE];
- u32 *iv32 = (u32 *)iv;
- __be32 *biv;
-
- biv = (void *)iv;
+ u32 iv32[AES_BLOCK_32];
+ u8 *iv = (u8 *)iv32;
+ __be32 *biv = (__be32 *)iv32;
memcpy(iv, cryp->areq->iv, AES_BLOCK_SIZE);
memset(iv + AES_BLOCK_SIZE - 1 - iv[0], 0, iv[0] + 1);
@@ -1165,39 +1189,18 @@ static int stm32_cryp_read_auth_tag(struct stm32_cryp *cryp)
}
if (is_encrypt(cryp)) {
+ u32 out_tag[AES_BLOCK_32];
+
/* Get and write tag */
- dst = sg_virt(cryp->out_sg) + _walked_out;
+ for (i = 0; i < AES_BLOCK_32; i++)
+ out_tag[i] = stm32_cryp_read(cryp, CRYP_DOUT);
- for (i = 0; i < AES_BLOCK_32; i++) {
- if (cryp->total_out >= sizeof(u32)) {
- /* Read a full u32 */
- *dst = stm32_cryp_read(cryp, CRYP_DOUT);
-
- dst = stm32_cryp_next_out(cryp, dst,
- sizeof(u32));
- cryp->total_out -= sizeof(u32);
- } else if (!cryp->total_out) {
- /* Empty fifo out (data from input padding) */
- stm32_cryp_read(cryp, CRYP_DOUT);
- } else {
- /* Read less than an u32 */
- d32 = stm32_cryp_read(cryp, CRYP_DOUT);
- d8 = (u8 *)&d32;
-
- for (j = 0; j < cryp->total_out; j++) {
- *((u8 *)dst) = *(d8++);
- dst = stm32_cryp_next_out(cryp, dst, 1);
- }
- cryp->total_out = 0;
- }
- }
+ scatterwalk_copychunks(out_tag, &cryp->out_walk, cryp->authsize, 1);
} else {
/* Get and check tag */
u32 in_tag[AES_BLOCK_32], out_tag[AES_BLOCK_32];
- scatterwalk_map_and_copy(in_tag, cryp->in_sg,
- cryp->total_in_save - cryp->authsize,
- cryp->authsize, 0);
+ scatterwalk_copychunks(in_tag, &cryp->in_walk, cryp->authsize, 0);
for (i = 0; i < AES_BLOCK_32; i++)
out_tag[i] = stm32_cryp_read(cryp, CRYP_DOUT);
@@ -1217,115 +1220,59 @@ static void stm32_cryp_check_ctr_counter(struct stm32_cryp *cryp)
{
u32 cr;
- if (unlikely(cryp->last_ctr[3] == 0xFFFFFFFF)) {
- cryp->last_ctr[3] = 0;
- cryp->last_ctr[2]++;
- if (!cryp->last_ctr[2]) {
- cryp->last_ctr[1]++;
- if (!cryp->last_ctr[1])
- cryp->last_ctr[0]++;
- }
+ if (unlikely(cryp->last_ctr[3] == cpu_to_be32(0xFFFFFFFF))) {
+ /*
+ * In this case, we need to increment manually the ctr counter,
+ * as HW doesn't handle the U32 carry.
+ */
+ crypto_inc((u8 *)cryp->last_ctr, sizeof(cryp->last_ctr));
cr = stm32_cryp_read(cryp, CRYP_CR);
stm32_cryp_write(cryp, CRYP_CR, cr & ~CR_CRYPEN);
- stm32_cryp_hw_write_iv(cryp, (__be32 *)cryp->last_ctr);
+ stm32_cryp_hw_write_iv(cryp, cryp->last_ctr);
stm32_cryp_write(cryp, CRYP_CR, cr);
}
- cryp->last_ctr[0] = stm32_cryp_read(cryp, CRYP_IV0LR);
- cryp->last_ctr[1] = stm32_cryp_read(cryp, CRYP_IV0RR);
- cryp->last_ctr[2] = stm32_cryp_read(cryp, CRYP_IV1LR);
- cryp->last_ctr[3] = stm32_cryp_read(cryp, CRYP_IV1RR);
+ /* The IV registers are BE */
+ cryp->last_ctr[0] = cpu_to_be32(stm32_cryp_read(cryp, CRYP_IV0LR));
+ cryp->last_ctr[1] = cpu_to_be32(stm32_cryp_read(cryp, CRYP_IV0RR));
+ cryp->last_ctr[2] = cpu_to_be32(stm32_cryp_read(cryp, CRYP_IV1LR));
+ cryp->last_ctr[3] = cpu_to_be32(stm32_cryp_read(cryp, CRYP_IV1RR));
}
-static bool stm32_cryp_irq_read_data(struct stm32_cryp *cryp)
+static void stm32_cryp_irq_read_data(struct stm32_cryp *cryp)
{
- unsigned int i, j;
- u32 d32, *dst;
- u8 *d8;
- size_t tag_size;
-
- /* Do no read tag now (if any) */
- if (is_encrypt(cryp) && (is_gcm(cryp) || is_ccm(cryp)))
- tag_size = cryp->authsize;
- else
- tag_size = 0;
-
- dst = sg_virt(cryp->out_sg) + _walked_out;
+ unsigned int i;
+ u32 block[AES_BLOCK_32];
- for (i = 0; i < cryp->hw_blocksize / sizeof(u32); i++) {
- if (likely(cryp->total_out - tag_size >= sizeof(u32))) {
- /* Read a full u32 */
- *dst = stm32_cryp_read(cryp, CRYP_DOUT);
+ for (i = 0; i < cryp->hw_blocksize / sizeof(u32); i++)
+ block[i] = stm32_cryp_read(cryp, CRYP_DOUT);
- dst = stm32_cryp_next_out(cryp, dst, sizeof(u32));
- cryp->total_out -= sizeof(u32);
- } else if (cryp->total_out == tag_size) {
- /* Empty fifo out (data from input padding) */
- d32 = stm32_cryp_read(cryp, CRYP_DOUT);
- } else {
- /* Read less than an u32 */
- d32 = stm32_cryp_read(cryp, CRYP_DOUT);
- d8 = (u8 *)&d32;
-
- for (j = 0; j < cryp->total_out - tag_size; j++) {
- *((u8 *)dst) = *(d8++);
- dst = stm32_cryp_next_out(cryp, dst, 1);
- }
- cryp->total_out = tag_size;
- }
- }
-
- return !(cryp->total_out - tag_size) || !cryp->total_in;
+ scatterwalk_copychunks(block, &cryp->out_walk, min_t(size_t, cryp->hw_blocksize,
+ cryp->payload_out), 1);
+ cryp->payload_out -= min_t(size_t, cryp->hw_blocksize,
+ cryp->payload_out);
}
static void stm32_cryp_irq_write_block(struct stm32_cryp *cryp)
{
- unsigned int i, j;
- u32 *src;
- u8 d8[4];
- size_t tag_size;
-
- /* Do no write tag (if any) */
- if (is_decrypt(cryp) && (is_gcm(cryp) || is_ccm(cryp)))
- tag_size = cryp->authsize;
- else
- tag_size = 0;
-
- src = sg_virt(cryp->in_sg) + _walked_in;
+ unsigned int i;
+ u32 block[AES_BLOCK_32] = {0};
- for (i = 0; i < cryp->hw_blocksize / sizeof(u32); i++) {
- if (likely(cryp->total_in - tag_size >= sizeof(u32))) {
- /* Write a full u32 */
- stm32_cryp_write(cryp, CRYP_DIN, *src);
+ scatterwalk_copychunks(block, &cryp->in_walk, min_t(size_t, cryp->hw_blocksize,
+ cryp->payload_in), 0);
+ for (i = 0; i < cryp->hw_blocksize / sizeof(u32); i++)
+ stm32_cryp_write(cryp, CRYP_DIN, block[i]);
- src = stm32_cryp_next_in(cryp, src, sizeof(u32));
- cryp->total_in -= sizeof(u32);
- } else if (cryp->total_in == tag_size) {
- /* Write padding data */
- stm32_cryp_write(cryp, CRYP_DIN, 0);
- } else {
- /* Write less than an u32 */
- memset(d8, 0, sizeof(u32));
- for (j = 0; j < cryp->total_in - tag_size; j++) {
- d8[j] = *((u8 *)src);
- src = stm32_cryp_next_in(cryp, src, 1);
- }
-
- stm32_cryp_write(cryp, CRYP_DIN, *(u32 *)d8);
- cryp->total_in = tag_size;
- }
- }
+ cryp->payload_in -= min_t(size_t, cryp->hw_blocksize, cryp->payload_in);
}
static void stm32_cryp_irq_write_gcm_padded_data(struct stm32_cryp *cryp)
{
int err;
- u32 cfg, tmp[AES_BLOCK_32];
- size_t total_in_ori = cryp->total_in;
- struct scatterlist *out_sg_ori = cryp->out_sg;
+ u32 cfg, block[AES_BLOCK_32] = {0};
unsigned int i;
/* 'Special workaround' procedure described in the datasheet */
@@ -1350,18 +1297,25 @@ static void stm32_cryp_irq_write_gcm_padded_data(struct stm32_cryp *cryp)
/* b) pad and write the last block */
stm32_cryp_irq_write_block(cryp);
- cryp->total_in = total_in_ori;
+ /* wait end of process */
err = stm32_cryp_wait_output(cryp);
if (err) {
- dev_err(cryp->dev, "Timeout (write gcm header)\n");
+ dev_err(cryp->dev, "Timeout (write gcm last data)\n");
return stm32_cryp_finish_req(cryp, err);
}
/* c) get and store encrypted data */
- stm32_cryp_irq_read_data(cryp);
- scatterwalk_map_and_copy(tmp, out_sg_ori,
- cryp->total_in_save - total_in_ori,
- total_in_ori, 0);
+ /*
+ * Same code as stm32_cryp_irq_read_data(), but we want to store
+ * block value
+ */
+ for (i = 0; i < cryp->hw_blocksize / sizeof(u32); i++)
+ block[i] = stm32_cryp_read(cryp, CRYP_DOUT);
+
+ scatterwalk_copychunks(block, &cryp->out_walk, min_t(size_t, cryp->hw_blocksize,
+ cryp->payload_out), 1);
+ cryp->payload_out -= min_t(size_t, cryp->hw_blocksize,
+ cryp->payload_out);
/* d) change mode back to AES GCM */
cfg &= ~CR_ALGO_MASK;
@@ -1374,19 +1328,13 @@ static void stm32_cryp_irq_write_gcm_padded_data(struct stm32_cryp *cryp)
stm32_cryp_write(cryp, CRYP_CR, cfg);
/* f) write padded data */
- for (i = 0; i < AES_BLOCK_32; i++) {
- if (cryp->total_in)
- stm32_cryp_write(cryp, CRYP_DIN, tmp[i]);
- else
- stm32_cryp_write(cryp, CRYP_DIN, 0);
-
- cryp->total_in -= min_t(size_t, sizeof(u32), cryp->total_in);
- }
+ for (i = 0; i < AES_BLOCK_32; i++)
+ stm32_cryp_write(cryp, CRYP_DIN, block[i]);
/* g) Empty fifo out */
err = stm32_cryp_wait_output(cryp);
if (err) {
- dev_err(cryp->dev, "Timeout (write gcm header)\n");
+ dev_err(cryp->dev, "Timeout (write gcm padded data)\n");
return stm32_cryp_finish_req(cryp, err);
}
@@ -1399,16 +1347,14 @@ static void stm32_cryp_irq_write_gcm_padded_data(struct stm32_cryp *cryp)
static void stm32_cryp_irq_set_npblb(struct stm32_cryp *cryp)
{
- u32 cfg, payload_bytes;
+ u32 cfg;
/* disable ip, set NPBLB and reneable ip */
cfg = stm32_cryp_read(cryp, CRYP_CR);
cfg &= ~CR_CRYPEN;
stm32_cryp_write(cryp, CRYP_CR, cfg);
- payload_bytes = is_decrypt(cryp) ? cryp->total_in - cryp->authsize :
- cryp->total_in;
- cfg |= (cryp->hw_blocksize - payload_bytes) << CR_NBPBL_SHIFT;
+ cfg |= (cryp->hw_blocksize - cryp->payload_in) << CR_NBPBL_SHIFT;
cfg |= CR_CRYPEN;
stm32_cryp_write(cryp, CRYP_CR, cfg);
}
@@ -1417,13 +1363,11 @@ static void stm32_cryp_irq_write_ccm_padded_data(struct stm32_cryp *cryp)
{
int err = 0;
u32 cfg, iv1tmp;
- u32 cstmp1[AES_BLOCK_32], cstmp2[AES_BLOCK_32], tmp[AES_BLOCK_32];
- size_t last_total_out, total_in_ori = cryp->total_in;
- struct scatterlist *out_sg_ori = cryp->out_sg;
+ u32 cstmp1[AES_BLOCK_32], cstmp2[AES_BLOCK_32];
+ u32 block[AES_BLOCK_32] = {0};
unsigned int i;
/* 'Special workaround' procedure described in the datasheet */
- cryp->flags |= FLG_CCM_PADDED_WA;
/* a) disable ip */
stm32_cryp_write(cryp, CRYP_IMSCR, 0);
@@ -1453,7 +1397,7 @@ static void stm32_cryp_irq_write_ccm_padded_data(struct stm32_cryp *cryp)
/* b) pad and write the last block */
stm32_cryp_irq_write_block(cryp);
- cryp->total_in = total_in_ori;
+ /* wait end of process */
err = stm32_cryp_wait_output(cryp);
if (err) {
dev_err(cryp->dev, "Timeout (wite ccm padded data)\n");
@@ -1461,13 +1405,16 @@ static void stm32_cryp_irq_write_ccm_padded_data(struct stm32_cryp *cryp)
}
/* c) get and store decrypted data */
- last_total_out = cryp->total_out;
- stm32_cryp_irq_read_data(cryp);
+ /*
+ * Same code as stm32_cryp_irq_read_data(), but we want to store
+ * block value
+ */
+ for (i = 0; i < cryp->hw_blocksize / sizeof(u32); i++)
+ block[i] = stm32_cryp_read(cryp, CRYP_DOUT);
- memset(tmp, 0, sizeof(tmp));
- scatterwalk_map_and_copy(tmp, out_sg_ori,
- cryp->total_out_save - last_total_out,
- last_total_out, 0);
+ scatterwalk_copychunks(block, &cryp->out_walk, min_t(size_t, cryp->hw_blocksize,
+ cryp->payload_out), 1);
+ cryp->payload_out -= min_t(size_t, cryp->hw_blocksize, cryp->payload_out);
/* d) Load again CRYP_CSGCMCCMxR */
for (i = 0; i < ARRAY_SIZE(cstmp2); i++)
@@ -1484,10 +1431,10 @@ static void stm32_cryp_irq_write_ccm_padded_data(struct stm32_cryp *cryp)
stm32_cryp_write(cryp, CRYP_CR, cfg);
/* g) XOR and write padded data */
- for (i = 0; i < ARRAY_SIZE(tmp); i++) {
- tmp[i] ^= cstmp1[i];
- tmp[i] ^= cstmp2[i];
- stm32_cryp_write(cryp, CRYP_DIN, tmp[i]);
+ for (i = 0; i < ARRAY_SIZE(block); i++) {
+ block[i] ^= cstmp1[i];
+ block[i] ^= cstmp2[i];
+ stm32_cryp_write(cryp, CRYP_DIN, block[i]);
}
/* h) wait for completion */
@@ -1501,30 +1448,34 @@ static void stm32_cryp_irq_write_ccm_padded_data(struct stm32_cryp *cryp)
static void stm32_cryp_irq_write_data(struct stm32_cryp *cryp)
{
- if (unlikely(!cryp->total_in)) {
+ if (unlikely(!cryp->payload_in)) {
dev_warn(cryp->dev, "No more data to process\n");
return;
}
- if (unlikely(cryp->total_in < AES_BLOCK_SIZE &&
+ if (unlikely(cryp->payload_in < AES_BLOCK_SIZE &&
(stm32_cryp_get_hw_mode(cryp) == CR_AES_GCM) &&
is_encrypt(cryp))) {
/* Padding for AES GCM encryption */
- if (cryp->caps->padding_wa)
+ if (cryp->caps->padding_wa) {
/* Special case 1 */
- return stm32_cryp_irq_write_gcm_padded_data(cryp);
+ stm32_cryp_irq_write_gcm_padded_data(cryp);
+ return;
+ }
/* Setting padding bytes (NBBLB) */
stm32_cryp_irq_set_npblb(cryp);
}
- if (unlikely((cryp->total_in - cryp->authsize < AES_BLOCK_SIZE) &&
+ if (unlikely((cryp->payload_in < AES_BLOCK_SIZE) &&
(stm32_cryp_get_hw_mode(cryp) == CR_AES_CCM) &&
is_decrypt(cryp))) {
/* Padding for AES CCM decryption */
- if (cryp->caps->padding_wa)
+ if (cryp->caps->padding_wa) {
/* Special case 2 */
- return stm32_cryp_irq_write_ccm_padded_data(cryp);
+ stm32_cryp_irq_write_ccm_padded_data(cryp);
+ return;
+ }
/* Setting padding bytes (NBBLB) */
stm32_cryp_irq_set_npblb(cryp);
@@ -1536,192 +1487,60 @@ static void stm32_cryp_irq_write_data(struct stm32_cryp *cryp)
stm32_cryp_irq_write_block(cryp);
}
-static void stm32_cryp_irq_write_gcm_header(struct stm32_cryp *cryp)
+static void stm32_cryp_irq_write_gcmccm_header(struct stm32_cryp *cryp)
{
- int err;
- unsigned int i, j;
- u32 cfg, *src;
-
- src = sg_virt(cryp->in_sg) + _walked_in;
-
- for (i = 0; i < AES_BLOCK_32; i++) {
- stm32_cryp_write(cryp, CRYP_DIN, *src);
-
- src = stm32_cryp_next_in(cryp, src, sizeof(u32));
- cryp->total_in -= min_t(size_t, sizeof(u32), cryp->total_in);
-
- /* Check if whole header written */
- if ((cryp->total_in_save - cryp->total_in) ==
- cryp->areq->assoclen) {
- /* Write padding if needed */
- for (j = i + 1; j < AES_BLOCK_32; j++)
- stm32_cryp_write(cryp, CRYP_DIN, 0);
-
- /* Wait for completion */
- err = stm32_cryp_wait_busy(cryp);
- if (err) {
- dev_err(cryp->dev, "Timeout (gcm header)\n");
- return stm32_cryp_finish_req(cryp, err);
- }
-
- if (stm32_cryp_get_input_text_len(cryp)) {
- /* Phase 3 : payload */
- cfg = stm32_cryp_read(cryp, CRYP_CR);
- cfg &= ~CR_CRYPEN;
- stm32_cryp_write(cryp, CRYP_CR, cfg);
-
- cfg &= ~CR_PH_MASK;
- cfg |= CR_PH_PAYLOAD;
- cfg |= CR_CRYPEN;
- stm32_cryp_write(cryp, CRYP_CR, cfg);
- } else {
- /* Phase 4 : tag */
- stm32_cryp_write(cryp, CRYP_IMSCR, 0);
- stm32_cryp_finish_req(cryp, 0);
- }
-
- break;
- }
-
- if (!cryp->total_in)
- break;
- }
-}
+ unsigned int i;
+ u32 block[AES_BLOCK_32] = {0};
+ size_t written;
-static void stm32_cryp_irq_write_ccm_header(struct stm32_cryp *cryp)
-{
- int err;
- unsigned int i = 0, j, k;
- u32 alen, cfg, *src;
- u8 d8[4];
-
- src = sg_virt(cryp->in_sg) + _walked_in;
- alen = cryp->areq->assoclen;
-
- if (!_walked_in) {
- if (cryp->areq->assoclen <= 65280) {
- /* Write first u32 of B1 */
- d8[0] = (alen >> 8) & 0xFF;
- d8[1] = alen & 0xFF;
- d8[2] = *((u8 *)src);
- src = stm32_cryp_next_in(cryp, src, 1);
- d8[3] = *((u8 *)src);
- src = stm32_cryp_next_in(cryp, src, 1);
-
- stm32_cryp_write(cryp, CRYP_DIN, *(u32 *)d8);
- i++;
-
- cryp->total_in -= min_t(size_t, 2, cryp->total_in);
- } else {
- /* Build the two first u32 of B1 */
- d8[0] = 0xFF;
- d8[1] = 0xFE;
- d8[2] = alen & 0xFF000000;
- d8[3] = alen & 0x00FF0000;
-
- stm32_cryp_write(cryp, CRYP_DIN, *(u32 *)d8);
- i++;
-
- d8[0] = alen & 0x0000FF00;
- d8[1] = alen & 0x000000FF;
- d8[2] = *((u8 *)src);
- src = stm32_cryp_next_in(cryp, src, 1);
- d8[3] = *((u8 *)src);
- src = stm32_cryp_next_in(cryp, src, 1);
-
- stm32_cryp_write(cryp, CRYP_DIN, *(u32 *)d8);
- i++;
-
- cryp->total_in -= min_t(size_t, 2, cryp->total_in);
- }
- }
+ written = min_t(size_t, AES_BLOCK_SIZE, cryp->header_in);
- /* Write next u32 */
- for (; i < AES_BLOCK_32; i++) {
- /* Build an u32 */
- memset(d8, 0, sizeof(u32));
- for (k = 0; k < sizeof(u32); k++) {
- d8[k] = *((u8 *)src);
- src = stm32_cryp_next_in(cryp, src, 1);
-
- cryp->total_in -= min_t(size_t, 1, cryp->total_in);
- if ((cryp->total_in_save - cryp->total_in) == alen)
- break;
- }
+ scatterwalk_copychunks(block, &cryp->in_walk, written, 0);
+ for (i = 0; i < AES_BLOCK_32; i++)
+ stm32_cryp_write(cryp, CRYP_DIN, block[i]);
- stm32_cryp_write(cryp, CRYP_DIN, *(u32 *)d8);
-
- if ((cryp->total_in_save - cryp->total_in) == alen) {
- /* Write padding if needed */
- for (j = i + 1; j < AES_BLOCK_32; j++)
- stm32_cryp_write(cryp, CRYP_DIN, 0);
-
- /* Wait for completion */
- err = stm32_cryp_wait_busy(cryp);
- if (err) {
- dev_err(cryp->dev, "Timeout (ccm header)\n");
- return stm32_cryp_finish_req(cryp, err);
- }
-
- if (stm32_cryp_get_input_text_len(cryp)) {
- /* Phase 3 : payload */
- cfg = stm32_cryp_read(cryp, CRYP_CR);
- cfg &= ~CR_CRYPEN;
- stm32_cryp_write(cryp, CRYP_CR, cfg);
-
- cfg &= ~CR_PH_MASK;
- cfg |= CR_PH_PAYLOAD;
- cfg |= CR_CRYPEN;
- stm32_cryp_write(cryp, CRYP_CR, cfg);
- } else {
- /* Phase 4 : tag */
- stm32_cryp_write(cryp, CRYP_IMSCR, 0);
- stm32_cryp_finish_req(cryp, 0);
- }
+ cryp->header_in -= written;
- break;
- }
- }
+ stm32_crypt_gcmccm_end_header(cryp);
}
static irqreturn_t stm32_cryp_irq_thread(int irq, void *arg)
{
struct stm32_cryp *cryp = arg;
u32 ph;
+ u32 it_mask = stm32_cryp_read(cryp, CRYP_IMSCR);
if (cryp->irq_status & MISR_OUT)
/* Output FIFO IRQ: read data */
- if (unlikely(stm32_cryp_irq_read_data(cryp))) {
- /* All bytes processed, finish */
- stm32_cryp_write(cryp, CRYP_IMSCR, 0);
- stm32_cryp_finish_req(cryp, 0);
- return IRQ_HANDLED;
- }
+ stm32_cryp_irq_read_data(cryp);
if (cryp->irq_status & MISR_IN) {
- if (is_gcm(cryp)) {
- ph = stm32_cryp_read(cryp, CRYP_CR) & CR_PH_MASK;
- if (unlikely(ph == CR_PH_HEADER))
- /* Write Header */
- stm32_cryp_irq_write_gcm_header(cryp);
- else
- /* Input FIFO IRQ: write data */
- stm32_cryp_irq_write_data(cryp);
- cryp->gcm_ctr++;
- } else if (is_ccm(cryp)) {
+ if (is_gcm(cryp) || is_ccm(cryp)) {
ph = stm32_cryp_read(cryp, CRYP_CR) & CR_PH_MASK;
if (unlikely(ph == CR_PH_HEADER))
/* Write Header */
- stm32_cryp_irq_write_ccm_header(cryp);
+ stm32_cryp_irq_write_gcmccm_header(cryp);
else
/* Input FIFO IRQ: write data */
stm32_cryp_irq_write_data(cryp);
+ if (is_gcm(cryp))
+ cryp->gcm_ctr++;
} else {
/* Input FIFO IRQ: write data */
stm32_cryp_irq_write_data(cryp);
}
}
+ /* Mask useless interrupts */
+ if (!cryp->payload_in && !cryp->header_in)
+ it_mask &= ~IMSCR_IN;
+ if (!cryp->payload_out)
+ it_mask &= ~IMSCR_OUT;
+ stm32_cryp_write(cryp, CRYP_IMSCR, it_mask);
+
+ if (!cryp->payload_in && !cryp->header_in && !cryp->payload_out)
+ stm32_cryp_finish_req(cryp, 0);
+
return IRQ_HANDLED;
}
@@ -1742,7 +1561,7 @@ static struct skcipher_alg crypto_algs[] = {
.base.cra_flags = CRYPTO_ALG_ASYNC,
.base.cra_blocksize = AES_BLOCK_SIZE,
.base.cra_ctxsize = sizeof(struct stm32_cryp_ctx),
- .base.cra_alignmask = 0xf,
+ .base.cra_alignmask = 0,
.base.cra_module = THIS_MODULE,
.init = stm32_cryp_init_tfm,
@@ -1759,7 +1578,7 @@ static struct skcipher_alg crypto_algs[] = {
.base.cra_flags = CRYPTO_ALG_ASYNC,
.base.cra_blocksize = AES_BLOCK_SIZE,
.base.cra_ctxsize = sizeof(struct stm32_cryp_ctx),
- .base.cra_alignmask = 0xf,
+ .base.cra_alignmask = 0,
.base.cra_module = THIS_MODULE,
.init = stm32_cryp_init_tfm,
@@ -1777,7 +1596,7 @@ static struct skcipher_alg crypto_algs[] = {
.base.cra_flags = CRYPTO_ALG_ASYNC,
.base.cra_blocksize = 1,
.base.cra_ctxsize = sizeof(struct stm32_cryp_ctx),
- .base.cra_alignmask = 0xf,
+ .base.cra_alignmask = 0,
.base.cra_module = THIS_MODULE,
.init = stm32_cryp_init_tfm,
@@ -1795,7 +1614,7 @@ static struct skcipher_alg crypto_algs[] = {
.base.cra_flags = CRYPTO_ALG_ASYNC,
.base.cra_blocksize = DES_BLOCK_SIZE,
.base.cra_ctxsize = sizeof(struct stm32_cryp_ctx),
- .base.cra_alignmask = 0xf,
+ .base.cra_alignmask = 0,
.base.cra_module = THIS_MODULE,
.init = stm32_cryp_init_tfm,
@@ -1812,7 +1631,7 @@ static struct skcipher_alg crypto_algs[] = {
.base.cra_flags = CRYPTO_ALG_ASYNC,
.base.cra_blocksize = DES_BLOCK_SIZE,
.base.cra_ctxsize = sizeof(struct stm32_cryp_ctx),
- .base.cra_alignmask = 0xf,
+ .base.cra_alignmask = 0,
.base.cra_module = THIS_MODULE,
.init = stm32_cryp_init_tfm,
@@ -1830,7 +1649,7 @@ static struct skcipher_alg crypto_algs[] = {
.base.cra_flags = CRYPTO_ALG_ASYNC,
.base.cra_blocksize = DES_BLOCK_SIZE,
.base.cra_ctxsize = sizeof(struct stm32_cryp_ctx),
- .base.cra_alignmask = 0xf,
+ .base.cra_alignmask = 0,
.base.cra_module = THIS_MODULE,
.init = stm32_cryp_init_tfm,
@@ -1847,7 +1666,7 @@ static struct skcipher_alg crypto_algs[] = {
.base.cra_flags = CRYPTO_ALG_ASYNC,
.base.cra_blocksize = DES_BLOCK_SIZE,
.base.cra_ctxsize = sizeof(struct stm32_cryp_ctx),
- .base.cra_alignmask = 0xf,
+ .base.cra_alignmask = 0,
.base.cra_module = THIS_MODULE,
.init = stm32_cryp_init_tfm,
@@ -1877,7 +1696,7 @@ static struct aead_alg aead_algs[] = {
.cra_flags = CRYPTO_ALG_ASYNC,
.cra_blocksize = 1,
.cra_ctxsize = sizeof(struct stm32_cryp_ctx),
- .cra_alignmask = 0xf,
+ .cra_alignmask = 0,
.cra_module = THIS_MODULE,
},
},
@@ -1897,7 +1716,7 @@ static struct aead_alg aead_algs[] = {
.cra_flags = CRYPTO_ALG_ASYNC,
.cra_blocksize = 1,
.cra_ctxsize = sizeof(struct stm32_cryp_ctx),
- .cra_alignmask = 0xf,
+ .cra_alignmask = 0,
.cra_module = THIS_MODULE,
},
},
@@ -1955,7 +1774,8 @@ static int stm32_cryp_probe(struct platform_device *pdev)
cryp->clk = devm_clk_get(dev, NULL);
if (IS_ERR(cryp->clk)) {
- dev_err(dev, "Could not get clock\n");
+ dev_err_probe(dev, PTR_ERR(cryp->clk), "Could not get clock\n");
+
return PTR_ERR(cryp->clk);
}
@@ -1973,7 +1793,11 @@ static int stm32_cryp_probe(struct platform_device *pdev)
pm_runtime_enable(dev);
rst = devm_reset_control_get(dev, NULL);
- if (!IS_ERR(rst)) {
+ if (IS_ERR(rst)) {
+ ret = PTR_ERR(rst);
+ if (ret == -EPROBE_DEFER)
+ goto err_rst;
+ } else {
reset_control_assert(rst);
udelay(2);
reset_control_deassert(rst);
@@ -2024,9 +1848,7 @@ err_engine1:
spin_lock(&cryp_list.lock);
list_del(&cryp->list);
spin_unlock(&cryp_list.lock);
-
- pm_runtime_disable(dev);
- pm_runtime_put_noidle(dev);
+err_rst:
pm_runtime_disable(dev);
pm_runtime_put_noidle(dev);
diff --git a/drivers/crypto/stm32/stm32-hash.c b/drivers/crypto/stm32/stm32-hash.c
index 389de9e3302d..d33006d43f76 100644
--- a/drivers/crypto/stm32/stm32-hash.c
+++ b/drivers/crypto/stm32/stm32-hash.c
@@ -813,7 +813,7 @@ static void stm32_hash_finish_req(struct ahash_request *req, int err)
static int stm32_hash_hw_init(struct stm32_hash_dev *hdev,
struct stm32_hash_request_ctx *rctx)
{
- pm_runtime_resume_and_get(hdev->dev);
+ pm_runtime_get_sync(hdev->dev);
if (!(HASH_FLAGS_INIT & hdev->flags)) {
stm32_hash_write(hdev, HASH_CR, HASH_CR_INIT);
@@ -962,7 +962,7 @@ static int stm32_hash_export(struct ahash_request *req, void *out)
u32 *preg;
unsigned int i;
- pm_runtime_resume_and_get(hdev->dev);
+ pm_runtime_get_sync(hdev->dev);
while ((stm32_hash_read(hdev, HASH_SR) & HASH_SR_BUSY))
cpu_relax();
@@ -1000,7 +1000,7 @@ static int stm32_hash_import(struct ahash_request *req, const void *in)
preg = rctx->hw_context;
- pm_runtime_resume_and_get(hdev->dev);
+ pm_runtime_get_sync(hdev->dev);
stm32_hash_write(hdev, HASH_IMR, *preg++);
stm32_hash_write(hdev, HASH_STR, *preg++);
diff --git a/drivers/crypto/ux500/cryp/cryp.h b/drivers/crypto/ux500/cryp/cryp.h
index db5713d7c940..59e1557a620a 100644
--- a/drivers/crypto/ux500/cryp/cryp.h
+++ b/drivers/crypto/ux500/cryp/cryp.h
@@ -224,6 +224,7 @@ struct cryp_dma {
* @phybase: Pointer to physical memory location of the cryp device.
* @dev: Pointer to the devices dev structure.
* @clk: Pointer to the device's clock control.
+ * @irq: IRQ number
* @pwr_regulator: Pointer to the device's power control.
* @power_status: Current status of the power.
* @ctx_lock: Lock for current_ctx.
@@ -239,6 +240,7 @@ struct cryp_device_data {
phys_addr_t phybase;
struct device *dev;
struct clk *clk;
+ int irq;
struct regulator *pwr_regulator;
int power_status;
spinlock_t ctx_lock;
diff --git a/drivers/crypto/ux500/cryp/cryp_core.c b/drivers/crypto/ux500/cryp/cryp_core.c
index 30cdd5253929..97277b7150cb 100644
--- a/drivers/crypto/ux500/cryp/cryp_core.c
+++ b/drivers/crypto/ux500/cryp/cryp_core.c
@@ -1257,7 +1257,6 @@ static int ux500_cryp_probe(struct platform_device *pdev)
{
int ret;
struct resource *res;
- struct resource *res_irq;
struct cryp_device_data *device_data;
struct cryp_protection_config prot = {
.privilege_access = CRYP_STATE_ENABLE
@@ -1341,15 +1340,13 @@ static int ux500_cryp_probe(struct platform_device *pdev)
goto out_power;
}
- res_irq = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
- if (!res_irq) {
- dev_err(dev, "[%s]: IORESOURCE_IRQ unavailable",
- __func__);
- ret = -ENODEV;
+ device_data->irq = platform_get_irq(pdev, 0);
+ if (device_data->irq <= 0) {
+ ret = device_data->irq ? device_data->irq : -ENXIO;
goto out_power;
}
- ret = devm_request_irq(&pdev->dev, res_irq->start,
+ ret = devm_request_irq(&pdev->dev, device_data->irq,
cryp_interrupt_handler, 0, "cryp1", device_data);
if (ret) {
dev_err(dev, "[%s]: Unable to request IRQ", __func__);
@@ -1489,7 +1486,6 @@ static int ux500_cryp_suspend(struct device *dev)
int ret;
struct platform_device *pdev = to_platform_device(dev);
struct cryp_device_data *device_data;
- struct resource *res_irq;
struct cryp_ctx *temp_ctx = NULL;
dev_dbg(dev, "[%s]", __func__);
@@ -1501,11 +1497,7 @@ static int ux500_cryp_suspend(struct device *dev)
return -ENOMEM;
}
- res_irq = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
- if (!res_irq)
- dev_err(dev, "[%s]: IORESOURCE_IRQ, unavailable", __func__);
- else
- disable_irq(res_irq->start);
+ disable_irq(device_data->irq);
spin_lock(&device_data->ctx_lock);
if (!device_data->current_ctx)
@@ -1532,7 +1524,6 @@ static int ux500_cryp_resume(struct device *dev)
int ret = 0;
struct platform_device *pdev = to_platform_device(dev);
struct cryp_device_data *device_data;
- struct resource *res_irq;
struct cryp_ctx *temp_ctx = NULL;
dev_dbg(dev, "[%s]", __func__);
@@ -1556,11 +1547,8 @@ static int ux500_cryp_resume(struct device *dev)
if (ret)
dev_err(dev, "[%s]: cryp_enable_power() failed!", __func__);
- else {
- res_irq = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
- if (res_irq)
- enable_irq(res_irq->start);
- }
+ else
+ enable_irq(device_data->irq);
return ret;
}
diff --git a/include/crypto/aead.h b/include/crypto/aead.h
index 5af914c1ab8e..14db3bee0519 100644
--- a/include/crypto/aead.h
+++ b/include/crypto/aead.h
@@ -8,9 +8,10 @@
#ifndef _CRYPTO_AEAD_H
#define _CRYPTO_AEAD_H
+#include <linux/container_of.h>
#include <linux/crypto.h>
-#include <linux/kernel.h>
#include <linux/slab.h>
+#include <linux/types.h>
/**
* DOC: Authenticated Encryption With Associated Data (AEAD) Cipher API
@@ -73,6 +74,7 @@
*/
struct crypto_aead;
+struct scatterlist;
/**
* struct aead_request - AEAD request
diff --git a/include/crypto/algapi.h b/include/crypto/algapi.h
index 5f6841c73e5a..f76ec723ceae 100644
--- a/include/crypto/algapi.h
+++ b/include/crypto/algapi.h
@@ -7,9 +7,11 @@
#ifndef _CRYPTO_ALGAPI_H
#define _CRYPTO_ALGAPI_H
+#include <linux/align.h>
#include <linux/crypto.h>
+#include <linux/kconfig.h>
#include <linux/list.h>
-#include <linux/kernel.h>
+#include <linux/types.h>
/*
* Maximum values for blocksize and alignmask, used to allocate
@@ -24,6 +26,7 @@
struct crypto_aead;
struct crypto_instance;
struct module;
+struct notifier_block;
struct rtattr;
struct seq_file;
struct sk_buff;
diff --git a/include/crypto/blake2b.h b/include/crypto/blake2b.h
index 18875f16f8ca..0c0176285349 100644
--- a/include/crypto/blake2b.h
+++ b/include/crypto/blake2b.h
@@ -5,7 +5,6 @@
#include <linux/bug.h>
#include <linux/types.h>
-#include <linux/kernel.h>
#include <linux/string.h>
enum blake2b_lengths {
diff --git a/include/crypto/blake2s.h b/include/crypto/blake2s.h
index bc3fb59442ce..df3c6c2f9553 100644
--- a/include/crypto/blake2s.h
+++ b/include/crypto/blake2s.h
@@ -7,8 +7,8 @@
#define _CRYPTO_BLAKE2S_H
#include <linux/bug.h>
+#include <linux/kconfig.h>
#include <linux/types.h>
-#include <linux/kernel.h>
#include <linux/string.h>
enum blake2s_lengths {
diff --git a/include/crypto/cryptd.h b/include/crypto/cryptd.h
index 23169f4d87e6..796d986e58e1 100644
--- a/include/crypto/cryptd.h
+++ b/include/crypto/cryptd.h
@@ -13,7 +13,8 @@
#ifndef _CRYPTO_CRYPT_H
#define _CRYPTO_CRYPT_H
-#include <linux/kernel.h>
+#include <linux/types.h>
+
#include <crypto/aead.h>
#include <crypto/hash.h>
#include <crypto/skcipher.h>
diff --git a/include/crypto/drbg.h b/include/crypto/drbg.h
index c4165126937e..af5ad51d3eef 100644
--- a/include/crypto/drbg.h
+++ b/include/crypto/drbg.h
@@ -105,6 +105,12 @@ struct drbg_test_data {
struct drbg_string *testentropy; /* TEST PARAMETER: test entropy */
};
+enum drbg_seed_state {
+ DRBG_SEED_STATE_UNSEEDED,
+ DRBG_SEED_STATE_PARTIAL, /* Seeded with !rng_is_initialized() */
+ DRBG_SEED_STATE_FULL,
+};
+
struct drbg_state {
struct mutex drbg_mutex; /* lock around DRBG */
unsigned char *V; /* internal state 10.1.1.1 1a) */
@@ -127,16 +133,15 @@ struct drbg_state {
struct crypto_wait ctr_wait; /* CTR mode async wait obj */
struct scatterlist sg_in, sg_out; /* CTR mode SGLs */
- bool seeded; /* DRBG fully seeded? */
+ enum drbg_seed_state seeded; /* DRBG fully seeded? */
+ unsigned long last_seed_time;
bool pr; /* Prediction resistance enabled? */
bool fips_primed; /* Continuous test primed? */
unsigned char *prev; /* FIPS 140-2 continuous test value */
- struct work_struct seed_work; /* asynchronous seeding support */
struct crypto_rng *jent;
const struct drbg_state_ops *d_ops;
const struct drbg_core *core;
struct drbg_string test_data;
- struct random_ready_callback random_ready;
};
static inline __u8 drbg_statelen(struct drbg_state *drbg)
diff --git a/include/crypto/engine.h b/include/crypto/engine.h
index fd4f2fa23f51..ae133e98d813 100644
--- a/include/crypto/engine.h
+++ b/include/crypto/engine.h
@@ -9,8 +9,10 @@
#include <linux/crypto.h>
#include <linux/list.h>
-#include <linux/kernel.h>
#include <linux/kthread.h>
+#include <linux/spinlock.h>
+#include <linux/types.h>
+
#include <crypto/algapi.h>
#include <crypto/aead.h>
#include <crypto/akcipher.h>
@@ -18,6 +20,8 @@
#include <crypto/skcipher.h>
#include <crypto/kpp.h>
+struct device;
+
#define ENGINE_NAME_LEN 30
/*
* struct crypto_engine - crypto hardware engine
diff --git a/include/crypto/internal/kdf_selftest.h b/include/crypto/internal/kdf_selftest.h
new file mode 100644
index 000000000000..4d03d2af57b7
--- /dev/null
+++ b/include/crypto/internal/kdf_selftest.h
@@ -0,0 +1,71 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+/*
+ * Copyright (C) 2021, Stephan Mueller <smueller@chronox.de>
+ */
+
+#ifndef _CRYPTO_KDF_SELFTEST_H
+#define _CRYPTO_KDF_SELFTEST_H
+
+#include <crypto/hash.h>
+#include <linux/uio.h>
+
+struct kdf_testvec {
+ unsigned char *key;
+ size_t keylen;
+ unsigned char *ikm;
+ size_t ikmlen;
+ struct kvec info;
+ unsigned char *expected;
+ size_t expectedlen;
+};
+
+static inline int
+kdf_test(const struct kdf_testvec *test, const char *name,
+ int (*crypto_kdf_setkey)(struct crypto_shash *kmd,
+ const u8 *key, size_t keylen,
+ const u8 *ikm, size_t ikmlen),
+ int (*crypto_kdf_generate)(struct crypto_shash *kmd,
+ const struct kvec *info,
+ unsigned int info_nvec,
+ u8 *dst, unsigned int dlen))
+{
+ struct crypto_shash *kmd;
+ int ret;
+ u8 *buf = kzalloc(test->expectedlen, GFP_KERNEL);
+
+ if (!buf)
+ return -ENOMEM;
+
+ kmd = crypto_alloc_shash(name, 0, 0);
+ if (IS_ERR(kmd)) {
+ pr_err("alg: kdf: could not allocate hash handle for %s\n",
+ name);
+ kfree(buf);
+ return -ENOMEM;
+ }
+
+ ret = crypto_kdf_setkey(kmd, test->key, test->keylen,
+ test->ikm, test->ikmlen);
+ if (ret) {
+ pr_err("alg: kdf: could not set key derivation key\n");
+ goto err;
+ }
+
+ ret = crypto_kdf_generate(kmd, &test->info, 1, buf, test->expectedlen);
+ if (ret) {
+ pr_err("alg: kdf: could not obtain key data\n");
+ goto err;
+ }
+
+ ret = memcmp(test->expected, buf, test->expectedlen);
+ if (ret)
+ ret = -EINVAL;
+
+err:
+ crypto_free_shash(kmd);
+ kfree(buf);
+ return ret;
+}
+
+#endif /* _CRYPTO_KDF_SELFTEST_H */
diff --git a/include/crypto/kdf_sp800108.h b/include/crypto/kdf_sp800108.h
new file mode 100644
index 000000000000..b7b20a778fb7
--- /dev/null
+++ b/include/crypto/kdf_sp800108.h
@@ -0,0 +1,61 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+/*
+ * Copyright (C) 2021, Stephan Mueller <smueller@chronox.de>
+ */
+
+#ifndef _CRYPTO_KDF108_H
+#define _CRYPTO_KDF108_H
+
+#include <crypto/hash.h>
+#include <linux/uio.h>
+
+/**
+ * Counter KDF generate operation according to SP800-108 section 5.1
+ * as well as SP800-56A section 5.8.1 (Single-step KDF).
+ *
+ * @kmd Keyed message digest whose key was set with crypto_kdf108_setkey or
+ * unkeyed message digest
+ * @info optional context and application specific information - this may be
+ * NULL
+ * @info_vec number of optional context/application specific information entries
+ * @dst destination buffer that the caller already allocated
+ * @dlen length of the destination buffer - the KDF derives that amount of
+ * bytes.
+ *
+ * To comply with SP800-108, the caller must provide Label || 0x00 || Context
+ * in the info parameter.
+ *
+ * @return 0 on success, < 0 on error
+ */
+int crypto_kdf108_ctr_generate(struct crypto_shash *kmd,
+ const struct kvec *info, unsigned int info_nvec,
+ u8 *dst, unsigned int dlen);
+
+/**
+ * Counter KDF setkey operation
+ *
+ * @kmd Keyed message digest allocated by the caller. The key should not have
+ * been set.
+ * @key Seed key to be used to initialize the keyed message digest context.
+ * @keylen This length of the key buffer.
+ * @ikm The SP800-108 KDF does not support IKM - this parameter must be NULL
+ * @ikmlen This parameter must be 0.
+ *
+ * According to SP800-108 section 7.2, the seed key must be at least as large as
+ * the message digest size of the used keyed message digest. This limitation
+ * is enforced by the implementation.
+ *
+ * SP800-108 allows the use of either a HMAC or a hash primitive. When
+ * the caller intends to use a hash primitive, the call to
+ * crypto_kdf108_setkey is not required and the key derivation operation can
+ * immediately performed using crypto_kdf108_ctr_generate after allocating
+ * a handle.
+ *
+ * @return 0 on success, < 0 on error
+ */
+int crypto_kdf108_setkey(struct crypto_shash *kmd,
+ const u8 *key, size_t keylen,
+ const u8 *ikm, size_t ikmlen);
+
+#endif /* _CRYPTO_KDF108_H */
diff --git a/include/crypto/pcrypt.h b/include/crypto/pcrypt.h
index b9bc3436196a..234d7cf3cf5e 100644
--- a/include/crypto/pcrypt.h
+++ b/include/crypto/pcrypt.h
@@ -9,8 +9,8 @@
#ifndef _CRYPTO_PCRYPT_H
#define _CRYPTO_PCRYPT_H
+#include <linux/container_of.h>
#include <linux/crypto.h>
-#include <linux/kernel.h>
#include <linux/padata.h>
struct pcrypt_request {
diff --git a/include/crypto/scatterwalk.h b/include/crypto/scatterwalk.h
index 7af08174a721..6407b4b61350 100644
--- a/include/crypto/scatterwalk.h
+++ b/include/crypto/scatterwalk.h
@@ -12,8 +12,9 @@
#define _CRYPTO_SCATTERWALK_H
#include <crypto/algapi.h>
+
#include <linux/highmem.h>
-#include <linux/kernel.h>
+#include <linux/mm.h>
#include <linux/scatterlist.h>
static inline void scatterwalk_crypto_chain(struct scatterlist *head,
diff --git a/include/crypto/skcipher.h b/include/crypto/skcipher.h
index ef0fc9ed4342..39f5b67c3069 100644
--- a/include/crypto/skcipher.h
+++ b/include/crypto/skcipher.h
@@ -8,9 +8,13 @@
#ifndef _CRYPTO_SKCIPHER_H
#define _CRYPTO_SKCIPHER_H
+#include <linux/container_of.h>
#include <linux/crypto.h>
-#include <linux/kernel.h>
#include <linux/slab.h>
+#include <linux/string.h>
+#include <linux/types.h>
+
+struct scatterlist;
/**
* struct skcipher_request - Symmetric key cipher request
diff --git a/include/linux/psp-sev.h b/include/linux/psp-sev.h
index d48a7192e881..1595088c428b 100644
--- a/include/linux/psp-sev.h
+++ b/include/linux/psp-sev.h
@@ -52,6 +52,7 @@ enum sev_cmd {
SEV_CMD_DF_FLUSH = 0x00A,
SEV_CMD_DOWNLOAD_FIRMWARE = 0x00B,
SEV_CMD_GET_ID = 0x00C,
+ SEV_CMD_INIT_EX = 0x00D,
/* Guest commands */
SEV_CMD_DECOMMISSION = 0x020,
@@ -102,6 +103,26 @@ struct sev_data_init {
u32 tmr_len; /* In */
} __packed;
+/**
+ * struct sev_data_init_ex - INIT_EX command parameters
+ *
+ * @length: len of the command buffer read by the PSP
+ * @flags: processing flags
+ * @tmr_address: system physical address used for SEV-ES
+ * @tmr_len: len of tmr_address
+ * @nv_address: system physical address used for PSP NV storage
+ * @nv_len: len of nv_address
+ */
+struct sev_data_init_ex {
+ u32 length; /* In */
+ u32 flags; /* In */
+ u64 tmr_address; /* In */
+ u32 tmr_len; /* In */
+ u32 reserved; /* In */
+ u64 nv_address; /* In/Out */
+ u32 nv_len; /* In */
+} __packed;
+
#define SEV_INIT_FLAGS_SEV_ES 0x01
/**
diff --git a/lib/mpi/mpi-mod.c b/lib/mpi/mpi-mod.c
index 47bc59edd4ff..54fcc01564d9 100644
--- a/lib/mpi/mpi-mod.c
+++ b/lib/mpi/mpi-mod.c
@@ -40,6 +40,8 @@ mpi_barrett_t mpi_barrett_init(MPI m, int copy)
mpi_normalize(m);
ctx = kcalloc(1, sizeof(*ctx), GFP_KERNEL);
+ if (!ctx)
+ return NULL;
if (copy) {
ctx->m = mpi_copy(m);
diff --git a/security/keys/Kconfig b/security/keys/Kconfig
index 64b81abd087e..969122c7b92f 100644
--- a/security/keys/Kconfig
+++ b/security/keys/Kconfig
@@ -109,7 +109,7 @@ config KEY_DH_OPERATIONS
bool "Diffie-Hellman operations on retained keys"
depends on KEYS
select CRYPTO
- select CRYPTO_HASH
+ select CRYPTO_KDF800108_CTR
select CRYPTO_DH
help
This option provides support for calculating Diffie-Hellman
diff --git a/security/keys/dh.c b/security/keys/dh.c
index 1abfa70ed6e1..4573fc15617d 100644
--- a/security/keys/dh.c
+++ b/security/keys/dh.c
@@ -11,6 +11,7 @@
#include <crypto/hash.h>
#include <crypto/kpp.h>
#include <crypto/dh.h>
+#include <crypto/kdf_sp800108.h>
#include <keys/user-type.h>
#include "internal.h"
@@ -79,17 +80,9 @@ static void dh_crypto_done(struct crypto_async_request *req, int err)
complete(&compl->completion);
}
-struct kdf_sdesc {
- struct shash_desc shash;
- char ctx[];
-};
-
-static int kdf_alloc(struct kdf_sdesc **sdesc_ret, char *hashname)
+static int kdf_alloc(struct crypto_shash **hash, char *hashname)
{
struct crypto_shash *tfm;
- struct kdf_sdesc *sdesc;
- int size;
- int err;
/* allocate synchronous hash */
tfm = crypto_alloc_shash(hashname, 0, 0);
@@ -98,112 +91,30 @@ static int kdf_alloc(struct kdf_sdesc **sdesc_ret, char *hashname)
return PTR_ERR(tfm);
}
- err = -EINVAL;
- if (crypto_shash_digestsize(tfm) == 0)
- goto out_free_tfm;
-
- err = -ENOMEM;
- size = sizeof(struct shash_desc) + crypto_shash_descsize(tfm);
- sdesc = kmalloc(size, GFP_KERNEL);
- if (!sdesc)
- goto out_free_tfm;
- sdesc->shash.tfm = tfm;
+ if (crypto_shash_digestsize(tfm) == 0) {
+ crypto_free_shash(tfm);
+ return -EINVAL;
+ }
- *sdesc_ret = sdesc;
+ *hash = tfm;
return 0;
-
-out_free_tfm:
- crypto_free_shash(tfm);
- return err;
}
-static void kdf_dealloc(struct kdf_sdesc *sdesc)
+static void kdf_dealloc(struct crypto_shash *hash)
{
- if (!sdesc)
- return;
-
- if (sdesc->shash.tfm)
- crypto_free_shash(sdesc->shash.tfm);
-
- kfree_sensitive(sdesc);
-}
-
-/*
- * Implementation of the KDF in counter mode according to SP800-108 section 5.1
- * as well as SP800-56A section 5.8.1 (Single-step KDF).
- *
- * SP800-56A:
- * The src pointer is defined as Z || other info where Z is the shared secret
- * from DH and other info is an arbitrary string (see SP800-56A section
- * 5.8.1.2).
- *
- * 'dlen' must be a multiple of the digest size.
- */
-static int kdf_ctr(struct kdf_sdesc *sdesc, const u8 *src, unsigned int slen,
- u8 *dst, unsigned int dlen, unsigned int zlen)
-{
- struct shash_desc *desc = &sdesc->shash;
- unsigned int h = crypto_shash_digestsize(desc->tfm);
- int err = 0;
- u8 *dst_orig = dst;
- __be32 counter = cpu_to_be32(1);
-
- while (dlen) {
- err = crypto_shash_init(desc);
- if (err)
- goto err;
-
- err = crypto_shash_update(desc, (u8 *)&counter, sizeof(__be32));
- if (err)
- goto err;
-
- if (zlen && h) {
- u8 tmpbuffer[32];
- size_t chunk = min_t(size_t, zlen, sizeof(tmpbuffer));
- memset(tmpbuffer, 0, chunk);
-
- do {
- err = crypto_shash_update(desc, tmpbuffer,
- chunk);
- if (err)
- goto err;
-
- zlen -= chunk;
- chunk = min_t(size_t, zlen, sizeof(tmpbuffer));
- } while (zlen);
- }
-
- if (src && slen) {
- err = crypto_shash_update(desc, src, slen);
- if (err)
- goto err;
- }
-
- err = crypto_shash_final(desc, dst);
- if (err)
- goto err;
-
- dlen -= h;
- dst += h;
- counter = cpu_to_be32(be32_to_cpu(counter) + 1);
- }
-
- return 0;
-
-err:
- memzero_explicit(dst_orig, dlen);
- return err;
+ if (hash)
+ crypto_free_shash(hash);
}
-static int keyctl_dh_compute_kdf(struct kdf_sdesc *sdesc,
+static int keyctl_dh_compute_kdf(struct crypto_shash *hash,
char __user *buffer, size_t buflen,
- uint8_t *kbuf, size_t kbuflen, size_t lzero)
+ uint8_t *kbuf, size_t kbuflen)
{
+ struct kvec kbuf_iov = { .iov_base = kbuf, .iov_len = kbuflen };
uint8_t *outbuf = NULL;
int ret;
- size_t outbuf_len = roundup(buflen,
- crypto_shash_digestsize(sdesc->shash.tfm));
+ size_t outbuf_len = roundup(buflen, crypto_shash_digestsize(hash));
outbuf = kmalloc(outbuf_len, GFP_KERNEL);
if (!outbuf) {
@@ -211,7 +122,7 @@ static int keyctl_dh_compute_kdf(struct kdf_sdesc *sdesc,
goto err;
}
- ret = kdf_ctr(sdesc, kbuf, kbuflen, outbuf, outbuf_len, lzero);
+ ret = crypto_kdf108_ctr_generate(hash, &kbuf_iov, 1, outbuf, outbuf_len);
if (ret)
goto err;
@@ -240,7 +151,7 @@ long __keyctl_dh_compute(struct keyctl_dh_params __user *params,
struct kpp_request *req;
uint8_t *secret;
uint8_t *outbuf;
- struct kdf_sdesc *sdesc = NULL;
+ struct crypto_shash *hash = NULL;
if (!params || (!buffer && buflen)) {
ret = -EINVAL;
@@ -273,7 +184,7 @@ long __keyctl_dh_compute(struct keyctl_dh_params __user *params,
}
/* allocate KDF from the kernel crypto API */
- ret = kdf_alloc(&sdesc, hashname);
+ ret = kdf_alloc(&hash, hashname);
kfree(hashname);
if (ret)
goto out1;
@@ -383,9 +294,8 @@ long __keyctl_dh_compute(struct keyctl_dh_params __user *params,
goto out6;
}
- ret = keyctl_dh_compute_kdf(sdesc, buffer, buflen, outbuf,
- req->dst_len + kdfcopy->otherinfolen,
- outlen - req->dst_len);
+ ret = keyctl_dh_compute_kdf(hash, buffer, buflen, outbuf,
+ req->dst_len + kdfcopy->otherinfolen);
} else if (copy_to_user(buffer, outbuf, req->dst_len) == 0) {
ret = req->dst_len;
} else {
@@ -403,7 +313,7 @@ out3:
out2:
dh_free_data(&dh_inputs);
out1:
- kdf_dealloc(sdesc);
+ kdf_dealloc(hash);
return ret;
}