From fbf31dd599875cb132d764cf4d05d7985e332c05 Mon Sep 17 00:00:00 2001 From: Tom Zanussi Date: Tue, 28 Mar 2023 10:39:49 -0500 Subject: crypto: keembay - Move driver to drivers/crypto/intel/keembay With the growing number of Intel crypto drivers, it makes sense to group them all into a single drivers/crypto/intel/ directory. Signed-off-by: Tom Zanussi Acked-by: Daniele Alessandrelli Signed-off-by: Herbert Xu --- MAINTAINERS | 26 +- drivers/crypto/Kconfig | 2 +- drivers/crypto/Makefile | 2 +- drivers/crypto/intel/Kconfig | 3 + drivers/crypto/intel/Makefile | 3 + drivers/crypto/intel/keembay/Kconfig | 90 ++ drivers/crypto/intel/keembay/Makefile | 10 + .../crypto/intel/keembay/keembay-ocs-aes-core.c | 1704 ++++++++++++++++++++ drivers/crypto/intel/keembay/keembay-ocs-ecc.c | 1016 ++++++++++++ .../crypto/intel/keembay/keembay-ocs-hcu-core.c | 1264 +++++++++++++++ drivers/crypto/intel/keembay/ocs-aes.c | 1489 +++++++++++++++++ drivers/crypto/intel/keembay/ocs-aes.h | 129 ++ drivers/crypto/intel/keembay/ocs-hcu.c | 840 ++++++++++ drivers/crypto/intel/keembay/ocs-hcu.h | 106 ++ drivers/crypto/keembay/Kconfig | 90 -- drivers/crypto/keembay/Makefile | 10 - drivers/crypto/keembay/keembay-ocs-aes-core.c | 1704 -------------------- drivers/crypto/keembay/keembay-ocs-ecc.c | 1016 ------------ drivers/crypto/keembay/keembay-ocs-hcu-core.c | 1264 --------------- drivers/crypto/keembay/ocs-aes.c | 1489 ----------------- drivers/crypto/keembay/ocs-aes.h | 129 -- drivers/crypto/keembay/ocs-hcu.c | 840 ---------- drivers/crypto/keembay/ocs-hcu.h | 106 -- 23 files changed, 6669 insertions(+), 6663 deletions(-) create mode 100644 drivers/crypto/intel/Kconfig create mode 100644 drivers/crypto/intel/Makefile create mode 100644 drivers/crypto/intel/keembay/Kconfig create mode 100644 drivers/crypto/intel/keembay/Makefile create mode 100644 drivers/crypto/intel/keembay/keembay-ocs-aes-core.c create mode 100644 drivers/crypto/intel/keembay/keembay-ocs-ecc.c create mode 100644 drivers/crypto/intel/keembay/keembay-ocs-hcu-core.c create mode 100644 drivers/crypto/intel/keembay/ocs-aes.c create mode 100644 drivers/crypto/intel/keembay/ocs-aes.h create mode 100644 drivers/crypto/intel/keembay/ocs-hcu.c create mode 100644 drivers/crypto/intel/keembay/ocs-hcu.h delete mode 100644 drivers/crypto/keembay/Kconfig delete mode 100644 drivers/crypto/keembay/Makefile delete mode 100644 drivers/crypto/keembay/keembay-ocs-aes-core.c delete mode 100644 drivers/crypto/keembay/keembay-ocs-ecc.c delete mode 100644 drivers/crypto/keembay/keembay-ocs-hcu-core.c delete mode 100644 drivers/crypto/keembay/ocs-aes.c delete mode 100644 drivers/crypto/keembay/ocs-aes.h delete mode 100644 drivers/crypto/keembay/ocs-hcu.c delete mode 100644 drivers/crypto/keembay/ocs-hcu.h diff --git a/MAINTAINERS b/MAINTAINERS index 77e8dc3a5cf6..b21027122ce8 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -10415,11 +10415,11 @@ INTEL KEEM BAY OCS AES/SM4 CRYPTO DRIVER M: Daniele Alessandrelli S: Maintained F: Documentation/devicetree/bindings/crypto/intel,keembay-ocs-aes.yaml -F: drivers/crypto/keembay/Kconfig -F: drivers/crypto/keembay/Makefile -F: drivers/crypto/keembay/keembay-ocs-aes-core.c -F: drivers/crypto/keembay/ocs-aes.c -F: drivers/crypto/keembay/ocs-aes.h +F: drivers/crypto/intel/keembay/Kconfig +F: drivers/crypto/intel/keembay/Makefile +F: drivers/crypto/intel/keembay/keembay-ocs-aes-core.c +F: drivers/crypto/intel/keembay/ocs-aes.c +F: drivers/crypto/intel/keembay/ocs-aes.h INTEL KEEM BAY OCS ECC CRYPTO DRIVER M: Daniele Alessandrelli @@ -10427,20 +10427,20 @@ M: Prabhjot Khurana M: Mark Gross S: Maintained F: Documentation/devicetree/bindings/crypto/intel,keembay-ocs-ecc.yaml -F: drivers/crypto/keembay/Kconfig -F: drivers/crypto/keembay/Makefile -F: drivers/crypto/keembay/keembay-ocs-ecc.c +F: drivers/crypto/intel/keembay/Kconfig +F: drivers/crypto/intel/keembay/Makefile +F: drivers/crypto/intel/keembay/keembay-ocs-ecc.c INTEL KEEM BAY OCS HCU CRYPTO DRIVER M: Daniele Alessandrelli M: Declan Murphy S: Maintained F: Documentation/devicetree/bindings/crypto/intel,keembay-ocs-hcu.yaml -F: drivers/crypto/keembay/Kconfig -F: drivers/crypto/keembay/Makefile -F: drivers/crypto/keembay/keembay-ocs-hcu-core.c -F: drivers/crypto/keembay/ocs-hcu.c -F: drivers/crypto/keembay/ocs-hcu.h +F: drivers/crypto/intel/keembay/Kconfig +F: drivers/crypto/intel/keembay/Makefile +F: drivers/crypto/intel/keembay/keembay-ocs-hcu-core.c +F: drivers/crypto/intel/keembay/ocs-hcu.c +F: drivers/crypto/intel/keembay/ocs-hcu.h INTEL THUNDER BAY EMMC PHY DRIVER M: Nandhini Srikandan diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig index 1571563e4ab9..fc00d31abd3c 100644 --- a/drivers/crypto/Kconfig +++ b/drivers/crypto/Kconfig @@ -506,6 +506,7 @@ source "drivers/crypto/qat/Kconfig" source "drivers/crypto/cavium/cpt/Kconfig" source "drivers/crypto/cavium/nitrox/Kconfig" source "drivers/crypto/marvell/Kconfig" +source "drivers/crypto/intel/Kconfig" config CRYPTO_DEV_CAVIUM_ZIP tristate "Cavium ZIP driver" @@ -821,7 +822,6 @@ config CRYPTO_DEV_SA2UL used for crypto offload. Select this if you want to use hardware acceleration for cryptographic algorithms on these devices. -source "drivers/crypto/keembay/Kconfig" source "drivers/crypto/aspeed/Kconfig" endif # CRYPTO_HW diff --git a/drivers/crypto/Makefile b/drivers/crypto/Makefile index 476f1a25ca32..eb8eb80aab3b 100644 --- a/drivers/crypto/Makefile +++ b/drivers/crypto/Makefile @@ -51,4 +51,4 @@ obj-$(CONFIG_CRYPTO_DEV_ARTPEC6) += axis/ obj-y += xilinx/ obj-y += hisilicon/ obj-$(CONFIG_CRYPTO_DEV_AMLOGIC_GXL) += amlogic/ -obj-y += keembay/ +obj-y += intel/ diff --git a/drivers/crypto/intel/Kconfig b/drivers/crypto/intel/Kconfig new file mode 100644 index 000000000000..b5ae1aa4a701 --- /dev/null +++ b/drivers/crypto/intel/Kconfig @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: GPL-2.0 + +source "drivers/crypto/intel/keembay/Kconfig" diff --git a/drivers/crypto/intel/Makefile b/drivers/crypto/intel/Makefile new file mode 100644 index 000000000000..c392189fd994 --- /dev/null +++ b/drivers/crypto/intel/Makefile @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: GPL-2.0 + +obj-y += keembay/ diff --git a/drivers/crypto/intel/keembay/Kconfig b/drivers/crypto/intel/keembay/Kconfig new file mode 100644 index 000000000000..1cd62f9c3e3a --- /dev/null +++ b/drivers/crypto/intel/keembay/Kconfig @@ -0,0 +1,90 @@ +config CRYPTO_DEV_KEEMBAY_OCS_AES_SM4 + tristate "Support for Intel Keem Bay OCS AES/SM4 HW acceleration" + depends on HAS_IOMEM + depends on ARCH_KEEMBAY || COMPILE_TEST + select CRYPTO_SKCIPHER + select CRYPTO_AEAD + select CRYPTO_ENGINE + help + Support for Intel Keem Bay Offload and Crypto Subsystem (OCS) AES and + SM4 cipher hardware acceleration for use with Crypto API. + + Provides HW acceleration for the following transformations: + cbc(aes), ctr(aes), ccm(aes), gcm(aes), cbc(sm4), ctr(sm4), ccm(sm4) + and gcm(sm4). + + Optionally, support for the following transformations can also be + enabled: ecb(aes), cts(cbc(aes)), ecb(sm4) and cts(cbc(sm4)). + +config CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB + bool "Support for Intel Keem Bay OCS AES/SM4 ECB HW acceleration" + depends on CRYPTO_DEV_KEEMBAY_OCS_AES_SM4 + help + Support for Intel Keem Bay Offload and Crypto Subsystem (OCS) + AES/SM4 ECB mode hardware acceleration for use with Crypto API. + + Provides OCS version of ecb(aes) and ecb(sm4) + + Intel does not recommend use of ECB mode with AES/SM4. + +config CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS + bool "Support for Intel Keem Bay OCS AES/SM4 CTS HW acceleration" + depends on CRYPTO_DEV_KEEMBAY_OCS_AES_SM4 + help + Support for Intel Keem Bay Offload and Crypto Subsystem (OCS) + AES/SM4 CBC with CTS mode hardware acceleration for use with + Crypto API. + + Provides OCS version of cts(cbc(aes)) and cts(cbc(sm4)). + + Intel does not recommend use of CTS mode with AES/SM4. + +config CRYPTO_DEV_KEEMBAY_OCS_ECC + tristate "Support for Intel Keem Bay OCS ECC HW acceleration" + depends on ARCH_KEEMBAY || COMPILE_TEST + depends on OF + depends on HAS_IOMEM + select CRYPTO_ECDH + select CRYPTO_ENGINE + help + Support for Intel Keem Bay Offload and Crypto Subsystem (OCS) + Elliptic Curve Cryptography (ECC) hardware acceleration for use with + Crypto API. + + Provides OCS acceleration for ECDH-256 and ECDH-384. + + Say Y or M if you are compiling for the Intel Keem Bay SoC. The + module will be called keembay-ocs-ecc. + + If unsure, say N. + +config CRYPTO_DEV_KEEMBAY_OCS_HCU + tristate "Support for Intel Keem Bay OCS HCU HW acceleration" + select CRYPTO_HASH + select CRYPTO_ENGINE + depends on HAS_IOMEM + depends on ARCH_KEEMBAY || COMPILE_TEST + depends on OF + help + Support for Intel Keem Bay Offload and Crypto Subsystem (OCS) Hash + Control Unit (HCU) hardware acceleration for use with Crypto API. + + Provides OCS HCU hardware acceleration of sha256, sha384, sha512, and + sm3, as well as the HMAC variant of these algorithms. + + Say Y or M if you're building for the Intel Keem Bay SoC. If compiled + as a module, the module will be called keembay-ocs-hcu. + + If unsure, say N. + +config CRYPTO_DEV_KEEMBAY_OCS_HCU_HMAC_SHA224 + bool "Enable sha224 and hmac(sha224) support in Intel Keem Bay OCS HCU" + depends on CRYPTO_DEV_KEEMBAY_OCS_HCU + help + Enables support for sha224 and hmac(sha224) algorithms in the Intel + Keem Bay OCS HCU driver. Intel recommends not to use these + algorithms. + + Provides OCS HCU hardware acceleration of sha224 and hmac(224). + + If unsure, say N. diff --git a/drivers/crypto/intel/keembay/Makefile b/drivers/crypto/intel/keembay/Makefile new file mode 100644 index 000000000000..7c12c3c138bd --- /dev/null +++ b/drivers/crypto/intel/keembay/Makefile @@ -0,0 +1,10 @@ +# +# Makefile for Intel Keem Bay OCS Crypto API Linux drivers +# +obj-$(CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4) += keembay-ocs-aes.o +keembay-ocs-aes-objs := keembay-ocs-aes-core.o ocs-aes.o + +obj-$(CONFIG_CRYPTO_DEV_KEEMBAY_OCS_ECC) += keembay-ocs-ecc.o + +obj-$(CONFIG_CRYPTO_DEV_KEEMBAY_OCS_HCU) += keembay-ocs-hcu.o +keembay-ocs-hcu-objs := keembay-ocs-hcu-core.o ocs-hcu.o diff --git a/drivers/crypto/intel/keembay/keembay-ocs-aes-core.c b/drivers/crypto/intel/keembay/keembay-ocs-aes-core.c new file mode 100644 index 000000000000..ae31be00357a --- /dev/null +++ b/drivers/crypto/intel/keembay/keembay-ocs-aes-core.c @@ -0,0 +1,1704 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Intel Keem Bay OCS AES Crypto Driver. + * + * Copyright (C) 2018-2020 Intel Corporation + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include + +#include "ocs-aes.h" + +#define KMB_OCS_PRIORITY 350 +#define DRV_NAME "keembay-ocs-aes" + +#define OCS_AES_MIN_KEY_SIZE 16 +#define OCS_AES_MAX_KEY_SIZE 32 +#define OCS_AES_KEYSIZE_128 16 +#define OCS_AES_KEYSIZE_192 24 +#define OCS_AES_KEYSIZE_256 32 +#define OCS_SM4_KEY_SIZE 16 + +/** + * struct ocs_aes_tctx - OCS AES Transform context + * @engine_ctx: Engine context. + * @aes_dev: The OCS AES device. + * @key: AES/SM4 key. + * @key_len: The length (in bytes) of @key. + * @cipher: OCS cipher to use (either AES or SM4). + * @sw_cipher: The cipher to use as fallback. + * @use_fallback: Whether or not fallback cipher should be used. + */ +struct ocs_aes_tctx { + struct crypto_engine_ctx engine_ctx; + struct ocs_aes_dev *aes_dev; + u8 key[OCS_AES_KEYSIZE_256]; + unsigned int key_len; + enum ocs_cipher cipher; + union { + struct crypto_sync_skcipher *sk; + struct crypto_aead *aead; + } sw_cipher; + bool use_fallback; +}; + +/** + * struct ocs_aes_rctx - OCS AES Request context. + * @instruction: Instruction to be executed (encrypt / decrypt). + * @mode: Mode to use (ECB, CBC, CTR, CCm, GCM, CTS) + * @src_nents: Number of source SG entries. + * @dst_nents: Number of destination SG entries. + * @src_dma_count: The number of DMA-mapped entries of the source SG. + * @dst_dma_count: The number of DMA-mapped entries of the destination SG. + * @in_place: Whether or not this is an in place request, i.e., + * src_sg == dst_sg. + * @src_dll: OCS DMA linked list for input data. + * @dst_dll: OCS DMA linked list for output data. + * @last_ct_blk: Buffer to hold last cipher text block (only used in CBC + * mode). + * @cts_swap: Whether or not CTS swap must be performed. + * @aad_src_dll: OCS DMA linked list for input AAD data. + * @aad_dst_dll: OCS DMA linked list for output AAD data. + * @in_tag: Buffer to hold input encrypted tag (only used for + * CCM/GCM decrypt). + * @out_tag: Buffer to hold output encrypted / decrypted tag (only + * used for GCM encrypt / decrypt). + */ +struct ocs_aes_rctx { + /* Fields common across all modes. */ + enum ocs_instruction instruction; + enum ocs_mode mode; + int src_nents; + int dst_nents; + int src_dma_count; + int dst_dma_count; + bool in_place; + struct ocs_dll_desc src_dll; + struct ocs_dll_desc dst_dll; + + /* CBC specific */ + u8 last_ct_blk[AES_BLOCK_SIZE]; + + /* CTS specific */ + int cts_swap; + + /* CCM/GCM specific */ + struct ocs_dll_desc aad_src_dll; + struct ocs_dll_desc aad_dst_dll; + u8 in_tag[AES_BLOCK_SIZE]; + + /* GCM specific */ + u8 out_tag[AES_BLOCK_SIZE]; +}; + +/* Driver data. */ +struct ocs_aes_drv { + struct list_head dev_list; + spinlock_t lock; /* Protects dev_list. */ +}; + +static struct ocs_aes_drv ocs_aes = { + .dev_list = LIST_HEAD_INIT(ocs_aes.dev_list), + .lock = __SPIN_LOCK_UNLOCKED(ocs_aes.lock), +}; + +static struct ocs_aes_dev *kmb_ocs_aes_find_dev(struct ocs_aes_tctx *tctx) +{ + struct ocs_aes_dev *aes_dev; + + spin_lock(&ocs_aes.lock); + + if (tctx->aes_dev) { + aes_dev = tctx->aes_dev; + goto exit; + } + + /* Only a single OCS device available */ + aes_dev = list_first_entry(&ocs_aes.dev_list, struct ocs_aes_dev, list); + tctx->aes_dev = aes_dev; + +exit: + spin_unlock(&ocs_aes.lock); + + return aes_dev; +} + +/* + * Ensure key is 128-bit or 256-bit for AES or 128-bit for SM4 and an actual + * key is being passed in. + * + * Return: 0 if key is valid, -EINVAL otherwise. + */ +static int check_key(const u8 *in_key, size_t key_len, enum ocs_cipher cipher) +{ + if (!in_key) + return -EINVAL; + + /* For AES, only 128-byte or 256-byte keys are supported. */ + if (cipher == OCS_AES && (key_len == OCS_AES_KEYSIZE_128 || + key_len == OCS_AES_KEYSIZE_256)) + return 0; + + /* For SM4, only 128-byte keys are supported. */ + if (cipher == OCS_SM4 && key_len == OCS_AES_KEYSIZE_128) + return 0; + + /* Everything else is unsupported. */ + return -EINVAL; +} + +/* Save key into transformation context. */ +static int save_key(struct ocs_aes_tctx *tctx, const u8 *in_key, size_t key_len, + enum ocs_cipher cipher) +{ + int ret; + + ret = check_key(in_key, key_len, cipher); + if (ret) + return ret; + + memcpy(tctx->key, in_key, key_len); + tctx->key_len = key_len; + tctx->cipher = cipher; + + return 0; +} + +/* Set key for symmetric cypher. */ +static int kmb_ocs_sk_set_key(struct crypto_skcipher *tfm, const u8 *in_key, + size_t key_len, enum ocs_cipher cipher) +{ + struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm); + + /* Fallback is used for AES with 192-bit key. */ + tctx->use_fallback = (cipher == OCS_AES && + key_len == OCS_AES_KEYSIZE_192); + + if (!tctx->use_fallback) + return save_key(tctx, in_key, key_len, cipher); + + crypto_sync_skcipher_clear_flags(tctx->sw_cipher.sk, + CRYPTO_TFM_REQ_MASK); + crypto_sync_skcipher_set_flags(tctx->sw_cipher.sk, + tfm->base.crt_flags & + CRYPTO_TFM_REQ_MASK); + + return crypto_sync_skcipher_setkey(tctx->sw_cipher.sk, in_key, key_len); +} + +/* Set key for AEAD cipher. */ +static int kmb_ocs_aead_set_key(struct crypto_aead *tfm, const u8 *in_key, + size_t key_len, enum ocs_cipher cipher) +{ + struct ocs_aes_tctx *tctx = crypto_aead_ctx(tfm); + + /* Fallback is used for AES with 192-bit key. */ + tctx->use_fallback = (cipher == OCS_AES && + key_len == OCS_AES_KEYSIZE_192); + + if (!tctx->use_fallback) + return save_key(tctx, in_key, key_len, cipher); + + crypto_aead_clear_flags(tctx->sw_cipher.aead, CRYPTO_TFM_REQ_MASK); + crypto_aead_set_flags(tctx->sw_cipher.aead, + crypto_aead_get_flags(tfm) & CRYPTO_TFM_REQ_MASK); + + return crypto_aead_setkey(tctx->sw_cipher.aead, in_key, key_len); +} + +/* Swap two AES blocks in SG lists. */ +static void sg_swap_blocks(struct scatterlist *sgl, unsigned int nents, + off_t blk1_offset, off_t blk2_offset) +{ + u8 tmp_buf1[AES_BLOCK_SIZE], tmp_buf2[AES_BLOCK_SIZE]; + + /* + * No easy way to copy within sg list, so copy both blocks to temporary + * buffers first. + */ + sg_pcopy_to_buffer(sgl, nents, tmp_buf1, AES_BLOCK_SIZE, blk1_offset); + sg_pcopy_to_buffer(sgl, nents, tmp_buf2, AES_BLOCK_SIZE, blk2_offset); + sg_pcopy_from_buffer(sgl, nents, tmp_buf1, AES_BLOCK_SIZE, blk2_offset); + sg_pcopy_from_buffer(sgl, nents, tmp_buf2, AES_BLOCK_SIZE, blk1_offset); +} + +/* Initialize request context to default values. */ +static void ocs_aes_init_rctx(struct ocs_aes_rctx *rctx) +{ + /* Zero everything. */ + memset(rctx, 0, sizeof(*rctx)); + + /* Set initial value for DMA addresses. */ + rctx->src_dll.dma_addr = DMA_MAPPING_ERROR; + rctx->dst_dll.dma_addr = DMA_MAPPING_ERROR; + rctx->aad_src_dll.dma_addr = DMA_MAPPING_ERROR; + rctx->aad_dst_dll.dma_addr = DMA_MAPPING_ERROR; +} + +static int kmb_ocs_sk_validate_input(struct skcipher_request *req, + enum ocs_mode mode) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + int iv_size = crypto_skcipher_ivsize(tfm); + + switch (mode) { + case OCS_MODE_ECB: + /* Ensure input length is multiple of block size */ + if (req->cryptlen % AES_BLOCK_SIZE != 0) + return -EINVAL; + + return 0; + + case OCS_MODE_CBC: + /* Ensure input length is multiple of block size */ + if (req->cryptlen % AES_BLOCK_SIZE != 0) + return -EINVAL; + + /* Ensure IV is present and block size in length */ + if (!req->iv || iv_size != AES_BLOCK_SIZE) + return -EINVAL; + /* + * NOTE: Since req->cryptlen == 0 case was already handled in + * kmb_ocs_sk_common(), the above two conditions also guarantee + * that: cryptlen >= iv_size + */ + return 0; + + case OCS_MODE_CTR: + /* Ensure IV is present and block size in length */ + if (!req->iv || iv_size != AES_BLOCK_SIZE) + return -EINVAL; + return 0; + + case OCS_MODE_CTS: + /* Ensure input length >= block size */ + if (req->cryptlen < AES_BLOCK_SIZE) + return -EINVAL; + + /* Ensure IV is present and block size in length */ + if (!req->iv || iv_size != AES_BLOCK_SIZE) + return -EINVAL; + + return 0; + default: + return -EINVAL; + } +} + +/* + * Called by encrypt() / decrypt() skcipher functions. + * + * Use fallback if needed, otherwise initialize context and enqueue request + * into engine. + */ +static int kmb_ocs_sk_common(struct skcipher_request *req, + enum ocs_cipher cipher, + enum ocs_instruction instruction, + enum ocs_mode mode) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct ocs_aes_rctx *rctx = skcipher_request_ctx(req); + struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm); + struct ocs_aes_dev *aes_dev; + int rc; + + if (tctx->use_fallback) { + SYNC_SKCIPHER_REQUEST_ON_STACK(subreq, tctx->sw_cipher.sk); + + skcipher_request_set_sync_tfm(subreq, tctx->sw_cipher.sk); + skcipher_request_set_callback(subreq, req->base.flags, NULL, + NULL); + skcipher_request_set_crypt(subreq, req->src, req->dst, + req->cryptlen, req->iv); + + if (instruction == OCS_ENCRYPT) + rc = crypto_skcipher_encrypt(subreq); + else + rc = crypto_skcipher_decrypt(subreq); + + skcipher_request_zero(subreq); + + return rc; + } + + /* + * If cryptlen == 0, no processing needed for ECB, CBC and CTR. + * + * For CTS continue: kmb_ocs_sk_validate_input() will return -EINVAL. + */ + if (!req->cryptlen && mode != OCS_MODE_CTS) + return 0; + + rc = kmb_ocs_sk_validate_input(req, mode); + if (rc) + return rc; + + aes_dev = kmb_ocs_aes_find_dev(tctx); + if (!aes_dev) + return -ENODEV; + + if (cipher != tctx->cipher) + return -EINVAL; + + ocs_aes_init_rctx(rctx); + rctx->instruction = instruction; + rctx->mode = mode; + + return crypto_transfer_skcipher_request_to_engine(aes_dev->engine, req); +} + +static void cleanup_ocs_dma_linked_list(struct device *dev, + struct ocs_dll_desc *dll) +{ + if (dll->vaddr) + dma_free_coherent(dev, dll->size, dll->vaddr, dll->dma_addr); + dll->vaddr = NULL; + dll->size = 0; + dll->dma_addr = DMA_MAPPING_ERROR; +} + +static void kmb_ocs_sk_dma_cleanup(struct skcipher_request *req) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct ocs_aes_rctx *rctx = skcipher_request_ctx(req); + struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm); + struct device *dev = tctx->aes_dev->dev; + + if (rctx->src_dma_count) { + dma_unmap_sg(dev, req->src, rctx->src_nents, DMA_TO_DEVICE); + rctx->src_dma_count = 0; + } + + if (rctx->dst_dma_count) { + dma_unmap_sg(dev, req->dst, rctx->dst_nents, rctx->in_place ? + DMA_BIDIRECTIONAL : + DMA_FROM_DEVICE); + rctx->dst_dma_count = 0; + } + + /* Clean up OCS DMA linked lists */ + cleanup_ocs_dma_linked_list(dev, &rctx->src_dll); + cleanup_ocs_dma_linked_list(dev, &rctx->dst_dll); +} + +static int kmb_ocs_sk_prepare_inplace(struct skcipher_request *req) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct ocs_aes_rctx *rctx = skcipher_request_ctx(req); + struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm); + int iv_size = crypto_skcipher_ivsize(tfm); + int rc; + + /* + * For CBC decrypt, save last block (iv) to last_ct_blk buffer. + * + * Note: if we are here, we already checked that cryptlen >= iv_size + * and iv_size == AES_BLOCK_SIZE (i.e., the size of last_ct_blk); see + * kmb_ocs_sk_validate_input(). + */ + if (rctx->mode == OCS_MODE_CBC && rctx->instruction == OCS_DECRYPT) + scatterwalk_map_and_copy(rctx->last_ct_blk, req->src, + req->cryptlen - iv_size, iv_size, 0); + + /* For CTS decrypt, swap last two blocks, if needed. */ + if (rctx->cts_swap && rctx->instruction == OCS_DECRYPT) + sg_swap_blocks(req->dst, rctx->dst_nents, + req->cryptlen - AES_BLOCK_SIZE, + req->cryptlen - (2 * AES_BLOCK_SIZE)); + + /* src and dst buffers are the same, use bidirectional DMA mapping. */ + rctx->dst_dma_count = dma_map_sg(tctx->aes_dev->dev, req->dst, + rctx->dst_nents, DMA_BIDIRECTIONAL); + if (rctx->dst_dma_count == 0) { + dev_err(tctx->aes_dev->dev, "Failed to map destination sg\n"); + return -ENOMEM; + } + + /* Create DST linked list */ + rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->dst, + rctx->dst_dma_count, &rctx->dst_dll, + req->cryptlen, 0); + if (rc) + return rc; + /* + * If descriptor creation was successful, set the src_dll.dma_addr to + * the value of dst_dll.dma_addr, as we do in-place AES operation on + * the src. + */ + rctx->src_dll.dma_addr = rctx->dst_dll.dma_addr; + + return 0; +} + +static int kmb_ocs_sk_prepare_notinplace(struct skcipher_request *req) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct ocs_aes_rctx *rctx = skcipher_request_ctx(req); + struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm); + int rc; + + rctx->src_nents = sg_nents_for_len(req->src, req->cryptlen); + if (rctx->src_nents < 0) + return -EBADMSG; + + /* Map SRC SG. */ + rctx->src_dma_count = dma_map_sg(tctx->aes_dev->dev, req->src, + rctx->src_nents, DMA_TO_DEVICE); + if (rctx->src_dma_count == 0) { + dev_err(tctx->aes_dev->dev, "Failed to map source sg\n"); + return -ENOMEM; + } + + /* Create SRC linked list */ + rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->src, + rctx->src_dma_count, &rctx->src_dll, + req->cryptlen, 0); + if (rc) + return rc; + + /* Map DST SG. */ + rctx->dst_dma_count = dma_map_sg(tctx->aes_dev->dev, req->dst, + rctx->dst_nents, DMA_FROM_DEVICE); + if (rctx->dst_dma_count == 0) { + dev_err(tctx->aes_dev->dev, "Failed to map destination sg\n"); + return -ENOMEM; + } + + /* Create DST linked list */ + rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->dst, + rctx->dst_dma_count, &rctx->dst_dll, + req->cryptlen, 0); + if (rc) + return rc; + + /* If this is not a CTS decrypt operation with swapping, we are done. */ + if (!(rctx->cts_swap && rctx->instruction == OCS_DECRYPT)) + return 0; + + /* + * Otherwise, we have to copy src to dst (as we cannot modify src). + * Use OCS AES bypass mode to copy src to dst via DMA. + * + * NOTE: for anything other than small data sizes this is rather + * inefficient. + */ + rc = ocs_aes_bypass_op(tctx->aes_dev, rctx->dst_dll.dma_addr, + rctx->src_dll.dma_addr, req->cryptlen); + if (rc) + return rc; + + /* + * Now dst == src, so clean up what we did so far and use in_place + * logic. + */ + kmb_ocs_sk_dma_cleanup(req); + rctx->in_place = true; + + return kmb_ocs_sk_prepare_inplace(req); +} + +static int kmb_ocs_sk_run(struct skcipher_request *req) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct ocs_aes_rctx *rctx = skcipher_request_ctx(req); + struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm); + struct ocs_aes_dev *aes_dev = tctx->aes_dev; + int iv_size = crypto_skcipher_ivsize(tfm); + int rc; + + rctx->dst_nents = sg_nents_for_len(req->dst, req->cryptlen); + if (rctx->dst_nents < 0) + return -EBADMSG; + + /* + * If 2 blocks or greater, and multiple of block size swap last two + * blocks to be compatible with other crypto API CTS implementations: + * OCS mode uses CBC-CS2, whereas other crypto API implementations use + * CBC-CS3. + * CBC-CS2 and CBC-CS3 defined by: + * https://nvlpubs.nist.gov/nistpubs/Legacy/SP/nistspecialpublication800-38a-add.pdf + */ + rctx->cts_swap = (rctx->mode == OCS_MODE_CTS && + req->cryptlen > AES_BLOCK_SIZE && + req->cryptlen % AES_BLOCK_SIZE == 0); + + rctx->in_place = (req->src == req->dst); + + if (rctx->in_place) + rc = kmb_ocs_sk_prepare_inplace(req); + else + rc = kmb_ocs_sk_prepare_notinplace(req); + + if (rc) + goto error; + + rc = ocs_aes_op(aes_dev, rctx->mode, tctx->cipher, rctx->instruction, + rctx->dst_dll.dma_addr, rctx->src_dll.dma_addr, + req->cryptlen, req->iv, iv_size); + if (rc) + goto error; + + /* Clean-up DMA before further processing output. */ + kmb_ocs_sk_dma_cleanup(req); + + /* For CTS Encrypt, swap last 2 blocks, if needed. */ + if (rctx->cts_swap && rctx->instruction == OCS_ENCRYPT) { + sg_swap_blocks(req->dst, rctx->dst_nents, + req->cryptlen - AES_BLOCK_SIZE, + req->cryptlen - (2 * AES_BLOCK_SIZE)); + return 0; + } + + /* For CBC copy IV to req->IV. */ + if (rctx->mode == OCS_MODE_CBC) { + /* CBC encrypt case. */ + if (rctx->instruction == OCS_ENCRYPT) { + scatterwalk_map_and_copy(req->iv, req->dst, + req->cryptlen - iv_size, + iv_size, 0); + return 0; + } + /* CBC decrypt case. */ + if (rctx->in_place) + memcpy(req->iv, rctx->last_ct_blk, iv_size); + else + scatterwalk_map_and_copy(req->iv, req->src, + req->cryptlen - iv_size, + iv_size, 0); + return 0; + } + /* For all other modes there's nothing to do. */ + + return 0; + +error: + kmb_ocs_sk_dma_cleanup(req); + + return rc; +} + +static int kmb_ocs_aead_validate_input(struct aead_request *req, + enum ocs_instruction instruction, + enum ocs_mode mode) +{ + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + int tag_size = crypto_aead_authsize(tfm); + int iv_size = crypto_aead_ivsize(tfm); + + /* For decrypt crytplen == len(PT) + len(tag). */ + if (instruction == OCS_DECRYPT && req->cryptlen < tag_size) + return -EINVAL; + + /* IV is mandatory. */ + if (!req->iv) + return -EINVAL; + + switch (mode) { + case OCS_MODE_GCM: + if (iv_size != GCM_AES_IV_SIZE) + return -EINVAL; + + return 0; + + case OCS_MODE_CCM: + /* Ensure IV is present and block size in length */ + if (iv_size != AES_BLOCK_SIZE) + return -EINVAL; + + return 0; + + default: + return -EINVAL; + } +} + +/* + * Called by encrypt() / decrypt() aead functions. + * + * Use fallback if needed, otherwise initialize context and enqueue request + * into engine. + */ +static int kmb_ocs_aead_common(struct aead_request *req, + enum ocs_cipher cipher, + enum ocs_instruction instruction, + enum ocs_mode mode) +{ + struct ocs_aes_tctx *tctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); + struct ocs_aes_rctx *rctx = aead_request_ctx(req); + struct ocs_aes_dev *dd; + int rc; + + if (tctx->use_fallback) { + struct aead_request *subreq = aead_request_ctx(req); + + aead_request_set_tfm(subreq, tctx->sw_cipher.aead); + aead_request_set_callback(subreq, req->base.flags, + req->base.complete, req->base.data); + aead_request_set_crypt(subreq, req->src, req->dst, + req->cryptlen, req->iv); + aead_request_set_ad(subreq, req->assoclen); + rc = crypto_aead_setauthsize(tctx->sw_cipher.aead, + crypto_aead_authsize(crypto_aead_reqtfm(req))); + if (rc) + return rc; + + return (instruction == OCS_ENCRYPT) ? + crypto_aead_encrypt(subreq) : + crypto_aead_decrypt(subreq); + } + + rc = kmb_ocs_aead_validate_input(req, instruction, mode); + if (rc) + return rc; + + dd = kmb_ocs_aes_find_dev(tctx); + if (!dd) + return -ENODEV; + + if (cipher != tctx->cipher) + return -EINVAL; + + ocs_aes_init_rctx(rctx); + rctx->instruction = instruction; + rctx->mode = mode; + + return crypto_transfer_aead_request_to_engine(dd->engine, req); +} + +static void kmb_ocs_aead_dma_cleanup(struct aead_request *req) +{ + struct ocs_aes_tctx *tctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); + struct ocs_aes_rctx *rctx = aead_request_ctx(req); + struct device *dev = tctx->aes_dev->dev; + + if (rctx->src_dma_count) { + dma_unmap_sg(dev, req->src, rctx->src_nents, DMA_TO_DEVICE); + rctx->src_dma_count = 0; + } + + if (rctx->dst_dma_count) { + dma_unmap_sg(dev, req->dst, rctx->dst_nents, rctx->in_place ? + DMA_BIDIRECTIONAL : + DMA_FROM_DEVICE); + rctx->dst_dma_count = 0; + } + /* Clean up OCS DMA linked lists */ + cleanup_ocs_dma_linked_list(dev, &rctx->src_dll); + cleanup_ocs_dma_linked_list(dev, &rctx->dst_dll); + cleanup_ocs_dma_linked_list(dev, &rctx->aad_src_dll); + cleanup_ocs_dma_linked_list(dev, &rctx->aad_dst_dll); +} + +/** + * kmb_ocs_aead_dma_prepare() - Do DMA mapping for AEAD processing. + * @req: The AEAD request being processed. + * @src_dll_size: Where to store the length of the data mapped into the + * src_dll OCS DMA list. + * + * Do the following: + * - DMA map req->src and req->dst + * - Initialize the following OCS DMA linked lists: rctx->src_dll, + * rctx->dst_dll, rctx->aad_src_dll and rxtc->aad_dst_dll. + * + * Return: 0 on success, negative error code otherwise. + */ +static int kmb_ocs_aead_dma_prepare(struct aead_request *req, u32 *src_dll_size) +{ + struct ocs_aes_tctx *tctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); + const int tag_size = crypto_aead_authsize(crypto_aead_reqtfm(req)); + struct ocs_aes_rctx *rctx = aead_request_ctx(req); + u32 in_size; /* The length of the data to be mapped by src_dll. */ + u32 out_size; /* The length of the data to be mapped by dst_dll. */ + u32 dst_size; /* The length of the data in dst_sg. */ + int rc; + + /* Get number of entries in input data SG list. */ + rctx->src_nents = sg_nents_for_len(req->src, + req->assoclen + req->cryptlen); + if (rctx->src_nents < 0) + return -EBADMSG; + + if (rctx->instruction == OCS_DECRYPT) { + /* + * For decrypt: + * - src sg list is: AAD|CT|tag + * - dst sg list expects: AAD|PT + * + * in_size == len(CT); out_size == len(PT) + */ + + /* req->cryptlen includes both CT and tag. */ + in_size = req->cryptlen - tag_size; + + /* out_size = PT size == CT size */ + out_size = in_size; + + /* len(dst_sg) == len(AAD) + len(PT) */ + dst_size = req->assoclen + out_size; + + /* + * Copy tag from source SG list to 'in_tag' buffer. + * + * Note: this needs to be done here, before DMA mapping src_sg. + */ + sg_pcopy_to_buffer(req->src, rctx->src_nents, rctx->in_tag, + tag_size, req->assoclen + in_size); + + } else { /* OCS_ENCRYPT */ + /* + * For encrypt: + * src sg list is: AAD|PT + * dst sg list expects: AAD|CT|tag + */ + /* in_size == len(PT) */ + in_size = req->cryptlen; + + /* + * In CCM mode the OCS engine appends the tag to the ciphertext, + * but in GCM mode the tag must be read from the tag registers + * and appended manually below + */ + out_size = (rctx->mode == OCS_MODE_CCM) ? in_size + tag_size : + in_size; + /* len(dst_sg) == len(AAD) + len(CT) + len(tag) */ + dst_size = req->assoclen + in_size + tag_size; + } + *src_dll_size = in_size; + + /* Get number of entries in output data SG list. */ + rctx->dst_nents = sg_nents_for_len(req->dst, dst_size); + if (rctx->dst_nents < 0) + return -EBADMSG; + + rctx->in_place = (req->src == req->dst) ? 1 : 0; + + /* Map destination; use bidirectional mapping for in-place case. */ + rctx->dst_dma_count = dma_map_sg(tctx->aes_dev->dev, req->dst, + rctx->dst_nents, + rctx->in_place ? DMA_BIDIRECTIONAL : + DMA_FROM_DEVICE); + if (rctx->dst_dma_count == 0 && rctx->dst_nents != 0) { + dev_err(tctx->aes_dev->dev, "Failed to map destination sg\n"); + return -ENOMEM; + } + + /* Create AAD DST list: maps dst[0:AAD_SIZE-1]. */ + rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->dst, + rctx->dst_dma_count, + &rctx->aad_dst_dll, req->assoclen, + 0); + if (rc) + return rc; + + /* Create DST list: maps dst[AAD_SIZE:out_size] */ + rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->dst, + rctx->dst_dma_count, &rctx->dst_dll, + out_size, req->assoclen); + if (rc) + return rc; + + if (rctx->in_place) { + /* If this is not CCM encrypt, we are done. */ + if (!(rctx->mode == OCS_MODE_CCM && + rctx->instruction == OCS_ENCRYPT)) { + /* + * SRC and DST are the same, so re-use the same DMA + * addresses (to avoid allocating new DMA lists + * identical to the dst ones). + */ + rctx->src_dll.dma_addr = rctx->dst_dll.dma_addr; + rctx->aad_src_dll.dma_addr = rctx->aad_dst_dll.dma_addr; + + return 0; + } + /* + * For CCM encrypt the input and output linked lists contain + * different amounts of data, so, we need to create different + * SRC and AAD SRC lists, even for the in-place case. + */ + rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->dst, + rctx->dst_dma_count, + &rctx->aad_src_dll, + req->assoclen, 0); + if (rc) + return rc; + rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->dst, + rctx->dst_dma_count, + &rctx->src_dll, in_size, + req->assoclen); + if (rc) + return rc; + + return 0; + } + /* Not in-place case. */ + + /* Map source SG. */ + rctx->src_dma_count = dma_map_sg(tctx->aes_dev->dev, req->src, + rctx->src_nents, DMA_TO_DEVICE); + if (rctx->src_dma_count == 0 && rctx->src_nents != 0) { + dev_err(tctx->aes_dev->dev, "Failed to map source sg\n"); + return -ENOMEM; + } + + /* Create AAD SRC list. */ + rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->src, + rctx->src_dma_count, + &rctx->aad_src_dll, + req->assoclen, 0); + if (rc) + return rc; + + /* Create SRC list. */ + rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->src, + rctx->src_dma_count, + &rctx->src_dll, in_size, + req->assoclen); + if (rc) + return rc; + + if (req->assoclen == 0) + return 0; + + /* Copy AAD from src sg to dst sg using OCS DMA. */ + rc = ocs_aes_bypass_op(tctx->aes_dev, rctx->aad_dst_dll.dma_addr, + rctx->aad_src_dll.dma_addr, req->cryptlen); + if (rc) + dev_err(tctx->aes_dev->dev, + "Failed to copy source AAD to destination AAD\n"); + + return rc; +} + +static int kmb_ocs_aead_run(struct aead_request *req) +{ + struct ocs_aes_tctx *tctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); + const int tag_size = crypto_aead_authsize(crypto_aead_reqtfm(req)); + struct ocs_aes_rctx *rctx = aead_request_ctx(req); + u32 in_size; /* The length of the data mapped by src_dll. */ + int rc; + + rc = kmb_ocs_aead_dma_prepare(req, &in_size); + if (rc) + goto exit; + + /* For CCM, we just call the OCS processing and we are done. */ + if (rctx->mode == OCS_MODE_CCM) { + rc = ocs_aes_ccm_op(tctx->aes_dev, tctx->cipher, + rctx->instruction, rctx->dst_dll.dma_addr, + rctx->src_dll.dma_addr, in_size, + req->iv, + rctx->aad_src_dll.dma_addr, req->assoclen, + rctx->in_tag, tag_size); + goto exit; + } + /* GCM case; invoke OCS processing. */ + rc = ocs_aes_gcm_op(tctx->aes_dev, tctx->cipher, + rctx->instruction, + rctx->dst_dll.dma_addr, + rctx->src_dll.dma_addr, in_size, + req->iv, + rctx->aad_src_dll.dma_addr, req->assoclen, + rctx->out_tag, tag_size); + if (rc) + goto exit; + + /* For GCM decrypt, we have to compare in_tag with out_tag. */ + if (rctx->instruction == OCS_DECRYPT) { + rc = memcmp(rctx->in_tag, rctx->out_tag, tag_size) ? + -EBADMSG : 0; + goto exit; + } + + /* For GCM encrypt, we must manually copy out_tag to DST sg. */ + + /* Clean-up must be called before the sg_pcopy_from_buffer() below. */ + kmb_ocs_aead_dma_cleanup(req); + + /* Copy tag to destination sg after AAD and CT. */ + sg_pcopy_from_buffer(req->dst, rctx->dst_nents, rctx->out_tag, + tag_size, req->assoclen + req->cryptlen); + + /* Return directly as DMA cleanup already done. */ + return 0; + +exit: + kmb_ocs_aead_dma_cleanup(req); + + return rc; +} + +static int kmb_ocs_aes_sk_do_one_request(struct crypto_engine *engine, + void *areq) +{ + struct skcipher_request *req = + container_of(areq, struct skcipher_request, base); + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm); + int err; + + if (!tctx->aes_dev) { + err = -ENODEV; + goto exit; + } + + err = ocs_aes_set_key(tctx->aes_dev, tctx->key_len, tctx->key, + tctx->cipher); + if (err) + goto exit; + + err = kmb_ocs_sk_run(req); + +exit: + crypto_finalize_skcipher_request(engine, req, err); + + return 0; +} + +static int kmb_ocs_aes_aead_do_one_request(struct crypto_engine *engine, + void *areq) +{ + struct aead_request *req = container_of(areq, + struct aead_request, base); + struct ocs_aes_tctx *tctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); + int err; + + if (!tctx->aes_dev) + return -ENODEV; + + err = ocs_aes_set_key(tctx->aes_dev, tctx->key_len, tctx->key, + tctx->cipher); + if (err) + goto exit; + + err = kmb_ocs_aead_run(req); + +exit: + crypto_finalize_aead_request(tctx->aes_dev->engine, req, err); + + return 0; +} + +static int kmb_ocs_aes_set_key(struct crypto_skcipher *tfm, const u8 *in_key, + unsigned int key_len) +{ + return kmb_ocs_sk_set_key(tfm, in_key, key_len, OCS_AES); +} + +static int kmb_ocs_aes_aead_set_key(struct crypto_aead *tfm, const u8 *in_key, + unsigned int key_len) +{ + return kmb_ocs_aead_set_key(tfm, in_key, key_len, OCS_AES); +} + +#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB +static int kmb_ocs_aes_ecb_encrypt(struct skcipher_request *req) +{ + return kmb_ocs_sk_common(req, OCS_AES, OCS_ENCRYPT, OCS_MODE_ECB); +} + +static int kmb_ocs_aes_ecb_decrypt(struct skcipher_request *req) +{ + return kmb_ocs_sk_common(req, OCS_AES, OCS_DECRYPT, OCS_MODE_ECB); +} +#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB */ + +static int kmb_ocs_aes_cbc_encrypt(struct skcipher_request *req) +{ + return kmb_ocs_sk_common(req, OCS_AES, OCS_ENCRYPT, OCS_MODE_CBC); +} + +static int kmb_ocs_aes_cbc_decrypt(struct skcipher_request *req) +{ + return kmb_ocs_sk_common(req, OCS_AES, OCS_DECRYPT, OCS_MODE_CBC); +} + +static int kmb_ocs_aes_ctr_encrypt(struct skcipher_request *req) +{ + return kmb_ocs_sk_common(req, OCS_AES, OCS_ENCRYPT, OCS_MODE_CTR); +} + +static int kmb_ocs_aes_ctr_decrypt(struct skcipher_request *req) +{ + return kmb_ocs_sk_common(req, OCS_AES, OCS_DECRYPT, OCS_MODE_CTR); +} + +#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS +static int kmb_ocs_aes_cts_encrypt(struct skcipher_request *req) +{ + return kmb_ocs_sk_common(req, OCS_AES, OCS_ENCRYPT, OCS_MODE_CTS); +} + +static int kmb_ocs_aes_cts_decrypt(struct skcipher_request *req) +{ + return kmb_ocs_sk_common(req, OCS_AES, OCS_DECRYPT, OCS_MODE_CTS); +} +#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS */ + +static int kmb_ocs_aes_gcm_encrypt(struct aead_request *req) +{ + return kmb_ocs_aead_common(req, OCS_AES, OCS_ENCRYPT, OCS_MODE_GCM); +} + +static int kmb_ocs_aes_gcm_decrypt(struct aead_request *req) +{ + return kmb_ocs_aead_common(req, OCS_AES, OCS_DECRYPT, OCS_MODE_GCM); +} + +static int kmb_ocs_aes_ccm_encrypt(struct aead_request *req) +{ + return kmb_ocs_aead_common(req, OCS_AES, OCS_ENCRYPT, OCS_MODE_CCM); +} + +static int kmb_ocs_aes_ccm_decrypt(struct aead_request *req) +{ + return kmb_ocs_aead_common(req, OCS_AES, OCS_DECRYPT, OCS_MODE_CCM); +} + +static int kmb_ocs_sm4_set_key(struct crypto_skcipher *tfm, const u8 *in_key, + unsigned int key_len) +{ + return kmb_ocs_sk_set_key(tfm, in_key, key_len, OCS_SM4); +} + +static int kmb_ocs_sm4_aead_set_key(struct crypto_aead *tfm, const u8 *in_key, + unsigned int key_len) +{ + return kmb_ocs_aead_set_key(tfm, in_key, key_len, OCS_SM4); +} + +#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB +static int kmb_ocs_sm4_ecb_encrypt(struct skcipher_request *req) +{ + return kmb_ocs_sk_common(req, OCS_SM4, OCS_ENCRYPT, OCS_MODE_ECB); +} + +static int kmb_ocs_sm4_ecb_decrypt(struct skcipher_request *req) +{ + return kmb_ocs_sk_common(req, OCS_SM4, OCS_DECRYPT, OCS_MODE_ECB); +} +#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB */ + +static int kmb_ocs_sm4_cbc_encrypt(struct skcipher_request *req) +{ + return kmb_ocs_sk_common(req, OCS_SM4, OCS_ENCRYPT, OCS_MODE_CBC); +} + +static int kmb_ocs_sm4_cbc_decrypt(struct skcipher_request *req) +{ + return kmb_ocs_sk_common(req, OCS_SM4, OCS_DECRYPT, OCS_MODE_CBC); +} + +static int kmb_ocs_sm4_ctr_encrypt(struct skcipher_request *req) +{ + return kmb_ocs_sk_common(req, OCS_SM4, OCS_ENCRYPT, OCS_MODE_CTR); +} + +static int kmb_ocs_sm4_ctr_decrypt(struct skcipher_request *req) +{ + return kmb_ocs_sk_common(req, OCS_SM4, OCS_DECRYPT, OCS_MODE_CTR); +} + +#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS +static int kmb_ocs_sm4_cts_encrypt(struct skcipher_request *req) +{ + return kmb_ocs_sk_common(req, OCS_SM4, OCS_ENCRYPT, OCS_MODE_CTS); +} + +static int kmb_ocs_sm4_cts_decrypt(struct skcipher_request *req) +{ + return kmb_ocs_sk_common(req, OCS_SM4, OCS_DECRYPT, OCS_MODE_CTS); +} +#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS */ + +static int kmb_ocs_sm4_gcm_encrypt(struct aead_request *req) +{ + return kmb_ocs_aead_common(req, OCS_SM4, OCS_ENCRYPT, OCS_MODE_GCM); +} + +static int kmb_ocs_sm4_gcm_decrypt(struct aead_request *req) +{ + return kmb_ocs_aead_common(req, OCS_SM4, OCS_DECRYPT, OCS_MODE_GCM); +} + +static int kmb_ocs_sm4_ccm_encrypt(struct aead_request *req) +{ + return kmb_ocs_aead_common(req, OCS_SM4, OCS_ENCRYPT, OCS_MODE_CCM); +} + +static int kmb_ocs_sm4_ccm_decrypt(struct aead_request *req) +{ + return kmb_ocs_aead_common(req, OCS_SM4, OCS_DECRYPT, OCS_MODE_CCM); +} + +static inline int ocs_common_init(struct ocs_aes_tctx *tctx) +{ + tctx->engine_ctx.op.prepare_request = NULL; + tctx->engine_ctx.op.do_one_request = kmb_ocs_aes_sk_do_one_request; + tctx->engine_ctx.op.unprepare_request = NULL; + + return 0; +} + +static int ocs_aes_init_tfm(struct crypto_skcipher *tfm) +{ + const char *alg_name = crypto_tfm_alg_name(&tfm->base); + struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm); + struct crypto_sync_skcipher *blk; + + /* set fallback cipher in case it will be needed */ + blk = crypto_alloc_sync_skcipher(alg_name, 0, CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(blk)) + return PTR_ERR(blk); + + tctx->sw_cipher.sk = blk; + + crypto_skcipher_set_reqsize(tfm, sizeof(struct ocs_aes_rctx)); + + return ocs_common_init(tctx); +} + +static int ocs_sm4_init_tfm(struct crypto_skcipher *tfm) +{ + struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm); + + crypto_skcipher_set_reqsize(tfm, sizeof(struct ocs_aes_rctx)); + + return ocs_common_init(tctx); +} + +static inline void clear_key(struct ocs_aes_tctx *tctx) +{ + memzero_explicit(tctx->key, OCS_AES_KEYSIZE_256); + + /* Zero key registers if set */ + if (tctx->aes_dev) + ocs_aes_set_key(tctx->aes_dev, OCS_AES_KEYSIZE_256, + tctx->key, OCS_AES); +} + +static void ocs_exit_tfm(struct crypto_skcipher *tfm) +{ + struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm); + + clear_key(tctx); + + if (tctx->sw_cipher.sk) { + crypto_free_sync_skcipher(tctx->sw_cipher.sk); + tctx->sw_cipher.sk = NULL; + } +} + +static inline int ocs_common_aead_init(struct ocs_aes_tctx *tctx) +{ + tctx->engine_ctx.op.prepare_request = NULL; + tctx->engine_ctx.op.do_one_request = kmb_ocs_aes_aead_do_one_request; + tctx->engine_ctx.op.unprepare_request = NULL; + + return 0; +} + +static int ocs_aes_aead_cra_init(struct crypto_aead *tfm) +{ + const char *alg_name = crypto_tfm_alg_name(&tfm->base); + struct ocs_aes_tctx *tctx = crypto_aead_ctx(tfm); + struct crypto_aead *blk; + + /* Set fallback cipher in case it will be needed */ + blk = crypto_alloc_aead(alg_name, 0, CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(blk)) + return PTR_ERR(blk); + + tctx->sw_cipher.aead = blk; + + crypto_aead_set_reqsize(tfm, + max(sizeof(struct ocs_aes_rctx), + (sizeof(struct aead_request) + + crypto_aead_reqsize(tctx->sw_cipher.aead)))); + + return ocs_common_aead_init(tctx); +} + +static int kmb_ocs_aead_ccm_setauthsize(struct crypto_aead *tfm, + unsigned int authsize) +{ + switch (authsize) { + case 4: + case 6: + case 8: + case 10: + case 12: + case 14: + case 16: + return 0; + default: + return -EINVAL; + } +} + +static int kmb_ocs_aead_gcm_setauthsize(struct crypto_aead *tfm, + unsigned int authsize) +{ + return crypto_gcm_check_authsize(authsize); +} + +static int ocs_sm4_aead_cra_init(struct crypto_aead *tfm) +{ + struct ocs_aes_tctx *tctx = crypto_aead_ctx(tfm); + + crypto_aead_set_reqsize(tfm, sizeof(struct ocs_aes_rctx)); + + return ocs_common_aead_init(tctx); +} + +static void ocs_aead_cra_exit(struct crypto_aead *tfm) +{ + struct ocs_aes_tctx *tctx = crypto_aead_ctx(tfm); + + clear_key(tctx); + + if (tctx->sw_cipher.aead) { + crypto_free_aead(tctx->sw_cipher.aead); + tctx->sw_cipher.aead = NULL; + } +} + +static struct skcipher_alg algs[] = { +#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB + { + .base.cra_name = "ecb(aes)", + .base.cra_driver_name = "ecb-aes-keembay-ocs", + .base.cra_priority = KMB_OCS_PRIORITY, + .base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct ocs_aes_tctx), + .base.cra_module = THIS_MODULE, + .base.cra_alignmask = 0, + + .min_keysize = OCS_AES_MIN_KEY_SIZE, + .max_keysize = OCS_AES_MAX_KEY_SIZE, + .setkey = kmb_ocs_aes_set_key, + .encrypt = kmb_ocs_aes_ecb_encrypt, + .decrypt = kmb_ocs_aes_ecb_decrypt, + .init = ocs_aes_init_tfm, + .exit = ocs_exit_tfm, + }, +#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB */ + { + .base.cra_name = "cbc(aes)", + .base.cra_driver_name = "cbc-aes-keembay-ocs", + .base.cra_priority = KMB_OCS_PRIORITY, + .base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct ocs_aes_tctx), + .base.cra_module = THIS_MODULE, + .base.cra_alignmask = 0, + + .min_keysize = OCS_AES_MIN_KEY_SIZE, + .max_keysize = OCS_AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = kmb_ocs_aes_set_key, + .encrypt = kmb_ocs_aes_cbc_encrypt, + .decrypt = kmb_ocs_aes_cbc_decrypt, + .init = ocs_aes_init_tfm, + .exit = ocs_exit_tfm, + }, + { + .base.cra_name = "ctr(aes)", + .base.cra_driver_name = "ctr-aes-keembay-ocs", + .base.cra_priority = KMB_OCS_PRIORITY, + .base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK, + .base.cra_blocksize = 1, + .base.cra_ctxsize = sizeof(struct ocs_aes_tctx), + .base.cra_module = THIS_MODULE, + .base.cra_alignmask = 0, + + .min_keysize = OCS_AES_MIN_KEY_SIZE, + .max_keysize = OCS_AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = kmb_ocs_aes_set_key, + .encrypt = kmb_ocs_aes_ctr_encrypt, + .decrypt = kmb_ocs_aes_ctr_decrypt, + .init = ocs_aes_init_tfm, + .exit = ocs_exit_tfm, + }, +#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS + { + .base.cra_name = "cts(cbc(aes))", + .base.cra_driver_name = "cts-aes-keembay-ocs", + .base.cra_priority = KMB_OCS_PRIORITY, + .base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct ocs_aes_tctx), + .base.cra_module = THIS_MODULE, + .base.cra_alignmask = 0, + + .min_keysize = OCS_AES_MIN_KEY_SIZE, + .max_keysize = OCS_AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = kmb_ocs_aes_set_key, + .encrypt = kmb_ocs_aes_cts_encrypt, + .decrypt = kmb_ocs_aes_cts_decrypt, + .init = ocs_aes_init_tfm, + .exit = ocs_exit_tfm, + }, +#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS */ +#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB + { + .base.cra_name = "ecb(sm4)", + .base.cra_driver_name = "ecb-sm4-keembay-ocs", + .base.cra_priority = KMB_OCS_PRIORITY, + .base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct ocs_aes_tctx), + .base.cra_module = THIS_MODULE, + .base.cra_alignmask = 0, + + .min_keysize = OCS_SM4_KEY_SIZE, + .max_keysize = OCS_SM4_KEY_SIZE, + .setkey = kmb_ocs_sm4_set_key, + .encrypt = kmb_ocs_sm4_ecb_encrypt, + .decrypt = kmb_ocs_sm4_ecb_decrypt, + .init = ocs_sm4_init_tfm, + .exit = ocs_exit_tfm, + }, +#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB */ + { + .base.cra_name = "cbc(sm4)", + .base.cra_driver_name = "cbc-sm4-keembay-ocs", + .base.cra_priority = KMB_OCS_PRIORITY, + .base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct ocs_aes_tctx), + .base.cra_module = THIS_MODULE, + .base.cra_alignmask = 0, + + .min_keysize = OCS_SM4_KEY_SIZE, + .max_keysize = OCS_SM4_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = kmb_ocs_sm4_set_key, + .encrypt = kmb_ocs_sm4_cbc_encrypt, + .decrypt = kmb_ocs_sm4_cbc_decrypt, + .init = ocs_sm4_init_tfm, + .exit = ocs_exit_tfm, + }, + { + .base.cra_name = "ctr(sm4)", + .base.cra_driver_name = "ctr-sm4-keembay-ocs", + .base.cra_priority = KMB_OCS_PRIORITY, + .base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .base.cra_blocksize = 1, + .base.cra_ctxsize = sizeof(struct ocs_aes_tctx), + .base.cra_module = THIS_MODULE, + .base.cra_alignmask = 0, + + .min_keysize = OCS_SM4_KEY_SIZE, + .max_keysize = OCS_SM4_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = kmb_ocs_sm4_set_key, + .encrypt = kmb_ocs_sm4_ctr_encrypt, + .decrypt = kmb_ocs_sm4_ctr_decrypt, + .init = ocs_sm4_init_tfm, + .exit = ocs_exit_tfm, + }, +#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS + { + .base.cra_name = "cts(cbc(sm4))", + .base.cra_driver_name = "cts-sm4-keembay-ocs", + .base.cra_priority = KMB_OCS_PRIORITY, + .base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct ocs_aes_tctx), + .base.cra_module = THIS_MODULE, + .base.cra_alignmask = 0, + + .min_keysize = OCS_SM4_KEY_SIZE, + .max_keysize = OCS_SM4_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = kmb_ocs_sm4_set_key, + .encrypt = kmb_ocs_sm4_cts_encrypt, + .decrypt = kmb_ocs_sm4_cts_decrypt, + .init = ocs_sm4_init_tfm, + .exit = ocs_exit_tfm, + } +#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS */ +}; + +static struct aead_alg algs_aead[] = { + { + .base = { + .cra_name = "gcm(aes)", + .cra_driver_name = "gcm-aes-keembay-ocs", + .cra_priority = KMB_OCS_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct ocs_aes_tctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .init = ocs_aes_aead_cra_init, + .exit = ocs_aead_cra_exit, + .ivsize = GCM_AES_IV_SIZE, + .maxauthsize = AES_BLOCK_SIZE, + .setauthsize = kmb_ocs_aead_gcm_setauthsize, + .setkey = kmb_ocs_aes_aead_set_key, + .encrypt = kmb_ocs_aes_gcm_encrypt, + .decrypt = kmb_ocs_aes_gcm_decrypt, + }, + { + .base = { + .cra_name = "ccm(aes)", + .cra_driver_name = "ccm-aes-keembay-ocs", + .cra_priority = KMB_OCS_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct ocs_aes_tctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .init = ocs_aes_aead_cra_init, + .exit = ocs_aead_cra_exit, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = AES_BLOCK_SIZE, + .setauthsize = kmb_ocs_aead_ccm_setauthsize, + .setkey = kmb_ocs_aes_aead_set_key, + .encrypt = kmb_ocs_aes_ccm_encrypt, + .decrypt = kmb_ocs_aes_ccm_decrypt, + }, + { + .base = { + .cra_name = "gcm(sm4)", + .cra_driver_name = "gcm-sm4-keembay-ocs", + .cra_priority = KMB_OCS_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct ocs_aes_tctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .init = ocs_sm4_aead_cra_init, + .exit = ocs_aead_cra_exit, + .ivsize = GCM_AES_IV_SIZE, + .maxauthsize = AES_BLOCK_SIZE, + .setauthsize = kmb_ocs_aead_gcm_setauthsize, + .setkey = kmb_ocs_sm4_aead_set_key, + .encrypt = kmb_ocs_sm4_gcm_encrypt, + .decrypt = kmb_ocs_sm4_gcm_decrypt, + }, + { + .base = { + .cra_name = "ccm(sm4)", + .cra_driver_name = "ccm-sm4-keembay-ocs", + .cra_priority = KMB_OCS_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct ocs_aes_tctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .init = ocs_sm4_aead_cra_init, + .exit = ocs_aead_cra_exit, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = AES_BLOCK_SIZE, + .setauthsize = kmb_ocs_aead_ccm_setauthsize, + .setkey = kmb_ocs_sm4_aead_set_key, + .encrypt = kmb_ocs_sm4_ccm_encrypt, + .decrypt = kmb_ocs_sm4_ccm_decrypt, + } +}; + +static void unregister_aes_algs(struct ocs_aes_dev *aes_dev) +{ + crypto_unregister_aeads(algs_aead, ARRAY_SIZE(algs_aead)); + crypto_unregister_skciphers(algs, ARRAY_SIZE(algs)); +} + +static int register_aes_algs(struct ocs_aes_dev *aes_dev) +{ + int ret; + + /* + * If any algorithm fails to register, all preceding algorithms that + * were successfully registered will be automatically unregistered. + */ + ret = crypto_register_aeads(algs_aead, ARRAY_SIZE(algs_aead)); + if (ret) + return ret; + + ret = crypto_register_skciphers(algs, ARRAY_SIZE(algs)); + if (ret) + crypto_unregister_aeads(algs_aead, ARRAY_SIZE(algs)); + + return ret; +} + +/* Device tree driver match. */ +static const struct of_device_id kmb_ocs_aes_of_match[] = { + { + .compatible = "intel,keembay-ocs-aes", + }, + {} +}; + +static int kmb_ocs_aes_remove(struct platform_device *pdev) +{ + struct ocs_aes_dev *aes_dev; + + aes_dev = platform_get_drvdata(pdev); + + unregister_aes_algs(aes_dev); + + spin_lock(&ocs_aes.lock); + list_del(&aes_dev->list); + spin_unlock(&ocs_aes.lock); + + crypto_engine_exit(aes_dev->engine); + + return 0; +} + +static int kmb_ocs_aes_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct ocs_aes_dev *aes_dev; + int rc; + + aes_dev = devm_kzalloc(dev, sizeof(*aes_dev), GFP_KERNEL); + if (!aes_dev) + return -ENOMEM; + + aes_dev->dev = dev; + + platform_set_drvdata(pdev, aes_dev); + + rc = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32)); + if (rc) { + dev_err(dev, "Failed to set 32 bit dma mask %d\n", rc); + return rc; + } + + /* Get base register address. */ + aes_dev->base_reg = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(aes_dev->base_reg)) + return PTR_ERR(aes_dev->base_reg); + + /* Get and request IRQ */ + aes_dev->irq = platform_get_irq(pdev, 0); + if (aes_dev->irq < 0) + return aes_dev->irq; + + rc = devm_request_threaded_irq(dev, aes_dev->irq, ocs_aes_irq_handler, + NULL, 0, "keembay-ocs-aes", aes_dev); + if (rc < 0) { + dev_err(dev, "Could not request IRQ\n"); + return rc; + } + + INIT_LIST_HEAD(&aes_dev->list); + spin_lock(&ocs_aes.lock); + list_add_tail(&aes_dev->list, &ocs_aes.dev_list); + spin_unlock(&ocs_aes.lock); + + init_completion(&aes_dev->irq_completion); + + /* Initialize crypto engine */ + aes_dev->engine = crypto_engine_alloc_init(dev, true); + if (!aes_dev->engine) { + rc = -ENOMEM; + goto list_del; + } + + rc = crypto_engine_start(aes_dev->engine); + if (rc) { + dev_err(dev, "Could not start crypto engine\n"); + goto cleanup; + } + + rc = register_aes_algs(aes_dev); + if (rc) { + dev_err(dev, + "Could not register OCS algorithms with Crypto API\n"); + goto cleanup; + } + + return 0; + +cleanup: + crypto_engine_exit(aes_dev->engine); +list_del: + spin_lock(&ocs_aes.lock); + list_del(&aes_dev->list); + spin_unlock(&ocs_aes.lock); + + return rc; +} + +/* The OCS driver is a platform device. */ +static struct platform_driver kmb_ocs_aes_driver = { + .probe = kmb_ocs_aes_probe, + .remove = kmb_ocs_aes_remove, + .driver = { + .name = DRV_NAME, + .of_match_table = kmb_ocs_aes_of_match, + }, +}; + +module_platform_driver(kmb_ocs_aes_driver); + +MODULE_DESCRIPTION("Intel Keem Bay Offload and Crypto Subsystem (OCS) AES/SM4 Driver"); +MODULE_LICENSE("GPL"); + +MODULE_ALIAS_CRYPTO("cbc-aes-keembay-ocs"); +MODULE_ALIAS_CRYPTO("ctr-aes-keembay-ocs"); +MODULE_ALIAS_CRYPTO("gcm-aes-keembay-ocs"); +MODULE_ALIAS_CRYPTO("ccm-aes-keembay-ocs"); + +MODULE_ALIAS_CRYPTO("cbc-sm4-keembay-ocs"); +MODULE_ALIAS_CRYPTO("ctr-sm4-keembay-ocs"); +MODULE_ALIAS_CRYPTO("gcm-sm4-keembay-ocs"); +MODULE_ALIAS_CRYPTO("ccm-sm4-keembay-ocs"); + +#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB +MODULE_ALIAS_CRYPTO("ecb-aes-keembay-ocs"); +MODULE_ALIAS_CRYPTO("ecb-sm4-keembay-ocs"); +#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB */ + +#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS +MODULE_ALIAS_CRYPTO("cts-aes-keembay-ocs"); +MODULE_ALIAS_CRYPTO("cts-sm4-keembay-ocs"); +#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS */ diff --git a/drivers/crypto/intel/keembay/keembay-ocs-ecc.c b/drivers/crypto/intel/keembay/keembay-ocs-ecc.c new file mode 100644 index 000000000000..2269df17514c --- /dev/null +++ b/drivers/crypto/intel/keembay/keembay-ocs-ecc.c @@ -0,0 +1,1016 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Intel Keem Bay OCS ECC Crypto Driver. + * + * Copyright (C) 2019-2021 Intel Corporation + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include +#include + +#define DRV_NAME "keembay-ocs-ecc" + +#define KMB_OCS_ECC_PRIORITY 350 + +#define HW_OFFS_OCS_ECC_COMMAND 0x00000000 +#define HW_OFFS_OCS_ECC_STATUS 0x00000004 +#define HW_OFFS_OCS_ECC_DATA_IN 0x00000080 +#define HW_OFFS_OCS_ECC_CX_DATA_OUT 0x00000100 +#define HW_OFFS_OCS_ECC_CY_DATA_OUT 0x00000180 +#define HW_OFFS_OCS_ECC_ISR 0x00000400 +#define HW_OFFS_OCS_ECC_IER 0x00000404 + +#define HW_OCS_ECC_ISR_INT_STATUS_DONE BIT(0) +#define HW_OCS_ECC_COMMAND_INS_BP BIT(0) + +#define HW_OCS_ECC_COMMAND_START_VAL BIT(0) + +#define OCS_ECC_OP_SIZE_384 BIT(8) +#define OCS_ECC_OP_SIZE_256 0 + +/* ECC Instruction : for ECC_COMMAND */ +#define OCS_ECC_INST_WRITE_AX (0x1 << HW_OCS_ECC_COMMAND_INS_BP) +#define OCS_ECC_INST_WRITE_AY (0x2 << HW_OCS_ECC_COMMAND_INS_BP) +#define OCS_ECC_INST_WRITE_BX_D (0x3 << HW_OCS_ECC_COMMAND_INS_BP) +#define OCS_ECC_INST_WRITE_BY_L (0x4 << HW_OCS_ECC_COMMAND_INS_BP) +#define OCS_ECC_INST_WRITE_P (0x5 << HW_OCS_ECC_COMMAND_INS_BP) +#define OCS_ECC_INST_WRITE_A (0x6 << HW_OCS_ECC_COMMAND_INS_BP) +#define OCS_ECC_INST_CALC_D_IDX_A (0x8 << HW_OCS_ECC_COMMAND_INS_BP) +#define OCS_ECC_INST_CALC_A_POW_B_MODP (0xB << HW_OCS_ECC_COMMAND_INS_BP) +#define OCS_ECC_INST_CALC_A_MUL_B_MODP (0xC << HW_OCS_ECC_COMMAND_INS_BP) +#define OCS_ECC_INST_CALC_A_ADD_B_MODP (0xD << HW_OCS_ECC_COMMAND_INS_BP) + +#define ECC_ENABLE_INTR 1 + +#define POLL_USEC 100 +#define TIMEOUT_USEC 10000 + +#define KMB_ECC_VLI_MAX_DIGITS ECC_CURVE_NIST_P384_DIGITS +#define KMB_ECC_VLI_MAX_BYTES (KMB_ECC_VLI_MAX_DIGITS \ + << ECC_DIGITS_TO_BYTES_SHIFT) + +#define POW_CUBE 3 + +/** + * struct ocs_ecc_dev - ECC device context + * @list: List of device contexts + * @dev: OCS ECC device + * @base_reg: IO base address of OCS ECC + * @engine: Crypto engine for the device + * @irq_done: IRQ done completion. + * @irq: IRQ number + */ +struct ocs_ecc_dev { + struct list_head list; + struct device *dev; + void __iomem *base_reg; + struct crypto_engine *engine; + struct completion irq_done; + int irq; +}; + +/** + * struct ocs_ecc_ctx - Transformation context. + * @engine_ctx: Crypto engine ctx. + * @ecc_dev: The ECC driver associated with this context. + * @curve: The elliptic curve used by this transformation. + * @private_key: The private key. + */ +struct ocs_ecc_ctx { + struct crypto_engine_ctx engine_ctx; + struct ocs_ecc_dev *ecc_dev; + const struct ecc_curve *curve; + u64 private_key[KMB_ECC_VLI_MAX_DIGITS]; +}; + +/* Driver data. */ +struct ocs_ecc_drv { + struct list_head dev_list; + spinlock_t lock; /* Protects dev_list. */ +}; + +/* Global variable holding the list of OCS ECC devices (only one expected). */ +static struct ocs_ecc_drv ocs_ecc = { + .dev_list = LIST_HEAD_INIT(ocs_ecc.dev_list), + .lock = __SPIN_LOCK_UNLOCKED(ocs_ecc.lock), +}; + +/* Get OCS ECC tfm context from kpp_request. */ +static inline struct ocs_ecc_ctx *kmb_ocs_ecc_tctx(struct kpp_request *req) +{ + return kpp_tfm_ctx(crypto_kpp_reqtfm(req)); +} + +/* Converts number of digits to number of bytes. */ +static inline unsigned int digits_to_bytes(unsigned int n) +{ + return n << ECC_DIGITS_TO_BYTES_SHIFT; +} + +/* + * Wait for ECC idle i.e when an operation (other than write operations) + * is done. + */ +static inline int ocs_ecc_wait_idle(struct ocs_ecc_dev *dev) +{ + u32 value; + + return readl_poll_timeout((dev->base_reg + HW_OFFS_OCS_ECC_STATUS), + value, + !(value & HW_OCS_ECC_ISR_INT_STATUS_DONE), + POLL_USEC, TIMEOUT_USEC); +} + +static void ocs_ecc_cmd_start(struct ocs_ecc_dev *ecc_dev, u32 op_size) +{ + iowrite32(op_size | HW_OCS_ECC_COMMAND_START_VAL, + ecc_dev->base_reg + HW_OFFS_OCS_ECC_COMMAND); +} + +/* Direct write of u32 buffer to ECC engine with associated instruction. */ +static void ocs_ecc_write_cmd_and_data(struct ocs_ecc_dev *dev, + u32 op_size, + u32 inst, + const void *data_in, + size_t data_size) +{ + iowrite32(op_size | inst, dev->base_reg + HW_OFFS_OCS_ECC_COMMAND); + + /* MMIO Write src uint32 to dst. */ + memcpy_toio(dev->base_reg + HW_OFFS_OCS_ECC_DATA_IN, data_in, + data_size); +} + +/* Start OCS ECC operation and wait for its completion. */ +static int ocs_ecc_trigger_op(struct ocs_ecc_dev *ecc_dev, u32 op_size, + u32 inst) +{ + reinit_completion(&ecc_dev->irq_done); + + iowrite32(ECC_ENABLE_INTR, ecc_dev->base_reg + HW_OFFS_OCS_ECC_IER); + iowrite32(op_size | inst, ecc_dev->base_reg + HW_OFFS_OCS_ECC_COMMAND); + + return wait_for_completion_interruptible(&ecc_dev->irq_done); +} + +/** + * ocs_ecc_read_cx_out() - Read the CX data output buffer. + * @dev: The OCS ECC device to read from. + * @cx_out: The buffer where to store the CX value. Must be at least + * @byte_count byte long. + * @byte_count: The amount of data to read. + */ +static inline void ocs_ecc_read_cx_out(struct ocs_ecc_dev *dev, void *cx_out, + size_t byte_count) +{ + memcpy_fromio(cx_out, dev->base_reg + HW_OFFS_OCS_ECC_CX_DATA_OUT, + byte_count); +} + +/** + * ocs_ecc_read_cy_out() - Read the CX data output buffer. + * @dev: The OCS ECC device to read from. + * @cy_out: The buffer where to store the CY value. Must be at least + * @byte_count byte long. + * @byte_count: The amount of data to read. + */ +static inline void ocs_ecc_read_cy_out(struct ocs_ecc_dev *dev, void *cy_out, + size_t byte_count) +{ + memcpy_fromio(cy_out, dev->base_reg + HW_OFFS_OCS_ECC_CY_DATA_OUT, + byte_count); +} + +static struct ocs_ecc_dev *kmb_ocs_ecc_find_dev(struct ocs_ecc_ctx *tctx) +{ + if (tctx->ecc_dev) + return tctx->ecc_dev; + + spin_lock(&ocs_ecc.lock); + + /* Only a single OCS device available. */ + tctx->ecc_dev = list_first_entry(&ocs_ecc.dev_list, struct ocs_ecc_dev, + list); + + spin_unlock(&ocs_ecc.lock); + + return tctx->ecc_dev; +} + +/* Do point multiplication using OCS ECC HW. */ +static int kmb_ecc_point_mult(struct ocs_ecc_dev *ecc_dev, + struct ecc_point *result, + const struct ecc_point *point, + u64 *scalar, + const struct ecc_curve *curve) +{ + u8 sca[KMB_ECC_VLI_MAX_BYTES]; /* Use the maximum data size. */ + u32 op_size = (curve->g.ndigits > ECC_CURVE_NIST_P256_DIGITS) ? + OCS_ECC_OP_SIZE_384 : OCS_ECC_OP_SIZE_256; + size_t nbytes = digits_to_bytes(curve->g.ndigits); + int rc = 0; + + /* Generate random nbytes for Simple and Differential SCA protection. */ + rc = crypto_get_default_rng(); + if (rc) + return rc; + + rc = crypto_rng_get_bytes(crypto_default_rng, sca, nbytes); + crypto_put_default_rng(); + if (rc) + return rc; + + /* Wait engine to be idle before starting new operation. */ + rc = ocs_ecc_wait_idle(ecc_dev); + if (rc) + return rc; + + /* Send ecc_start pulse as well as indicating operation size. */ + ocs_ecc_cmd_start(ecc_dev, op_size); + + /* Write ax param; Base point (Gx). */ + ocs_ecc_write_cmd_and_data(ecc_dev, op_size, OCS_ECC_INST_WRITE_AX, + point->x, nbytes); + + /* Write ay param; Base point (Gy). */ + ocs_ecc_write_cmd_and_data(ecc_dev, op_size, OCS_ECC_INST_WRITE_AY, + point->y, nbytes); + + /* + * Write the private key into DATA_IN reg. + * + * Since DATA_IN register is used to write different values during the + * computation private Key value is overwritten with + * side-channel-resistance value. + */ + ocs_ecc_write_cmd_and_data(ecc_dev, op_size, OCS_ECC_INST_WRITE_BX_D, + scalar, nbytes); + + /* Write operand by/l. */ + ocs_ecc_write_cmd_and_data(ecc_dev, op_size, OCS_ECC_INST_WRITE_BY_L, + sca, nbytes); + memzero_explicit(sca, sizeof(sca)); + + /* Write p = curve prime(GF modulus). */ + ocs_ecc_write_cmd_and_data(ecc_dev, op_size, OCS_ECC_INST_WRITE_P, + curve->p, nbytes); + + /* Write a = curve coefficient. */ + ocs_ecc_write_cmd_and_data(ecc_dev, op_size, OCS_ECC_INST_WRITE_A, + curve->a, nbytes); + + /* Make hardware perform the multiplication. */ + rc = ocs_ecc_trigger_op(ecc_dev, op_size, OCS_ECC_INST_CALC_D_IDX_A); + if (rc) + return rc; + + /* Read result. */ + ocs_ecc_read_cx_out(ecc_dev, result->x, nbytes); + ocs_ecc_read_cy_out(ecc_dev, result->y, nbytes); + + return 0; +} + +/** + * kmb_ecc_do_scalar_op() - Perform Scalar operation using OCS ECC HW. + * @ecc_dev: The OCS ECC device to use. + * @scalar_out: Where to store the output scalar. + * @scalar_a: Input scalar operand 'a'. + * @scalar_b: Input scalar operand 'b' + * @curve: The curve on which the operation is performed. + * @ndigits: The size of the operands (in digits). + * @inst: The operation to perform (as an OCS ECC instruction). + * + * Return: 0 on success, negative error code otherwise. + */ +static int kmb_ecc_do_scalar_op(struct ocs_ecc_dev *ecc_dev, u64 *scalar_out, + const u64 *scalar_a, const u64 *scalar_b, + const struct ecc_curve *curve, + unsigned int ndigits, const u32 inst) +{ + u32 op_size = (ndigits > ECC_CURVE_NIST_P256_DIGITS) ? + OCS_ECC_OP_SIZE_384 : OCS_ECC_OP_SIZE_256; + size_t nbytes = digits_to_bytes(ndigits); + int rc; + + /* Wait engine to be idle before starting new operation. */ + rc = ocs_ecc_wait_idle(ecc_dev); + if (rc) + return rc; + + /* Send ecc_start pulse as well as indicating operation size. */ + ocs_ecc_cmd_start(ecc_dev, op_size); + + /* Write ax param (Base point (Gx).*/ + ocs_ecc_write_cmd_and_data(ecc_dev, op_size, OCS_ECC_INST_WRITE_AX, + scalar_a, nbytes); + + /* Write ay param Base point (Gy).*/ + ocs_ecc_write_cmd_and_data(ecc_dev, op_size, OCS_ECC_INST_WRITE_AY, + scalar_b, nbytes); + + /* Write p = curve prime(GF modulus).*/ + ocs_ecc_write_cmd_and_data(ecc_dev, op_size, OCS_ECC_INST_WRITE_P, + curve->p, nbytes); + + /* Give instruction A.B or A+B to ECC engine. */ + rc = ocs_ecc_trigger_op(ecc_dev, op_size, inst); + if (rc) + return rc; + + ocs_ecc_read_cx_out(ecc_dev, scalar_out, nbytes); + + if (vli_is_zero(scalar_out, ndigits)) + return -EINVAL; + + return 0; +} + +/* SP800-56A section 5.6.2.3.4 partial verification: ephemeral keys only */ +static int kmb_ocs_ecc_is_pubkey_valid_partial(struct ocs_ecc_dev *ecc_dev, + const struct ecc_curve *curve, + struct ecc_point *pk) +{ + u64 xxx[KMB_ECC_VLI_MAX_DIGITS] = { 0 }; + u64 yy[KMB_ECC_VLI_MAX_DIGITS] = { 0 }; + u64 w[KMB_ECC_VLI_MAX_DIGITS] = { 0 }; + int rc; + + if (WARN_ON(pk->ndigits != curve->g.ndigits)) + return -EINVAL; + + /* Check 1: Verify key is not the zero point. */ + if (ecc_point_is_zero(pk)) + return -EINVAL; + + /* Check 2: Verify key is in the range [0, p-1]. */ + if (vli_cmp(curve->p, pk->x, pk->ndigits) != 1) + return -EINVAL; + + if (vli_cmp(curve->p, pk->y, pk->ndigits) != 1) + return -EINVAL; + + /* Check 3: Verify that y^2 == (x^3 + a·x + b) mod p */ + + /* y^2 */ + /* Compute y^2 -> store in yy */ + rc = kmb_ecc_do_scalar_op(ecc_dev, yy, pk->y, pk->y, curve, pk->ndigits, + OCS_ECC_INST_CALC_A_MUL_B_MODP); + if (rc) + goto exit; + + /* x^3 */ + /* Assigning w = 3, used for calculating x^3. */ + w[0] = POW_CUBE; + /* Load the next stage.*/ + rc = kmb_ecc_do_scalar_op(ecc_dev, xxx, pk->x, w, curve, pk->ndigits, + OCS_ECC_INST_CALC_A_POW_B_MODP); + if (rc) + goto exit; + + /* Do a*x -> store in w. */ + rc = kmb_ecc_do_scalar_op(ecc_dev, w, curve->a, pk->x, curve, + pk->ndigits, + OCS_ECC_INST_CALC_A_MUL_B_MODP); + if (rc) + goto exit; + + /* Do ax + b == w + b; store in w. */ + rc = kmb_ecc_do_scalar_op(ecc_dev, w, w, curve->b, curve, + pk->ndigits, + OCS_ECC_INST_CALC_A_ADD_B_MODP); + if (rc) + goto exit; + + /* x^3 + ax + b == x^3 + w -> store in w. */ + rc = kmb_ecc_do_scalar_op(ecc_dev, w, xxx, w, curve, pk->ndigits, + OCS_ECC_INST_CALC_A_ADD_B_MODP); + if (rc) + goto exit; + + /* Compare y^2 == x^3 + a·x + b. */ + rc = vli_cmp(yy, w, pk->ndigits); + if (rc) + rc = -EINVAL; + +exit: + memzero_explicit(xxx, sizeof(xxx)); + memzero_explicit(yy, sizeof(yy)); + memzero_explicit(w, sizeof(w)); + + return rc; +} + +/* SP800-56A section 5.6.2.3.3 full verification */ +static int kmb_ocs_ecc_is_pubkey_valid_full(struct ocs_ecc_dev *ecc_dev, + const struct ecc_curve *curve, + struct ecc_point *pk) +{ + struct ecc_point *nQ; + int rc; + + /* Checks 1 through 3 */ + rc = kmb_ocs_ecc_is_pubkey_valid_partial(ecc_dev, curve, pk); + if (rc) + return rc; + + /* Check 4: Verify that nQ is the zero point. */ + nQ = ecc_alloc_point(pk->ndigits); + if (!nQ) + return -ENOMEM; + + rc = kmb_ecc_point_mult(ecc_dev, nQ, pk, curve->n, curve); + if (rc) + goto exit; + + if (!ecc_point_is_zero(nQ)) + rc = -EINVAL; + +exit: + ecc_free_point(nQ); + + return rc; +} + +static int kmb_ecc_is_key_valid(const struct ecc_curve *curve, + const u64 *private_key, size_t private_key_len) +{ + size_t ndigits = curve->g.ndigits; + u64 one[KMB_ECC_VLI_MAX_DIGITS] = {1}; + u64 res[KMB_ECC_VLI_MAX_DIGITS]; + + if (private_key_len != digits_to_bytes(ndigits)) + return -EINVAL; + + if (!private_key) + return -EINVAL; + + /* Make sure the private key is in the range [2, n-3]. */ + if (vli_cmp(one, private_key, ndigits) != -1) + return -EINVAL; + + vli_sub(res, curve->n, one, ndigits); + vli_sub(res, res, one, ndigits); + if (vli_cmp(res, private_key, ndigits) != 1) + return -EINVAL; + + return 0; +} + +/* + * ECC private keys are generated using the method of extra random bits, + * equivalent to that described in FIPS 186-4, Appendix B.4.1. + * + * d = (c mod(n–1)) + 1 where c is a string of random bits, 64 bits longer + * than requested + * 0 <= c mod(n-1) <= n-2 and implies that + * 1 <= d <= n-1 + * + * This method generates a private key uniformly distributed in the range + * [1, n-1]. + */ +static int kmb_ecc_gen_privkey(const struct ecc_curve *curve, u64 *privkey) +{ + size_t nbytes = digits_to_bytes(curve->g.ndigits); + u64 priv[KMB_ECC_VLI_MAX_DIGITS]; + size_t nbits; + int rc; + + nbits = vli_num_bits(curve->n, curve->g.ndigits); + + /* Check that N is included in Table 1 of FIPS 186-4, section 6.1.1 */ + if (nbits < 160 || curve->g.ndigits > ARRAY_SIZE(priv)) + return -EINVAL; + + /* + * FIPS 186-4 recommends that the private key should be obtained from a + * RBG with a security strength equal to or greater than the security + * strength associated with N. + * + * The maximum security strength identified by NIST SP800-57pt1r4 for + * ECC is 256 (N >= 512). + * + * This condition is met by the default RNG because it selects a favored + * DRBG with a security strength of 256. + */ + if (crypto_get_default_rng()) + return -EFAULT; + + rc = crypto_rng_get_bytes(crypto_default_rng, (u8 *)priv, nbytes); + crypto_put_default_rng(); + if (rc) + goto cleanup; + + rc = kmb_ecc_is_key_valid(curve, priv, nbytes); + if (rc) + goto cleanup; + + ecc_swap_digits(priv, privkey, curve->g.ndigits); + +cleanup: + memzero_explicit(&priv, sizeof(priv)); + + return rc; +} + +static int kmb_ocs_ecdh_set_secret(struct crypto_kpp *tfm, const void *buf, + unsigned int len) +{ + struct ocs_ecc_ctx *tctx = kpp_tfm_ctx(tfm); + struct ecdh params; + int rc = 0; + + rc = crypto_ecdh_decode_key(buf, len, ¶ms); + if (rc) + goto cleanup; + + /* Ensure key size is not bigger then expected. */ + if (params.key_size > digits_to_bytes(tctx->curve->g.ndigits)) { + rc = -EINVAL; + goto cleanup; + } + + /* Auto-generate private key is not provided. */ + if (!params.key || !params.key_size) { + rc = kmb_ecc_gen_privkey(tctx->curve, tctx->private_key); + goto cleanup; + } + + rc = kmb_ecc_is_key_valid(tctx->curve, (const u64 *)params.key, + params.key_size); + if (rc) + goto cleanup; + + ecc_swap_digits((const u64 *)params.key, tctx->private_key, + tctx->curve->g.ndigits); +cleanup: + memzero_explicit(¶ms, sizeof(params)); + + if (rc) + tctx->curve = NULL; + + return rc; +} + +/* Compute shared secret. */ +static int kmb_ecc_do_shared_secret(struct ocs_ecc_ctx *tctx, + struct kpp_request *req) +{ + struct ocs_ecc_dev *ecc_dev = tctx->ecc_dev; + const struct ecc_curve *curve = tctx->curve; + u64 shared_secret[KMB_ECC_VLI_MAX_DIGITS]; + u64 pubk_buf[KMB_ECC_VLI_MAX_DIGITS * 2]; + size_t copied, nbytes, pubk_len; + struct ecc_point *pk, *result; + int rc; + + nbytes = digits_to_bytes(curve->g.ndigits); + + /* Public key is a point, thus it has two coordinates */ + pubk_len = 2 * nbytes; + + /* Copy public key from SG list to pubk_buf. */ + copied = sg_copy_to_buffer(req->src, + sg_nents_for_len(req->src, pubk_len), + pubk_buf, pubk_len); + if (copied != pubk_len) + return -EINVAL; + + /* Allocate and initialize public key point. */ + pk = ecc_alloc_point(curve->g.ndigits); + if (!pk) + return -ENOMEM; + + ecc_swap_digits(pubk_buf, pk->x, curve->g.ndigits); + ecc_swap_digits(&pubk_buf[curve->g.ndigits], pk->y, curve->g.ndigits); + + /* + * Check the public key for following + * Check 1: Verify key is not the zero point. + * Check 2: Verify key is in the range [1, p-1]. + * Check 3: Verify that y^2 == (x^3 + a·x + b) mod p + */ + rc = kmb_ocs_ecc_is_pubkey_valid_partial(ecc_dev, curve, pk); + if (rc) + goto exit_free_pk; + + /* Allocate point for storing computed shared secret. */ + result = ecc_alloc_point(pk->ndigits); + if (!result) { + rc = -ENOMEM; + goto exit_free_pk; + } + + /* Calculate the shared secret.*/ + rc = kmb_ecc_point_mult(ecc_dev, result, pk, tctx->private_key, curve); + if (rc) + goto exit_free_result; + + if (ecc_point_is_zero(result)) { + rc = -EFAULT; + goto exit_free_result; + } + + /* Copy shared secret from point to buffer. */ + ecc_swap_digits(result->x, shared_secret, result->ndigits); + + /* Request might ask for less bytes than what we have. */ + nbytes = min_t(size_t, nbytes, req->dst_len); + + copied = sg_copy_from_buffer(req->dst, + sg_nents_for_len(req->dst, nbytes), + shared_secret, nbytes); + + if (copied != nbytes) + rc = -EINVAL; + + memzero_explicit(shared_secret, sizeof(shared_secret)); + +exit_free_result: + ecc_free_point(result); + +exit_free_pk: + ecc_free_point(pk); + + return rc; +} + +/* Compute public key. */ +static int kmb_ecc_do_public_key(struct ocs_ecc_ctx *tctx, + struct kpp_request *req) +{ + const struct ecc_curve *curve = tctx->curve; + u64 pubk_buf[KMB_ECC_VLI_MAX_DIGITS * 2]; + struct ecc_point *pk; + size_t pubk_len; + size_t copied; + int rc; + + /* Public key is a point, so it has double the digits. */ + pubk_len = 2 * digits_to_bytes(curve->g.ndigits); + + pk = ecc_alloc_point(curve->g.ndigits); + if (!pk) + return -ENOMEM; + + /* Public Key(pk) = priv * G. */ + rc = kmb_ecc_point_mult(tctx->ecc_dev, pk, &curve->g, tctx->private_key, + curve); + if (rc) + goto exit; + + /* SP800-56A rev 3 5.6.2.1.3 key check */ + if (kmb_ocs_ecc_is_pubkey_valid_full(tctx->ecc_dev, curve, pk)) { + rc = -EAGAIN; + goto exit; + } + + /* Copy public key from point to buffer. */ + ecc_swap_digits(pk->x, pubk_buf, pk->ndigits); + ecc_swap_digits(pk->y, &pubk_buf[pk->ndigits], pk->ndigits); + + /* Copy public key to req->dst. */ + copied = sg_copy_from_buffer(req->dst, + sg_nents_for_len(req->dst, pubk_len), + pubk_buf, pubk_len); + + if (copied != pubk_len) + rc = -EINVAL; + +exit: + ecc_free_point(pk); + + return rc; +} + +static int kmb_ocs_ecc_do_one_request(struct crypto_engine *engine, + void *areq) +{ + struct kpp_request *req = container_of(areq, struct kpp_request, base); + struct ocs_ecc_ctx *tctx = kmb_ocs_ecc_tctx(req); + struct ocs_ecc_dev *ecc_dev = tctx->ecc_dev; + int rc; + + if (req->src) + rc = kmb_ecc_do_shared_secret(tctx, req); + else + rc = kmb_ecc_do_public_key(tctx, req); + + crypto_finalize_kpp_request(ecc_dev->engine, req, rc); + + return 0; +} + +static int kmb_ocs_ecdh_generate_public_key(struct kpp_request *req) +{ + struct ocs_ecc_ctx *tctx = kmb_ocs_ecc_tctx(req); + const struct ecc_curve *curve = tctx->curve; + + /* Ensure kmb_ocs_ecdh_set_secret() has been successfully called. */ + if (!tctx->curve) + return -EINVAL; + + /* Ensure dst is present. */ + if (!req->dst) + return -EINVAL; + + /* Check the request dst is big enough to hold the public key. */ + if (req->dst_len < (2 * digits_to_bytes(curve->g.ndigits))) + return -EINVAL; + + /* 'src' is not supposed to be present when generate pubk is called. */ + if (req->src) + return -EINVAL; + + return crypto_transfer_kpp_request_to_engine(tctx->ecc_dev->engine, + req); +} + +static int kmb_ocs_ecdh_compute_shared_secret(struct kpp_request *req) +{ + struct ocs_ecc_ctx *tctx = kmb_ocs_ecc_tctx(req); + const struct ecc_curve *curve = tctx->curve; + + /* Ensure kmb_ocs_ecdh_set_secret() has been successfully called. */ + if (!tctx->curve) + return -EINVAL; + + /* Ensure dst is present. */ + if (!req->dst) + return -EINVAL; + + /* Ensure src is present. */ + if (!req->src) + return -EINVAL; + + /* + * req->src is expected to the (other-side) public key, so its length + * must be 2 * coordinate size (in bytes). + */ + if (req->src_len != 2 * digits_to_bytes(curve->g.ndigits)) + return -EINVAL; + + return crypto_transfer_kpp_request_to_engine(tctx->ecc_dev->engine, + req); +} + +static int kmb_ecc_tctx_init(struct ocs_ecc_ctx *tctx, unsigned int curve_id) +{ + memset(tctx, 0, sizeof(*tctx)); + + tctx->ecc_dev = kmb_ocs_ecc_find_dev(tctx); + + if (IS_ERR(tctx->ecc_dev)) { + pr_err("Failed to find the device : %ld\n", + PTR_ERR(tctx->ecc_dev)); + return PTR_ERR(tctx->ecc_dev); + } + + tctx->curve = ecc_get_curve(curve_id); + if (!tctx->curve) + return -EOPNOTSUPP; + + tctx->engine_ctx.op.prepare_request = NULL; + tctx->engine_ctx.op.do_one_request = kmb_ocs_ecc_do_one_request; + tctx->engine_ctx.op.unprepare_request = NULL; + + return 0; +} + +static int kmb_ocs_ecdh_nist_p256_init_tfm(struct crypto_kpp *tfm) +{ + struct ocs_ecc_ctx *tctx = kpp_tfm_ctx(tfm); + + return kmb_ecc_tctx_init(tctx, ECC_CURVE_NIST_P256); +} + +static int kmb_ocs_ecdh_nist_p384_init_tfm(struct crypto_kpp *tfm) +{ + struct ocs_ecc_ctx *tctx = kpp_tfm_ctx(tfm); + + return kmb_ecc_tctx_init(tctx, ECC_CURVE_NIST_P384); +} + +static void kmb_ocs_ecdh_exit_tfm(struct crypto_kpp *tfm) +{ + struct ocs_ecc_ctx *tctx = kpp_tfm_ctx(tfm); + + memzero_explicit(tctx->private_key, sizeof(*tctx->private_key)); +} + +static unsigned int kmb_ocs_ecdh_max_size(struct crypto_kpp *tfm) +{ + struct ocs_ecc_ctx *tctx = kpp_tfm_ctx(tfm); + + /* Public key is made of two coordinates, so double the digits. */ + return digits_to_bytes(tctx->curve->g.ndigits) * 2; +} + +static struct kpp_alg ocs_ecdh_p256 = { + .set_secret = kmb_ocs_ecdh_set_secret, + .generate_public_key = kmb_ocs_ecdh_generate_public_key, + .compute_shared_secret = kmb_ocs_ecdh_compute_shared_secret, + .init = kmb_ocs_ecdh_nist_p256_init_tfm, + .exit = kmb_ocs_ecdh_exit_tfm, + .max_size = kmb_ocs_ecdh_max_size, + .base = { + .cra_name = "ecdh-nist-p256", + .cra_driver_name = "ecdh-nist-p256-keembay-ocs", + .cra_priority = KMB_OCS_ECC_PRIORITY, + .cra_module = THIS_MODULE, + .cra_ctxsize = sizeof(struct ocs_ecc_ctx), + }, +}; + +static struct kpp_alg ocs_ecdh_p384 = { + .set_secret = kmb_ocs_ecdh_set_secret, + .generate_public_key = kmb_ocs_ecdh_generate_public_key, + .compute_shared_secret = kmb_ocs_ecdh_compute_shared_secret, + .init = kmb_ocs_ecdh_nist_p384_init_tfm, + .exit = kmb_ocs_ecdh_exit_tfm, + .max_size = kmb_ocs_ecdh_max_size, + .base = { + .cra_name = "ecdh-nist-p384", + .cra_driver_name = "ecdh-nist-p384-keembay-ocs", + .cra_priority = KMB_OCS_ECC_PRIORITY, + .cra_module = THIS_MODULE, + .cra_ctxsize = sizeof(struct ocs_ecc_ctx), + }, +}; + +static irqreturn_t ocs_ecc_irq_handler(int irq, void *dev_id) +{ + struct ocs_ecc_dev *ecc_dev = dev_id; + u32 status; + + /* + * Read the status register and write it back to clear the + * DONE_INT_STATUS bit. + */ + status = ioread32(ecc_dev->base_reg + HW_OFFS_OCS_ECC_ISR); + iowrite32(status, ecc_dev->base_reg + HW_OFFS_OCS_ECC_ISR); + + if (!(status & HW_OCS_ECC_ISR_INT_STATUS_DONE)) + return IRQ_NONE; + + complete(&ecc_dev->irq_done); + + return IRQ_HANDLED; +} + +static int kmb_ocs_ecc_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct ocs_ecc_dev *ecc_dev; + int rc; + + ecc_dev = devm_kzalloc(dev, sizeof(*ecc_dev), GFP_KERNEL); + if (!ecc_dev) + return -ENOMEM; + + ecc_dev->dev = dev; + + platform_set_drvdata(pdev, ecc_dev); + + INIT_LIST_HEAD(&ecc_dev->list); + init_completion(&ecc_dev->irq_done); + + /* Get base register address. */ + ecc_dev->base_reg = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(ecc_dev->base_reg)) { + dev_err(dev, "Failed to get base address\n"); + rc = PTR_ERR(ecc_dev->base_reg); + goto list_del; + } + + /* Get and request IRQ */ + ecc_dev->irq = platform_get_irq(pdev, 0); + if (ecc_dev->irq < 0) { + rc = ecc_dev->irq; + goto list_del; + } + + rc = devm_request_threaded_irq(dev, ecc_dev->irq, ocs_ecc_irq_handler, + NULL, 0, "keembay-ocs-ecc", ecc_dev); + if (rc < 0) { + dev_err(dev, "Could not request IRQ\n"); + goto list_del; + } + + /* Add device to the list of OCS ECC devices. */ + spin_lock(&ocs_ecc.lock); + list_add_tail(&ecc_dev->list, &ocs_ecc.dev_list); + spin_unlock(&ocs_ecc.lock); + + /* Initialize crypto engine. */ + ecc_dev->engine = crypto_engine_alloc_init(dev, 1); + if (!ecc_dev->engine) { + dev_err(dev, "Could not allocate crypto engine\n"); + rc = -ENOMEM; + goto list_del; + } + + rc = crypto_engine_start(ecc_dev->engine); + if (rc) { + dev_err(dev, "Could not start crypto engine\n"); + goto cleanup; + } + + /* Register the KPP algo. */ + rc = crypto_register_kpp(&ocs_ecdh_p256); + if (rc) { + dev_err(dev, + "Could not register OCS algorithms with Crypto API\n"); + goto cleanup; + } + + rc = crypto_register_kpp(&ocs_ecdh_p384); + if (rc) { + dev_err(dev, + "Could not register OCS algorithms with Crypto API\n"); + goto ocs_ecdh_p384_error; + } + + return 0; + +ocs_ecdh_p384_error: + crypto_unregister_kpp(&ocs_ecdh_p256); + +cleanup: + crypto_engine_exit(ecc_dev->engine); + +list_del: + spin_lock(&ocs_ecc.lock); + list_del(&ecc_dev->list); + spin_unlock(&ocs_ecc.lock); + + return rc; +} + +static int kmb_ocs_ecc_remove(struct platform_device *pdev) +{ + struct ocs_ecc_dev *ecc_dev; + + ecc_dev = platform_get_drvdata(pdev); + + crypto_unregister_kpp(&ocs_ecdh_p384); + crypto_unregister_kpp(&ocs_ecdh_p256); + + spin_lock(&ocs_ecc.lock); + list_del(&ecc_dev->list); + spin_unlock(&ocs_ecc.lock); + + crypto_engine_exit(ecc_dev->engine); + + return 0; +} + +/* Device tree driver match. */ +static const struct of_device_id kmb_ocs_ecc_of_match[] = { + { + .compatible = "intel,keembay-ocs-ecc", + }, + {} +}; + +/* The OCS driver is a platform device. */ +static struct platform_driver kmb_ocs_ecc_driver = { + .probe = kmb_ocs_ecc_probe, + .remove = kmb_ocs_ecc_remove, + .driver = { + .name = DRV_NAME, + .of_match_table = kmb_ocs_ecc_of_match, + }, +}; +module_platform_driver(kmb_ocs_ecc_driver); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("Intel Keem Bay OCS ECC Driver"); +MODULE_ALIAS_CRYPTO("ecdh-nist-p256"); +MODULE_ALIAS_CRYPTO("ecdh-nist-p384"); +MODULE_ALIAS_CRYPTO("ecdh-nist-p256-keembay-ocs"); +MODULE_ALIAS_CRYPTO("ecdh-nist-p384-keembay-ocs"); diff --git a/drivers/crypto/intel/keembay/keembay-ocs-hcu-core.c b/drivers/crypto/intel/keembay/keembay-ocs-hcu-core.c new file mode 100644 index 000000000000..d4bcbed1f546 --- /dev/null +++ b/drivers/crypto/intel/keembay/keembay-ocs-hcu-core.c @@ -0,0 +1,1264 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Intel Keem Bay OCS HCU Crypto Driver. + * + * Copyright (C) 2018-2020 Intel Corporation + */ + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "ocs-hcu.h" + +#define DRV_NAME "keembay-ocs-hcu" + +/* Flag marking a final request. */ +#define REQ_FINAL BIT(0) +/* Flag marking a HMAC request. */ +#define REQ_FLAGS_HMAC BIT(1) +/* Flag set when HW HMAC is being used. */ +#define REQ_FLAGS_HMAC_HW BIT(2) +/* Flag set when SW HMAC is being used. */ +#define REQ_FLAGS_HMAC_SW BIT(3) + +/** + * struct ocs_hcu_ctx: OCS HCU Transform context. + * @engine_ctx: Crypto Engine context. + * @hcu_dev: The OCS HCU device used by the transformation. + * @key: The key (used only for HMAC transformations). + * @key_len: The length of the key. + * @is_sm3_tfm: Whether or not this is an SM3 transformation. + * @is_hmac_tfm: Whether or not this is a HMAC transformation. + */ +struct ocs_hcu_ctx { + struct crypto_engine_ctx engine_ctx; + struct ocs_hcu_dev *hcu_dev; + u8 key[SHA512_BLOCK_SIZE]; + size_t key_len; + bool is_sm3_tfm; + bool is_hmac_tfm; +}; + +/** + * struct ocs_hcu_rctx - Context for the request. + * @hcu_dev: OCS HCU device to be used to service the request. + * @flags: Flags tracking request status. + * @algo: Algorithm to use for the request. + * @blk_sz: Block size of the transformation / request. + * @dig_sz: Digest size of the transformation / request. + * @dma_list: OCS DMA linked list. + * @hash_ctx: OCS HCU hashing context. + * @buffer: Buffer to store: partial block of data and SW HMAC + * artifacts (ipad, opad, etc.). + * @buf_cnt: Number of bytes currently stored in the buffer. + * @buf_dma_addr: The DMA address of @buffer (when mapped). + * @buf_dma_count: The number of bytes in @buffer currently DMA-mapped. + * @sg: Head of the scatterlist entries containing data. + * @sg_data_total: Total data in the SG list at any time. + * @sg_data_offset: Offset into the data of the current individual SG node. + * @sg_dma_nents: Number of sg entries mapped in dma_list. + */ +struct ocs_hcu_rctx { + struct ocs_hcu_dev *hcu_dev; + u32 flags; + enum ocs_hcu_algo algo; + size_t blk_sz; + size_t dig_sz; + struct ocs_hcu_dma_list *dma_list; + struct ocs_hcu_hash_ctx hash_ctx; + /* + * Buffer is double the block size because we need space for SW HMAC + * artifacts, i.e: + * - ipad (1 block) + a possible partial block of data. + * - opad (1 block) + digest of H(k ^ ipad || m) + */ + u8 buffer[2 * SHA512_BLOCK_SIZE]; + size_t buf_cnt; + dma_addr_t buf_dma_addr; + size_t buf_dma_count; + struct scatterlist *sg; + unsigned int sg_data_total; + unsigned int sg_data_offset; + unsigned int sg_dma_nents; +}; + +/** + * struct ocs_hcu_drv - Driver data + * @dev_list: The list of HCU devices. + * @lock: The lock protecting dev_list. + */ +struct ocs_hcu_drv { + struct list_head dev_list; + spinlock_t lock; /* Protects dev_list. */ +}; + +static struct ocs_hcu_drv ocs_hcu = { + .dev_list = LIST_HEAD_INIT(ocs_hcu.dev_list), + .lock = __SPIN_LOCK_UNLOCKED(ocs_hcu.lock), +}; + +/* + * Return the total amount of data in the request; that is: the data in the + * request buffer + the data in the sg list. + */ +static inline unsigned int kmb_get_total_data(struct ocs_hcu_rctx *rctx) +{ + return rctx->sg_data_total + rctx->buf_cnt; +} + +/* Move remaining content of scatter-gather list to context buffer. */ +static int flush_sg_to_ocs_buffer(struct ocs_hcu_rctx *rctx) +{ + size_t count; + + if (rctx->sg_data_total > (sizeof(rctx->buffer) - rctx->buf_cnt)) { + WARN(1, "%s: sg data does not fit in buffer\n", __func__); + return -EINVAL; + } + + while (rctx->sg_data_total) { + if (!rctx->sg) { + WARN(1, "%s: unexpected NULL sg\n", __func__); + return -EINVAL; + } + /* + * If current sg has been fully processed, skip to the next + * one. + */ + if (rctx->sg_data_offset == rctx->sg->length) { + rctx->sg = sg_next(rctx->sg); + rctx->sg_data_offset = 0; + continue; + } + /* + * Determine the maximum data available to copy from the node. + * Minimum of the length left in the sg node, or the total data + * in the request. + */ + count = min(rctx->sg->length - rctx->sg_data_offset, + rctx->sg_data_total); + /* Copy from scatter-list entry to context buffer. */ + scatterwalk_map_and_copy(&rctx->buffer[rctx->buf_cnt], + rctx->sg, rctx->sg_data_offset, + count, 0); + + rctx->sg_data_offset += count; + rctx->sg_data_total -= count; + rctx->buf_cnt += count; + } + + return 0; +} + +static struct ocs_hcu_dev *kmb_ocs_hcu_find_dev(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct ocs_hcu_ctx *tctx = crypto_ahash_ctx(tfm); + + /* If the HCU device for the request was previously set, return it. */ + if (tctx->hcu_dev) + return tctx->hcu_dev; + + /* + * Otherwise, get the first HCU device available (there should be one + * and only one device). + */ + spin_lock_bh(&ocs_hcu.lock); + tctx->hcu_dev = list_first_entry_or_null(&ocs_hcu.dev_list, + struct ocs_hcu_dev, + list); + spin_unlock_bh(&ocs_hcu.lock); + + return tctx->hcu_dev; +} + +/* Free OCS DMA linked list and DMA-able context buffer. */ +static void kmb_ocs_hcu_dma_cleanup(struct ahash_request *req, + struct ocs_hcu_rctx *rctx) +{ + struct ocs_hcu_dev *hcu_dev = rctx->hcu_dev; + struct device *dev = hcu_dev->dev; + + /* Unmap rctx->buffer (if mapped). */ + if (rctx->buf_dma_count) { + dma_unmap_single(dev, rctx->buf_dma_addr, rctx->buf_dma_count, + DMA_TO_DEVICE); + rctx->buf_dma_count = 0; + } + + /* Unmap req->src (if mapped). */ + if (rctx->sg_dma_nents) { + dma_unmap_sg(dev, req->src, rctx->sg_dma_nents, DMA_TO_DEVICE); + rctx->sg_dma_nents = 0; + } + + /* Free dma_list (if allocated). */ + if (rctx->dma_list) { + ocs_hcu_dma_list_free(hcu_dev, rctx->dma_list); + rctx->dma_list = NULL; + } +} + +/* + * Prepare for DMA operation: + * - DMA-map request context buffer (if needed) + * - DMA-map SG list (only the entries to be processed, see note below) + * - Allocate OCS HCU DMA linked list (number of elements = SG entries to + * process + context buffer (if not empty)). + * - Add DMA-mapped request context buffer to OCS HCU DMA list. + * - Add SG entries to DMA list. + * + * Note: if this is a final request, we process all the data in the SG list, + * otherwise we can only process up to the maximum amount of block-aligned data + * (the remainder will be put into the context buffer and processed in the next + * request). + */ +static int kmb_ocs_dma_prepare(struct ahash_request *req) +{ + struct ocs_hcu_rctx *rctx = ahash_request_ctx_dma(req); + struct device *dev = rctx->hcu_dev->dev; + unsigned int remainder = 0; + unsigned int total; + size_t nents; + size_t count; + int rc; + int i; + + /* This function should be called only when there is data to process. */ + total = kmb_get_total_data(rctx); + if (!total) + return -EINVAL; + + /* + * If this is not a final DMA (terminated DMA), the data passed to the + * HCU must be aligned to the block size; compute the remainder data to + * be processed in the next request. + */ + if (!(rctx->flags & REQ_FINAL)) + remainder = total % rctx->blk_sz; + + /* Determine the number of scatter gather list entries to process. */ + nents = sg_nents_for_len(req->src, rctx->sg_data_total - remainder); + + /* If there are entries to process, map them. */ + if (nents) { + rctx->sg_dma_nents = dma_map_sg(dev, req->src, nents, + DMA_TO_DEVICE); + if (!rctx->sg_dma_nents) { + dev_err(dev, "Failed to MAP SG\n"); + rc = -ENOMEM; + goto cleanup; + } + /* + * The value returned by dma_map_sg() can be < nents; so update + * nents accordingly. + */ + nents = rctx->sg_dma_nents; + } + + /* + * If context buffer is not empty, map it and add extra DMA entry for + * it. + */ + if (rctx->buf_cnt) { + rctx->buf_dma_addr = dma_map_single(dev, rctx->buffer, + rctx->buf_cnt, + DMA_TO_DEVICE); + if (dma_mapping_error(dev, rctx->buf_dma_addr)) { + dev_err(dev, "Failed to map request context buffer\n"); + rc = -ENOMEM; + goto cleanup; + } + rctx->buf_dma_count = rctx->buf_cnt; + /* Increase number of dma entries. */ + nents++; + } + + /* Allocate OCS HCU DMA list. */ + rctx->dma_list = ocs_hcu_dma_list_alloc(rctx->hcu_dev, nents); + if (!rctx->dma_list) { + rc = -ENOMEM; + goto cleanup; + } + + /* Add request context buffer (if previously DMA-mapped) */ + if (rctx->buf_dma_count) { + rc = ocs_hcu_dma_list_add_tail(rctx->hcu_dev, rctx->dma_list, + rctx->buf_dma_addr, + rctx->buf_dma_count); + if (rc) + goto cleanup; + } + + /* Add the SG nodes to be processed to the DMA linked list. */ + for_each_sg(req->src, rctx->sg, rctx->sg_dma_nents, i) { + /* + * The number of bytes to add to the list entry is the minimum + * between: + * - The DMA length of the SG entry. + * - The data left to be processed. + */ + count = min(rctx->sg_data_total - remainder, + sg_dma_len(rctx->sg) - rctx->sg_data_offset); + /* + * Do not create a zero length DMA descriptor. Check in case of + * zero length SG node. + */ + if (count == 0) + continue; + /* Add sg to HCU DMA list. */ + rc = ocs_hcu_dma_list_add_tail(rctx->hcu_dev, + rctx->dma_list, + rctx->sg->dma_address, + count); + if (rc) + goto cleanup; + + /* Update amount of data remaining in SG list. */ + rctx->sg_data_total -= count; + + /* + * If remaining data is equal to remainder (note: 'less than' + * case should never happen in practice), we are done: update + * offset and exit the loop. + */ + if (rctx->sg_data_total <= remainder) { + WARN_ON(rctx->sg_data_total < remainder); + rctx->sg_data_offset += count; + break; + } + + /* + * If we get here is because we need to process the next sg in + * the list; set offset within the sg to 0. + */ + rctx->sg_data_offset = 0; + } + + return 0; +cleanup: + dev_err(dev, "Failed to prepare DMA.\n"); + kmb_ocs_hcu_dma_cleanup(req, rctx); + + return rc; +} + +static void kmb_ocs_hcu_secure_cleanup(struct ahash_request *req) +{ + struct ocs_hcu_rctx *rctx = ahash_request_ctx_dma(req); + + /* Clear buffer of any data. */ + memzero_explicit(rctx->buffer, sizeof(rctx->buffer)); +} + +static int kmb_ocs_hcu_handle_queue(struct ahash_request *req) +{ + struct ocs_hcu_dev *hcu_dev = kmb_ocs_hcu_find_dev(req); + + if (!hcu_dev) + return -ENOENT; + + return crypto_transfer_hash_request_to_engine(hcu_dev->engine, req); +} + +static int prepare_ipad(struct ahash_request *req) +{ + struct ocs_hcu_rctx *rctx = ahash_request_ctx_dma(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct ocs_hcu_ctx *ctx = crypto_ahash_ctx(tfm); + int i; + + WARN(rctx->buf_cnt, "%s: Context buffer is not empty\n", __func__); + WARN(!(rctx->flags & REQ_FLAGS_HMAC_SW), + "%s: HMAC_SW flag is not set\n", __func__); + /* + * Key length must be equal to block size. If key is shorter, + * we pad it with zero (note: key cannot be longer, since + * longer keys are hashed by kmb_ocs_hcu_setkey()). + */ + if (ctx->key_len > rctx->blk_sz) { + WARN(1, "%s: Invalid key length in tfm context\n", __func__); + return -EINVAL; + } + memzero_explicit(&ctx->key[ctx->key_len], + rctx->blk_sz - ctx->key_len); + ctx->key_len = rctx->blk_sz; + /* + * Prepare IPAD for HMAC. Only done for first block. + * HMAC(k,m) = H(k ^ opad || H(k ^ ipad || m)) + * k ^ ipad will be first hashed block. + * k ^ opad will be calculated in the final request. + * Only needed if not using HW HMAC. + */ + for (i = 0; i < rctx->blk_sz; i++) + rctx->buffer[i] = ctx->key[i] ^ HMAC_IPAD_VALUE; + rctx->buf_cnt = rctx->blk_sz; + + return 0; +} + +static int kmb_ocs_hcu_do_one_request(struct crypto_engine *engine, void *areq) +{ + struct ahash_request *req = container_of(areq, struct ahash_request, + base); + struct ocs_hcu_dev *hcu_dev = kmb_ocs_hcu_find_dev(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct ocs_hcu_rctx *rctx = ahash_request_ctx_dma(req); + struct ocs_hcu_ctx *tctx = crypto_ahash_ctx(tfm); + int rc; + int i; + + if (!hcu_dev) { + rc = -ENOENT; + goto error; + } + + /* + * If hardware HMAC flag is set, perform HMAC in hardware. + * + * NOTE: this flag implies REQ_FINAL && kmb_get_total_data(rctx) + */ + if (rctx->flags & REQ_FLAGS_HMAC_HW) { + /* Map input data into the HCU DMA linked list. */ + rc = kmb_ocs_dma_prepare(req); + if (rc) + goto error; + + rc = ocs_hcu_hmac(hcu_dev, rctx->algo, tctx->key, tctx->key_len, + rctx->dma_list, req->result, rctx->dig_sz); + + /* Unmap data and free DMA list regardless of return code. */ + kmb_ocs_hcu_dma_cleanup(req, rctx); + + /* Process previous return code. */ + if (rc) + goto error; + + goto done; + } + + /* Handle update request case. */ + if (!(rctx->flags & REQ_FINAL)) { + /* Update should always have input data. */ + if (!kmb_get_total_data(rctx)) + return -EINVAL; + + /* Map input data into the HCU DMA linked list. */ + rc = kmb_ocs_dma_prepare(req); + if (rc) + goto error; + + /* Do hashing step. */ + rc = ocs_hcu_hash_update(hcu_dev, &rctx->hash_ctx, + rctx->dma_list); + + /* Unmap data and free DMA list regardless of return code. */ + kmb_ocs_hcu_dma_cleanup(req, rctx); + + /* Process previous return code. */ + if (rc) + goto error; + + /* + * Reset request buffer count (data in the buffer was just + * processed). + */ + rctx->buf_cnt = 0; + /* + * Move remaining sg data into the request buffer, so that it + * will be processed during the next request. + * + * NOTE: we have remaining data if kmb_get_total_data() was not + * a multiple of block size. + */ + rc = flush_sg_to_ocs_buffer(rctx); + if (rc) + goto error; + + goto done; + } + + /* If we get here, this is a final request. */ + + /* If there is data to process, use finup. */ + if (kmb_get_total_data(rctx)) { + /* Map input data into the HCU DMA linked list. */ + rc = kmb_ocs_dma_prepare(req); + if (rc) + goto error; + + /* Do hashing step. */ + rc = ocs_hcu_hash_finup(hcu_dev, &rctx->hash_ctx, + rctx->dma_list, + req->result, rctx->dig_sz); + /* Free DMA list regardless of return code. */ + kmb_ocs_hcu_dma_cleanup(req, rctx); + + /* Process previous return code. */ + if (rc) + goto error; + + } else { /* Otherwise (if we have no data), use final. */ + rc = ocs_hcu_hash_final(hcu_dev, &rctx->hash_ctx, req->result, + rctx->dig_sz); + if (rc) + goto error; + } + + /* + * If we are finalizing a SW HMAC request, we just computed the result + * of: H(k ^ ipad || m). + * + * We now need to complete the HMAC calculation with the OPAD step, + * that is, we need to compute H(k ^ opad || digest), where digest is + * the digest we just obtained, i.e., H(k ^ ipad || m). + */ + if (rctx->flags & REQ_FLAGS_HMAC_SW) { + /* + * Compute k ^ opad and store it in the request buffer (which + * is not used anymore at this point). + * Note: key has been padded / hashed already (so keylen == + * blksz) . + */ + WARN_ON(tctx->key_len != rctx->blk_sz); + for (i = 0; i < rctx->blk_sz; i++) + rctx->buffer[i] = tctx->key[i] ^ HMAC_OPAD_VALUE; + /* Now append the digest to the rest of the buffer. */ + for (i = 0; (i < rctx->dig_sz); i++) + rctx->buffer[rctx->blk_sz + i] = req->result[i]; + + /* Now hash the buffer to obtain the final HMAC. */ + rc = ocs_hcu_digest(hcu_dev, rctx->algo, rctx->buffer, + rctx->blk_sz + rctx->dig_sz, req->result, + rctx->dig_sz); + if (rc) + goto error; + } + + /* Perform secure clean-up. */ + kmb_ocs_hcu_secure_cleanup(req); +done: + crypto_finalize_hash_request(hcu_dev->engine, req, 0); + + return 0; + +error: + kmb_ocs_hcu_secure_cleanup(req); + return rc; +} + +static int kmb_ocs_hcu_init(struct ahash_request *req) +{ + struct ocs_hcu_dev *hcu_dev = kmb_ocs_hcu_find_dev(req); + struct ocs_hcu_rctx *rctx = ahash_request_ctx_dma(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct ocs_hcu_ctx *ctx = crypto_ahash_ctx(tfm); + + if (!hcu_dev) + return -ENOENT; + + /* Initialize entire request context to zero. */ + memset(rctx, 0, sizeof(*rctx)); + + rctx->hcu_dev = hcu_dev; + rctx->dig_sz = crypto_ahash_digestsize(tfm); + + switch (rctx->dig_sz) { +#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_HCU_HMAC_SHA224 + case SHA224_DIGEST_SIZE: + rctx->blk_sz = SHA224_BLOCK_SIZE; + rctx->algo = OCS_HCU_ALGO_SHA224; + break; +#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_HCU_HMAC_SHA224 */ + case SHA256_DIGEST_SIZE: + rctx->blk_sz = SHA256_BLOCK_SIZE; + /* + * SHA256 and SM3 have the same digest size: use info from tfm + * context to find out which one we should use. + */ + rctx->algo = ctx->is_sm3_tfm ? OCS_HCU_ALGO_SM3 : + OCS_HCU_ALGO_SHA256; + break; + case SHA384_DIGEST_SIZE: + rctx->blk_sz = SHA384_BLOCK_SIZE; + rctx->algo = OCS_HCU_ALGO_SHA384; + break; + case SHA512_DIGEST_SIZE: + rctx->blk_sz = SHA512_BLOCK_SIZE; + rctx->algo = OCS_HCU_ALGO_SHA512; + break; + default: + return -EINVAL; + } + + /* Initialize intermediate data. */ + ocs_hcu_hash_init(&rctx->hash_ctx, rctx->algo); + + /* If this a HMAC request, set HMAC flag. */ + if (ctx->is_hmac_tfm) + rctx->flags |= REQ_FLAGS_HMAC; + + return 0; +} + +static int kmb_ocs_hcu_update(struct ahash_request *req) +{ + struct ocs_hcu_rctx *rctx = ahash_request_ctx_dma(req); + int rc; + + if (!req->nbytes) + return 0; + + rctx->sg_data_total = req->nbytes; + rctx->sg_data_offset = 0; + rctx->sg = req->src; + + /* + * If we are doing HMAC, then we must use SW-assisted HMAC, since HW + * HMAC does not support context switching (there it can only be used + * with finup() or digest()). + */ + if (rctx->flags & REQ_FLAGS_HMAC && + !(rctx->flags & REQ_FLAGS_HMAC_SW)) { + rctx->flags |= REQ_FLAGS_HMAC_SW; + rc = prepare_ipad(req); + if (rc) + return rc; + } + + /* + * If remaining sg_data fits into ctx buffer, just copy it there; we'll + * process it at the next update() or final(). + */ + if (rctx->sg_data_total <= (sizeof(rctx->buffer) - rctx->buf_cnt)) + return flush_sg_to_ocs_buffer(rctx); + + return kmb_ocs_hcu_handle_queue(req); +} + +/* Common logic for kmb_ocs_hcu_final() and kmb_ocs_hcu_finup(). */ +static int kmb_ocs_hcu_fin_common(struct ahash_request *req) +{ + struct ocs_hcu_rctx *rctx = ahash_request_ctx_dma(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct ocs_hcu_ctx *ctx = crypto_ahash_ctx(tfm); + int rc; + + rctx->flags |= REQ_FINAL; + + /* + * If this is a HMAC request and, so far, we didn't have to switch to + * SW HMAC, check if we can use HW HMAC. + */ + if (rctx->flags & REQ_FLAGS_HMAC && + !(rctx->flags & REQ_FLAGS_HMAC_SW)) { + /* + * If we are here, it means we never processed any data so far, + * so we can use HW HMAC, but only if there is some data to + * process (since OCS HW MAC does not support zero-length + * messages) and the key length is supported by the hardware + * (OCS HCU HW only supports length <= 64); if HW HMAC cannot + * be used, fall back to SW-assisted HMAC. + */ + if (kmb_get_total_data(rctx) && + ctx->key_len <= OCS_HCU_HW_KEY_LEN) { + rctx->flags |= REQ_FLAGS_HMAC_HW; + } else { + rctx->flags |= REQ_FLAGS_HMAC_SW; + rc = prepare_ipad(req); + if (rc) + return rc; + } + } + + return kmb_ocs_hcu_handle_queue(req); +} + +static int kmb_ocs_hcu_final(struct ahash_request *req) +{ + struct ocs_hcu_rctx *rctx = ahash_request_ctx_dma(req); + + rctx->sg_data_total = 0; + rctx->sg_data_offset = 0; + rctx->sg = NULL; + + return kmb_ocs_hcu_fin_common(req); +} + +static int kmb_ocs_hcu_finup(struct ahash_request *req) +{ + struct ocs_hcu_rctx *rctx = ahash_request_ctx_dma(req); + + rctx->sg_data_total = req->nbytes; + rctx->sg_data_offset = 0; + rctx->sg = req->src; + + return kmb_ocs_hcu_fin_common(req); +} + +static int kmb_ocs_hcu_digest(struct ahash_request *req) +{ + int rc = 0; + struct ocs_hcu_dev *hcu_dev = kmb_ocs_hcu_find_dev(req); + + if (!hcu_dev) + return -ENOENT; + + rc = kmb_ocs_hcu_init(req); + if (rc) + return rc; + + rc = kmb_ocs_hcu_finup(req); + + return rc; +} + +static int kmb_ocs_hcu_export(struct ahash_request *req, void *out) +{ + struct ocs_hcu_rctx *rctx = ahash_request_ctx_dma(req); + + /* Intermediate data is always stored and applied per request. */ + memcpy(out, rctx, sizeof(*rctx)); + + return 0; +} + +static int kmb_ocs_hcu_import(struct ahash_request *req, const void *in) +{ + struct ocs_hcu_rctx *rctx = ahash_request_ctx_dma(req); + + /* Intermediate data is always stored and applied per request. */ + memcpy(rctx, in, sizeof(*rctx)); + + return 0; +} + +static int kmb_ocs_hcu_setkey(struct crypto_ahash *tfm, const u8 *key, + unsigned int keylen) +{ + unsigned int digestsize = crypto_ahash_digestsize(tfm); + struct ocs_hcu_ctx *ctx = crypto_ahash_ctx(tfm); + size_t blk_sz = crypto_ahash_blocksize(tfm); + struct crypto_ahash *ahash_tfm; + struct ahash_request *req; + struct crypto_wait wait; + struct scatterlist sg; + const char *alg_name; + int rc; + + /* + * Key length must be equal to block size: + * - If key is shorter, we are done for now (the key will be padded + * later on); this is to maximize the use of HW HMAC (which works + * only for keys <= 64 bytes). + * - If key is longer, we hash it. + */ + if (keylen <= blk_sz) { + memcpy(ctx->key, key, keylen); + ctx->key_len = keylen; + return 0; + } + + switch (digestsize) { +#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_HCU_HMAC_SHA224 + case SHA224_DIGEST_SIZE: + alg_name = "sha224-keembay-ocs"; + break; +#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_HCU_HMAC_SHA224 */ + case SHA256_DIGEST_SIZE: + alg_name = ctx->is_sm3_tfm ? "sm3-keembay-ocs" : + "sha256-keembay-ocs"; + break; + case SHA384_DIGEST_SIZE: + alg_name = "sha384-keembay-ocs"; + break; + case SHA512_DIGEST_SIZE: + alg_name = "sha512-keembay-ocs"; + break; + default: + return -EINVAL; + } + + ahash_tfm = crypto_alloc_ahash(alg_name, 0, 0); + if (IS_ERR(ahash_tfm)) + return PTR_ERR(ahash_tfm); + + req = ahash_request_alloc(ahash_tfm, GFP_KERNEL); + if (!req) { + rc = -ENOMEM; + goto err_free_ahash; + } + + crypto_init_wait(&wait); + ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, + crypto_req_done, &wait); + crypto_ahash_clear_flags(ahash_tfm, ~0); + + sg_init_one(&sg, key, keylen); + ahash_request_set_crypt(req, &sg, ctx->key, keylen); + + rc = crypto_wait_req(crypto_ahash_digest(req), &wait); + if (rc == 0) + ctx->key_len = digestsize; + + ahash_request_free(req); +err_free_ahash: + crypto_free_ahash(ahash_tfm); + + return rc; +} + +/* Set request size and initialize tfm context. */ +static void __cra_init(struct crypto_tfm *tfm, struct ocs_hcu_ctx *ctx) +{ + crypto_ahash_set_reqsize_dma(__crypto_ahash_cast(tfm), + sizeof(struct ocs_hcu_rctx)); + + /* Init context to 0. */ + memzero_explicit(ctx, sizeof(*ctx)); + /* Set engine ops. */ + ctx->engine_ctx.op.do_one_request = kmb_ocs_hcu_do_one_request; +} + +static int kmb_ocs_hcu_sha_cra_init(struct crypto_tfm *tfm) +{ + struct ocs_hcu_ctx *ctx = crypto_tfm_ctx(tfm); + + __cra_init(tfm, ctx); + + return 0; +} + +static int kmb_ocs_hcu_sm3_cra_init(struct crypto_tfm *tfm) +{ + struct ocs_hcu_ctx *ctx = crypto_tfm_ctx(tfm); + + __cra_init(tfm, ctx); + + ctx->is_sm3_tfm = true; + + return 0; +} + +static int kmb_ocs_hcu_hmac_sm3_cra_init(struct crypto_tfm *tfm) +{ + struct ocs_hcu_ctx *ctx = crypto_tfm_ctx(tfm); + + __cra_init(tfm, ctx); + + ctx->is_sm3_tfm = true; + ctx->is_hmac_tfm = true; + + return 0; +} + +static int kmb_ocs_hcu_hmac_cra_init(struct crypto_tfm *tfm) +{ + struct ocs_hcu_ctx *ctx = crypto_tfm_ctx(tfm); + + __cra_init(tfm, ctx); + + ctx->is_hmac_tfm = true; + + return 0; +} + +/* Function called when 'tfm' is de-initialized. */ +static void kmb_ocs_hcu_hmac_cra_exit(struct crypto_tfm *tfm) +{ + struct ocs_hcu_ctx *ctx = crypto_tfm_ctx(tfm); + + /* Clear the key. */ + memzero_explicit(ctx->key, sizeof(ctx->key)); +} + +static struct ahash_alg ocs_hcu_algs[] = { +#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_HCU_HMAC_SHA224 +{ + .init = kmb_ocs_hcu_init, + .update = kmb_ocs_hcu_update, + .final = kmb_ocs_hcu_final, + .finup = kmb_ocs_hcu_finup, + .digest = kmb_ocs_hcu_digest, + .export = kmb_ocs_hcu_export, + .import = kmb_ocs_hcu_import, + .halg = { + .digestsize = SHA224_DIGEST_SIZE, + .statesize = sizeof(struct ocs_hcu_rctx), + .base = { + .cra_name = "sha224", + .cra_driver_name = "sha224-keembay-ocs", + .cra_priority = 255, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = SHA224_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct ocs_hcu_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = kmb_ocs_hcu_sha_cra_init, + } + } +}, +{ + .init = kmb_ocs_hcu_init, + .update = kmb_ocs_hcu_update, + .final = kmb_ocs_hcu_final, + .finup = kmb_ocs_hcu_finup, + .digest = kmb_ocs_hcu_digest, + .export = kmb_ocs_hcu_export, + .import = kmb_ocs_hcu_import, + .setkey = kmb_ocs_hcu_setkey, + .halg = { + .digestsize = SHA224_DIGEST_SIZE, + .statesize = sizeof(struct ocs_hcu_rctx), + .base = { + .cra_name = "hmac(sha224)", + .cra_driver_name = "hmac-sha224-keembay-ocs", + .cra_priority = 255, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = SHA224_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct ocs_hcu_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = kmb_ocs_hcu_hmac_cra_init, + .cra_exit = kmb_ocs_hcu_hmac_cra_exit, + } + } +}, +#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_HCU_HMAC_SHA224 */ +{ + .init = kmb_ocs_hcu_init, + .update = kmb_ocs_hcu_update, + .final = kmb_ocs_hcu_final, + .finup = kmb_ocs_hcu_finup, + .digest = kmb_ocs_hcu_digest, + .export = kmb_ocs_hcu_export, + .import = kmb_ocs_hcu_import, + .halg = { + .digestsize = SHA256_DIGEST_SIZE, + .statesize = sizeof(struct ocs_hcu_rctx), + .base = { + .cra_name = "sha256", + .cra_driver_name = "sha256-keembay-ocs", + .cra_priority = 255, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = SHA256_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct ocs_hcu_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = kmb_ocs_hcu_sha_cra_init, + } + } +}, +{ + .init = kmb_ocs_hcu_init, + .update = kmb_ocs_hcu_update, + .final = kmb_ocs_hcu_final, + .finup = kmb_ocs_hcu_finup, + .digest = kmb_ocs_hcu_digest, + .export = kmb_ocs_hcu_export, + .import = kmb_ocs_hcu_import, + .setkey = kmb_ocs_hcu_setkey, + .halg = { + .digestsize = SHA256_DIGEST_SIZE, + .statesize = sizeof(struct ocs_hcu_rctx), + .base = { + .cra_name = "hmac(sha256)", + .cra_driver_name = "hmac-sha256-keembay-ocs", + .cra_priority = 255, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = SHA256_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct ocs_hcu_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = kmb_ocs_hcu_hmac_cra_init, + .cra_exit = kmb_ocs_hcu_hmac_cra_exit, + } + } +}, +{ + .init = kmb_ocs_hcu_init, + .update = kmb_ocs_hcu_update, + .final = kmb_ocs_hcu_final, + .finup = kmb_ocs_hcu_finup, + .digest = kmb_ocs_hcu_digest, + .export = kmb_ocs_hcu_export, + .import = kmb_ocs_hcu_import, + .halg = { + .digestsize = SM3_DIGEST_SIZE, + .statesize = sizeof(struct ocs_hcu_rctx), + .base = { + .cra_name = "sm3", + .cra_driver_name = "sm3-keembay-ocs", + .cra_priority = 255, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = SM3_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct ocs_hcu_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = kmb_ocs_hcu_sm3_cra_init, + } + } +}, +{ + .init = kmb_ocs_hcu_init, + .update = kmb_ocs_hcu_update, + .final = kmb_ocs_hcu_final, + .finup = kmb_ocs_hcu_finup, + .digest = kmb_ocs_hcu_digest, + .export = kmb_ocs_hcu_export, + .import = kmb_ocs_hcu_import, + .setkey = kmb_ocs_hcu_setkey, + .halg = { + .digestsize = SM3_DIGEST_SIZE, + .statesize = sizeof(struct ocs_hcu_rctx), + .base = { + .cra_name = "hmac(sm3)", + .cra_driver_name = "hmac-sm3-keembay-ocs", + .cra_priority = 255, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = SM3_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct ocs_hcu_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = kmb_ocs_hcu_hmac_sm3_cra_init, + .cra_exit = kmb_ocs_hcu_hmac_cra_exit, + } + } +}, +{ + .init = kmb_ocs_hcu_init, + .update = kmb_ocs_hcu_update, + .final = kmb_ocs_hcu_final, + .finup = kmb_ocs_hcu_finup, + .digest = kmb_ocs_hcu_digest, + .export = kmb_ocs_hcu_export, + .import = kmb_ocs_hcu_import, + .halg = { + .digestsize = SHA384_DIGEST_SIZE, + .statesize = sizeof(struct ocs_hcu_rctx), + .base = { + .cra_name = "sha384", + .cra_driver_name = "sha384-keembay-ocs", + .cra_priority = 255, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = SHA384_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct ocs_hcu_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = kmb_ocs_hcu_sha_cra_init, + } + } +}, +{ + .init = kmb_ocs_hcu_init, + .update = kmb_ocs_hcu_update, + .final = kmb_ocs_hcu_final, + .finup = kmb_ocs_hcu_finup, + .digest = kmb_ocs_hcu_digest, + .export = kmb_ocs_hcu_export, + .import = kmb_ocs_hcu_import, + .setkey = kmb_ocs_hcu_setkey, + .halg = { + .digestsize = SHA384_DIGEST_SIZE, + .statesize = sizeof(struct ocs_hcu_rctx), + .base = { + .cra_name = "hmac(sha384)", + .cra_driver_name = "hmac-sha384-keembay-ocs", + .cra_priority = 255, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = SHA384_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct ocs_hcu_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = kmb_ocs_hcu_hmac_cra_init, + .cra_exit = kmb_ocs_hcu_hmac_cra_exit, + } + } +}, +{ + .init = kmb_ocs_hcu_init, + .update = kmb_ocs_hcu_update, + .final = kmb_ocs_hcu_final, + .finup = kmb_ocs_hcu_finup, + .digest = kmb_ocs_hcu_digest, + .export = kmb_ocs_hcu_export, + .import = kmb_ocs_hcu_import, + .halg = { + .digestsize = SHA512_DIGEST_SIZE, + .statesize = sizeof(struct ocs_hcu_rctx), + .base = { + .cra_name = "sha512", + .cra_driver_name = "sha512-keembay-ocs", + .cra_priority = 255, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = SHA512_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct ocs_hcu_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = kmb_ocs_hcu_sha_cra_init, + } + } +}, +{ + .init = kmb_ocs_hcu_init, + .update = kmb_ocs_hcu_update, + .final = kmb_ocs_hcu_final, + .finup = kmb_ocs_hcu_finup, + .digest = kmb_ocs_hcu_digest, + .export = kmb_ocs_hcu_export, + .import = kmb_ocs_hcu_import, + .setkey = kmb_ocs_hcu_setkey, + .halg = { + .digestsize = SHA512_DIGEST_SIZE, + .statesize = sizeof(struct ocs_hcu_rctx), + .base = { + .cra_name = "hmac(sha512)", + .cra_driver_name = "hmac-sha512-keembay-ocs", + .cra_priority = 255, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = SHA512_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct ocs_hcu_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = kmb_ocs_hcu_hmac_cra_init, + .cra_exit = kmb_ocs_hcu_hmac_cra_exit, + } + } +}, +}; + +/* Device tree driver match. */ +static const struct of_device_id kmb_ocs_hcu_of_match[] = { + { + .compatible = "intel,keembay-ocs-hcu", + }, + {} +}; + +static int kmb_ocs_hcu_remove(struct platform_device *pdev) +{ + struct ocs_hcu_dev *hcu_dev; + int rc; + + hcu_dev = platform_get_drvdata(pdev); + if (!hcu_dev) + return -ENODEV; + + crypto_unregister_ahashes(ocs_hcu_algs, ARRAY_SIZE(ocs_hcu_algs)); + + rc = crypto_engine_exit(hcu_dev->engine); + + spin_lock_bh(&ocs_hcu.lock); + list_del(&hcu_dev->list); + spin_unlock_bh(&ocs_hcu.lock); + + return rc; +} + +static int kmb_ocs_hcu_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct ocs_hcu_dev *hcu_dev; + struct resource *hcu_mem; + int rc; + + hcu_dev = devm_kzalloc(dev, sizeof(*hcu_dev), GFP_KERNEL); + if (!hcu_dev) + return -ENOMEM; + + hcu_dev->dev = dev; + + platform_set_drvdata(pdev, hcu_dev); + rc = dma_set_mask_and_coherent(&pdev->dev, OCS_HCU_DMA_BIT_MASK); + if (rc) + return rc; + + /* Get the memory address and remap. */ + hcu_mem = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (!hcu_mem) { + dev_err(dev, "Could not retrieve io mem resource.\n"); + return -ENODEV; + } + + hcu_dev->io_base = devm_ioremap_resource(dev, hcu_mem); + if (IS_ERR(hcu_dev->io_base)) + return PTR_ERR(hcu_dev->io_base); + + init_completion(&hcu_dev->irq_done); + + /* Get and request IRQ. */ + hcu_dev->irq = platform_get_irq(pdev, 0); + if (hcu_dev->irq < 0) + return hcu_dev->irq; + + rc = devm_request_threaded_irq(&pdev->dev, hcu_dev->irq, + ocs_hcu_irq_handler, NULL, 0, + "keembay-ocs-hcu", hcu_dev); + if (rc < 0) { + dev_err(dev, "Could not request IRQ.\n"); + return rc; + } + + INIT_LIST_HEAD(&hcu_dev->list); + + spin_lock_bh(&ocs_hcu.lock); + list_add_tail(&hcu_dev->list, &ocs_hcu.dev_list); + spin_unlock_bh(&ocs_hcu.lock); + + /* Initialize crypto engine */ + hcu_dev->engine = crypto_engine_alloc_init(dev, 1); + if (!hcu_dev->engine) { + rc = -ENOMEM; + goto list_del; + } + + rc = crypto_engine_start(hcu_dev->engine); + if (rc) { + dev_err(dev, "Could not start engine.\n"); + goto cleanup; + } + + /* Security infrastructure guarantees OCS clock is enabled. */ + + rc = crypto_register_ahashes(ocs_hcu_algs, ARRAY_SIZE(ocs_hcu_algs)); + if (rc) { + dev_err(dev, "Could not register algorithms.\n"); + goto cleanup; + } + + return 0; + +cleanup: + crypto_engine_exit(hcu_dev->engine); +list_del: + spin_lock_bh(&ocs_hcu.lock); + list_del(&hcu_dev->list); + spin_unlock_bh(&ocs_hcu.lock); + + return rc; +} + +/* The OCS driver is a platform device. */ +static struct platform_driver kmb_ocs_hcu_driver = { + .probe = kmb_ocs_hcu_probe, + .remove = kmb_ocs_hcu_remove, + .driver = { + .name = DRV_NAME, + .of_match_table = kmb_ocs_hcu_of_match, + }, +}; + +module_platform_driver(kmb_ocs_hcu_driver); + +MODULE_LICENSE("GPL"); diff --git a/drivers/crypto/intel/keembay/ocs-aes.c b/drivers/crypto/intel/keembay/ocs-aes.c new file mode 100644 index 000000000000..be9f32fc8f42 --- /dev/null +++ b/drivers/crypto/intel/keembay/ocs-aes.c @@ -0,0 +1,1489 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Intel Keem Bay OCS AES Crypto Driver. + * + * Copyright (C) 2018-2020 Intel Corporation + */ + +#include +#include +#include +#include +#include + +#include +#include + +#include +#include + +#include "ocs-aes.h" + +#define AES_COMMAND_OFFSET 0x0000 +#define AES_KEY_0_OFFSET 0x0004 +#define AES_KEY_1_OFFSET 0x0008 +#define AES_KEY_2_OFFSET 0x000C +#define AES_KEY_3_OFFSET 0x0010 +#define AES_KEY_4_OFFSET 0x0014 +#define AES_KEY_5_OFFSET 0x0018 +#define AES_KEY_6_OFFSET 0x001C +#define AES_KEY_7_OFFSET 0x0020 +#define AES_IV_0_OFFSET 0x0024 +#define AES_IV_1_OFFSET 0x0028 +#define AES_IV_2_OFFSET 0x002C +#define AES_IV_3_OFFSET 0x0030 +#define AES_ACTIVE_OFFSET 0x0034 +#define AES_STATUS_OFFSET 0x0038 +#define AES_KEY_SIZE_OFFSET 0x0044 +#define AES_IER_OFFSET 0x0048 +#define AES_ISR_OFFSET 0x005C +#define AES_MULTIPURPOSE1_0_OFFSET 0x0200 +#define AES_MULTIPURPOSE1_1_OFFSET 0x0204 +#define AES_MULTIPURPOSE1_2_OFFSET 0x0208 +#define AES_MULTIPURPOSE1_3_OFFSET 0x020C +#define AES_MULTIPURPOSE2_0_OFFSET 0x0220 +#define AES_MULTIPURPOSE2_1_OFFSET 0x0224 +#define AES_MULTIPURPOSE2_2_OFFSET 0x0228 +#define AES_MULTIPURPOSE2_3_OFFSET 0x022C +#define AES_BYTE_ORDER_CFG_OFFSET 0x02C0 +#define AES_TLEN_OFFSET 0x0300 +#define AES_T_MAC_0_OFFSET 0x0304 +#define AES_T_MAC_1_OFFSET 0x0308 +#define AES_T_MAC_2_OFFSET 0x030C +#define AES_T_MAC_3_OFFSET 0x0310 +#define AES_PLEN_OFFSET 0x0314 +#define AES_A_DMA_SRC_ADDR_OFFSET 0x0400 +#define AES_A_DMA_DST_ADDR_OFFSET 0x0404 +#define AES_A_DMA_SRC_SIZE_OFFSET 0x0408 +#define AES_A_DMA_DST_SIZE_OFFSET 0x040C +#define AES_A_DMA_DMA_MODE_OFFSET 0x0410 +#define AES_A_DMA_NEXT_SRC_DESCR_OFFSET 0x0418 +#define AES_A_DMA_NEXT_DST_DESCR_OFFSET 0x041C +#define AES_A_DMA_WHILE_ACTIVE_MODE_OFFSET 0x0420 +#define AES_A_DMA_LOG_OFFSET 0x0424 +#define AES_A_DMA_STATUS_OFFSET 0x0428 +#define AES_A_DMA_PERF_CNTR_OFFSET 0x042C +#define AES_A_DMA_MSI_ISR_OFFSET 0x0480 +#define AES_A_DMA_MSI_IER_OFFSET 0x0484 +#define AES_A_DMA_MSI_MASK_OFFSET 0x0488 +#define AES_A_DMA_INBUFFER_WRITE_FIFO_OFFSET 0x0600 +#define AES_A_DMA_OUTBUFFER_READ_FIFO_OFFSET 0x0700 + +/* + * AES_A_DMA_DMA_MODE register. + * Default: 0x00000000. + * bit[31] ACTIVE + * This bit activates the DMA. When the DMA finishes, it resets + * this bit to zero. + * bit[30:26] Unused by this driver. + * bit[25] SRC_LINK_LIST_EN + * Source link list enable bit. When the linked list is terminated + * this bit is reset by the DMA. + * bit[24] DST_LINK_LIST_EN + * Destination link list enable bit. When the linked list is + * terminated this bit is reset by the DMA. + * bit[23:0] Unused by this driver. + */ +#define AES_A_DMA_DMA_MODE_ACTIVE BIT(31) +#define AES_A_DMA_DMA_MODE_SRC_LINK_LIST_EN BIT(25) +#define AES_A_DMA_DMA_MODE_DST_LINK_LIST_EN BIT(24) + +/* + * AES_ACTIVE register + * default 0x00000000 + * bit[31:10] Reserved + * bit[9] LAST_ADATA + * bit[8] LAST_GCX + * bit[7:2] Reserved + * bit[1] TERMINATION + * bit[0] TRIGGER + */ +#define AES_ACTIVE_LAST_ADATA BIT(9) +#define AES_ACTIVE_LAST_CCM_GCM BIT(8) +#define AES_ACTIVE_TERMINATION BIT(1) +#define AES_ACTIVE_TRIGGER BIT(0) + +#define AES_DISABLE_INT 0x00000000 +#define AES_DMA_CPD_ERR_INT BIT(8) +#define AES_DMA_OUTBUF_RD_ERR_INT BIT(7) +#define AES_DMA_OUTBUF_WR_ERR_INT BIT(6) +#define AES_DMA_INBUF_RD_ERR_INT BIT(5) +#define AES_DMA_INBUF_WR_ERR_INT BIT(4) +#define AES_DMA_BAD_COMP_INT BIT(3) +#define AES_DMA_SAI_INT BIT(2) +#define AES_DMA_SRC_DONE_INT BIT(0) +#define AES_COMPLETE_INT BIT(1) + +#define AES_DMA_MSI_MASK_CLEAR BIT(0) + +#define AES_128_BIT_KEY 0x00000000 +#define AES_256_BIT_KEY BIT(0) + +#define AES_DEACTIVATE_PERF_CNTR 0x00000000 +#define AES_ACTIVATE_PERF_CNTR BIT(0) + +#define AES_MAX_TAG_SIZE_U32 4 + +#define OCS_LL_DMA_FLAG_TERMINATE BIT(31) + +/* + * There is an inconsistency in the documentation. This is documented as a + * 11-bit value, but it is actually 10-bits. + */ +#define AES_DMA_STATUS_INPUT_BUFFER_OCCUPANCY_MASK 0x3FF + +/* + * During CCM decrypt, the OCS block needs to finish processing the ciphertext + * before the tag is written. For 128-bit mode this required delay is 28 OCS + * clock cycles. For 256-bit mode it is 36 OCS clock cycles. + */ +#define CCM_DECRYPT_DELAY_TAG_CLK_COUNT 36UL + +/* + * During CCM decrypt there must be a delay of at least 42 OCS clock cycles + * between setting the TRIGGER bit in AES_ACTIVE and setting the LAST_CCM_GCM + * bit in the same register (as stated in the OCS databook) + */ +#define CCM_DECRYPT_DELAY_LAST_GCX_CLK_COUNT 42UL + +/* See RFC3610 section 2.2 */ +#define L_PRIME_MIN (1) +#define L_PRIME_MAX (7) +/* + * CCM IV format from RFC 3610 section 2.3 + * + * Octet Number Contents + * ------------ --------- + * 0 Flags + * 1 ... 15-L Nonce N + * 16-L ... 15 Counter i + * + * Flags = L' = L - 1 + */ +#define L_PRIME_IDX 0 +#define COUNTER_START(lprime) (16 - ((lprime) + 1)) +#define COUNTER_LEN(lprime) ((lprime) + 1) + +enum aes_counter_mode { + AES_CTR_M_NO_INC = 0, + AES_CTR_M_32_INC = 1, + AES_CTR_M_64_INC = 2, + AES_CTR_M_128_INC = 3, +}; + +/** + * struct ocs_dma_linked_list - OCS DMA linked list entry. + * @src_addr: Source address of the data. + * @src_len: Length of data to be fetched. + * @next: Next dma_list to fetch. + * @ll_flags: Flags (Freeze @ terminate) for the DMA engine. + */ +struct ocs_dma_linked_list { + u32 src_addr; + u32 src_len; + u32 next; + u32 ll_flags; +} __packed; + +/* + * Set endianness of inputs and outputs + * AES_BYTE_ORDER_CFG + * default 0x00000000 + * bit [10] - KEY_HI_LO_SWAP + * bit [9] - KEY_HI_SWAP_DWORDS_IN_OCTWORD + * bit [8] - KEY_HI_SWAP_BYTES_IN_DWORD + * bit [7] - KEY_LO_SWAP_DWORDS_IN_OCTWORD + * bit [6] - KEY_LO_SWAP_BYTES_IN_DWORD + * bit [5] - IV_SWAP_DWORDS_IN_OCTWORD + * bit [4] - IV_SWAP_BYTES_IN_DWORD + * bit [3] - DOUT_SWAP_DWORDS_IN_OCTWORD + * bit [2] - DOUT_SWAP_BYTES_IN_DWORD + * bit [1] - DOUT_SWAP_DWORDS_IN_OCTWORD + * bit [0] - DOUT_SWAP_BYTES_IN_DWORD + */ +static inline void aes_a_set_endianness(const struct ocs_aes_dev *aes_dev) +{ + iowrite32(0x7FF, aes_dev->base_reg + AES_BYTE_ORDER_CFG_OFFSET); +} + +/* Trigger AES process start. */ +static inline void aes_a_op_trigger(const struct ocs_aes_dev *aes_dev) +{ + iowrite32(AES_ACTIVE_TRIGGER, aes_dev->base_reg + AES_ACTIVE_OFFSET); +} + +/* Indicate last bulk of data. */ +static inline void aes_a_op_termination(const struct ocs_aes_dev *aes_dev) +{ + iowrite32(AES_ACTIVE_TERMINATION, + aes_dev->base_reg + AES_ACTIVE_OFFSET); +} + +/* + * Set LAST_CCM_GCM in AES_ACTIVE register and clear all other bits. + * + * Called when DMA is programmed to fetch the last batch of data. + * - For AES-CCM it is called for the last batch of Payload data and Ciphertext + * data. + * - For AES-GCM, it is called for the last batch of Plaintext data and + * Ciphertext data. + */ +static inline void aes_a_set_last_gcx(const struct ocs_aes_dev *aes_dev) +{ + iowrite32(AES_ACTIVE_LAST_CCM_GCM, + aes_dev->base_reg + AES_ACTIVE_OFFSET); +} + +/* Wait for LAST_CCM_GCM bit to be unset. */ +static inline void aes_a_wait_last_gcx(const struct ocs_aes_dev *aes_dev) +{ + u32 aes_active_reg; + + do { + aes_active_reg = ioread32(aes_dev->base_reg + + AES_ACTIVE_OFFSET); + } while (aes_active_reg & AES_ACTIVE_LAST_CCM_GCM); +} + +/* Wait for 10 bits of input occupancy. */ +static void aes_a_dma_wait_input_buffer_occupancy(const struct ocs_aes_dev *aes_dev) +{ + u32 reg; + + do { + reg = ioread32(aes_dev->base_reg + AES_A_DMA_STATUS_OFFSET); + } while (reg & AES_DMA_STATUS_INPUT_BUFFER_OCCUPANCY_MASK); +} + + /* + * Set LAST_CCM_GCM and LAST_ADATA bits in AES_ACTIVE register (and clear all + * other bits). + * + * Called when DMA is programmed to fetch the last batch of Associated Data + * (CCM case) or Additional Authenticated Data (GCM case). + */ +static inline void aes_a_set_last_gcx_and_adata(const struct ocs_aes_dev *aes_dev) +{ + iowrite32(AES_ACTIVE_LAST_ADATA | AES_ACTIVE_LAST_CCM_GCM, + aes_dev->base_reg + AES_ACTIVE_OFFSET); +} + +/* Set DMA src and dst transfer size to 0 */ +static inline void aes_a_dma_set_xfer_size_zero(const struct ocs_aes_dev *aes_dev) +{ + iowrite32(0, aes_dev->base_reg + AES_A_DMA_SRC_SIZE_OFFSET); + iowrite32(0, aes_dev->base_reg + AES_A_DMA_DST_SIZE_OFFSET); +} + +/* Activate DMA for zero-byte transfer case. */ +static inline void aes_a_dma_active(const struct ocs_aes_dev *aes_dev) +{ + iowrite32(AES_A_DMA_DMA_MODE_ACTIVE, + aes_dev->base_reg + AES_A_DMA_DMA_MODE_OFFSET); +} + +/* Activate DMA and enable src linked list */ +static inline void aes_a_dma_active_src_ll_en(const struct ocs_aes_dev *aes_dev) +{ + iowrite32(AES_A_DMA_DMA_MODE_ACTIVE | + AES_A_DMA_DMA_MODE_SRC_LINK_LIST_EN, + aes_dev->base_reg + AES_A_DMA_DMA_MODE_OFFSET); +} + +/* Activate DMA and enable dst linked list */ +static inline void aes_a_dma_active_dst_ll_en(const struct ocs_aes_dev *aes_dev) +{ + iowrite32(AES_A_DMA_DMA_MODE_ACTIVE | + AES_A_DMA_DMA_MODE_DST_LINK_LIST_EN, + aes_dev->base_reg + AES_A_DMA_DMA_MODE_OFFSET); +} + +/* Activate DMA and enable src and dst linked lists */ +static inline void aes_a_dma_active_src_dst_ll_en(const struct ocs_aes_dev *aes_dev) +{ + iowrite32(AES_A_DMA_DMA_MODE_ACTIVE | + AES_A_DMA_DMA_MODE_SRC_LINK_LIST_EN | + AES_A_DMA_DMA_MODE_DST_LINK_LIST_EN, + aes_dev->base_reg + AES_A_DMA_DMA_MODE_OFFSET); +} + +/* Reset PERF_CNTR to 0 and activate it */ +static inline void aes_a_dma_reset_and_activate_perf_cntr(const struct ocs_aes_dev *aes_dev) +{ + iowrite32(0x00000000, aes_dev->base_reg + AES_A_DMA_PERF_CNTR_OFFSET); + iowrite32(AES_ACTIVATE_PERF_CNTR, + aes_dev->base_reg + AES_A_DMA_WHILE_ACTIVE_MODE_OFFSET); +} + +/* Wait until PERF_CNTR is > delay, then deactivate it */ +static inline void aes_a_dma_wait_and_deactivate_perf_cntr(const struct ocs_aes_dev *aes_dev, + int delay) +{ + while (ioread32(aes_dev->base_reg + AES_A_DMA_PERF_CNTR_OFFSET) < delay) + ; + iowrite32(AES_DEACTIVATE_PERF_CNTR, + aes_dev->base_reg + AES_A_DMA_WHILE_ACTIVE_MODE_OFFSET); +} + +/* Disable AES and DMA IRQ. */ +static void aes_irq_disable(struct ocs_aes_dev *aes_dev) +{ + u32 isr_val = 0; + + /* Disable interrupts */ + iowrite32(AES_DISABLE_INT, + aes_dev->base_reg + AES_A_DMA_MSI_IER_OFFSET); + iowrite32(AES_DISABLE_INT, aes_dev->base_reg + AES_IER_OFFSET); + + /* Clear any pending interrupt */ + isr_val = ioread32(aes_dev->base_reg + AES_A_DMA_MSI_ISR_OFFSET); + if (isr_val) + iowrite32(isr_val, + aes_dev->base_reg + AES_A_DMA_MSI_ISR_OFFSET); + + isr_val = ioread32(aes_dev->base_reg + AES_A_DMA_MSI_MASK_OFFSET); + if (isr_val) + iowrite32(isr_val, + aes_dev->base_reg + AES_A_DMA_MSI_MASK_OFFSET); + + isr_val = ioread32(aes_dev->base_reg + AES_ISR_OFFSET); + if (isr_val) + iowrite32(isr_val, aes_dev->base_reg + AES_ISR_OFFSET); +} + +/* Enable AES or DMA IRQ. IRQ is disabled once fired. */ +static void aes_irq_enable(struct ocs_aes_dev *aes_dev, u8 irq) +{ + if (irq == AES_COMPLETE_INT) { + /* Ensure DMA error interrupts are enabled */ + iowrite32(AES_DMA_CPD_ERR_INT | + AES_DMA_OUTBUF_RD_ERR_INT | + AES_DMA_OUTBUF_WR_ERR_INT | + AES_DMA_INBUF_RD_ERR_INT | + AES_DMA_INBUF_WR_ERR_INT | + AES_DMA_BAD_COMP_INT | + AES_DMA_SAI_INT, + aes_dev->base_reg + AES_A_DMA_MSI_IER_OFFSET); + /* + * AES_IER + * default 0x00000000 + * bits [31:3] - reserved + * bit [2] - EN_SKS_ERR + * bit [1] - EN_AES_COMPLETE + * bit [0] - reserved + */ + iowrite32(AES_COMPLETE_INT, aes_dev->base_reg + AES_IER_OFFSET); + return; + } + if (irq == AES_DMA_SRC_DONE_INT) { + /* Ensure AES interrupts are disabled */ + iowrite32(AES_DISABLE_INT, aes_dev->base_reg + AES_IER_OFFSET); + /* + * DMA_MSI_IER + * default 0x00000000 + * bits [31:9] - reserved + * bit [8] - CPD_ERR_INT_EN + * bit [7] - OUTBUF_RD_ERR_INT_EN + * bit [6] - OUTBUF_WR_ERR_INT_EN + * bit [5] - INBUF_RD_ERR_INT_EN + * bit [4] - INBUF_WR_ERR_INT_EN + * bit [3] - BAD_COMP_INT_EN + * bit [2] - SAI_INT_EN + * bit [1] - DST_DONE_INT_EN + * bit [0] - SRC_DONE_INT_EN + */ + iowrite32(AES_DMA_CPD_ERR_INT | + AES_DMA_OUTBUF_RD_ERR_INT | + AES_DMA_OUTBUF_WR_ERR_INT | + AES_DMA_INBUF_RD_ERR_INT | + AES_DMA_INBUF_WR_ERR_INT | + AES_DMA_BAD_COMP_INT | + AES_DMA_SAI_INT | + AES_DMA_SRC_DONE_INT, + aes_dev->base_reg + AES_A_DMA_MSI_IER_OFFSET); + } +} + +/* Enable and wait for IRQ (either from OCS AES engine or DMA) */ +static int ocs_aes_irq_enable_and_wait(struct ocs_aes_dev *aes_dev, u8 irq) +{ + int rc; + + reinit_completion(&aes_dev->irq_completion); + aes_irq_enable(aes_dev, irq); + rc = wait_for_completion_interruptible(&aes_dev->irq_completion); + if (rc) + return rc; + + return aes_dev->dma_err_mask ? -EIO : 0; +} + +/* Configure DMA to OCS, linked list mode */ +static inline void dma_to_ocs_aes_ll(struct ocs_aes_dev *aes_dev, + dma_addr_t dma_list) +{ + iowrite32(0, aes_dev->base_reg + AES_A_DMA_SRC_SIZE_OFFSET); + iowrite32(dma_list, + aes_dev->base_reg + AES_A_DMA_NEXT_SRC_DESCR_OFFSET); +} + +/* Configure DMA from OCS, linked list mode */ +static inline void dma_from_ocs_aes_ll(struct ocs_aes_dev *aes_dev, + dma_addr_t dma_list) +{ + iowrite32(0, aes_dev->base_reg + AES_A_DMA_DST_SIZE_OFFSET); + iowrite32(dma_list, + aes_dev->base_reg + AES_A_DMA_NEXT_DST_DESCR_OFFSET); +} + +irqreturn_t ocs_aes_irq_handler(int irq, void *dev_id) +{ + struct ocs_aes_dev *aes_dev = dev_id; + u32 aes_dma_isr; + + /* Read DMA ISR status. */ + aes_dma_isr = ioread32(aes_dev->base_reg + AES_A_DMA_MSI_ISR_OFFSET); + + /* Disable and clear interrupts. */ + aes_irq_disable(aes_dev); + + /* Save DMA error status. */ + aes_dev->dma_err_mask = aes_dma_isr & + (AES_DMA_CPD_ERR_INT | + AES_DMA_OUTBUF_RD_ERR_INT | + AES_DMA_OUTBUF_WR_ERR_INT | + AES_DMA_INBUF_RD_ERR_INT | + AES_DMA_INBUF_WR_ERR_INT | + AES_DMA_BAD_COMP_INT | + AES_DMA_SAI_INT); + + /* Signal IRQ completion. */ + complete(&aes_dev->irq_completion); + + return IRQ_HANDLED; +} + +/** + * ocs_aes_set_key() - Write key into OCS AES hardware. + * @aes_dev: The OCS AES device to write the key to. + * @key_size: The size of the key (in bytes). + * @key: The key to write. + * @cipher: The cipher the key is for. + * + * For AES @key_size must be either 16 or 32. For SM4 @key_size must be 16. + * + * Return: 0 on success, negative error code otherwise. + */ +int ocs_aes_set_key(struct ocs_aes_dev *aes_dev, u32 key_size, const u8 *key, + enum ocs_cipher cipher) +{ + const u32 *key_u32; + u32 val; + int i; + + /* OCS AES supports 128-bit and 256-bit keys only. */ + if (cipher == OCS_AES && !(key_size == 32 || key_size == 16)) { + dev_err(aes_dev->dev, + "%d-bit keys not supported by AES cipher\n", + key_size * 8); + return -EINVAL; + } + /* OCS SM4 supports 128-bit keys only. */ + if (cipher == OCS_SM4 && key_size != 16) { + dev_err(aes_dev->dev, + "%d-bit keys not supported for SM4 cipher\n", + key_size * 8); + return -EINVAL; + } + + if (!key) + return -EINVAL; + + key_u32 = (const u32 *)key; + + /* Write key to AES_KEY[0-7] registers */ + for (i = 0; i < (key_size / sizeof(u32)); i++) { + iowrite32(key_u32[i], + aes_dev->base_reg + AES_KEY_0_OFFSET + + (i * sizeof(u32))); + } + /* + * Write key size + * bits [31:1] - reserved + * bit [0] - AES_KEY_SIZE + * 0 - 128 bit key + * 1 - 256 bit key + */ + val = (key_size == 16) ? AES_128_BIT_KEY : AES_256_BIT_KEY; + iowrite32(val, aes_dev->base_reg + AES_KEY_SIZE_OFFSET); + + return 0; +} + +/* Write AES_COMMAND */ +static inline void set_ocs_aes_command(struct ocs_aes_dev *aes_dev, + enum ocs_cipher cipher, + enum ocs_mode mode, + enum ocs_instruction instruction) +{ + u32 val; + + /* AES_COMMAND + * default 0x000000CC + * bit [14] - CIPHER_SELECT + * 0 - AES + * 1 - SM4 + * bits [11:8] - OCS_AES_MODE + * 0000 - ECB + * 0001 - CBC + * 0010 - CTR + * 0110 - CCM + * 0111 - GCM + * 1001 - CTS + * bits [7:6] - AES_INSTRUCTION + * 00 - ENCRYPT + * 01 - DECRYPT + * 10 - EXPAND + * 11 - BYPASS + * bits [3:2] - CTR_M_BITS + * 00 - No increment + * 01 - Least significant 32 bits are incremented + * 10 - Least significant 64 bits are incremented + * 11 - Full 128 bits are incremented + */ + val = (cipher << 14) | (mode << 8) | (instruction << 6) | + (AES_CTR_M_128_INC << 2); + iowrite32(val, aes_dev->base_reg + AES_COMMAND_OFFSET); +} + +static void ocs_aes_init(struct ocs_aes_dev *aes_dev, + enum ocs_mode mode, + enum ocs_cipher cipher, + enum ocs_instruction instruction) +{ + /* Ensure interrupts are disabled and pending interrupts cleared. */ + aes_irq_disable(aes_dev); + + /* Set endianness recommended by data-sheet. */ + aes_a_set_endianness(aes_dev); + + /* Set AES_COMMAND register. */ + set_ocs_aes_command(aes_dev, cipher, mode, instruction); +} + +/* + * Write the byte length of the last AES/SM4 block of Payload data (without + * zero padding and without the length of the MAC) in register AES_PLEN. + */ +static inline void ocs_aes_write_last_data_blk_len(struct ocs_aes_dev *aes_dev, + u32 size) +{ + u32 val; + + if (size == 0) { + val = 0; + goto exit; + } + + val = size % AES_BLOCK_SIZE; + if (val == 0) + val = AES_BLOCK_SIZE; + +exit: + iowrite32(val, aes_dev->base_reg + AES_PLEN_OFFSET); +} + +/* + * Validate inputs according to mode. + * If OK return 0; else return -EINVAL. + */ +static int ocs_aes_validate_inputs(dma_addr_t src_dma_list, u32 src_size, + const u8 *iv, u32 iv_size, + dma_addr_t aad_dma_list, u32 aad_size, + const u8 *tag, u32 tag_size, + enum ocs_cipher cipher, enum ocs_mode mode, + enum ocs_instruction instruction, + dma_addr_t dst_dma_list) +{ + /* Ensure cipher, mode and instruction are valid. */ + if (!(cipher == OCS_AES || cipher == OCS_SM4)) + return -EINVAL; + + if (mode != OCS_MODE_ECB && mode != OCS_MODE_CBC && + mode != OCS_MODE_CTR && mode != OCS_MODE_CCM && + mode != OCS_MODE_GCM && mode != OCS_MODE_CTS) + return -EINVAL; + + if (instruction != OCS_ENCRYPT && instruction != OCS_DECRYPT && + instruction != OCS_EXPAND && instruction != OCS_BYPASS) + return -EINVAL; + + /* + * When instruction is OCS_BYPASS, OCS simply copies data from source + * to destination using DMA. + * + * AES mode is irrelevant, but both source and destination DMA + * linked-list must be defined. + */ + if (instruction == OCS_BYPASS) { + if (src_dma_list == DMA_MAPPING_ERROR || + dst_dma_list == DMA_MAPPING_ERROR) + return -EINVAL; + + return 0; + } + + /* + * For performance reasons switch based on mode to limit unnecessary + * conditionals for each mode + */ + switch (mode) { + case OCS_MODE_ECB: + /* Ensure input length is multiple of block size */ + if (src_size % AES_BLOCK_SIZE != 0) + return -EINVAL; + + /* Ensure source and destination linked lists are created */ + if (src_dma_list == DMA_MAPPING_ERROR || + dst_dma_list == DMA_MAPPING_ERROR) + return -EINVAL; + + return 0; + + case OCS_MODE_CBC: + /* Ensure input length is multiple of block size */ + if (src_size % AES_BLOCK_SIZE != 0) + return -EINVAL; + + /* Ensure source and destination linked lists are created */ + if (src_dma_list == DMA_MAPPING_ERROR || + dst_dma_list == DMA_MAPPING_ERROR) + return -EINVAL; + + /* Ensure IV is present and block size in length */ + if (!iv || iv_size != AES_BLOCK_SIZE) + return -EINVAL; + + return 0; + + case OCS_MODE_CTR: + /* Ensure input length of 1 byte or greater */ + if (src_size == 0) + return -EINVAL; + + /* Ensure source and destination linked lists are created */ + if (src_dma_list == DMA_MAPPING_ERROR || + dst_dma_list == DMA_MAPPING_ERROR) + return -EINVAL; + + /* Ensure IV is present and block size in length */ + if (!iv || iv_size != AES_BLOCK_SIZE) + return -EINVAL; + + return 0; + + case OCS_MODE_CTS: + /* Ensure input length >= block size */ + if (src_size < AES_BLOCK_SIZE) + return -EINVAL; + + /* Ensure source and destination linked lists are created */ + if (src_dma_list == DMA_MAPPING_ERROR || + dst_dma_list == DMA_MAPPING_ERROR) + return -EINVAL; + + /* Ensure IV is present and block size in length */ + if (!iv || iv_size != AES_BLOCK_SIZE) + return -EINVAL; + + return 0; + + case OCS_MODE_GCM: + /* Ensure IV is present and GCM_AES_IV_SIZE in length */ + if (!iv || iv_size != GCM_AES_IV_SIZE) + return -EINVAL; + + /* + * If input data present ensure source and destination linked + * lists are created + */ + if (src_size && (src_dma_list == DMA_MAPPING_ERROR || + dst_dma_list == DMA_MAPPING_ERROR)) + return -EINVAL; + + /* If aad present ensure aad linked list is created */ + if (aad_size && aad_dma_list == DMA_MAPPING_ERROR) + return -EINVAL; + + /* Ensure tag destination is set */ + if (!tag) + return -EINVAL; + + /* Just ensure that tag_size doesn't cause overflows. */ + if (tag_size > (AES_MAX_TAG_SIZE_U32 * sizeof(u32))) + return -EINVAL; + + return 0; + + case OCS_MODE_CCM: + /* Ensure IV is present and block size in length */ + if (!iv || iv_size != AES_BLOCK_SIZE) + return -EINVAL; + + /* 2 <= L <= 8, so 1 <= L' <= 7 */ + if (iv[L_PRIME_IDX] < L_PRIME_MIN || + iv[L_PRIME_IDX] > L_PRIME_MAX) + return -EINVAL; + + /* If aad present ensure aad linked list is created */ + if (aad_size && aad_dma_list == DMA_MAPPING_ERROR) + return -EINVAL; + + /* Just ensure that tag_size doesn't cause overflows. */ + if (tag_size > (AES_MAX_TAG_SIZE_U32 * sizeof(u32))) + return -EINVAL; + + if (instruction == OCS_DECRYPT) { + /* + * If input data present ensure source and destination + * linked lists are created + */ + if (src_size && (src_dma_list == DMA_MAPPING_ERROR || + dst_dma_list == DMA_MAPPING_ERROR)) + return -EINVAL; + + /* Ensure input tag is present */ + if (!tag) + return -EINVAL; + + return 0; + } + + /* Instruction == OCS_ENCRYPT */ + + /* + * Destination linked list always required (for tag even if no + * input data) + */ + if (dst_dma_list == DMA_MAPPING_ERROR) + return -EINVAL; + + /* If input data present ensure src linked list is created */ + if (src_size && src_dma_list == DMA_MAPPING_ERROR) + return -EINVAL; + + return 0; + + default: + return -EINVAL; + } +} + +/** + * ocs_aes_op() - Perform AES/SM4 operation. + * @aes_dev: The OCS AES device to use. + * @mode: The mode to use (ECB, CBC, CTR, or CTS). + * @cipher: The cipher to use (AES or SM4). + * @instruction: The instruction to perform (encrypt or decrypt). + * @dst_dma_list: The OCS DMA list mapping output memory. + * @src_dma_list: The OCS DMA list mapping input payload data. + * @src_size: The amount of data mapped by @src_dma_list. + * @iv: The IV vector. + * @iv_size: The size (in bytes) of @iv. + * + * Return: 0 on success, negative error code otherwise. + */ +int ocs_aes_op(struct ocs_aes_dev *aes_dev, + enum ocs_mode mode, + enum ocs_cipher cipher, + enum ocs_instruction instruction, + dma_addr_t dst_dma_list, + dma_addr_t src_dma_list, + u32 src_size, + u8 *iv, + u32 iv_size) +{ + u32 *iv32; + int rc; + + rc = ocs_aes_validate_inputs(src_dma_list, src_size, iv, iv_size, 0, 0, + NULL, 0, cipher, mode, instruction, + dst_dma_list); + if (rc) + return rc; + /* + * ocs_aes_validate_inputs() is a generic check, now ensure mode is not + * GCM or CCM. + */ + if (mode == OCS_MODE_GCM || mode == OCS_MODE_CCM) + return -EINVAL; + + /* Cast IV to u32 array. */ + iv32 = (u32 *)iv; + + ocs_aes_init(aes_dev, mode, cipher, instruction); + + if (mode == OCS_MODE_CTS) { + /* Write the byte length of the last data block to engine. */ + ocs_aes_write_last_data_blk_len(aes_dev, src_size); + } + + /* ECB is the only mode that doesn't use IV. */ + if (mode != OCS_MODE_ECB) { + iowrite32(iv32[0], aes_dev->base_reg + AES_IV_0_OFFSET); + iowrite32(iv32[1], aes_dev->base_reg + AES_IV_1_OFFSET); + iowrite32(iv32[2], aes_dev->base_reg + AES_IV_2_OFFSET); + iowrite32(iv32[3], aes_dev->base_reg + AES_IV_3_OFFSET); + } + + /* Set AES_ACTIVE.TRIGGER to start the operation. */ + aes_a_op_trigger(aes_dev); + + /* Configure and activate input / output DMA. */ + dma_to_ocs_aes_ll(aes_dev, src_dma_list); + dma_from_ocs_aes_ll(aes_dev, dst_dma_list); + aes_a_dma_active_src_dst_ll_en(aes_dev); + + if (mode == OCS_MODE_CTS) { + /* + * For CTS mode, instruct engine to activate ciphertext + * stealing if last block of data is incomplete. + */ + aes_a_set_last_gcx(aes_dev); + } else { + /* For all other modes, just write the 'termination' bit. */ + aes_a_op_termination(aes_dev); + } + + /* Wait for engine to complete processing. */ + rc = ocs_aes_irq_enable_and_wait(aes_dev, AES_COMPLETE_INT); + if (rc) + return rc; + + if (mode == OCS_MODE_CTR) { + /* Read back IV for streaming mode */ + iv32[0] = ioread32(aes_dev->base_reg + AES_IV_0_OFFSET); + iv32[1] = ioread32(aes_dev->base_reg + AES_IV_1_OFFSET); + iv32[2] = ioread32(aes_dev->base_reg + AES_IV_2_OFFSET); + iv32[3] = ioread32(aes_dev->base_reg + AES_IV_3_OFFSET); + } + + return 0; +} + +/* Compute and write J0 to engine registers. */ +static void ocs_aes_gcm_write_j0(const struct ocs_aes_dev *aes_dev, + const u8 *iv) +{ + const u32 *j0 = (u32 *)iv; + + /* + * IV must be 12 bytes; Other sizes not supported as Linux crypto API + * does only expects/allows 12 byte IV for GCM + */ + iowrite32(0x00000001, aes_dev->base_reg + AES_IV_0_OFFSET); + iowrite32(__swab32(j0[2]), aes_dev->base_reg + AES_IV_1_OFFSET); + iowrite32(__swab32(j0[1]), aes_dev->base_reg + AES_IV_2_OFFSET); + iowrite32(__swab32(j0[0]), aes_dev->base_reg + AES_IV_3_OFFSET); +} + +/* Read GCM tag from engine registers. */ +static inline void ocs_aes_gcm_read_tag(struct ocs_aes_dev *aes_dev, + u8 *tag, u32 tag_size) +{ + u32 tag_u32[AES_MAX_TAG_SIZE_U32]; + + /* + * The Authentication Tag T is stored in Little Endian order in the + * registers with the most significant bytes stored from AES_T_MAC[3] + * downward. + */ + tag_u32[0] = __swab32(ioread32(aes_dev->base_reg + AES_T_MAC_3_OFFSET)); + tag_u32[1] = __swab32(ioread32(aes_dev->base_reg + AES_T_MAC_2_OFFSET)); + tag_u32[2] = __swab32(ioread32(aes_dev->base_reg + AES_T_MAC_1_OFFSET)); + tag_u32[3] = __swab32(ioread32(aes_dev->base_reg + AES_T_MAC_0_OFFSET)); + + memcpy(tag, tag_u32, tag_size); +} + +/** + * ocs_aes_gcm_op() - Perform GCM operation. + * @aes_dev: The OCS AES device to use. + * @cipher: The Cipher to use (AES or SM4). + * @instruction: The instruction to perform (encrypt or decrypt). + * @dst_dma_list: The OCS DMA list mapping output memory. + * @src_dma_list: The OCS DMA list mapping input payload data. + * @src_size: The amount of data mapped by @src_dma_list. + * @iv: The input IV vector. + * @aad_dma_list: The OCS DMA list mapping input AAD data. + * @aad_size: The amount of data mapped by @aad_dma_list. + * @out_tag: Where to store computed tag. + * @tag_size: The size (in bytes) of @out_tag. + * + * Return: 0 on success, negative error code otherwise. + */ +int ocs_aes_gcm_op(struct ocs_aes_dev *aes_dev, + enum ocs_cipher cipher, + enum ocs_instruction instruction, + dma_addr_t dst_dma_list, + dma_addr_t src_dma_list, + u32 src_size, + const u8 *iv, + dma_addr_t aad_dma_list, + u32 aad_size, + u8 *out_tag, + u32 tag_size) +{ + u64 bit_len; + u32 val; + int rc; + + rc = ocs_aes_validate_inputs(src_dma_list, src_size, iv, + GCM_AES_IV_SIZE, aad_dma_list, + aad_size, out_tag, tag_size, cipher, + OCS_MODE_GCM, instruction, + dst_dma_list); + if (rc) + return rc; + + ocs_aes_init(aes_dev, OCS_MODE_GCM, cipher, instruction); + + /* Compute and write J0 to OCS HW. */ + ocs_aes_gcm_write_j0(aes_dev, iv); + + /* Write out_tag byte length */ + iowrite32(tag_size, aes_dev->base_reg + AES_TLEN_OFFSET); + + /* Write the byte length of the last plaintext / ciphertext block. */ + ocs_aes_write_last_data_blk_len(aes_dev, src_size); + + /* Write ciphertext bit length */ + bit_len = (u64)src_size * 8; + val = bit_len & 0xFFFFFFFF; + iowrite32(val, aes_dev->base_reg + AES_MULTIPURPOSE2_0_OFFSET); + val = bit_len >> 32; + iowrite32(val, aes_dev->base_reg + AES_MULTIPURPOSE2_1_OFFSET); + + /* Write aad bit length */ + bit_len = (u64)aad_size * 8; + val = bit_len & 0xFFFFFFFF; + iowrite32(val, aes_dev->base_reg + AES_MULTIPURPOSE2_2_OFFSET); + val = bit_len >> 32; + iowrite32(val, aes_dev->base_reg + AES_MULTIPURPOSE2_3_OFFSET); + + /* Set AES_ACTIVE.TRIGGER to start the operation. */ + aes_a_op_trigger(aes_dev); + + /* Process AAD. */ + if (aad_size) { + /* If aad present, configure DMA to feed it to the engine. */ + dma_to_ocs_aes_ll(aes_dev, aad_dma_list); + aes_a_dma_active_src_ll_en(aes_dev); + + /* Instructs engine to pad last block of aad, if needed. */ + aes_a_set_last_gcx_and_adata(aes_dev); + + /* Wait for DMA transfer to complete. */ + rc = ocs_aes_irq_enable_and_wait(aes_dev, AES_DMA_SRC_DONE_INT); + if (rc) + return rc; + } else { + aes_a_set_last_gcx_and_adata(aes_dev); + } + + /* Wait until adata (if present) has been processed. */ + aes_a_wait_last_gcx(aes_dev); + aes_a_dma_wait_input_buffer_occupancy(aes_dev); + + /* Now process payload. */ + if (src_size) { + /* Configure and activate DMA for both input and output data. */ + dma_to_ocs_aes_ll(aes_dev, src_dma_list); + dma_from_ocs_aes_ll(aes_dev, dst_dma_list); + aes_a_dma_active_src_dst_ll_en(aes_dev); + } else { + aes_a_dma_set_xfer_size_zero(aes_dev); + aes_a_dma_active(aes_dev); + } + + /* Instruct AES/SMA4 engine payload processing is over. */ + aes_a_set_last_gcx(aes_dev); + + /* Wait for OCS AES engine to complete processing. */ + rc = ocs_aes_irq_enable_and_wait(aes_dev, AES_COMPLETE_INT); + if (rc) + return rc; + + ocs_aes_gcm_read_tag(aes_dev, out_tag, tag_size); + + return 0; +} + +/* Write encrypted tag to AES/SM4 engine. */ +static void ocs_aes_ccm_write_encrypted_tag(struct ocs_aes_dev *aes_dev, + const u8 *in_tag, u32 tag_size) +{ + int i; + + /* Ensure DMA input buffer is empty */ + aes_a_dma_wait_input_buffer_occupancy(aes_dev); + + /* + * During CCM decrypt, the OCS block needs to finish processing the + * ciphertext before the tag is written. So delay needed after DMA has + * completed writing the ciphertext + */ + aes_a_dma_reset_and_activate_perf_cntr(aes_dev); + aes_a_dma_wait_and_deactivate_perf_cntr(aes_dev, + CCM_DECRYPT_DELAY_TAG_CLK_COUNT); + + /* Write encrypted tag to AES/SM4 engine. */ + for (i = 0; i < tag_size; i++) { + iowrite8(in_tag[i], aes_dev->base_reg + + AES_A_DMA_INBUFFER_WRITE_FIFO_OFFSET); + } +} + +/* + * Write B0 CCM block to OCS AES HW. + * + * Note: B0 format is documented in NIST Special Publication 800-38C + * https://nvlpubs.nist.gov/nistpubs/Legacy/SP/nistspecialpublication800-38c.pdf + * (see Section A.2.1) + */ +static int ocs_aes_ccm_write_b0(const struct ocs_aes_dev *aes_dev, + const u8 *iv, u32 adata_size, u32 tag_size, + u32 cryptlen) +{ + u8 b0[16]; /* CCM B0 block is 16 bytes long. */ + int i, q; + + /* Initialize B0 to 0. */ + memset(b0, 0, sizeof(b0)); + + /* + * B0[0] is the 'Flags Octet' and has the following structure: + * bit 7: Reserved + * bit 6: Adata flag + * bit 5-3: t value encoded as (t-2)/2 + * bit 2-0: q value encoded as q - 1 + */ + /* If there is AAD data, set the Adata flag. */ + if (adata_size) + b0[0] |= BIT(6); + /* + * t denotes the octet length of T. + * t can only be an element of { 4, 6, 8, 10, 12, 14, 16} and is + * encoded as (t - 2) / 2 + */ + b0[0] |= (((tag_size - 2) / 2) & 0x7) << 3; + /* + * q is the octet length of Q. + * q can only be an element of {2, 3, 4, 5, 6, 7, 8} and is encoded as + * q - 1 == iv[0] & 0x7; + */ + b0[0] |= iv[0] & 0x7; + /* + * Copy the Nonce N from IV to B0; N is located in iv[1]..iv[15 - q] + * and must be copied to b0[1]..b0[15-q]. + * q == (iv[0] & 0x7) + 1 + */ + q = (iv[0] & 0x7) + 1; + for (i = 1; i <= 15 - q; i++) + b0[i] = iv[i]; + /* + * The rest of B0 must contain Q, i.e., the message length. + * Q is encoded in q octets, in big-endian order, so to write it, we + * start from the end of B0 and we move backward. + */ + i = sizeof(b0) - 1; + while (q) { + b0[i] = cryptlen & 0xff; + cryptlen >>= 8; + i--; + q--; + } + /* + * If cryptlen is not zero at this point, it means that its original + * value was too big. + */ + if (cryptlen) + return -EOVERFLOW; + /* Now write B0 to OCS AES input buffer. */ + for (i = 0; i < sizeof(b0); i++) + iowrite8(b0[i], aes_dev->base_reg + + AES_A_DMA_INBUFFER_WRITE_FIFO_OFFSET); + return 0; +} + +/* + * Write adata length to OCS AES HW. + * + * Note: adata len encoding is documented in NIST Special Publication 800-38C + * https://nvlpubs.nist.gov/nistpubs/Legacy/SP/nistspecialpublication800-38c.pdf + * (see Section A.2.2) + */ +static void ocs_aes_ccm_write_adata_len(const struct ocs_aes_dev *aes_dev, + u64 adata_len) +{ + u8 enc_a[10]; /* Maximum encoded size: 10 octets. */ + int i, len; + + /* + * adata_len ('a') is encoded as follows: + * If 0 < a < 2^16 - 2^8 ==> 'a' encoded as [a]16, i.e., two octets + * (big endian). + * If 2^16 - 2^8 ≤ a < 2^32 ==> 'a' encoded as 0xff || 0xfe || [a]32, + * i.e., six octets (big endian). + * If 2^32 ≤ a < 2^64 ==> 'a' encoded as 0xff || 0xff || [a]64, + * i.e., ten octets (big endian). + */ + if (adata_len < 65280) { + len = 2; + *(__be16 *)enc_a = cpu_to_be16(adata_len); + } else if (adata_len <= 0xFFFFFFFF) { + len = 6; + *(__be16 *)enc_a = cpu_to_be16(0xfffe); + *(__be32 *)&enc_a[2] = cpu_to_be32(adata_len); + } else { /* adata_len >= 2^32 */ + len = 10; + *(__be16 *)enc_a = cpu_to_be16(0xffff); + *(__be64 *)&enc_a[2] = cpu_to_be64(adata_len); + } + for (i = 0; i < len; i++) + iowrite8(enc_a[i], + aes_dev->base_reg + + AES_A_DMA_INBUFFER_WRITE_FIFO_OFFSET); +} + +static int ocs_aes_ccm_do_adata(struct ocs_aes_dev *aes_dev, + dma_addr_t adata_dma_list, u32 adata_size) +{ + int rc; + + if (!adata_size) { + /* Since no aad the LAST_GCX bit can be set now */ + aes_a_set_last_gcx_and_adata(aes_dev); + goto exit; + } + + /* Adata case. */ + + /* + * Form the encoding of the Associated data length and write it + * to the AES/SM4 input buffer. + */ + ocs_aes_ccm_write_adata_len(aes_dev, adata_size); + + /* Configure the AES/SM4 DMA to fetch the Associated Data */ + dma_to_ocs_aes_ll(aes_dev, adata_dma_list); + + /* Activate DMA to fetch Associated data. */ + aes_a_dma_active_src_ll_en(aes_dev); + + /* Set LAST_GCX and LAST_ADATA in AES ACTIVE register. */ + aes_a_set_last_gcx_and_adata(aes_dev); + + /* Wait for DMA transfer to complete. */ + rc = ocs_aes_irq_enable_and_wait(aes_dev, AES_DMA_SRC_DONE_INT); + if (rc) + return rc; + +exit: + /* Wait until adata (if present) has been processed. */ + aes_a_wait_last_gcx(aes_dev); + aes_a_dma_wait_input_buffer_occupancy(aes_dev); + + return 0; +} + +static int ocs_aes_ccm_encrypt_do_payload(struct ocs_aes_dev *aes_dev, + dma_addr_t dst_dma_list, + dma_addr_t src_dma_list, + u32 src_size) +{ + if (src_size) { + /* + * Configure and activate DMA for both input and output + * data. + */ + dma_to_ocs_aes_ll(aes_dev, src_dma_list); + dma_from_ocs_aes_ll(aes_dev, dst_dma_list); + aes_a_dma_active_src_dst_ll_en(aes_dev); + } else { + /* Configure and activate DMA for output data only. */ + dma_from_ocs_aes_ll(aes_dev, dst_dma_list); + aes_a_dma_active_dst_ll_en(aes_dev); + } + + /* + * Set the LAST GCX bit in AES_ACTIVE Register to instruct + * AES/SM4 engine to pad the last block of data. + */ + aes_a_set_last_gcx(aes_dev); + + /* We are done, wait for IRQ and return. */ + return ocs_aes_irq_enable_and_wait(aes_dev, AES_COMPLETE_INT); +} + +static int ocs_aes_ccm_decrypt_do_payload(struct ocs_aes_dev *aes_dev, + dma_addr_t dst_dma_list, + dma_addr_t src_dma_list, + u32 src_size) +{ + if (!src_size) { + /* Let engine process 0-length input. */ + aes_a_dma_set_xfer_size_zero(aes_dev); + aes_a_dma_active(aes_dev); + aes_a_set_last_gcx(aes_dev); + + return 0; + } + + /* + * Configure and activate DMA for both input and output + * data. + */ + dma_to_ocs_aes_ll(aes_dev, src_dma_list); + dma_from_ocs_aes_ll(aes_dev, dst_dma_list); + aes_a_dma_active_src_dst_ll_en(aes_dev); + /* + * Set the LAST GCX bit in AES_ACTIVE Register; this allows the + * AES/SM4 engine to differentiate between encrypted data and + * encrypted MAC. + */ + aes_a_set_last_gcx(aes_dev); + /* + * Enable DMA DONE interrupt; once DMA transfer is over, + * interrupt handler will process the MAC/tag. + */ + return ocs_aes_irq_enable_and_wait(aes_dev, AES_DMA_SRC_DONE_INT); +} + +/* + * Compare Tag to Yr. + * + * Only used at the end of CCM decrypt. If tag == yr, message authentication + * has succeeded. + */ +static inline int ccm_compare_tag_to_yr(struct ocs_aes_dev *aes_dev, + u8 tag_size_bytes) +{ + u32 tag[AES_MAX_TAG_SIZE_U32]; + u32 yr[AES_MAX_TAG_SIZE_U32]; + u8 i; + + /* Read Tag and Yr from AES registers. */ + for (i = 0; i < AES_MAX_TAG_SIZE_U32; i++) { + tag[i] = ioread32(aes_dev->base_reg + + AES_T_MAC_0_OFFSET + (i * sizeof(u32))); + yr[i] = ioread32(aes_dev->base_reg + + AES_MULTIPURPOSE2_0_OFFSET + + (i * sizeof(u32))); + } + + return memcmp(tag, yr, tag_size_bytes) ? -EBADMSG : 0; +} + +/** + * ocs_aes_ccm_op() - Perform CCM operation. + * @aes_dev: The OCS AES device to use. + * @cipher: The Cipher to use (AES or SM4). + * @instruction: The instruction to perform (encrypt or decrypt). + * @dst_dma_list: The OCS DMA list mapping output memory. + * @src_dma_list: The OCS DMA list mapping input payload data. + * @src_size: The amount of data mapped by @src_dma_list. + * @iv: The input IV vector. + * @adata_dma_list: The OCS DMA list mapping input A-data. + * @adata_size: The amount of data mapped by @adata_dma_list. + * @in_tag: Input tag. + * @tag_size: The size (in bytes) of @in_tag. + * + * Note: for encrypt the tag is appended to the ciphertext (in the memory + * mapped by @dst_dma_list). + * + * Return: 0 on success, negative error code otherwise. + */ +int ocs_aes_ccm_op(struct ocs_aes_dev *aes_dev, + enum ocs_cipher cipher, + enum ocs_instruction instruction, + dma_addr_t dst_dma_list, + dma_addr_t src_dma_list, + u32 src_size, + u8 *iv, + dma_addr_t adata_dma_list, + u32 adata_size, + u8 *in_tag, + u32 tag_size) +{ + u32 *iv_32; + u8 lprime; + int rc; + + rc = ocs_aes_validate_inputs(src_dma_list, src_size, iv, + AES_BLOCK_SIZE, adata_dma_list, adata_size, + in_tag, tag_size, cipher, OCS_MODE_CCM, + instruction, dst_dma_list); + if (rc) + return rc; + + ocs_aes_init(aes_dev, OCS_MODE_CCM, cipher, instruction); + + /* + * Note: rfc 3610 and NIST 800-38C require counter of zero to encrypt + * auth tag so ensure this is the case + */ + lprime = iv[L_PRIME_IDX]; + memset(&iv[COUNTER_START(lprime)], 0, COUNTER_LEN(lprime)); + + /* + * Nonce is already converted to ctr0 before being passed into this + * function as iv. + */ + iv_32 = (u32 *)iv; + iowrite32(__swab32(iv_32[0]), + aes_dev->base_reg + AES_MULTIPURPOSE1_3_OFFSET); + iowrite32(__swab32(iv_32[1]), + aes_dev->base_reg + AES_MULTIPURPOSE1_2_OFFSET); + iowrite32(__swab32(iv_32[2]), + aes_dev->base_reg + AES_MULTIPURPOSE1_1_OFFSET); + iowrite32(__swab32(iv_32[3]), + aes_dev->base_reg + AES_MULTIPURPOSE1_0_OFFSET); + + /* Write MAC/tag length in register AES_TLEN */ + iowrite32(tag_size, aes_dev->base_reg + AES_TLEN_OFFSET); + /* + * Write the byte length of the last AES/SM4 block of Payload data + * (without zero padding and without the length of the MAC) in register + * AES_PLEN. + */ + ocs_aes_write_last_data_blk_len(aes_dev, src_size); + + /* Set AES_ACTIVE.TRIGGER to start the operation. */ + aes_a_op_trigger(aes_dev); + + aes_a_dma_reset_and_activate_perf_cntr(aes_dev); + + /* Form block B0 and write it to the AES/SM4 input buffer. */ + rc = ocs_aes_ccm_write_b0(aes_dev, iv, adata_size, tag_size, src_size); + if (rc) + return rc; + /* + * Ensure there has been at least CCM_DECRYPT_DELAY_LAST_GCX_CLK_COUNT + * clock cycles since TRIGGER bit was set + */ + aes_a_dma_wait_and_deactivate_perf_cntr(aes_dev, + CCM_DECRYPT_DELAY_LAST_GCX_CLK_COUNT); + + /* Process Adata. */ + ocs_aes_ccm_do_adata(aes_dev, adata_dma_list, adata_size); + + /* For Encrypt case we just process the payload and return. */ + if (instruction == OCS_ENCRYPT) { + return ocs_aes_ccm_encrypt_do_payload(aes_dev, dst_dma_list, + src_dma_list, src_size); + } + /* For Decypt we need to process the payload and then the tag. */ + rc = ocs_aes_ccm_decrypt_do_payload(aes_dev, dst_dma_list, + src_dma_list, src_size); + if (rc) + return rc; + + /* Process MAC/tag directly: feed tag to engine and wait for IRQ. */ + ocs_aes_ccm_write_encrypted_tag(aes_dev, in_tag, tag_size); + rc = ocs_aes_irq_enable_and_wait(aes_dev, AES_COMPLETE_INT); + if (rc) + return rc; + + return ccm_compare_tag_to_yr(aes_dev, tag_size); +} + +/** + * ocs_create_linked_list_from_sg() - Create OCS DMA linked list from SG list. + * @aes_dev: The OCS AES device the list will be created for. + * @sg: The SG list OCS DMA linked list will be created from. When + * passed to this function, @sg must have been already mapped + * with dma_map_sg(). + * @sg_dma_count: The number of DMA-mapped entries in @sg. This must be the + * value returned by dma_map_sg() when @sg was mapped. + * @dll_desc: The OCS DMA dma_list to use to store information about the + * created linked list. + * @data_size: The size of the data (from the SG list) to be mapped into the + * OCS DMA linked list. + * @data_offset: The offset (within the SG list) of the data to be mapped. + * + * Return: 0 on success, negative error code otherwise. + */ +int ocs_create_linked_list_from_sg(const struct ocs_aes_dev *aes_dev, + struct scatterlist *sg, + int sg_dma_count, + struct ocs_dll_desc *dll_desc, + size_t data_size, size_t data_offset) +{ + struct ocs_dma_linked_list *ll = NULL; + struct scatterlist *sg_tmp; + unsigned int tmp; + int dma_nents; + int i; + + if (!dll_desc || !sg || !aes_dev) + return -EINVAL; + + /* Default values for when no ddl_desc is created. */ + dll_desc->vaddr = NULL; + dll_desc->dma_addr = DMA_MAPPING_ERROR; + dll_desc->size = 0; + + if (data_size == 0) + return 0; + + /* Loop over sg_list until we reach entry at specified offset. */ + while (data_offset >= sg_dma_len(sg)) { + data_offset -= sg_dma_len(sg); + sg_dma_count--; + sg = sg_next(sg); + /* If we reach the end of the list, offset was invalid. */ + if (!sg || sg_dma_count == 0) + return -EINVAL; + } + + /* Compute number of DMA-mapped SG entries to add into OCS DMA list. */ + dma_nents = 0; + tmp = 0; + sg_tmp = sg; + while (tmp < data_offset + data_size) { + /* If we reach the end of the list, data_size was invalid. */ + if (!sg_tmp) + return -EINVAL; + tmp += sg_dma_len(sg_tmp); + dma_nents++; + sg_tmp = sg_next(sg_tmp); + } + if (dma_nents > sg_dma_count) + return -EINVAL; + + /* Allocate the DMA list, one entry for each SG entry. */ + dll_desc->size = sizeof(struct ocs_dma_linked_list) * dma_nents; + dll_desc->vaddr = dma_alloc_coherent(aes_dev->dev, dll_desc->size, + &dll_desc->dma_addr, GFP_KERNEL); + if (!dll_desc->vaddr) + return -ENOMEM; + + /* Populate DMA linked list entries. */ + ll = dll_desc->vaddr; + for (i = 0; i < dma_nents; i++, sg = sg_next(sg)) { + ll[i].src_addr = sg_dma_address(sg) + data_offset; + ll[i].src_len = (sg_dma_len(sg) - data_offset) < data_size ? + (sg_dma_len(sg) - data_offset) : data_size; + data_offset = 0; + data_size -= ll[i].src_len; + /* Current element points to the DMA address of the next one. */ + ll[i].next = dll_desc->dma_addr + (sizeof(*ll) * (i + 1)); + ll[i].ll_flags = 0; + } + /* Terminate last element. */ + ll[i - 1].next = 0; + ll[i - 1].ll_flags = OCS_LL_DMA_FLAG_TERMINATE; + + return 0; +} diff --git a/drivers/crypto/intel/keembay/ocs-aes.h b/drivers/crypto/intel/keembay/ocs-aes.h new file mode 100644 index 000000000000..c035fc48b7ed --- /dev/null +++ b/drivers/crypto/intel/keembay/ocs-aes.h @@ -0,0 +1,129 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Intel Keem Bay OCS AES Crypto Driver. + * + * Copyright (C) 2018-2020 Intel Corporation + */ + +#ifndef _CRYPTO_OCS_AES_H +#define _CRYPTO_OCS_AES_H + +#include + +enum ocs_cipher { + OCS_AES = 0, + OCS_SM4 = 1, +}; + +enum ocs_mode { + OCS_MODE_ECB = 0, + OCS_MODE_CBC = 1, + OCS_MODE_CTR = 2, + OCS_MODE_CCM = 6, + OCS_MODE_GCM = 7, + OCS_MODE_CTS = 9, +}; + +enum ocs_instruction { + OCS_ENCRYPT = 0, + OCS_DECRYPT = 1, + OCS_EXPAND = 2, + OCS_BYPASS = 3, +}; + +/** + * struct ocs_aes_dev - AES device context. + * @list: List head for insertion into device list hold + * by driver. + * @dev: OCS AES device. + * @irq: IRQ number. + * @base_reg: IO base address of OCS AES. + * @irq_copy_completion: Completion to indicate IRQ has been triggered. + * @dma_err_mask: Error reported by OCS DMA interrupts. + * @engine: Crypto engine for the device. + */ +struct ocs_aes_dev { + struct list_head list; + struct device *dev; + int irq; + void __iomem *base_reg; + struct completion irq_completion; + u32 dma_err_mask; + struct crypto_engine *engine; +}; + +/** + * struct ocs_dll_desc - Descriptor of an OCS DMA Linked List. + * @vaddr: Virtual address of the linked list head. + * @dma_addr: DMA address of the linked list head. + * @size: Size (in bytes) of the linked list. + */ +struct ocs_dll_desc { + void *vaddr; + dma_addr_t dma_addr; + size_t size; +}; + +int ocs_aes_set_key(struct ocs_aes_dev *aes_dev, const u32 key_size, + const u8 *key, const enum ocs_cipher cipher); + +int ocs_aes_op(struct ocs_aes_dev *aes_dev, + enum ocs_mode mode, + enum ocs_cipher cipher, + enum ocs_instruction instruction, + dma_addr_t dst_dma_list, + dma_addr_t src_dma_list, + u32 src_size, + u8 *iv, + u32 iv_size); + +/** + * ocs_aes_bypass_op() - Use OCS DMA to copy data. + * @aes_dev: The OCS AES device to use. + * @dst_dma_list: The OCS DMA list mapping the memory where input data + * will be copied to. + * @src_dma_list: The OCS DMA list mapping input data. + * @src_size: The amount of data to copy. + */ +static inline int ocs_aes_bypass_op(struct ocs_aes_dev *aes_dev, + dma_addr_t dst_dma_list, + dma_addr_t src_dma_list, u32 src_size) +{ + return ocs_aes_op(aes_dev, OCS_MODE_ECB, OCS_AES, OCS_BYPASS, + dst_dma_list, src_dma_list, src_size, NULL, 0); +} + +int ocs_aes_gcm_op(struct ocs_aes_dev *aes_dev, + enum ocs_cipher cipher, + enum ocs_instruction instruction, + dma_addr_t dst_dma_list, + dma_addr_t src_dma_list, + u32 src_size, + const u8 *iv, + dma_addr_t aad_dma_list, + u32 aad_size, + u8 *out_tag, + u32 tag_size); + +int ocs_aes_ccm_op(struct ocs_aes_dev *aes_dev, + enum ocs_cipher cipher, + enum ocs_instruction instruction, + dma_addr_t dst_dma_list, + dma_addr_t src_dma_list, + u32 src_size, + u8 *iv, + dma_addr_t adata_dma_list, + u32 adata_size, + u8 *in_tag, + u32 tag_size); + +int ocs_create_linked_list_from_sg(const struct ocs_aes_dev *aes_dev, + struct scatterlist *sg, + int sg_dma_count, + struct ocs_dll_desc *dll_desc, + size_t data_size, + size_t data_offset); + +irqreturn_t ocs_aes_irq_handler(int irq, void *dev_id); + +#endif diff --git a/drivers/crypto/intel/keembay/ocs-hcu.c b/drivers/crypto/intel/keembay/ocs-hcu.c new file mode 100644 index 000000000000..deb9bd460ee6 --- /dev/null +++ b/drivers/crypto/intel/keembay/ocs-hcu.c @@ -0,0 +1,840 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Intel Keem Bay OCS HCU Crypto Driver. + * + * Copyright (C) 2018-2020 Intel Corporation + */ + +#include +#include +#include +#include +#include + +#include + +#include "ocs-hcu.h" + +/* Registers. */ +#define OCS_HCU_MODE 0x00 +#define OCS_HCU_CHAIN 0x04 +#define OCS_HCU_OPERATION 0x08 +#define OCS_HCU_KEY_0 0x0C +#define OCS_HCU_ISR 0x50 +#define OCS_HCU_IER 0x54 +#define OCS_HCU_STATUS 0x58 +#define OCS_HCU_MSG_LEN_LO 0x60 +#define OCS_HCU_MSG_LEN_HI 0x64 +#define OCS_HCU_KEY_BYTE_ORDER_CFG 0x80 +#define OCS_HCU_DMA_SRC_ADDR 0x400 +#define OCS_HCU_DMA_SRC_SIZE 0x408 +#define OCS_HCU_DMA_DST_SIZE 0x40C +#define OCS_HCU_DMA_DMA_MODE 0x410 +#define OCS_HCU_DMA_NEXT_SRC_DESCR 0x418 +#define OCS_HCU_DMA_MSI_ISR 0x480 +#define OCS_HCU_DMA_MSI_IER 0x484 +#define OCS_HCU_DMA_MSI_MASK 0x488 + +/* Register bit definitions. */ +#define HCU_MODE_ALGO_SHIFT 16 +#define HCU_MODE_HMAC_SHIFT 22 + +#define HCU_STATUS_BUSY BIT(0) + +#define HCU_BYTE_ORDER_SWAP BIT(0) + +#define HCU_IRQ_HASH_DONE BIT(2) +#define HCU_IRQ_HASH_ERR_MASK (BIT(3) | BIT(1) | BIT(0)) + +#define HCU_DMA_IRQ_SRC_DONE BIT(0) +#define HCU_DMA_IRQ_SAI_ERR BIT(2) +#define HCU_DMA_IRQ_BAD_COMP_ERR BIT(3) +#define HCU_DMA_IRQ_INBUF_RD_ERR BIT(4) +#define HCU_DMA_IRQ_INBUF_WD_ERR BIT(5) +#define HCU_DMA_IRQ_OUTBUF_WR_ERR BIT(6) +#define HCU_DMA_IRQ_OUTBUF_RD_ERR BIT(7) +#define HCU_DMA_IRQ_CRD_ERR BIT(8) +#define HCU_DMA_IRQ_ERR_MASK (HCU_DMA_IRQ_SAI_ERR | \ + HCU_DMA_IRQ_BAD_COMP_ERR | \ + HCU_DMA_IRQ_INBUF_RD_ERR | \ + HCU_DMA_IRQ_INBUF_WD_ERR | \ + HCU_DMA_IRQ_OUTBUF_WR_ERR | \ + HCU_DMA_IRQ_OUTBUF_RD_ERR | \ + HCU_DMA_IRQ_CRD_ERR) + +#define HCU_DMA_SNOOP_MASK (0x7 << 28) +#define HCU_DMA_SRC_LL_EN BIT(25) +#define HCU_DMA_EN BIT(31) + +#define OCS_HCU_ENDIANNESS_VALUE 0x2A + +#define HCU_DMA_MSI_UNMASK BIT(0) +#define HCU_DMA_MSI_DISABLE 0 +#define HCU_IRQ_DISABLE 0 + +#define OCS_HCU_START BIT(0) +#define OCS_HCU_TERMINATE BIT(1) + +#define OCS_LL_DMA_FLAG_TERMINATE BIT(31) + +#define OCS_HCU_HW_KEY_LEN_U32 (OCS_HCU_HW_KEY_LEN / sizeof(u32)) + +#define HCU_DATA_WRITE_ENDIANNESS_OFFSET 26 + +#define OCS_HCU_NUM_CHAINS_SHA256_224_SM3 (SHA256_DIGEST_SIZE / sizeof(u32)) +#define OCS_HCU_NUM_CHAINS_SHA384_512 (SHA512_DIGEST_SIZE / sizeof(u32)) + +/* + * While polling on a busy HCU, wait maximum 200us between one check and the + * other. + */ +#define OCS_HCU_WAIT_BUSY_RETRY_DELAY_US 200 +/* Wait on a busy HCU for maximum 1 second. */ +#define OCS_HCU_WAIT_BUSY_TIMEOUT_US 1000000 + +/** + * struct ocs_hcu_dma_entry - An entry in an OCS DMA linked list. + * @src_addr: Source address of the data. + * @src_len: Length of data to be fetched. + * @nxt_desc: Next descriptor to fetch. + * @ll_flags: Flags (Freeze @ terminate) for the DMA engine. + */ +struct ocs_hcu_dma_entry { + u32 src_addr; + u32 src_len; + u32 nxt_desc; + u32 ll_flags; +}; + +/** + * struct ocs_hcu_dma_list - OCS-specific DMA linked list. + * @head: The head of the list (points to the array backing the list). + * @tail: The current tail of the list; NULL if the list is empty. + * @dma_addr: The DMA address of @head (i.e., the DMA address of the backing + * array). + * @max_nents: Maximum number of entries in the list (i.e., number of elements + * in the backing array). + * + * The OCS DMA list is an array-backed list of OCS DMA descriptors. The array + * backing the list is allocated with dma_alloc_coherent() and pointed by + * @head. + */ +struct ocs_hcu_dma_list { + struct ocs_hcu_dma_entry *head; + struct ocs_hcu_dma_entry *tail; + dma_addr_t dma_addr; + size_t max_nents; +}; + +static inline u32 ocs_hcu_num_chains(enum ocs_hcu_algo algo) +{ + switch (algo) { + case OCS_HCU_ALGO_SHA224: + case OCS_HCU_ALGO_SHA256: + case OCS_HCU_ALGO_SM3: + return OCS_HCU_NUM_CHAINS_SHA256_224_SM3; + case OCS_HCU_ALGO_SHA384: + case OCS_HCU_ALGO_SHA512: + return OCS_HCU_NUM_CHAINS_SHA384_512; + default: + return 0; + }; +} + +static inline u32 ocs_hcu_digest_size(enum ocs_hcu_algo algo) +{ + switch (algo) { + case OCS_HCU_ALGO_SHA224: + return SHA224_DIGEST_SIZE; + case OCS_HCU_ALGO_SHA256: + case OCS_HCU_ALGO_SM3: + /* SM3 shares the same block size. */ + return SHA256_DIGEST_SIZE; + case OCS_HCU_ALGO_SHA384: + return SHA384_DIGEST_SIZE; + case OCS_HCU_ALGO_SHA512: + return SHA512_DIGEST_SIZE; + default: + return 0; + } +} + +/** + * ocs_hcu_wait_busy() - Wait for HCU OCS hardware to became usable. + * @hcu_dev: OCS HCU device to wait for. + * + * Return: 0 if device free, -ETIMEOUT if device busy and internal timeout has + * expired. + */ +static int ocs_hcu_wait_busy(struct ocs_hcu_dev *hcu_dev) +{ + long val; + + return readl_poll_timeout(hcu_dev->io_base + OCS_HCU_STATUS, val, + !(val & HCU_STATUS_BUSY), + OCS_HCU_WAIT_BUSY_RETRY_DELAY_US, + OCS_HCU_WAIT_BUSY_TIMEOUT_US); +} + +static void ocs_hcu_done_irq_en(struct ocs_hcu_dev *hcu_dev) +{ + /* Clear any pending interrupts. */ + writel(0xFFFFFFFF, hcu_dev->io_base + OCS_HCU_ISR); + hcu_dev->irq_err = false; + /* Enable error and HCU done interrupts. */ + writel(HCU_IRQ_HASH_DONE | HCU_IRQ_HASH_ERR_MASK, + hcu_dev->io_base + OCS_HCU_IER); +} + +static void ocs_hcu_dma_irq_en(struct ocs_hcu_dev *hcu_dev) +{ + /* Clear any pending interrupts. */ + writel(0xFFFFFFFF, hcu_dev->io_base + OCS_HCU_DMA_MSI_ISR); + hcu_dev->irq_err = false; + /* Only operating on DMA source completion and error interrupts. */ + writel(HCU_DMA_IRQ_ERR_MASK | HCU_DMA_IRQ_SRC_DONE, + hcu_dev->io_base + OCS_HCU_DMA_MSI_IER); + /* Unmask */ + writel(HCU_DMA_MSI_UNMASK, hcu_dev->io_base + OCS_HCU_DMA_MSI_MASK); +} + +static void ocs_hcu_irq_dis(struct ocs_hcu_dev *hcu_dev) +{ + writel(HCU_IRQ_DISABLE, hcu_dev->io_base + OCS_HCU_IER); + writel(HCU_DMA_MSI_DISABLE, hcu_dev->io_base + OCS_HCU_DMA_MSI_IER); +} + +static int ocs_hcu_wait_and_disable_irq(struct ocs_hcu_dev *hcu_dev) +{ + int rc; + + rc = wait_for_completion_interruptible(&hcu_dev->irq_done); + if (rc) + goto exit; + + if (hcu_dev->irq_err) { + /* Unset flag and return error. */ + hcu_dev->irq_err = false; + rc = -EIO; + goto exit; + } + +exit: + ocs_hcu_irq_dis(hcu_dev); + + return rc; +} + +/** + * ocs_hcu_get_intermediate_data() - Get intermediate data. + * @hcu_dev: The target HCU device. + * @data: Where to store the intermediate. + * @algo: The algorithm being used. + * + * This function is used to save the current hashing process state in order to + * continue it in the future. + * + * Note: once all data has been processed, the intermediate data actually + * contains the hashing result. So this function is also used to retrieve the + * final result of a hashing process. + * + * Return: 0 on success, negative error code otherwise. + */ +static int ocs_hcu_get_intermediate_data(struct ocs_hcu_dev *hcu_dev, + struct ocs_hcu_idata *data, + enum ocs_hcu_algo algo) +{ + const int n = ocs_hcu_num_chains(algo); + u32 *chain; + int rc; + int i; + + /* Data not requested. */ + if (!data) + return -EINVAL; + + chain = (u32 *)data->digest; + + /* Ensure that the OCS is no longer busy before reading the chains. */ + rc = ocs_hcu_wait_busy(hcu_dev); + if (rc) + return rc; + + /* + * This loops is safe because data->digest is an array of + * SHA512_DIGEST_SIZE bytes and the maximum value returned by + * ocs_hcu_num_chains() is OCS_HCU_NUM_CHAINS_SHA384_512 which is equal + * to SHA512_DIGEST_SIZE / sizeof(u32). + */ + for (i = 0; i < n; i++) + chain[i] = readl(hcu_dev->io_base + OCS_HCU_CHAIN); + + data->msg_len_lo = readl(hcu_dev->io_base + OCS_HCU_MSG_LEN_LO); + data->msg_len_hi = readl(hcu_dev->io_base + OCS_HCU_MSG_LEN_HI); + + return 0; +} + +/** + * ocs_hcu_set_intermediate_data() - Set intermediate data. + * @hcu_dev: The target HCU device. + * @data: The intermediate data to be set. + * @algo: The algorithm being used. + * + * This function is used to continue a previous hashing process. + */ +static void ocs_hcu_set_intermediate_data(struct ocs_hcu_dev *hcu_dev, + const struct ocs_hcu_idata *data, + enum ocs_hcu_algo algo) +{ + const int n = ocs_hcu_num_chains(algo); + u32 *chain = (u32 *)data->digest; + int i; + + /* + * This loops is safe because data->digest is an array of + * SHA512_DIGEST_SIZE bytes and the maximum value returned by + * ocs_hcu_num_chains() is OCS_HCU_NUM_CHAINS_SHA384_512 which is equal + * to SHA512_DIGEST_SIZE / sizeof(u32). + */ + for (i = 0; i < n; i++) + writel(chain[i], hcu_dev->io_base + OCS_HCU_CHAIN); + + writel(data->msg_len_lo, hcu_dev->io_base + OCS_HCU_MSG_LEN_LO); + writel(data->msg_len_hi, hcu_dev->io_base + OCS_HCU_MSG_LEN_HI); +} + +static int ocs_hcu_get_digest(struct ocs_hcu_dev *hcu_dev, + enum ocs_hcu_algo algo, u8 *dgst, size_t dgst_len) +{ + u32 *chain; + int rc; + int i; + + if (!dgst) + return -EINVAL; + + /* Length of the output buffer must match the algo digest size. */ + if (dgst_len != ocs_hcu_digest_size(algo)) + return -EINVAL; + + /* Ensure that the OCS is no longer busy before reading the chains. */ + rc = ocs_hcu_wait_busy(hcu_dev); + if (rc) + return rc; + + chain = (u32 *)dgst; + for (i = 0; i < dgst_len / sizeof(u32); i++) + chain[i] = readl(hcu_dev->io_base + OCS_HCU_CHAIN); + + return 0; +} + +/** + * ocs_hcu_hw_cfg() - Configure the HCU hardware. + * @hcu_dev: The HCU device to configure. + * @algo: The algorithm to be used by the HCU device. + * @use_hmac: Whether or not HW HMAC should be used. + * + * Return: 0 on success, negative error code otherwise. + */ +static int ocs_hcu_hw_cfg(struct ocs_hcu_dev *hcu_dev, enum ocs_hcu_algo algo, + bool use_hmac) +{ + u32 cfg; + int rc; + + if (algo != OCS_HCU_ALGO_SHA256 && algo != OCS_HCU_ALGO_SHA224 && + algo != OCS_HCU_ALGO_SHA384 && algo != OCS_HCU_ALGO_SHA512 && + algo != OCS_HCU_ALGO_SM3) + return -EINVAL; + + rc = ocs_hcu_wait_busy(hcu_dev); + if (rc) + return rc; + + /* Ensure interrupts are disabled. */ + ocs_hcu_irq_dis(hcu_dev); + + /* Configure endianness, hashing algorithm and HW HMAC (if needed) */ + cfg = OCS_HCU_ENDIANNESS_VALUE << HCU_DATA_WRITE_ENDIANNESS_OFFSET; + cfg |= algo << HCU_MODE_ALGO_SHIFT; + if (use_hmac) + cfg |= BIT(HCU_MODE_HMAC_SHIFT); + + writel(cfg, hcu_dev->io_base + OCS_HCU_MODE); + + return 0; +} + +/** + * ocs_hcu_clear_key() - Clear key stored in OCS HMAC KEY registers. + * @hcu_dev: The OCS HCU device whose key registers should be cleared. + */ +static void ocs_hcu_clear_key(struct ocs_hcu_dev *hcu_dev) +{ + int reg_off; + + /* Clear OCS_HCU_KEY_[0..15] */ + for (reg_off = 0; reg_off < OCS_HCU_HW_KEY_LEN; reg_off += sizeof(u32)) + writel(0, hcu_dev->io_base + OCS_HCU_KEY_0 + reg_off); +} + +/** + * ocs_hcu_write_key() - Write key to OCS HMAC KEY registers. + * @hcu_dev: The OCS HCU device the key should be written to. + * @key: The key to be written. + * @len: The size of the key to write. It must be OCS_HCU_HW_KEY_LEN. + * + * Return: 0 on success, negative error code otherwise. + */ +static int ocs_hcu_write_key(struct ocs_hcu_dev *hcu_dev, const u8 *key, size_t len) +{ + u32 key_u32[OCS_HCU_HW_KEY_LEN_U32]; + int i; + + if (len > OCS_HCU_HW_KEY_LEN) + return -EINVAL; + + /* Copy key into temporary u32 array. */ + memcpy(key_u32, key, len); + + /* + * Hardware requires all the bytes of the HW Key vector to be + * written. So pad with zero until we reach OCS_HCU_HW_KEY_LEN. + */ + memzero_explicit((u8 *)key_u32 + len, OCS_HCU_HW_KEY_LEN - len); + + /* + * OCS hardware expects the MSB of the key to be written at the highest + * address of the HCU Key vector; in other word, the key must be + * written in reverse order. + * + * Therefore, we first enable byte swapping for the HCU key vector; + * so that bytes of 32-bit word written to OCS_HCU_KEY_[0..15] will be + * swapped: + * 3 <---> 0, 2 <---> 1. + */ + writel(HCU_BYTE_ORDER_SWAP, + hcu_dev->io_base + OCS_HCU_KEY_BYTE_ORDER_CFG); + /* + * And then we write the 32-bit words composing the key starting from + * the end of the key. + */ + for (i = 0; i < OCS_HCU_HW_KEY_LEN_U32; i++) + writel(key_u32[OCS_HCU_HW_KEY_LEN_U32 - 1 - i], + hcu_dev->io_base + OCS_HCU_KEY_0 + (sizeof(u32) * i)); + + memzero_explicit(key_u32, OCS_HCU_HW_KEY_LEN); + + return 0; +} + +/** + * ocs_hcu_ll_dma_start() - Start OCS HCU hashing via DMA + * @hcu_dev: The OCS HCU device to use. + * @dma_list: The OCS DMA list mapping the data to hash. + * @finalize: Whether or not this is the last hashing operation and therefore + * the final hash should be compute even if data is not + * block-aligned. + * + * Return: 0 on success, negative error code otherwise. + */ +static int ocs_hcu_ll_dma_start(struct ocs_hcu_dev *hcu_dev, + const struct ocs_hcu_dma_list *dma_list, + bool finalize) +{ + u32 cfg = HCU_DMA_SNOOP_MASK | HCU_DMA_SRC_LL_EN | HCU_DMA_EN; + int rc; + + if (!dma_list) + return -EINVAL; + + /* + * For final requests we use HCU_DONE IRQ to be notified when all input + * data has been processed by the HCU; however, we cannot do so for + * non-final requests, because we don't get a HCU_DONE IRQ when we + * don't terminate the operation. + * + * Therefore, for non-final requests, we use the DMA IRQ, which + * triggers when DMA has finishing feeding all the input data to the + * HCU, but the HCU may still be processing it. This is fine, since we + * will wait for the HCU processing to be completed when we try to read + * intermediate results, in ocs_hcu_get_intermediate_data(). + */ + if (finalize) + ocs_hcu_done_irq_en(hcu_dev); + else + ocs_hcu_dma_irq_en(hcu_dev); + + reinit_completion(&hcu_dev->irq_done); + writel(dma_list->dma_addr, hcu_dev->io_base + OCS_HCU_DMA_NEXT_SRC_DESCR); + writel(0, hcu_dev->io_base + OCS_HCU_DMA_SRC_SIZE); + writel(0, hcu_dev->io_base + OCS_HCU_DMA_DST_SIZE); + + writel(OCS_HCU_START, hcu_dev->io_base + OCS_HCU_OPERATION); + + writel(cfg, hcu_dev->io_base + OCS_HCU_DMA_DMA_MODE); + + if (finalize) + writel(OCS_HCU_TERMINATE, hcu_dev->io_base + OCS_HCU_OPERATION); + + rc = ocs_hcu_wait_and_disable_irq(hcu_dev); + if (rc) + return rc; + + return 0; +} + +struct ocs_hcu_dma_list *ocs_hcu_dma_list_alloc(struct ocs_hcu_dev *hcu_dev, + int max_nents) +{ + struct ocs_hcu_dma_list *dma_list; + + dma_list = kmalloc(sizeof(*dma_list), GFP_KERNEL); + if (!dma_list) + return NULL; + + /* Total size of the DMA list to allocate. */ + dma_list->head = dma_alloc_coherent(hcu_dev->dev, + sizeof(*dma_list->head) * max_nents, + &dma_list->dma_addr, GFP_KERNEL); + if (!dma_list->head) { + kfree(dma_list); + return NULL; + } + dma_list->max_nents = max_nents; + dma_list->tail = NULL; + + return dma_list; +} + +void ocs_hcu_dma_list_free(struct ocs_hcu_dev *hcu_dev, + struct ocs_hcu_dma_list *dma_list) +{ + if (!dma_list) + return; + + dma_free_coherent(hcu_dev->dev, + sizeof(*dma_list->head) * dma_list->max_nents, + dma_list->head, dma_list->dma_addr); + + kfree(dma_list); +} + +/* Add a new DMA entry at the end of the OCS DMA list. */ +int ocs_hcu_dma_list_add_tail(struct ocs_hcu_dev *hcu_dev, + struct ocs_hcu_dma_list *dma_list, + dma_addr_t addr, u32 len) +{ + struct device *dev = hcu_dev->dev; + struct ocs_hcu_dma_entry *old_tail; + struct ocs_hcu_dma_entry *new_tail; + + if (!len) + return 0; + + if (!dma_list) + return -EINVAL; + + if (addr & ~OCS_HCU_DMA_BIT_MASK) { + dev_err(dev, + "Unexpected error: Invalid DMA address for OCS HCU\n"); + return -EINVAL; + } + + old_tail = dma_list->tail; + new_tail = old_tail ? old_tail + 1 : dma_list->head; + + /* Check if list is full. */ + if (new_tail - dma_list->head >= dma_list->max_nents) + return -ENOMEM; + + /* + * If there was an old tail (i.e., this is not the first element we are + * adding), un-terminate the old tail and make it point to the new one. + */ + if (old_tail) { + old_tail->ll_flags &= ~OCS_LL_DMA_FLAG_TERMINATE; + /* + * The old tail 'nxt_desc' must point to the DMA address of the + * new tail. + */ + old_tail->nxt_desc = dma_list->dma_addr + + sizeof(*dma_list->tail) * (new_tail - + dma_list->head); + } + + new_tail->src_addr = (u32)addr; + new_tail->src_len = (u32)len; + new_tail->ll_flags = OCS_LL_DMA_FLAG_TERMINATE; + new_tail->nxt_desc = 0; + + /* Update list tail with new tail. */ + dma_list->tail = new_tail; + + return 0; +} + +/** + * ocs_hcu_hash_init() - Initialize hash operation context. + * @ctx: The context to initialize. + * @algo: The hashing algorithm to use. + * + * Return: 0 on success, negative error code otherwise. + */ +int ocs_hcu_hash_init(struct ocs_hcu_hash_ctx *ctx, enum ocs_hcu_algo algo) +{ + if (!ctx) + return -EINVAL; + + ctx->algo = algo; + ctx->idata.msg_len_lo = 0; + ctx->idata.msg_len_hi = 0; + /* No need to set idata.digest to 0. */ + + return 0; +} + +/** + * ocs_hcu_hash_update() - Perform a hashing iteration. + * @hcu_dev: The OCS HCU device to use. + * @ctx: The OCS HCU hashing context. + * @dma_list: The OCS DMA list mapping the input data to process. + * + * Return: 0 on success; negative error code otherwise. + */ +int ocs_hcu_hash_update(struct ocs_hcu_dev *hcu_dev, + struct ocs_hcu_hash_ctx *ctx, + const struct ocs_hcu_dma_list *dma_list) +{ + int rc; + + if (!hcu_dev || !ctx) + return -EINVAL; + + /* Configure the hardware for the current request. */ + rc = ocs_hcu_hw_cfg(hcu_dev, ctx->algo, false); + if (rc) + return rc; + + /* If we already processed some data, idata needs to be set. */ + if (ctx->idata.msg_len_lo || ctx->idata.msg_len_hi) + ocs_hcu_set_intermediate_data(hcu_dev, &ctx->idata, ctx->algo); + + /* Start linked-list DMA hashing. */ + rc = ocs_hcu_ll_dma_start(hcu_dev, dma_list, false); + if (rc) + return rc; + + /* Update idata and return. */ + return ocs_hcu_get_intermediate_data(hcu_dev, &ctx->idata, ctx->algo); +} + +/** + * ocs_hcu_hash_finup() - Update and finalize hash computation. + * @hcu_dev: The OCS HCU device to use. + * @ctx: The OCS HCU hashing context. + * @dma_list: The OCS DMA list mapping the input data to process. + * @dgst: The buffer where to save the computed digest. + * @dgst_len: The length of @dgst. + * + * Return: 0 on success; negative error code otherwise. + */ +int ocs_hcu_hash_finup(struct ocs_hcu_dev *hcu_dev, + const struct ocs_hcu_hash_ctx *ctx, + const struct ocs_hcu_dma_list *dma_list, + u8 *dgst, size_t dgst_len) +{ + int rc; + + if (!hcu_dev || !ctx) + return -EINVAL; + + /* Configure the hardware for the current request. */ + rc = ocs_hcu_hw_cfg(hcu_dev, ctx->algo, false); + if (rc) + return rc; + + /* If we already processed some data, idata needs to be set. */ + if (ctx->idata.msg_len_lo || ctx->idata.msg_len_hi) + ocs_hcu_set_intermediate_data(hcu_dev, &ctx->idata, ctx->algo); + + /* Start linked-list DMA hashing. */ + rc = ocs_hcu_ll_dma_start(hcu_dev, dma_list, true); + if (rc) + return rc; + + /* Get digest and return. */ + return ocs_hcu_get_digest(hcu_dev, ctx->algo, dgst, dgst_len); +} + +/** + * ocs_hcu_hash_final() - Finalize hash computation. + * @hcu_dev: The OCS HCU device to use. + * @ctx: The OCS HCU hashing context. + * @dgst: The buffer where to save the computed digest. + * @dgst_len: The length of @dgst. + * + * Return: 0 on success; negative error code otherwise. + */ +int ocs_hcu_hash_final(struct ocs_hcu_dev *hcu_dev, + const struct ocs_hcu_hash_ctx *ctx, u8 *dgst, + size_t dgst_len) +{ + int rc; + + if (!hcu_dev || !ctx) + return -EINVAL; + + /* Configure the hardware for the current request. */ + rc = ocs_hcu_hw_cfg(hcu_dev, ctx->algo, false); + if (rc) + return rc; + + /* If we already processed some data, idata needs to be set. */ + if (ctx->idata.msg_len_lo || ctx->idata.msg_len_hi) + ocs_hcu_set_intermediate_data(hcu_dev, &ctx->idata, ctx->algo); + + /* + * Enable HCU interrupts, so that HCU_DONE will be triggered once the + * final hash is computed. + */ + ocs_hcu_done_irq_en(hcu_dev); + reinit_completion(&hcu_dev->irq_done); + writel(OCS_HCU_TERMINATE, hcu_dev->io_base + OCS_HCU_OPERATION); + + rc = ocs_hcu_wait_and_disable_irq(hcu_dev); + if (rc) + return rc; + + /* Get digest and return. */ + return ocs_hcu_get_digest(hcu_dev, ctx->algo, dgst, dgst_len); +} + +/** + * ocs_hcu_digest() - Compute hash digest. + * @hcu_dev: The OCS HCU device to use. + * @algo: The hash algorithm to use. + * @data: The input data to process. + * @data_len: The length of @data. + * @dgst: The buffer where to save the computed digest. + * @dgst_len: The length of @dgst. + * + * Return: 0 on success; negative error code otherwise. + */ +int ocs_hcu_digest(struct ocs_hcu_dev *hcu_dev, enum ocs_hcu_algo algo, + void *data, size_t data_len, u8 *dgst, size_t dgst_len) +{ + struct device *dev = hcu_dev->dev; + dma_addr_t dma_handle; + u32 reg; + int rc; + + /* Configure the hardware for the current request. */ + rc = ocs_hcu_hw_cfg(hcu_dev, algo, false); + if (rc) + return rc; + + dma_handle = dma_map_single(dev, data, data_len, DMA_TO_DEVICE); + if (dma_mapping_error(dev, dma_handle)) + return -EIO; + + reg = HCU_DMA_SNOOP_MASK | HCU_DMA_EN; + + ocs_hcu_done_irq_en(hcu_dev); + + reinit_completion(&hcu_dev->irq_done); + + writel(dma_handle, hcu_dev->io_base + OCS_HCU_DMA_SRC_ADDR); + writel(data_len, hcu_dev->io_base + OCS_HCU_DMA_SRC_SIZE); + writel(OCS_HCU_START, hcu_dev->io_base + OCS_HCU_OPERATION); + writel(reg, hcu_dev->io_base + OCS_HCU_DMA_DMA_MODE); + + writel(OCS_HCU_TERMINATE, hcu_dev->io_base + OCS_HCU_OPERATION); + + rc = ocs_hcu_wait_and_disable_irq(hcu_dev); + if (rc) + return rc; + + dma_unmap_single(dev, dma_handle, data_len, DMA_TO_DEVICE); + + return ocs_hcu_get_digest(hcu_dev, algo, dgst, dgst_len); +} + +/** + * ocs_hcu_hmac() - Compute HMAC. + * @hcu_dev: The OCS HCU device to use. + * @algo: The hash algorithm to use with HMAC. + * @key: The key to use. + * @dma_list: The OCS DMA list mapping the input data to process. + * @key_len: The length of @key. + * @dgst: The buffer where to save the computed HMAC. + * @dgst_len: The length of @dgst. + * + * Return: 0 on success; negative error code otherwise. + */ +int ocs_hcu_hmac(struct ocs_hcu_dev *hcu_dev, enum ocs_hcu_algo algo, + const u8 *key, size_t key_len, + const struct ocs_hcu_dma_list *dma_list, + u8 *dgst, size_t dgst_len) +{ + int rc; + + /* Ensure 'key' is not NULL. */ + if (!key || key_len == 0) + return -EINVAL; + + /* Configure the hardware for the current request. */ + rc = ocs_hcu_hw_cfg(hcu_dev, algo, true); + if (rc) + return rc; + + rc = ocs_hcu_write_key(hcu_dev, key, key_len); + if (rc) + return rc; + + rc = ocs_hcu_ll_dma_start(hcu_dev, dma_list, true); + + /* Clear HW key before processing return code. */ + ocs_hcu_clear_key(hcu_dev); + + if (rc) + return rc; + + return ocs_hcu_get_digest(hcu_dev, algo, dgst, dgst_len); +} + +irqreturn_t ocs_hcu_irq_handler(int irq, void *dev_id) +{ + struct ocs_hcu_dev *hcu_dev = dev_id; + u32 hcu_irq; + u32 dma_irq; + + /* Read and clear the HCU interrupt. */ + hcu_irq = readl(hcu_dev->io_base + OCS_HCU_ISR); + writel(hcu_irq, hcu_dev->io_base + OCS_HCU_ISR); + + /* Read and clear the HCU DMA interrupt. */ + dma_irq = readl(hcu_dev->io_base + OCS_HCU_DMA_MSI_ISR); + writel(dma_irq, hcu_dev->io_base + OCS_HCU_DMA_MSI_ISR); + + /* Check for errors. */ + if (hcu_irq & HCU_IRQ_HASH_ERR_MASK || dma_irq & HCU_DMA_IRQ_ERR_MASK) { + hcu_dev->irq_err = true; + goto complete; + } + + /* Check for DONE IRQs. */ + if (hcu_irq & HCU_IRQ_HASH_DONE || dma_irq & HCU_DMA_IRQ_SRC_DONE) + goto complete; + + return IRQ_NONE; + +complete: + complete(&hcu_dev->irq_done); + + return IRQ_HANDLED; +} + +MODULE_LICENSE("GPL"); diff --git a/drivers/crypto/intel/keembay/ocs-hcu.h b/drivers/crypto/intel/keembay/ocs-hcu.h new file mode 100644 index 000000000000..fbbbb92a0592 --- /dev/null +++ b/drivers/crypto/intel/keembay/ocs-hcu.h @@ -0,0 +1,106 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Intel Keem Bay OCS HCU Crypto Driver. + * + * Copyright (C) 2018-2020 Intel Corporation + */ + +#include + +#ifndef _CRYPTO_OCS_HCU_H +#define _CRYPTO_OCS_HCU_H + +#define OCS_HCU_DMA_BIT_MASK DMA_BIT_MASK(32) + +#define OCS_HCU_HW_KEY_LEN 64 + +struct ocs_hcu_dma_list; + +enum ocs_hcu_algo { + OCS_HCU_ALGO_SHA256 = 2, + OCS_HCU_ALGO_SHA224 = 3, + OCS_HCU_ALGO_SHA384 = 4, + OCS_HCU_ALGO_SHA512 = 5, + OCS_HCU_ALGO_SM3 = 6, +}; + +/** + * struct ocs_hcu_dev - OCS HCU device context. + * @list: List of device contexts. + * @dev: OCS HCU device. + * @io_base: Base address of OCS HCU registers. + * @engine: Crypto engine for the device. + * @irq: IRQ number. + * @irq_done: Completion for IRQ. + * @irq_err: Flag indicating an IRQ error has happened. + */ +struct ocs_hcu_dev { + struct list_head list; + struct device *dev; + void __iomem *io_base; + struct crypto_engine *engine; + int irq; + struct completion irq_done; + bool irq_err; +}; + +/** + * struct ocs_hcu_idata - Intermediate data generated by the HCU. + * @msg_len_lo: Length of data the HCU has operated on in bits, low 32b. + * @msg_len_hi: Length of data the HCU has operated on in bits, high 32b. + * @digest: The digest read from the HCU. If the HCU is terminated, it will + * contain the actual hash digest. Otherwise it is the intermediate + * state. + */ +struct ocs_hcu_idata { + u32 msg_len_lo; + u32 msg_len_hi; + u8 digest[SHA512_DIGEST_SIZE]; +}; + +/** + * struct ocs_hcu_hash_ctx - Context for OCS HCU hashing operation. + * @algo: The hashing algorithm being used. + * @idata: The current intermediate data. + */ +struct ocs_hcu_hash_ctx { + enum ocs_hcu_algo algo; + struct ocs_hcu_idata idata; +}; + +irqreturn_t ocs_hcu_irq_handler(int irq, void *dev_id); + +struct ocs_hcu_dma_list *ocs_hcu_dma_list_alloc(struct ocs_hcu_dev *hcu_dev, + int max_nents); + +void ocs_hcu_dma_list_free(struct ocs_hcu_dev *hcu_dev, + struct ocs_hcu_dma_list *dma_list); + +int ocs_hcu_dma_list_add_tail(struct ocs_hcu_dev *hcu_dev, + struct ocs_hcu_dma_list *dma_list, + dma_addr_t addr, u32 len); + +int ocs_hcu_hash_init(struct ocs_hcu_hash_ctx *ctx, enum ocs_hcu_algo algo); + +int ocs_hcu_hash_update(struct ocs_hcu_dev *hcu_dev, + struct ocs_hcu_hash_ctx *ctx, + const struct ocs_hcu_dma_list *dma_list); + +int ocs_hcu_hash_finup(struct ocs_hcu_dev *hcu_dev, + const struct ocs_hcu_hash_ctx *ctx, + const struct ocs_hcu_dma_list *dma_list, + u8 *dgst, size_t dgst_len); + +int ocs_hcu_hash_final(struct ocs_hcu_dev *hcu_dev, + const struct ocs_hcu_hash_ctx *ctx, u8 *dgst, + size_t dgst_len); + +int ocs_hcu_digest(struct ocs_hcu_dev *hcu_dev, enum ocs_hcu_algo algo, + void *data, size_t data_len, u8 *dgst, size_t dgst_len); + +int ocs_hcu_hmac(struct ocs_hcu_dev *hcu_dev, enum ocs_hcu_algo algo, + const u8 *key, size_t key_len, + const struct ocs_hcu_dma_list *dma_list, + u8 *dgst, size_t dgst_len); + +#endif /* _CRYPTO_OCS_HCU_H */ diff --git a/drivers/crypto/keembay/Kconfig b/drivers/crypto/keembay/Kconfig deleted file mode 100644 index 1cd62f9c3e3a..000000000000 --- a/drivers/crypto/keembay/Kconfig +++ /dev/null @@ -1,90 +0,0 @@ -config CRYPTO_DEV_KEEMBAY_OCS_AES_SM4 - tristate "Support for Intel Keem Bay OCS AES/SM4 HW acceleration" - depends on HAS_IOMEM - depends on ARCH_KEEMBAY || COMPILE_TEST - select CRYPTO_SKCIPHER - select CRYPTO_AEAD - select CRYPTO_ENGINE - help - Support for Intel Keem Bay Offload and Crypto Subsystem (OCS) AES and - SM4 cipher hardware acceleration for use with Crypto API. - - Provides HW acceleration for the following transformations: - cbc(aes), ctr(aes), ccm(aes), gcm(aes), cbc(sm4), ctr(sm4), ccm(sm4) - and gcm(sm4). - - Optionally, support for the following transformations can also be - enabled: ecb(aes), cts(cbc(aes)), ecb(sm4) and cts(cbc(sm4)). - -config CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB - bool "Support for Intel Keem Bay OCS AES/SM4 ECB HW acceleration" - depends on CRYPTO_DEV_KEEMBAY_OCS_AES_SM4 - help - Support for Intel Keem Bay Offload and Crypto Subsystem (OCS) - AES/SM4 ECB mode hardware acceleration for use with Crypto API. - - Provides OCS version of ecb(aes) and ecb(sm4) - - Intel does not recommend use of ECB mode with AES/SM4. - -config CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS - bool "Support for Intel Keem Bay OCS AES/SM4 CTS HW acceleration" - depends on CRYPTO_DEV_KEEMBAY_OCS_AES_SM4 - help - Support for Intel Keem Bay Offload and Crypto Subsystem (OCS) - AES/SM4 CBC with CTS mode hardware acceleration for use with - Crypto API. - - Provides OCS version of cts(cbc(aes)) and cts(cbc(sm4)). - - Intel does not recommend use of CTS mode with AES/SM4. - -config CRYPTO_DEV_KEEMBAY_OCS_ECC - tristate "Support for Intel Keem Bay OCS ECC HW acceleration" - depends on ARCH_KEEMBAY || COMPILE_TEST - depends on OF - depends on HAS_IOMEM - select CRYPTO_ECDH - select CRYPTO_ENGINE - help - Support for Intel Keem Bay Offload and Crypto Subsystem (OCS) - Elliptic Curve Cryptography (ECC) hardware acceleration for use with - Crypto API. - - Provides OCS acceleration for ECDH-256 and ECDH-384. - - Say Y or M if you are compiling for the Intel Keem Bay SoC. The - module will be called keembay-ocs-ecc. - - If unsure, say N. - -config CRYPTO_DEV_KEEMBAY_OCS_HCU - tristate "Support for Intel Keem Bay OCS HCU HW acceleration" - select CRYPTO_HASH - select CRYPTO_ENGINE - depends on HAS_IOMEM - depends on ARCH_KEEMBAY || COMPILE_TEST - depends on OF - help - Support for Intel Keem Bay Offload and Crypto Subsystem (OCS) Hash - Control Unit (HCU) hardware acceleration for use with Crypto API. - - Provides OCS HCU hardware acceleration of sha256, sha384, sha512, and - sm3, as well as the HMAC variant of these algorithms. - - Say Y or M if you're building for the Intel Keem Bay SoC. If compiled - as a module, the module will be called keembay-ocs-hcu. - - If unsure, say N. - -config CRYPTO_DEV_KEEMBAY_OCS_HCU_HMAC_SHA224 - bool "Enable sha224 and hmac(sha224) support in Intel Keem Bay OCS HCU" - depends on CRYPTO_DEV_KEEMBAY_OCS_HCU - help - Enables support for sha224 and hmac(sha224) algorithms in the Intel - Keem Bay OCS HCU driver. Intel recommends not to use these - algorithms. - - Provides OCS HCU hardware acceleration of sha224 and hmac(224). - - If unsure, say N. diff --git a/drivers/crypto/keembay/Makefile b/drivers/crypto/keembay/Makefile deleted file mode 100644 index 7c12c3c138bd..000000000000 --- a/drivers/crypto/keembay/Makefile +++ /dev/null @@ -1,10 +0,0 @@ -# -# Makefile for Intel Keem Bay OCS Crypto API Linux drivers -# -obj-$(CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4) += keembay-ocs-aes.o -keembay-ocs-aes-objs := keembay-ocs-aes-core.o ocs-aes.o - -obj-$(CONFIG_CRYPTO_DEV_KEEMBAY_OCS_ECC) += keembay-ocs-ecc.o - -obj-$(CONFIG_CRYPTO_DEV_KEEMBAY_OCS_HCU) += keembay-ocs-hcu.o -keembay-ocs-hcu-objs := keembay-ocs-hcu-core.o ocs-hcu.o diff --git a/drivers/crypto/keembay/keembay-ocs-aes-core.c b/drivers/crypto/keembay/keembay-ocs-aes-core.c deleted file mode 100644 index ae31be00357a..000000000000 --- a/drivers/crypto/keembay/keembay-ocs-aes-core.c +++ /dev/null @@ -1,1704 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Intel Keem Bay OCS AES Crypto Driver. - * - * Copyright (C) 2018-2020 Intel Corporation - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -#include -#include - -#include "ocs-aes.h" - -#define KMB_OCS_PRIORITY 350 -#define DRV_NAME "keembay-ocs-aes" - -#define OCS_AES_MIN_KEY_SIZE 16 -#define OCS_AES_MAX_KEY_SIZE 32 -#define OCS_AES_KEYSIZE_128 16 -#define OCS_AES_KEYSIZE_192 24 -#define OCS_AES_KEYSIZE_256 32 -#define OCS_SM4_KEY_SIZE 16 - -/** - * struct ocs_aes_tctx - OCS AES Transform context - * @engine_ctx: Engine context. - * @aes_dev: The OCS AES device. - * @key: AES/SM4 key. - * @key_len: The length (in bytes) of @key. - * @cipher: OCS cipher to use (either AES or SM4). - * @sw_cipher: The cipher to use as fallback. - * @use_fallback: Whether or not fallback cipher should be used. - */ -struct ocs_aes_tctx { - struct crypto_engine_ctx engine_ctx; - struct ocs_aes_dev *aes_dev; - u8 key[OCS_AES_KEYSIZE_256]; - unsigned int key_len; - enum ocs_cipher cipher; - union { - struct crypto_sync_skcipher *sk; - struct crypto_aead *aead; - } sw_cipher; - bool use_fallback; -}; - -/** - * struct ocs_aes_rctx - OCS AES Request context. - * @instruction: Instruction to be executed (encrypt / decrypt). - * @mode: Mode to use (ECB, CBC, CTR, CCm, GCM, CTS) - * @src_nents: Number of source SG entries. - * @dst_nents: Number of destination SG entries. - * @src_dma_count: The number of DMA-mapped entries of the source SG. - * @dst_dma_count: The number of DMA-mapped entries of the destination SG. - * @in_place: Whether or not this is an in place request, i.e., - * src_sg == dst_sg. - * @src_dll: OCS DMA linked list for input data. - * @dst_dll: OCS DMA linked list for output data. - * @last_ct_blk: Buffer to hold last cipher text block (only used in CBC - * mode). - * @cts_swap: Whether or not CTS swap must be performed. - * @aad_src_dll: OCS DMA linked list for input AAD data. - * @aad_dst_dll: OCS DMA linked list for output AAD data. - * @in_tag: Buffer to hold input encrypted tag (only used for - * CCM/GCM decrypt). - * @out_tag: Buffer to hold output encrypted / decrypted tag (only - * used for GCM encrypt / decrypt). - */ -struct ocs_aes_rctx { - /* Fields common across all modes. */ - enum ocs_instruction instruction; - enum ocs_mode mode; - int src_nents; - int dst_nents; - int src_dma_count; - int dst_dma_count; - bool in_place; - struct ocs_dll_desc src_dll; - struct ocs_dll_desc dst_dll; - - /* CBC specific */ - u8 last_ct_blk[AES_BLOCK_SIZE]; - - /* CTS specific */ - int cts_swap; - - /* CCM/GCM specific */ - struct ocs_dll_desc aad_src_dll; - struct ocs_dll_desc aad_dst_dll; - u8 in_tag[AES_BLOCK_SIZE]; - - /* GCM specific */ - u8 out_tag[AES_BLOCK_SIZE]; -}; - -/* Driver data. */ -struct ocs_aes_drv { - struct list_head dev_list; - spinlock_t lock; /* Protects dev_list. */ -}; - -static struct ocs_aes_drv ocs_aes = { - .dev_list = LIST_HEAD_INIT(ocs_aes.dev_list), - .lock = __SPIN_LOCK_UNLOCKED(ocs_aes.lock), -}; - -static struct ocs_aes_dev *kmb_ocs_aes_find_dev(struct ocs_aes_tctx *tctx) -{ - struct ocs_aes_dev *aes_dev; - - spin_lock(&ocs_aes.lock); - - if (tctx->aes_dev) { - aes_dev = tctx->aes_dev; - goto exit; - } - - /* Only a single OCS device available */ - aes_dev = list_first_entry(&ocs_aes.dev_list, struct ocs_aes_dev, list); - tctx->aes_dev = aes_dev; - -exit: - spin_unlock(&ocs_aes.lock); - - return aes_dev; -} - -/* - * Ensure key is 128-bit or 256-bit for AES or 128-bit for SM4 and an actual - * key is being passed in. - * - * Return: 0 if key is valid, -EINVAL otherwise. - */ -static int check_key(const u8 *in_key, size_t key_len, enum ocs_cipher cipher) -{ - if (!in_key) - return -EINVAL; - - /* For AES, only 128-byte or 256-byte keys are supported. */ - if (cipher == OCS_AES && (key_len == OCS_AES_KEYSIZE_128 || - key_len == OCS_AES_KEYSIZE_256)) - return 0; - - /* For SM4, only 128-byte keys are supported. */ - if (cipher == OCS_SM4 && key_len == OCS_AES_KEYSIZE_128) - return 0; - - /* Everything else is unsupported. */ - return -EINVAL; -} - -/* Save key into transformation context. */ -static int save_key(struct ocs_aes_tctx *tctx, const u8 *in_key, size_t key_len, - enum ocs_cipher cipher) -{ - int ret; - - ret = check_key(in_key, key_len, cipher); - if (ret) - return ret; - - memcpy(tctx->key, in_key, key_len); - tctx->key_len = key_len; - tctx->cipher = cipher; - - return 0; -} - -/* Set key for symmetric cypher. */ -static int kmb_ocs_sk_set_key(struct crypto_skcipher *tfm, const u8 *in_key, - size_t key_len, enum ocs_cipher cipher) -{ - struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm); - - /* Fallback is used for AES with 192-bit key. */ - tctx->use_fallback = (cipher == OCS_AES && - key_len == OCS_AES_KEYSIZE_192); - - if (!tctx->use_fallback) - return save_key(tctx, in_key, key_len, cipher); - - crypto_sync_skcipher_clear_flags(tctx->sw_cipher.sk, - CRYPTO_TFM_REQ_MASK); - crypto_sync_skcipher_set_flags(tctx->sw_cipher.sk, - tfm->base.crt_flags & - CRYPTO_TFM_REQ_MASK); - - return crypto_sync_skcipher_setkey(tctx->sw_cipher.sk, in_key, key_len); -} - -/* Set key for AEAD cipher. */ -static int kmb_ocs_aead_set_key(struct crypto_aead *tfm, const u8 *in_key, - size_t key_len, enum ocs_cipher cipher) -{ - struct ocs_aes_tctx *tctx = crypto_aead_ctx(tfm); - - /* Fallback is used for AES with 192-bit key. */ - tctx->use_fallback = (cipher == OCS_AES && - key_len == OCS_AES_KEYSIZE_192); - - if (!tctx->use_fallback) - return save_key(tctx, in_key, key_len, cipher); - - crypto_aead_clear_flags(tctx->sw_cipher.aead, CRYPTO_TFM_REQ_MASK); - crypto_aead_set_flags(tctx->sw_cipher.aead, - crypto_aead_get_flags(tfm) & CRYPTO_TFM_REQ_MASK); - - return crypto_aead_setkey(tctx->sw_cipher.aead, in_key, key_len); -} - -/* Swap two AES blocks in SG lists. */ -static void sg_swap_blocks(struct scatterlist *sgl, unsigned int nents, - off_t blk1_offset, off_t blk2_offset) -{ - u8 tmp_buf1[AES_BLOCK_SIZE], tmp_buf2[AES_BLOCK_SIZE]; - - /* - * No easy way to copy within sg list, so copy both blocks to temporary - * buffers first. - */ - sg_pcopy_to_buffer(sgl, nents, tmp_buf1, AES_BLOCK_SIZE, blk1_offset); - sg_pcopy_to_buffer(sgl, nents, tmp_buf2, AES_BLOCK_SIZE, blk2_offset); - sg_pcopy_from_buffer(sgl, nents, tmp_buf1, AES_BLOCK_SIZE, blk2_offset); - sg_pcopy_from_buffer(sgl, nents, tmp_buf2, AES_BLOCK_SIZE, blk1_offset); -} - -/* Initialize request context to default values. */ -static void ocs_aes_init_rctx(struct ocs_aes_rctx *rctx) -{ - /* Zero everything. */ - memset(rctx, 0, sizeof(*rctx)); - - /* Set initial value for DMA addresses. */ - rctx->src_dll.dma_addr = DMA_MAPPING_ERROR; - rctx->dst_dll.dma_addr = DMA_MAPPING_ERROR; - rctx->aad_src_dll.dma_addr = DMA_MAPPING_ERROR; - rctx->aad_dst_dll.dma_addr = DMA_MAPPING_ERROR; -} - -static int kmb_ocs_sk_validate_input(struct skcipher_request *req, - enum ocs_mode mode) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - int iv_size = crypto_skcipher_ivsize(tfm); - - switch (mode) { - case OCS_MODE_ECB: - /* Ensure input length is multiple of block size */ - if (req->cryptlen % AES_BLOCK_SIZE != 0) - return -EINVAL; - - return 0; - - case OCS_MODE_CBC: - /* Ensure input length is multiple of block size */ - if (req->cryptlen % AES_BLOCK_SIZE != 0) - return -EINVAL; - - /* Ensure IV is present and block size in length */ - if (!req->iv || iv_size != AES_BLOCK_SIZE) - return -EINVAL; - /* - * NOTE: Since req->cryptlen == 0 case was already handled in - * kmb_ocs_sk_common(), the above two conditions also guarantee - * that: cryptlen >= iv_size - */ - return 0; - - case OCS_MODE_CTR: - /* Ensure IV is present and block size in length */ - if (!req->iv || iv_size != AES_BLOCK_SIZE) - return -EINVAL; - return 0; - - case OCS_MODE_CTS: - /* Ensure input length >= block size */ - if (req->cryptlen < AES_BLOCK_SIZE) - return -EINVAL; - - /* Ensure IV is present and block size in length */ - if (!req->iv || iv_size != AES_BLOCK_SIZE) - return -EINVAL; - - return 0; - default: - return -EINVAL; - } -} - -/* - * Called by encrypt() / decrypt() skcipher functions. - * - * Use fallback if needed, otherwise initialize context and enqueue request - * into engine. - */ -static int kmb_ocs_sk_common(struct skcipher_request *req, - enum ocs_cipher cipher, - enum ocs_instruction instruction, - enum ocs_mode mode) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct ocs_aes_rctx *rctx = skcipher_request_ctx(req); - struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm); - struct ocs_aes_dev *aes_dev; - int rc; - - if (tctx->use_fallback) { - SYNC_SKCIPHER_REQUEST_ON_STACK(subreq, tctx->sw_cipher.sk); - - skcipher_request_set_sync_tfm(subreq, tctx->sw_cipher.sk); - skcipher_request_set_callback(subreq, req->base.flags, NULL, - NULL); - skcipher_request_set_crypt(subreq, req->src, req->dst, - req->cryptlen, req->iv); - - if (instruction == OCS_ENCRYPT) - rc = crypto_skcipher_encrypt(subreq); - else - rc = crypto_skcipher_decrypt(subreq); - - skcipher_request_zero(subreq); - - return rc; - } - - /* - * If cryptlen == 0, no processing needed for ECB, CBC and CTR. - * - * For CTS continue: kmb_ocs_sk_validate_input() will return -EINVAL. - */ - if (!req->cryptlen && mode != OCS_MODE_CTS) - return 0; - - rc = kmb_ocs_sk_validate_input(req, mode); - if (rc) - return rc; - - aes_dev = kmb_ocs_aes_find_dev(tctx); - if (!aes_dev) - return -ENODEV; - - if (cipher != tctx->cipher) - return -EINVAL; - - ocs_aes_init_rctx(rctx); - rctx->instruction = instruction; - rctx->mode = mode; - - return crypto_transfer_skcipher_request_to_engine(aes_dev->engine, req); -} - -static void cleanup_ocs_dma_linked_list(struct device *dev, - struct ocs_dll_desc *dll) -{ - if (dll->vaddr) - dma_free_coherent(dev, dll->size, dll->vaddr, dll->dma_addr); - dll->vaddr = NULL; - dll->size = 0; - dll->dma_addr = DMA_MAPPING_ERROR; -} - -static void kmb_ocs_sk_dma_cleanup(struct skcipher_request *req) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct ocs_aes_rctx *rctx = skcipher_request_ctx(req); - struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm); - struct device *dev = tctx->aes_dev->dev; - - if (rctx->src_dma_count) { - dma_unmap_sg(dev, req->src, rctx->src_nents, DMA_TO_DEVICE); - rctx->src_dma_count = 0; - } - - if (rctx->dst_dma_count) { - dma_unmap_sg(dev, req->dst, rctx->dst_nents, rctx->in_place ? - DMA_BIDIRECTIONAL : - DMA_FROM_DEVICE); - rctx->dst_dma_count = 0; - } - - /* Clean up OCS DMA linked lists */ - cleanup_ocs_dma_linked_list(dev, &rctx->src_dll); - cleanup_ocs_dma_linked_list(dev, &rctx->dst_dll); -} - -static int kmb_ocs_sk_prepare_inplace(struct skcipher_request *req) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct ocs_aes_rctx *rctx = skcipher_request_ctx(req); - struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm); - int iv_size = crypto_skcipher_ivsize(tfm); - int rc; - - /* - * For CBC decrypt, save last block (iv) to last_ct_blk buffer. - * - * Note: if we are here, we already checked that cryptlen >= iv_size - * and iv_size == AES_BLOCK_SIZE (i.e., the size of last_ct_blk); see - * kmb_ocs_sk_validate_input(). - */ - if (rctx->mode == OCS_MODE_CBC && rctx->instruction == OCS_DECRYPT) - scatterwalk_map_and_copy(rctx->last_ct_blk, req->src, - req->cryptlen - iv_size, iv_size, 0); - - /* For CTS decrypt, swap last two blocks, if needed. */ - if (rctx->cts_swap && rctx->instruction == OCS_DECRYPT) - sg_swap_blocks(req->dst, rctx->dst_nents, - req->cryptlen - AES_BLOCK_SIZE, - req->cryptlen - (2 * AES_BLOCK_SIZE)); - - /* src and dst buffers are the same, use bidirectional DMA mapping. */ - rctx->dst_dma_count = dma_map_sg(tctx->aes_dev->dev, req->dst, - rctx->dst_nents, DMA_BIDIRECTIONAL); - if (rctx->dst_dma_count == 0) { - dev_err(tctx->aes_dev->dev, "Failed to map destination sg\n"); - return -ENOMEM; - } - - /* Create DST linked list */ - rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->dst, - rctx->dst_dma_count, &rctx->dst_dll, - req->cryptlen, 0); - if (rc) - return rc; - /* - * If descriptor creation was successful, set the src_dll.dma_addr to - * the value of dst_dll.dma_addr, as we do in-place AES operation on - * the src. - */ - rctx->src_dll.dma_addr = rctx->dst_dll.dma_addr; - - return 0; -} - -static int kmb_ocs_sk_prepare_notinplace(struct skcipher_request *req) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct ocs_aes_rctx *rctx = skcipher_request_ctx(req); - struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm); - int rc; - - rctx->src_nents = sg_nents_for_len(req->src, req->cryptlen); - if (rctx->src_nents < 0) - return -EBADMSG; - - /* Map SRC SG. */ - rctx->src_dma_count = dma_map_sg(tctx->aes_dev->dev, req->src, - rctx->src_nents, DMA_TO_DEVICE); - if (rctx->src_dma_count == 0) { - dev_err(tctx->aes_dev->dev, "Failed to map source sg\n"); - return -ENOMEM; - } - - /* Create SRC linked list */ - rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->src, - rctx->src_dma_count, &rctx->src_dll, - req->cryptlen, 0); - if (rc) - return rc; - - /* Map DST SG. */ - rctx->dst_dma_count = dma_map_sg(tctx->aes_dev->dev, req->dst, - rctx->dst_nents, DMA_FROM_DEVICE); - if (rctx->dst_dma_count == 0) { - dev_err(tctx->aes_dev->dev, "Failed to map destination sg\n"); - return -ENOMEM; - } - - /* Create DST linked list */ - rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->dst, - rctx->dst_dma_count, &rctx->dst_dll, - req->cryptlen, 0); - if (rc) - return rc; - - /* If this is not a CTS decrypt operation with swapping, we are done. */ - if (!(rctx->cts_swap && rctx->instruction == OCS_DECRYPT)) - return 0; - - /* - * Otherwise, we have to copy src to dst (as we cannot modify src). - * Use OCS AES bypass mode to copy src to dst via DMA. - * - * NOTE: for anything other than small data sizes this is rather - * inefficient. - */ - rc = ocs_aes_bypass_op(tctx->aes_dev, rctx->dst_dll.dma_addr, - rctx->src_dll.dma_addr, req->cryptlen); - if (rc) - return rc; - - /* - * Now dst == src, so clean up what we did so far and use in_place - * logic. - */ - kmb_ocs_sk_dma_cleanup(req); - rctx->in_place = true; - - return kmb_ocs_sk_prepare_inplace(req); -} - -static int kmb_ocs_sk_run(struct skcipher_request *req) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct ocs_aes_rctx *rctx = skcipher_request_ctx(req); - struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm); - struct ocs_aes_dev *aes_dev = tctx->aes_dev; - int iv_size = crypto_skcipher_ivsize(tfm); - int rc; - - rctx->dst_nents = sg_nents_for_len(req->dst, req->cryptlen); - if (rctx->dst_nents < 0) - return -EBADMSG; - - /* - * If 2 blocks or greater, and multiple of block size swap last two - * blocks to be compatible with other crypto API CTS implementations: - * OCS mode uses CBC-CS2, whereas other crypto API implementations use - * CBC-CS3. - * CBC-CS2 and CBC-CS3 defined by: - * https://nvlpubs.nist.gov/nistpubs/Legacy/SP/nistspecialpublication800-38a-add.pdf - */ - rctx->cts_swap = (rctx->mode == OCS_MODE_CTS && - req->cryptlen > AES_BLOCK_SIZE && - req->cryptlen % AES_BLOCK_SIZE == 0); - - rctx->in_place = (req->src == req->dst); - - if (rctx->in_place) - rc = kmb_ocs_sk_prepare_inplace(req); - else - rc = kmb_ocs_sk_prepare_notinplace(req); - - if (rc) - goto error; - - rc = ocs_aes_op(aes_dev, rctx->mode, tctx->cipher, rctx->instruction, - rctx->dst_dll.dma_addr, rctx->src_dll.dma_addr, - req->cryptlen, req->iv, iv_size); - if (rc) - goto error; - - /* Clean-up DMA before further processing output. */ - kmb_ocs_sk_dma_cleanup(req); - - /* For CTS Encrypt, swap last 2 blocks, if needed. */ - if (rctx->cts_swap && rctx->instruction == OCS_ENCRYPT) { - sg_swap_blocks(req->dst, rctx->dst_nents, - req->cryptlen - AES_BLOCK_SIZE, - req->cryptlen - (2 * AES_BLOCK_SIZE)); - return 0; - } - - /* For CBC copy IV to req->IV. */ - if (rctx->mode == OCS_MODE_CBC) { - /* CBC encrypt case. */ - if (rctx->instruction == OCS_ENCRYPT) { - scatterwalk_map_and_copy(req->iv, req->dst, - req->cryptlen - iv_size, - iv_size, 0); - return 0; - } - /* CBC decrypt case. */ - if (rctx->in_place) - memcpy(req->iv, rctx->last_ct_blk, iv_size); - else - scatterwalk_map_and_copy(req->iv, req->src, - req->cryptlen - iv_size, - iv_size, 0); - return 0; - } - /* For all other modes there's nothing to do. */ - - return 0; - -error: - kmb_ocs_sk_dma_cleanup(req); - - return rc; -} - -static int kmb_ocs_aead_validate_input(struct aead_request *req, - enum ocs_instruction instruction, - enum ocs_mode mode) -{ - struct crypto_aead *tfm = crypto_aead_reqtfm(req); - int tag_size = crypto_aead_authsize(tfm); - int iv_size = crypto_aead_ivsize(tfm); - - /* For decrypt crytplen == len(PT) + len(tag). */ - if (instruction == OCS_DECRYPT && req->cryptlen < tag_size) - return -EINVAL; - - /* IV is mandatory. */ - if (!req->iv) - return -EINVAL; - - switch (mode) { - case OCS_MODE_GCM: - if (iv_size != GCM_AES_IV_SIZE) - return -EINVAL; - - return 0; - - case OCS_MODE_CCM: - /* Ensure IV is present and block size in length */ - if (iv_size != AES_BLOCK_SIZE) - return -EINVAL; - - return 0; - - default: - return -EINVAL; - } -} - -/* - * Called by encrypt() / decrypt() aead functions. - * - * Use fallback if needed, otherwise initialize context and enqueue request - * into engine. - */ -static int kmb_ocs_aead_common(struct aead_request *req, - enum ocs_cipher cipher, - enum ocs_instruction instruction, - enum ocs_mode mode) -{ - struct ocs_aes_tctx *tctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); - struct ocs_aes_rctx *rctx = aead_request_ctx(req); - struct ocs_aes_dev *dd; - int rc; - - if (tctx->use_fallback) { - struct aead_request *subreq = aead_request_ctx(req); - - aead_request_set_tfm(subreq, tctx->sw_cipher.aead); - aead_request_set_callback(subreq, req->base.flags, - req->base.complete, req->base.data); - aead_request_set_crypt(subreq, req->src, req->dst, - req->cryptlen, req->iv); - aead_request_set_ad(subreq, req->assoclen); - rc = crypto_aead_setauthsize(tctx->sw_cipher.aead, - crypto_aead_authsize(crypto_aead_reqtfm(req))); - if (rc) - return rc; - - return (instruction == OCS_ENCRYPT) ? - crypto_aead_encrypt(subreq) : - crypto_aead_decrypt(subreq); - } - - rc = kmb_ocs_aead_validate_input(req, instruction, mode); - if (rc) - return rc; - - dd = kmb_ocs_aes_find_dev(tctx); - if (!dd) - return -ENODEV; - - if (cipher != tctx->cipher) - return -EINVAL; - - ocs_aes_init_rctx(rctx); - rctx->instruction = instruction; - rctx->mode = mode; - - return crypto_transfer_aead_request_to_engine(dd->engine, req); -} - -static void kmb_ocs_aead_dma_cleanup(struct aead_request *req) -{ - struct ocs_aes_tctx *tctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); - struct ocs_aes_rctx *rctx = aead_request_ctx(req); - struct device *dev = tctx->aes_dev->dev; - - if (rctx->src_dma_count) { - dma_unmap_sg(dev, req->src, rctx->src_nents, DMA_TO_DEVICE); - rctx->src_dma_count = 0; - } - - if (rctx->dst_dma_count) { - dma_unmap_sg(dev, req->dst, rctx->dst_nents, rctx->in_place ? - DMA_BIDIRECTIONAL : - DMA_FROM_DEVICE); - rctx->dst_dma_count = 0; - } - /* Clean up OCS DMA linked lists */ - cleanup_ocs_dma_linked_list(dev, &rctx->src_dll); - cleanup_ocs_dma_linked_list(dev, &rctx->dst_dll); - cleanup_ocs_dma_linked_list(dev, &rctx->aad_src_dll); - cleanup_ocs_dma_linked_list(dev, &rctx->aad_dst_dll); -} - -/** - * kmb_ocs_aead_dma_prepare() - Do DMA mapping for AEAD processing. - * @req: The AEAD request being processed. - * @src_dll_size: Where to store the length of the data mapped into the - * src_dll OCS DMA list. - * - * Do the following: - * - DMA map req->src and req->dst - * - Initialize the following OCS DMA linked lists: rctx->src_dll, - * rctx->dst_dll, rctx->aad_src_dll and rxtc->aad_dst_dll. - * - * Return: 0 on success, negative error code otherwise. - */ -static int kmb_ocs_aead_dma_prepare(struct aead_request *req, u32 *src_dll_size) -{ - struct ocs_aes_tctx *tctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); - const int tag_size = crypto_aead_authsize(crypto_aead_reqtfm(req)); - struct ocs_aes_rctx *rctx = aead_request_ctx(req); - u32 in_size; /* The length of the data to be mapped by src_dll. */ - u32 out_size; /* The length of the data to be mapped by dst_dll. */ - u32 dst_size; /* The length of the data in dst_sg. */ - int rc; - - /* Get number of entries in input data SG list. */ - rctx->src_nents = sg_nents_for_len(req->src, - req->assoclen + req->cryptlen); - if (rctx->src_nents < 0) - return -EBADMSG; - - if (rctx->instruction == OCS_DECRYPT) { - /* - * For decrypt: - * - src sg list is: AAD|CT|tag - * - dst sg list expects: AAD|PT - * - * in_size == len(CT); out_size == len(PT) - */ - - /* req->cryptlen includes both CT and tag. */ - in_size = req->cryptlen - tag_size; - - /* out_size = PT size == CT size */ - out_size = in_size; - - /* len(dst_sg) == len(AAD) + len(PT) */ - dst_size = req->assoclen + out_size; - - /* - * Copy tag from source SG list to 'in_tag' buffer. - * - * Note: this needs to be done here, before DMA mapping src_sg. - */ - sg_pcopy_to_buffer(req->src, rctx->src_nents, rctx->in_tag, - tag_size, req->assoclen + in_size); - - } else { /* OCS_ENCRYPT */ - /* - * For encrypt: - * src sg list is: AAD|PT - * dst sg list expects: AAD|CT|tag - */ - /* in_size == len(PT) */ - in_size = req->cryptlen; - - /* - * In CCM mode the OCS engine appends the tag to the ciphertext, - * but in GCM mode the tag must be read from the tag registers - * and appended manually below - */ - out_size = (rctx->mode == OCS_MODE_CCM) ? in_size + tag_size : - in_size; - /* len(dst_sg) == len(AAD) + len(CT) + len(tag) */ - dst_size = req->assoclen + in_size + tag_size; - } - *src_dll_size = in_size; - - /* Get number of entries in output data SG list. */ - rctx->dst_nents = sg_nents_for_len(req->dst, dst_size); - if (rctx->dst_nents < 0) - return -EBADMSG; - - rctx->in_place = (req->src == req->dst) ? 1 : 0; - - /* Map destination; use bidirectional mapping for in-place case. */ - rctx->dst_dma_count = dma_map_sg(tctx->aes_dev->dev, req->dst, - rctx->dst_nents, - rctx->in_place ? DMA_BIDIRECTIONAL : - DMA_FROM_DEVICE); - if (rctx->dst_dma_count == 0 && rctx->dst_nents != 0) { - dev_err(tctx->aes_dev->dev, "Failed to map destination sg\n"); - return -ENOMEM; - } - - /* Create AAD DST list: maps dst[0:AAD_SIZE-1]. */ - rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->dst, - rctx->dst_dma_count, - &rctx->aad_dst_dll, req->assoclen, - 0); - if (rc) - return rc; - - /* Create DST list: maps dst[AAD_SIZE:out_size] */ - rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->dst, - rctx->dst_dma_count, &rctx->dst_dll, - out_size, req->assoclen); - if (rc) - return rc; - - if (rctx->in_place) { - /* If this is not CCM encrypt, we are done. */ - if (!(rctx->mode == OCS_MODE_CCM && - rctx->instruction == OCS_ENCRYPT)) { - /* - * SRC and DST are the same, so re-use the same DMA - * addresses (to avoid allocating new DMA lists - * identical to the dst ones). - */ - rctx->src_dll.dma_addr = rctx->dst_dll.dma_addr; - rctx->aad_src_dll.dma_addr = rctx->aad_dst_dll.dma_addr; - - return 0; - } - /* - * For CCM encrypt the input and output linked lists contain - * different amounts of data, so, we need to create different - * SRC and AAD SRC lists, even for the in-place case. - */ - rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->dst, - rctx->dst_dma_count, - &rctx->aad_src_dll, - req->assoclen, 0); - if (rc) - return rc; - rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->dst, - rctx->dst_dma_count, - &rctx->src_dll, in_size, - req->assoclen); - if (rc) - return rc; - - return 0; - } - /* Not in-place case. */ - - /* Map source SG. */ - rctx->src_dma_count = dma_map_sg(tctx->aes_dev->dev, req->src, - rctx->src_nents, DMA_TO_DEVICE); - if (rctx->src_dma_count == 0 && rctx->src_nents != 0) { - dev_err(tctx->aes_dev->dev, "Failed to map source sg\n"); - return -ENOMEM; - } - - /* Create AAD SRC list. */ - rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->src, - rctx->src_dma_count, - &rctx->aad_src_dll, - req->assoclen, 0); - if (rc) - return rc; - - /* Create SRC list. */ - rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->src, - rctx->src_dma_count, - &rctx->src_dll, in_size, - req->assoclen); - if (rc) - return rc; - - if (req->assoclen == 0) - return 0; - - /* Copy AAD from src sg to dst sg using OCS DMA. */ - rc = ocs_aes_bypass_op(tctx->aes_dev, rctx->aad_dst_dll.dma_addr, - rctx->aad_src_dll.dma_addr, req->cryptlen); - if (rc) - dev_err(tctx->aes_dev->dev, - "Failed to copy source AAD to destination AAD\n"); - - return rc; -} - -static int kmb_ocs_aead_run(struct aead_request *req) -{ - struct ocs_aes_tctx *tctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); - const int tag_size = crypto_aead_authsize(crypto_aead_reqtfm(req)); - struct ocs_aes_rctx *rctx = aead_request_ctx(req); - u32 in_size; /* The length of the data mapped by src_dll. */ - int rc; - - rc = kmb_ocs_aead_dma_prepare(req, &in_size); - if (rc) - goto exit; - - /* For CCM, we just call the OCS processing and we are done. */ - if (rctx->mode == OCS_MODE_CCM) { - rc = ocs_aes_ccm_op(tctx->aes_dev, tctx->cipher, - rctx->instruction, rctx->dst_dll.dma_addr, - rctx->src_dll.dma_addr, in_size, - req->iv, - rctx->aad_src_dll.dma_addr, req->assoclen, - rctx->in_tag, tag_size); - goto exit; - } - /* GCM case; invoke OCS processing. */ - rc = ocs_aes_gcm_op(tctx->aes_dev, tctx->cipher, - rctx->instruction, - rctx->dst_dll.dma_addr, - rctx->src_dll.dma_addr, in_size, - req->iv, - rctx->aad_src_dll.dma_addr, req->assoclen, - rctx->out_tag, tag_size); - if (rc) - goto exit; - - /* For GCM decrypt, we have to compare in_tag with out_tag. */ - if (rctx->instruction == OCS_DECRYPT) { - rc = memcmp(rctx->in_tag, rctx->out_tag, tag_size) ? - -EBADMSG : 0; - goto exit; - } - - /* For GCM encrypt, we must manually copy out_tag to DST sg. */ - - /* Clean-up must be called before the sg_pcopy_from_buffer() below. */ - kmb_ocs_aead_dma_cleanup(req); - - /* Copy tag to destination sg after AAD and CT. */ - sg_pcopy_from_buffer(req->dst, rctx->dst_nents, rctx->out_tag, - tag_size, req->assoclen + req->cryptlen); - - /* Return directly as DMA cleanup already done. */ - return 0; - -exit: - kmb_ocs_aead_dma_cleanup(req); - - return rc; -} - -static int kmb_ocs_aes_sk_do_one_request(struct crypto_engine *engine, - void *areq) -{ - struct skcipher_request *req = - container_of(areq, struct skcipher_request, base); - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm); - int err; - - if (!tctx->aes_dev) { - err = -ENODEV; - goto exit; - } - - err = ocs_aes_set_key(tctx->aes_dev, tctx->key_len, tctx->key, - tctx->cipher); - if (err) - goto exit; - - err = kmb_ocs_sk_run(req); - -exit: - crypto_finalize_skcipher_request(engine, req, err); - - return 0; -} - -static int kmb_ocs_aes_aead_do_one_request(struct crypto_engine *engine, - void *areq) -{ - struct aead_request *req = container_of(areq, - struct aead_request, base); - struct ocs_aes_tctx *tctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); - int err; - - if (!tctx->aes_dev) - return -ENODEV; - - err = ocs_aes_set_key(tctx->aes_dev, tctx->key_len, tctx->key, - tctx->cipher); - if (err) - goto exit; - - err = kmb_ocs_aead_run(req); - -exit: - crypto_finalize_aead_request(tctx->aes_dev->engine, req, err); - - return 0; -} - -static int kmb_ocs_aes_set_key(struct crypto_skcipher *tfm, const u8 *in_key, - unsigned int key_len) -{ - return kmb_ocs_sk_set_key(tfm, in_key, key_len, OCS_AES); -} - -static int kmb_ocs_aes_aead_set_key(struct crypto_aead *tfm, const u8 *in_key, - unsigned int key_len) -{ - return kmb_ocs_aead_set_key(tfm, in_key, key_len, OCS_AES); -} - -#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB -static int kmb_ocs_aes_ecb_encrypt(struct skcipher_request *req) -{ - return kmb_ocs_sk_common(req, OCS_AES, OCS_ENCRYPT, OCS_MODE_ECB); -} - -static int kmb_ocs_aes_ecb_decrypt(struct skcipher_request *req) -{ - return kmb_ocs_sk_common(req, OCS_AES, OCS_DECRYPT, OCS_MODE_ECB); -} -#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB */ - -static int kmb_ocs_aes_cbc_encrypt(struct skcipher_request *req) -{ - return kmb_ocs_sk_common(req, OCS_AES, OCS_ENCRYPT, OCS_MODE_CBC); -} - -static int kmb_ocs_aes_cbc_decrypt(struct skcipher_request *req) -{ - return kmb_ocs_sk_common(req, OCS_AES, OCS_DECRYPT, OCS_MODE_CBC); -} - -static int kmb_ocs_aes_ctr_encrypt(struct skcipher_request *req) -{ - return kmb_ocs_sk_common(req, OCS_AES, OCS_ENCRYPT, OCS_MODE_CTR); -} - -static int kmb_ocs_aes_ctr_decrypt(struct skcipher_request *req) -{ - return kmb_ocs_sk_common(req, OCS_AES, OCS_DECRYPT, OCS_MODE_CTR); -} - -#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS -static int kmb_ocs_aes_cts_encrypt(struct skcipher_request *req) -{ - return kmb_ocs_sk_common(req, OCS_AES, OCS_ENCRYPT, OCS_MODE_CTS); -} - -static int kmb_ocs_aes_cts_decrypt(struct skcipher_request *req) -{ - return kmb_ocs_sk_common(req, OCS_AES, OCS_DECRYPT, OCS_MODE_CTS); -} -#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS */ - -static int kmb_ocs_aes_gcm_encrypt(struct aead_request *req) -{ - return kmb_ocs_aead_common(req, OCS_AES, OCS_ENCRYPT, OCS_MODE_GCM); -} - -static int kmb_ocs_aes_gcm_decrypt(struct aead_request *req) -{ - return kmb_ocs_aead_common(req, OCS_AES, OCS_DECRYPT, OCS_MODE_GCM); -} - -static int kmb_ocs_aes_ccm_encrypt(struct aead_request *req) -{ - return kmb_ocs_aead_common(req, OCS_AES, OCS_ENCRYPT, OCS_MODE_CCM); -} - -static int kmb_ocs_aes_ccm_decrypt(struct aead_request *req) -{ - return kmb_ocs_aead_common(req, OCS_AES, OCS_DECRYPT, OCS_MODE_CCM); -} - -static int kmb_ocs_sm4_set_key(struct crypto_skcipher *tfm, const u8 *in_key, - unsigned int key_len) -{ - return kmb_ocs_sk_set_key(tfm, in_key, key_len, OCS_SM4); -} - -static int kmb_ocs_sm4_aead_set_key(struct crypto_aead *tfm, const u8 *in_key, - unsigned int key_len) -{ - return kmb_ocs_aead_set_key(tfm, in_key, key_len, OCS_SM4); -} - -#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB -static int kmb_ocs_sm4_ecb_encrypt(struct skcipher_request *req) -{ - return kmb_ocs_sk_common(req, OCS_SM4, OCS_ENCRYPT, OCS_MODE_ECB); -} - -static int kmb_ocs_sm4_ecb_decrypt(struct skcipher_request *req) -{ - return kmb_ocs_sk_common(req, OCS_SM4, OCS_DECRYPT, OCS_MODE_ECB); -} -#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB */ - -static int kmb_ocs_sm4_cbc_encrypt(struct skcipher_request *req) -{ - return kmb_ocs_sk_common(req, OCS_SM4, OCS_ENCRYPT, OCS_MODE_CBC); -} - -static int kmb_ocs_sm4_cbc_decrypt(struct skcipher_request *req) -{ - return kmb_ocs_sk_common(req, OCS_SM4, OCS_DECRYPT, OCS_MODE_CBC); -} - -static int kmb_ocs_sm4_ctr_encrypt(struct skcipher_request *req) -{ - return kmb_ocs_sk_common(req, OCS_SM4, OCS_ENCRYPT, OCS_MODE_CTR); -} - -static int kmb_ocs_sm4_ctr_decrypt(struct skcipher_request *req) -{ - return kmb_ocs_sk_common(req, OCS_SM4, OCS_DECRYPT, OCS_MODE_CTR); -} - -#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS -static int kmb_ocs_sm4_cts_encrypt(struct skcipher_request *req) -{ - return kmb_ocs_sk_common(req, OCS_SM4, OCS_ENCRYPT, OCS_MODE_CTS); -} - -static int kmb_ocs_sm4_cts_decrypt(struct skcipher_request *req) -{ - return kmb_ocs_sk_common(req, OCS_SM4, OCS_DECRYPT, OCS_MODE_CTS); -} -#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS */ - -static int kmb_ocs_sm4_gcm_encrypt(struct aead_request *req) -{ - return kmb_ocs_aead_common(req, OCS_SM4, OCS_ENCRYPT, OCS_MODE_GCM); -} - -static int kmb_ocs_sm4_gcm_decrypt(struct aead_request *req) -{ - return kmb_ocs_aead_common(req, OCS_SM4, OCS_DECRYPT, OCS_MODE_GCM); -} - -static int kmb_ocs_sm4_ccm_encrypt(struct aead_request *req) -{ - return kmb_ocs_aead_common(req, OCS_SM4, OCS_ENCRYPT, OCS_MODE_CCM); -} - -static int kmb_ocs_sm4_ccm_decrypt(struct aead_request *req) -{ - return kmb_ocs_aead_common(req, OCS_SM4, OCS_DECRYPT, OCS_MODE_CCM); -} - -static inline int ocs_common_init(struct ocs_aes_tctx *tctx) -{ - tctx->engine_ctx.op.prepare_request = NULL; - tctx->engine_ctx.op.do_one_request = kmb_ocs_aes_sk_do_one_request; - tctx->engine_ctx.op.unprepare_request = NULL; - - return 0; -} - -static int ocs_aes_init_tfm(struct crypto_skcipher *tfm) -{ - const char *alg_name = crypto_tfm_alg_name(&tfm->base); - struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm); - struct crypto_sync_skcipher *blk; - - /* set fallback cipher in case it will be needed */ - blk = crypto_alloc_sync_skcipher(alg_name, 0, CRYPTO_ALG_NEED_FALLBACK); - if (IS_ERR(blk)) - return PTR_ERR(blk); - - tctx->sw_cipher.sk = blk; - - crypto_skcipher_set_reqsize(tfm, sizeof(struct ocs_aes_rctx)); - - return ocs_common_init(tctx); -} - -static int ocs_sm4_init_tfm(struct crypto_skcipher *tfm) -{ - struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm); - - crypto_skcipher_set_reqsize(tfm, sizeof(struct ocs_aes_rctx)); - - return ocs_common_init(tctx); -} - -static inline void clear_key(struct ocs_aes_tctx *tctx) -{ - memzero_explicit(tctx->key, OCS_AES_KEYSIZE_256); - - /* Zero key registers if set */ - if (tctx->aes_dev) - ocs_aes_set_key(tctx->aes_dev, OCS_AES_KEYSIZE_256, - tctx->key, OCS_AES); -} - -static void ocs_exit_tfm(struct crypto_skcipher *tfm) -{ - struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm); - - clear_key(tctx); - - if (tctx->sw_cipher.sk) { - crypto_free_sync_skcipher(tctx->sw_cipher.sk); - tctx->sw_cipher.sk = NULL; - } -} - -static inline int ocs_common_aead_init(struct ocs_aes_tctx *tctx) -{ - tctx->engine_ctx.op.prepare_request = NULL; - tctx->engine_ctx.op.do_one_request = kmb_ocs_aes_aead_do_one_request; - tctx->engine_ctx.op.unprepare_request = NULL; - - return 0; -} - -static int ocs_aes_aead_cra_init(struct crypto_aead *tfm) -{ - const char *alg_name = crypto_tfm_alg_name(&tfm->base); - struct ocs_aes_tctx *tctx = crypto_aead_ctx(tfm); - struct crypto_aead *blk; - - /* Set fallback cipher in case it will be needed */ - blk = crypto_alloc_aead(alg_name, 0, CRYPTO_ALG_NEED_FALLBACK); - if (IS_ERR(blk)) - return PTR_ERR(blk); - - tctx->sw_cipher.aead = blk; - - crypto_aead_set_reqsize(tfm, - max(sizeof(struct ocs_aes_rctx), - (sizeof(struct aead_request) + - crypto_aead_reqsize(tctx->sw_cipher.aead)))); - - return ocs_common_aead_init(tctx); -} - -static int kmb_ocs_aead_ccm_setauthsize(struct crypto_aead *tfm, - unsigned int authsize) -{ - switch (authsize) { - case 4: - case 6: - case 8: - case 10: - case 12: - case 14: - case 16: - return 0; - default: - return -EINVAL; - } -} - -static int kmb_ocs_aead_gcm_setauthsize(struct crypto_aead *tfm, - unsigned int authsize) -{ - return crypto_gcm_check_authsize(authsize); -} - -static int ocs_sm4_aead_cra_init(struct crypto_aead *tfm) -{ - struct ocs_aes_tctx *tctx = crypto_aead_ctx(tfm); - - crypto_aead_set_reqsize(tfm, sizeof(struct ocs_aes_rctx)); - - return ocs_common_aead_init(tctx); -} - -static void ocs_aead_cra_exit(struct crypto_aead *tfm) -{ - struct ocs_aes_tctx *tctx = crypto_aead_ctx(tfm); - - clear_key(tctx); - - if (tctx->sw_cipher.aead) { - crypto_free_aead(tctx->sw_cipher.aead); - tctx->sw_cipher.aead = NULL; - } -} - -static struct skcipher_alg algs[] = { -#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB - { - .base.cra_name = "ecb(aes)", - .base.cra_driver_name = "ecb-aes-keembay-ocs", - .base.cra_priority = KMB_OCS_PRIORITY, - .base.cra_flags = CRYPTO_ALG_ASYNC | - CRYPTO_ALG_KERN_DRIVER_ONLY | - CRYPTO_ALG_NEED_FALLBACK, - .base.cra_blocksize = AES_BLOCK_SIZE, - .base.cra_ctxsize = sizeof(struct ocs_aes_tctx), - .base.cra_module = THIS_MODULE, - .base.cra_alignmask = 0, - - .min_keysize = OCS_AES_MIN_KEY_SIZE, - .max_keysize = OCS_AES_MAX_KEY_SIZE, - .setkey = kmb_ocs_aes_set_key, - .encrypt = kmb_ocs_aes_ecb_encrypt, - .decrypt = kmb_ocs_aes_ecb_decrypt, - .init = ocs_aes_init_tfm, - .exit = ocs_exit_tfm, - }, -#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB */ - { - .base.cra_name = "cbc(aes)", - .base.cra_driver_name = "cbc-aes-keembay-ocs", - .base.cra_priority = KMB_OCS_PRIORITY, - .base.cra_flags = CRYPTO_ALG_ASYNC | - CRYPTO_ALG_KERN_DRIVER_ONLY | - CRYPTO_ALG_NEED_FALLBACK, - .base.cra_blocksize = AES_BLOCK_SIZE, - .base.cra_ctxsize = sizeof(struct ocs_aes_tctx), - .base.cra_module = THIS_MODULE, - .base.cra_alignmask = 0, - - .min_keysize = OCS_AES_MIN_KEY_SIZE, - .max_keysize = OCS_AES_MAX_KEY_SIZE, - .ivsize = AES_BLOCK_SIZE, - .setkey = kmb_ocs_aes_set_key, - .encrypt = kmb_ocs_aes_cbc_encrypt, - .decrypt = kmb_ocs_aes_cbc_decrypt, - .init = ocs_aes_init_tfm, - .exit = ocs_exit_tfm, - }, - { - .base.cra_name = "ctr(aes)", - .base.cra_driver_name = "ctr-aes-keembay-ocs", - .base.cra_priority = KMB_OCS_PRIORITY, - .base.cra_flags = CRYPTO_ALG_ASYNC | - CRYPTO_ALG_KERN_DRIVER_ONLY | - CRYPTO_ALG_NEED_FALLBACK, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct ocs_aes_tctx), - .base.cra_module = THIS_MODULE, - .base.cra_alignmask = 0, - - .min_keysize = OCS_AES_MIN_KEY_SIZE, - .max_keysize = OCS_AES_MAX_KEY_SIZE, - .ivsize = AES_BLOCK_SIZE, - .setkey = kmb_ocs_aes_set_key, - .encrypt = kmb_ocs_aes_ctr_encrypt, - .decrypt = kmb_ocs_aes_ctr_decrypt, - .init = ocs_aes_init_tfm, - .exit = ocs_exit_tfm, - }, -#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS - { - .base.cra_name = "cts(cbc(aes))", - .base.cra_driver_name = "cts-aes-keembay-ocs", - .base.cra_priority = KMB_OCS_PRIORITY, - .base.cra_flags = CRYPTO_ALG_ASYNC | - CRYPTO_ALG_KERN_DRIVER_ONLY | - CRYPTO_ALG_NEED_FALLBACK, - .base.cra_blocksize = AES_BLOCK_SIZE, - .base.cra_ctxsize = sizeof(struct ocs_aes_tctx), - .base.cra_module = THIS_MODULE, - .base.cra_alignmask = 0, - - .min_keysize = OCS_AES_MIN_KEY_SIZE, - .max_keysize = OCS_AES_MAX_KEY_SIZE, - .ivsize = AES_BLOCK_SIZE, - .setkey = kmb_ocs_aes_set_key, - .encrypt = kmb_ocs_aes_cts_encrypt, - .decrypt = kmb_ocs_aes_cts_decrypt, - .init = ocs_aes_init_tfm, - .exit = ocs_exit_tfm, - }, -#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS */ -#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB - { - .base.cra_name = "ecb(sm4)", - .base.cra_driver_name = "ecb-sm4-keembay-ocs", - .base.cra_priority = KMB_OCS_PRIORITY, - .base.cra_flags = CRYPTO_ALG_ASYNC | - CRYPTO_ALG_KERN_DRIVER_ONLY, - .base.cra_blocksize = AES_BLOCK_SIZE, - .base.cra_ctxsize = sizeof(struct ocs_aes_tctx), - .base.cra_module = THIS_MODULE, - .base.cra_alignmask = 0, - - .min_keysize = OCS_SM4_KEY_SIZE, - .max_keysize = OCS_SM4_KEY_SIZE, - .setkey = kmb_ocs_sm4_set_key, - .encrypt = kmb_ocs_sm4_ecb_encrypt, - .decrypt = kmb_ocs_sm4_ecb_decrypt, - .init = ocs_sm4_init_tfm, - .exit = ocs_exit_tfm, - }, -#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB */ - { - .base.cra_name = "cbc(sm4)", - .base.cra_driver_name = "cbc-sm4-keembay-ocs", - .base.cra_priority = KMB_OCS_PRIORITY, - .base.cra_flags = CRYPTO_ALG_ASYNC | - CRYPTO_ALG_KERN_DRIVER_ONLY, - .base.cra_blocksize = AES_BLOCK_SIZE, - .base.cra_ctxsize = sizeof(struct ocs_aes_tctx), - .base.cra_module = THIS_MODULE, - .base.cra_alignmask = 0, - - .min_keysize = OCS_SM4_KEY_SIZE, - .max_keysize = OCS_SM4_KEY_SIZE, - .ivsize = AES_BLOCK_SIZE, - .setkey = kmb_ocs_sm4_set_key, - .encrypt = kmb_ocs_sm4_cbc_encrypt, - .decrypt = kmb_ocs_sm4_cbc_decrypt, - .init = ocs_sm4_init_tfm, - .exit = ocs_exit_tfm, - }, - { - .base.cra_name = "ctr(sm4)", - .base.cra_driver_name = "ctr-sm4-keembay-ocs", - .base.cra_priority = KMB_OCS_PRIORITY, - .base.cra_flags = CRYPTO_ALG_ASYNC | - CRYPTO_ALG_KERN_DRIVER_ONLY, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct ocs_aes_tctx), - .base.cra_module = THIS_MODULE, - .base.cra_alignmask = 0, - - .min_keysize = OCS_SM4_KEY_SIZE, - .max_keysize = OCS_SM4_KEY_SIZE, - .ivsize = AES_BLOCK_SIZE, - .setkey = kmb_ocs_sm4_set_key, - .encrypt = kmb_ocs_sm4_ctr_encrypt, - .decrypt = kmb_ocs_sm4_ctr_decrypt, - .init = ocs_sm4_init_tfm, - .exit = ocs_exit_tfm, - }, -#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS - { - .base.cra_name = "cts(cbc(sm4))", - .base.cra_driver_name = "cts-sm4-keembay-ocs", - .base.cra_priority = KMB_OCS_PRIORITY, - .base.cra_flags = CRYPTO_ALG_ASYNC | - CRYPTO_ALG_KERN_DRIVER_ONLY, - .base.cra_blocksize = AES_BLOCK_SIZE, - .base.cra_ctxsize = sizeof(struct ocs_aes_tctx), - .base.cra_module = THIS_MODULE, - .base.cra_alignmask = 0, - - .min_keysize = OCS_SM4_KEY_SIZE, - .max_keysize = OCS_SM4_KEY_SIZE, - .ivsize = AES_BLOCK_SIZE, - .setkey = kmb_ocs_sm4_set_key, - .encrypt = kmb_ocs_sm4_cts_encrypt, - .decrypt = kmb_ocs_sm4_cts_decrypt, - .init = ocs_sm4_init_tfm, - .exit = ocs_exit_tfm, - } -#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS */ -}; - -static struct aead_alg algs_aead[] = { - { - .base = { - .cra_name = "gcm(aes)", - .cra_driver_name = "gcm-aes-keembay-ocs", - .cra_priority = KMB_OCS_PRIORITY, - .cra_flags = CRYPTO_ALG_ASYNC | - CRYPTO_ALG_KERN_DRIVER_ONLY | - CRYPTO_ALG_NEED_FALLBACK, - .cra_blocksize = 1, - .cra_ctxsize = sizeof(struct ocs_aes_tctx), - .cra_alignmask = 0, - .cra_module = THIS_MODULE, - }, - .init = ocs_aes_aead_cra_init, - .exit = ocs_aead_cra_exit, - .ivsize = GCM_AES_IV_SIZE, - .maxauthsize = AES_BLOCK_SIZE, - .setauthsize = kmb_ocs_aead_gcm_setauthsize, - .setkey = kmb_ocs_aes_aead_set_key, - .encrypt = kmb_ocs_aes_gcm_encrypt, - .decrypt = kmb_ocs_aes_gcm_decrypt, - }, - { - .base = { - .cra_name = "ccm(aes)", - .cra_driver_name = "ccm-aes-keembay-ocs", - .cra_priority = KMB_OCS_PRIORITY, - .cra_flags = CRYPTO_ALG_ASYNC | - CRYPTO_ALG_KERN_DRIVER_ONLY | - CRYPTO_ALG_NEED_FALLBACK, - .cra_blocksize = 1, - .cra_ctxsize = sizeof(struct ocs_aes_tctx), - .cra_alignmask = 0, - .cra_module = THIS_MODULE, - }, - .init = ocs_aes_aead_cra_init, - .exit = ocs_aead_cra_exit, - .ivsize = AES_BLOCK_SIZE, - .maxauthsize = AES_BLOCK_SIZE, - .setauthsize = kmb_ocs_aead_ccm_setauthsize, - .setkey = kmb_ocs_aes_aead_set_key, - .encrypt = kmb_ocs_aes_ccm_encrypt, - .decrypt = kmb_ocs_aes_ccm_decrypt, - }, - { - .base = { - .cra_name = "gcm(sm4)", - .cra_driver_name = "gcm-sm4-keembay-ocs", - .cra_priority = KMB_OCS_PRIORITY, - .cra_flags = CRYPTO_ALG_ASYNC | - CRYPTO_ALG_KERN_DRIVER_ONLY, - .cra_blocksize = 1, - .cra_ctxsize = sizeof(struct ocs_aes_tctx), - .cra_alignmask = 0, - .cra_module = THIS_MODULE, - }, - .init = ocs_sm4_aead_cra_init, - .exit = ocs_aead_cra_exit, - .ivsize = GCM_AES_IV_SIZE, - .maxauthsize = AES_BLOCK_SIZE, - .setauthsize = kmb_ocs_aead_gcm_setauthsize, - .setkey = kmb_ocs_sm4_aead_set_key, - .encrypt = kmb_ocs_sm4_gcm_encrypt, - .decrypt = kmb_ocs_sm4_gcm_decrypt, - }, - { - .base = { - .cra_name = "ccm(sm4)", - .cra_driver_name = "ccm-sm4-keembay-ocs", - .cra_priority = KMB_OCS_PRIORITY, - .cra_flags = CRYPTO_ALG_ASYNC | - CRYPTO_ALG_KERN_DRIVER_ONLY, - .cra_blocksize = 1, - .cra_ctxsize = sizeof(struct ocs_aes_tctx), - .cra_alignmask = 0, - .cra_module = THIS_MODULE, - }, - .init = ocs_sm4_aead_cra_init, - .exit = ocs_aead_cra_exit, - .ivsize = AES_BLOCK_SIZE, - .maxauthsize = AES_BLOCK_SIZE, - .setauthsize = kmb_ocs_aead_ccm_setauthsize, - .setkey = kmb_ocs_sm4_aead_set_key, - .encrypt = kmb_ocs_sm4_ccm_encrypt, - .decrypt = kmb_ocs_sm4_ccm_decrypt, - } -}; - -static void unregister_aes_algs(struct ocs_aes_dev *aes_dev) -{ - crypto_unregister_aeads(algs_aead, ARRAY_SIZE(algs_aead)); - crypto_unregister_skciphers(algs, ARRAY_SIZE(algs)); -} - -static int register_aes_algs(struct ocs_aes_dev *aes_dev) -{ - int ret; - - /* - * If any algorithm fails to register, all preceding algorithms that - * were successfully registered will be automatically unregistered. - */ - ret = crypto_register_aeads(algs_aead, ARRAY_SIZE(algs_aead)); - if (ret) - return ret; - - ret = crypto_register_skciphers(algs, ARRAY_SIZE(algs)); - if (ret) - crypto_unregister_aeads(algs_aead, ARRAY_SIZE(algs)); - - return ret; -} - -/* Device tree driver match. */ -static const struct of_device_id kmb_ocs_aes_of_match[] = { - { - .compatible = "intel,keembay-ocs-aes", - }, - {} -}; - -static int kmb_ocs_aes_remove(struct platform_device *pdev) -{ - struct ocs_aes_dev *aes_dev; - - aes_dev = platform_get_drvdata(pdev); - - unregister_aes_algs(aes_dev); - - spin_lock(&ocs_aes.lock); - list_del(&aes_dev->list); - spin_unlock(&ocs_aes.lock); - - crypto_engine_exit(aes_dev->engine); - - return 0; -} - -static int kmb_ocs_aes_probe(struct platform_device *pdev) -{ - struct device *dev = &pdev->dev; - struct ocs_aes_dev *aes_dev; - int rc; - - aes_dev = devm_kzalloc(dev, sizeof(*aes_dev), GFP_KERNEL); - if (!aes_dev) - return -ENOMEM; - - aes_dev->dev = dev; - - platform_set_drvdata(pdev, aes_dev); - - rc = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32)); - if (rc) { - dev_err(dev, "Failed to set 32 bit dma mask %d\n", rc); - return rc; - } - - /* Get base register address. */ - aes_dev->base_reg = devm_platform_ioremap_resource(pdev, 0); - if (IS_ERR(aes_dev->base_reg)) - return PTR_ERR(aes_dev->base_reg); - - /* Get and request IRQ */ - aes_dev->irq = platform_get_irq(pdev, 0); - if (aes_dev->irq < 0) - return aes_dev->irq; - - rc = devm_request_threaded_irq(dev, aes_dev->irq, ocs_aes_irq_handler, - NULL, 0, "keembay-ocs-aes", aes_dev); - if (rc < 0) { - dev_err(dev, "Could not request IRQ\n"); - return rc; - } - - INIT_LIST_HEAD(&aes_dev->list); - spin_lock(&ocs_aes.lock); - list_add_tail(&aes_dev->list, &ocs_aes.dev_list); - spin_unlock(&ocs_aes.lock); - - init_completion(&aes_dev->irq_completion); - - /* Initialize crypto engine */ - aes_dev->engine = crypto_engine_alloc_init(dev, true); - if (!aes_dev->engine) { - rc = -ENOMEM; - goto list_del; - } - - rc = crypto_engine_start(aes_dev->engine); - if (rc) { - dev_err(dev, "Could not start crypto engine\n"); - goto cleanup; - } - - rc = register_aes_algs(aes_dev); - if (rc) { - dev_err(dev, - "Could not register OCS algorithms with Crypto API\n"); - goto cleanup; - } - - return 0; - -cleanup: - crypto_engine_exit(aes_dev->engine); -list_del: - spin_lock(&ocs_aes.lock); - list_del(&aes_dev->list); - spin_unlock(&ocs_aes.lock); - - return rc; -} - -/* The OCS driver is a platform device. */ -static struct platform_driver kmb_ocs_aes_driver = { - .probe = kmb_ocs_aes_probe, - .remove = kmb_ocs_aes_remove, - .driver = { - .name = DRV_NAME, - .of_match_table = kmb_ocs_aes_of_match, - }, -}; - -module_platform_driver(kmb_ocs_aes_driver); - -MODULE_DESCRIPTION("Intel Keem Bay Offload and Crypto Subsystem (OCS) AES/SM4 Driver"); -MODULE_LICENSE("GPL"); - -MODULE_ALIAS_CRYPTO("cbc-aes-keembay-ocs"); -MODULE_ALIAS_CRYPTO("ctr-aes-keembay-ocs"); -MODULE_ALIAS_CRYPTO("gcm-aes-keembay-ocs"); -MODULE_ALIAS_CRYPTO("ccm-aes-keembay-ocs"); - -MODULE_ALIAS_CRYPTO("cbc-sm4-keembay-ocs"); -MODULE_ALIAS_CRYPTO("ctr-sm4-keembay-ocs"); -MODULE_ALIAS_CRYPTO("gcm-sm4-keembay-ocs"); -MODULE_ALIAS_CRYPTO("ccm-sm4-keembay-ocs"); - -#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB -MODULE_ALIAS_CRYPTO("ecb-aes-keembay-ocs"); -MODULE_ALIAS_CRYPTO("ecb-sm4-keembay-ocs"); -#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB */ - -#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS -MODULE_ALIAS_CRYPTO("cts-aes-keembay-ocs"); -MODULE_ALIAS_CRYPTO("cts-sm4-keembay-ocs"); -#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS */ diff --git a/drivers/crypto/keembay/keembay-ocs-ecc.c b/drivers/crypto/keembay/keembay-ocs-ecc.c deleted file mode 100644 index 2269df17514c..000000000000 --- a/drivers/crypto/keembay/keembay-ocs-ecc.c +++ /dev/null @@ -1,1016 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Intel Keem Bay OCS ECC Crypto Driver. - * - * Copyright (C) 2019-2021 Intel Corporation - */ - -#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include - -#include -#include - -#define DRV_NAME "keembay-ocs-ecc" - -#define KMB_OCS_ECC_PRIORITY 350 - -#define HW_OFFS_OCS_ECC_COMMAND 0x00000000 -#define HW_OFFS_OCS_ECC_STATUS 0x00000004 -#define HW_OFFS_OCS_ECC_DATA_IN 0x00000080 -#define HW_OFFS_OCS_ECC_CX_DATA_OUT 0x00000100 -#define HW_OFFS_OCS_ECC_CY_DATA_OUT 0x00000180 -#define HW_OFFS_OCS_ECC_ISR 0x00000400 -#define HW_OFFS_OCS_ECC_IER 0x00000404 - -#define HW_OCS_ECC_ISR_INT_STATUS_DONE BIT(0) -#define HW_OCS_ECC_COMMAND_INS_BP BIT(0) - -#define HW_OCS_ECC_COMMAND_START_VAL BIT(0) - -#define OCS_ECC_OP_SIZE_384 BIT(8) -#define OCS_ECC_OP_SIZE_256 0 - -/* ECC Instruction : for ECC_COMMAND */ -#define OCS_ECC_INST_WRITE_AX (0x1 << HW_OCS_ECC_COMMAND_INS_BP) -#define OCS_ECC_INST_WRITE_AY (0x2 << HW_OCS_ECC_COMMAND_INS_BP) -#define OCS_ECC_INST_WRITE_BX_D (0x3 << HW_OCS_ECC_COMMAND_INS_BP) -#define OCS_ECC_INST_WRITE_BY_L (0x4 << HW_OCS_ECC_COMMAND_INS_BP) -#define OCS_ECC_INST_WRITE_P (0x5 << HW_OCS_ECC_COMMAND_INS_BP) -#define OCS_ECC_INST_WRITE_A (0x6 << HW_OCS_ECC_COMMAND_INS_BP) -#define OCS_ECC_INST_CALC_D_IDX_A (0x8 << HW_OCS_ECC_COMMAND_INS_BP) -#define OCS_ECC_INST_CALC_A_POW_B_MODP (0xB << HW_OCS_ECC_COMMAND_INS_BP) -#define OCS_ECC_INST_CALC_A_MUL_B_MODP (0xC << HW_OCS_ECC_COMMAND_INS_BP) -#define OCS_ECC_INST_CALC_A_ADD_B_MODP (0xD << HW_OCS_ECC_COMMAND_INS_BP) - -#define ECC_ENABLE_INTR 1 - -#define POLL_USEC 100 -#define TIMEOUT_USEC 10000 - -#define KMB_ECC_VLI_MAX_DIGITS ECC_CURVE_NIST_P384_DIGITS -#define KMB_ECC_VLI_MAX_BYTES (KMB_ECC_VLI_MAX_DIGITS \ - << ECC_DIGITS_TO_BYTES_SHIFT) - -#define POW_CUBE 3 - -/** - * struct ocs_ecc_dev - ECC device context - * @list: List of device contexts - * @dev: OCS ECC device - * @base_reg: IO base address of OCS ECC - * @engine: Crypto engine for the device - * @irq_done: IRQ done completion. - * @irq: IRQ number - */ -struct ocs_ecc_dev { - struct list_head list; - struct device *dev; - void __iomem *base_reg; - struct crypto_engine *engine; - struct completion irq_done; - int irq; -}; - -/** - * struct ocs_ecc_ctx - Transformation context. - * @engine_ctx: Crypto engine ctx. - * @ecc_dev: The ECC driver associated with this context. - * @curve: The elliptic curve used by this transformation. - * @private_key: The private key. - */ -struct ocs_ecc_ctx { - struct crypto_engine_ctx engine_ctx; - struct ocs_ecc_dev *ecc_dev; - const struct ecc_curve *curve; - u64 private_key[KMB_ECC_VLI_MAX_DIGITS]; -}; - -/* Driver data. */ -struct ocs_ecc_drv { - struct list_head dev_list; - spinlock_t lock; /* Protects dev_list. */ -}; - -/* Global variable holding the list of OCS ECC devices (only one expected). */ -static struct ocs_ecc_drv ocs_ecc = { - .dev_list = LIST_HEAD_INIT(ocs_ecc.dev_list), - .lock = __SPIN_LOCK_UNLOCKED(ocs_ecc.lock), -}; - -/* Get OCS ECC tfm context from kpp_request. */ -static inline struct ocs_ecc_ctx *kmb_ocs_ecc_tctx(struct kpp_request *req) -{ - return kpp_tfm_ctx(crypto_kpp_reqtfm(req)); -} - -/* Converts number of digits to number of bytes. */ -static inline unsigned int digits_to_bytes(unsigned int n) -{ - return n << ECC_DIGITS_TO_BYTES_SHIFT; -} - -/* - * Wait for ECC idle i.e when an operation (other than write operations) - * is done. - */ -static inline int ocs_ecc_wait_idle(struct ocs_ecc_dev *dev) -{ - u32 value; - - return readl_poll_timeout((dev->base_reg + HW_OFFS_OCS_ECC_STATUS), - value, - !(value & HW_OCS_ECC_ISR_INT_STATUS_DONE), - POLL_USEC, TIMEOUT_USEC); -} - -static void ocs_ecc_cmd_start(struct ocs_ecc_dev *ecc_dev, u32 op_size) -{ - iowrite32(op_size | HW_OCS_ECC_COMMAND_START_VAL, - ecc_dev->base_reg + HW_OFFS_OCS_ECC_COMMAND); -} - -/* Direct write of u32 buffer to ECC engine with associated instruction. */ -static void ocs_ecc_write_cmd_and_data(struct ocs_ecc_dev *dev, - u32 op_size, - u32 inst, - const void *data_in, - size_t data_size) -{ - iowrite32(op_size | inst, dev->base_reg + HW_OFFS_OCS_ECC_COMMAND); - - /* MMIO Write src uint32 to dst. */ - memcpy_toio(dev->base_reg + HW_OFFS_OCS_ECC_DATA_IN, data_in, - data_size); -} - -/* Start OCS ECC operation and wait for its completion. */ -static int ocs_ecc_trigger_op(struct ocs_ecc_dev *ecc_dev, u32 op_size, - u32 inst) -{ - reinit_completion(&ecc_dev->irq_done); - - iowrite32(ECC_ENABLE_INTR, ecc_dev->base_reg + HW_OFFS_OCS_ECC_IER); - iowrite32(op_size | inst, ecc_dev->base_reg + HW_OFFS_OCS_ECC_COMMAND); - - return wait_for_completion_interruptible(&ecc_dev->irq_done); -} - -/** - * ocs_ecc_read_cx_out() - Read the CX data output buffer. - * @dev: The OCS ECC device to read from. - * @cx_out: The buffer where to store the CX value. Must be at least - * @byte_count byte long. - * @byte_count: The amount of data to read. - */ -static inline void ocs_ecc_read_cx_out(struct ocs_ecc_dev *dev, void *cx_out, - size_t byte_count) -{ - memcpy_fromio(cx_out, dev->base_reg + HW_OFFS_OCS_ECC_CX_DATA_OUT, - byte_count); -} - -/** - * ocs_ecc_read_cy_out() - Read the CX data output buffer. - * @dev: The OCS ECC device to read from. - * @cy_out: The buffer where to store the CY value. Must be at least - * @byte_count byte long. - * @byte_count: The amount of data to read. - */ -static inline void ocs_ecc_read_cy_out(struct ocs_ecc_dev *dev, void *cy_out, - size_t byte_count) -{ - memcpy_fromio(cy_out, dev->base_reg + HW_OFFS_OCS_ECC_CY_DATA_OUT, - byte_count); -} - -static struct ocs_ecc_dev *kmb_ocs_ecc_find_dev(struct ocs_ecc_ctx *tctx) -{ - if (tctx->ecc_dev) - return tctx->ecc_dev; - - spin_lock(&ocs_ecc.lock); - - /* Only a single OCS device available. */ - tctx->ecc_dev = list_first_entry(&ocs_ecc.dev_list, struct ocs_ecc_dev, - list); - - spin_unlock(&ocs_ecc.lock); - - return tctx->ecc_dev; -} - -/* Do point multiplication using OCS ECC HW. */ -static int kmb_ecc_point_mult(struct ocs_ecc_dev *ecc_dev, - struct ecc_point *result, - const struct ecc_point *point, - u64 *scalar, - const struct ecc_curve *curve) -{ - u8 sca[KMB_ECC_VLI_MAX_BYTES]; /* Use the maximum data size. */ - u32 op_size = (curve->g.ndigits > ECC_CURVE_NIST_P256_DIGITS) ? - OCS_ECC_OP_SIZE_384 : OCS_ECC_OP_SIZE_256; - size_t nbytes = digits_to_bytes(curve->g.ndigits); - int rc = 0; - - /* Generate random nbytes for Simple and Differential SCA protection. */ - rc = crypto_get_default_rng(); - if (rc) - return rc; - - rc = crypto_rng_get_bytes(crypto_default_rng, sca, nbytes); - crypto_put_default_rng(); - if (rc) - return rc; - - /* Wait engine to be idle before starting new operation. */ - rc = ocs_ecc_wait_idle(ecc_dev); - if (rc) - return rc; - - /* Send ecc_start pulse as well as indicating operation size. */ - ocs_ecc_cmd_start(ecc_dev, op_size); - - /* Write ax param; Base point (Gx). */ - ocs_ecc_write_cmd_and_data(ecc_dev, op_size, OCS_ECC_INST_WRITE_AX, - point->x, nbytes); - - /* Write ay param; Base point (Gy). */ - ocs_ecc_write_cmd_and_data(ecc_dev, op_size, OCS_ECC_INST_WRITE_AY, - point->y, nbytes); - - /* - * Write the private key into DATA_IN reg. - * - * Since DATA_IN register is used to write different values during the - * computation private Key value is overwritten with - * side-channel-resistance value. - */ - ocs_ecc_write_cmd_and_data(ecc_dev, op_size, OCS_ECC_INST_WRITE_BX_D, - scalar, nbytes); - - /* Write operand by/l. */ - ocs_ecc_write_cmd_and_data(ecc_dev, op_size, OCS_ECC_INST_WRITE_BY_L, - sca, nbytes); - memzero_explicit(sca, sizeof(sca)); - - /* Write p = curve prime(GF modulus). */ - ocs_ecc_write_cmd_and_data(ecc_dev, op_size, OCS_ECC_INST_WRITE_P, - curve->p, nbytes); - - /* Write a = curve coefficient. */ - ocs_ecc_write_cmd_and_data(ecc_dev, op_size, OCS_ECC_INST_WRITE_A, - curve->a, nbytes); - - /* Make hardware perform the multiplication. */ - rc = ocs_ecc_trigger_op(ecc_dev, op_size, OCS_ECC_INST_CALC_D_IDX_A); - if (rc) - return rc; - - /* Read result. */ - ocs_ecc_read_cx_out(ecc_dev, result->x, nbytes); - ocs_ecc_read_cy_out(ecc_dev, result->y, nbytes); - - return 0; -} - -/** - * kmb_ecc_do_scalar_op() - Perform Scalar operation using OCS ECC HW. - * @ecc_dev: The OCS ECC device to use. - * @scalar_out: Where to store the output scalar. - * @scalar_a: Input scalar operand 'a'. - * @scalar_b: Input scalar operand 'b' - * @curve: The curve on which the operation is performed. - * @ndigits: The size of the operands (in digits). - * @inst: The operation to perform (as an OCS ECC instruction). - * - * Return: 0 on success, negative error code otherwise. - */ -static int kmb_ecc_do_scalar_op(struct ocs_ecc_dev *ecc_dev, u64 *scalar_out, - const u64 *scalar_a, const u64 *scalar_b, - const struct ecc_curve *curve, - unsigned int ndigits, const u32 inst) -{ - u32 op_size = (ndigits > ECC_CURVE_NIST_P256_DIGITS) ? - OCS_ECC_OP_SIZE_384 : OCS_ECC_OP_SIZE_256; - size_t nbytes = digits_to_bytes(ndigits); - int rc; - - /* Wait engine to be idle before starting new operation. */ - rc = ocs_ecc_wait_idle(ecc_dev); - if (rc) - return rc; - - /* Send ecc_start pulse as well as indicating operation size. */ - ocs_ecc_cmd_start(ecc_dev, op_size); - - /* Write ax param (Base point (Gx).*/ - ocs_ecc_write_cmd_and_data(ecc_dev, op_size, OCS_ECC_INST_WRITE_AX, - scalar_a, nbytes); - - /* Write ay param Base point (Gy).*/ - ocs_ecc_write_cmd_and_data(ecc_dev, op_size, OCS_ECC_INST_WRITE_AY, - scalar_b, nbytes); - - /* Write p = curve prime(GF modulus).*/ - ocs_ecc_write_cmd_and_data(ecc_dev, op_size, OCS_ECC_INST_WRITE_P, - curve->p, nbytes); - - /* Give instruction A.B or A+B to ECC engine. */ - rc = ocs_ecc_trigger_op(ecc_dev, op_size, inst); - if (rc) - return rc; - - ocs_ecc_read_cx_out(ecc_dev, scalar_out, nbytes); - - if (vli_is_zero(scalar_out, ndigits)) - return -EINVAL; - - return 0; -} - -/* SP800-56A section 5.6.2.3.4 partial verification: ephemeral keys only */ -static int kmb_ocs_ecc_is_pubkey_valid_partial(struct ocs_ecc_dev *ecc_dev, - const struct ecc_curve *curve, - struct ecc_point *pk) -{ - u64 xxx[KMB_ECC_VLI_MAX_DIGITS] = { 0 }; - u64 yy[KMB_ECC_VLI_MAX_DIGITS] = { 0 }; - u64 w[KMB_ECC_VLI_MAX_DIGITS] = { 0 }; - int rc; - - if (WARN_ON(pk->ndigits != curve->g.ndigits)) - return -EINVAL; - - /* Check 1: Verify key is not the zero point. */ - if (ecc_point_is_zero(pk)) - return -EINVAL; - - /* Check 2: Verify key is in the range [0, p-1]. */ - if (vli_cmp(curve->p, pk->x, pk->ndigits) != 1) - return -EINVAL; - - if (vli_cmp(curve->p, pk->y, pk->ndigits) != 1) - return -EINVAL; - - /* Check 3: Verify that y^2 == (x^3 + a·x + b) mod p */ - - /* y^2 */ - /* Compute y^2 -> store in yy */ - rc = kmb_ecc_do_scalar_op(ecc_dev, yy, pk->y, pk->y, curve, pk->ndigits, - OCS_ECC_INST_CALC_A_MUL_B_MODP); - if (rc) - goto exit; - - /* x^3 */ - /* Assigning w = 3, used for calculating x^3. */ - w[0] = POW_CUBE; - /* Load the next stage.*/ - rc = kmb_ecc_do_scalar_op(ecc_dev, xxx, pk->x, w, curve, pk->ndigits, - OCS_ECC_INST_CALC_A_POW_B_MODP); - if (rc) - goto exit; - - /* Do a*x -> store in w. */ - rc = kmb_ecc_do_scalar_op(ecc_dev, w, curve->a, pk->x, curve, - pk->ndigits, - OCS_ECC_INST_CALC_A_MUL_B_MODP); - if (rc) - goto exit; - - /* Do ax + b == w + b; store in w. */ - rc = kmb_ecc_do_scalar_op(ecc_dev, w, w, curve->b, curve, - pk->ndigits, - OCS_ECC_INST_CALC_A_ADD_B_MODP); - if (rc) - goto exit; - - /* x^3 + ax + b == x^3 + w -> store in w. */ - rc = kmb_ecc_do_scalar_op(ecc_dev, w, xxx, w, curve, pk->ndigits, - OCS_ECC_INST_CALC_A_ADD_B_MODP); - if (rc) - goto exit; - - /* Compare y^2 == x^3 + a·x + b. */ - rc = vli_cmp(yy, w, pk->ndigits); - if (rc) - rc = -EINVAL; - -exit: - memzero_explicit(xxx, sizeof(xxx)); - memzero_explicit(yy, sizeof(yy)); - memzero_explicit(w, sizeof(w)); - - return rc; -} - -/* SP800-56A section 5.6.2.3.3 full verification */ -static int kmb_ocs_ecc_is_pubkey_valid_full(struct ocs_ecc_dev *ecc_dev, - const struct ecc_curve *curve, - struct ecc_point *pk) -{ - struct ecc_point *nQ; - int rc; - - /* Checks 1 through 3 */ - rc = kmb_ocs_ecc_is_pubkey_valid_partial(ecc_dev, curve, pk); - if (rc) - return rc; - - /* Check 4: Verify that nQ is the zero point. */ - nQ = ecc_alloc_point(pk->ndigits); - if (!nQ) - return -ENOMEM; - - rc = kmb_ecc_point_mult(ecc_dev, nQ, pk, curve->n, curve); - if (rc) - goto exit; - - if (!ecc_point_is_zero(nQ)) - rc = -EINVAL; - -exit: - ecc_free_point(nQ); - - return rc; -} - -static int kmb_ecc_is_key_valid(const struct ecc_curve *curve, - const u64 *private_key, size_t private_key_len) -{ - size_t ndigits = curve->g.ndigits; - u64 one[KMB_ECC_VLI_MAX_DIGITS] = {1}; - u64 res[KMB_ECC_VLI_MAX_DIGITS]; - - if (private_key_len != digits_to_bytes(ndigits)) - return -EINVAL; - - if (!private_key) - return -EINVAL; - - /* Make sure the private key is in the range [2, n-3]. */ - if (vli_cmp(one, private_key, ndigits) != -1) - return -EINVAL; - - vli_sub(res, curve->n, one, ndigits); - vli_sub(res, res, one, ndigits); - if (vli_cmp(res, private_key, ndigits) != 1) - return -EINVAL; - - return 0; -} - -/* - * ECC private keys are generated using the method of extra random bits, - * equivalent to that described in FIPS 186-4, Appendix B.4.1. - * - * d = (c mod(n–1)) + 1 where c is a string of random bits, 64 bits longer - * than requested - * 0 <= c mod(n-1) <= n-2 and implies that - * 1 <= d <= n-1 - * - * This method generates a private key uniformly distributed in the range - * [1, n-1]. - */ -static int kmb_ecc_gen_privkey(const struct ecc_curve *curve, u64 *privkey) -{ - size_t nbytes = digits_to_bytes(curve->g.ndigits); - u64 priv[KMB_ECC_VLI_MAX_DIGITS]; - size_t nbits; - int rc; - - nbits = vli_num_bits(curve->n, curve->g.ndigits); - - /* Check that N is included in Table 1 of FIPS 186-4, section 6.1.1 */ - if (nbits < 160 || curve->g.ndigits > ARRAY_SIZE(priv)) - return -EINVAL; - - /* - * FIPS 186-4 recommends that the private key should be obtained from a - * RBG with a security strength equal to or greater than the security - * strength associated with N. - * - * The maximum security strength identified by NIST SP800-57pt1r4 for - * ECC is 256 (N >= 512). - * - * This condition is met by the default RNG because it selects a favored - * DRBG with a security strength of 256. - */ - if (crypto_get_default_rng()) - return -EFAULT; - - rc = crypto_rng_get_bytes(crypto_default_rng, (u8 *)priv, nbytes); - crypto_put_default_rng(); - if (rc) - goto cleanup; - - rc = kmb_ecc_is_key_valid(curve, priv, nbytes); - if (rc) - goto cleanup; - - ecc_swap_digits(priv, privkey, curve->g.ndigits); - -cleanup: - memzero_explicit(&priv, sizeof(priv)); - - return rc; -} - -static int kmb_ocs_ecdh_set_secret(struct crypto_kpp *tfm, const void *buf, - unsigned int len) -{ - struct ocs_ecc_ctx *tctx = kpp_tfm_ctx(tfm); - struct ecdh params; - int rc = 0; - - rc = crypto_ecdh_decode_key(buf, len, ¶ms); - if (rc) - goto cleanup; - - /* Ensure key size is not bigger then expected. */ - if (params.key_size > digits_to_bytes(tctx->curve->g.ndigits)) { - rc = -EINVAL; - goto cleanup; - } - - /* Auto-generate private key is not provided. */ - if (!params.key || !params.key_size) { - rc = kmb_ecc_gen_privkey(tctx->curve, tctx->private_key); - goto cleanup; - } - - rc = kmb_ecc_is_key_valid(tctx->curve, (const u64 *)params.key, - params.key_size); - if (rc) - goto cleanup; - - ecc_swap_digits((const u64 *)params.key, tctx->private_key, - tctx->curve->g.ndigits); -cleanup: - memzero_explicit(¶ms, sizeof(params)); - - if (rc) - tctx->curve = NULL; - - return rc; -} - -/* Compute shared secret. */ -static int kmb_ecc_do_shared_secret(struct ocs_ecc_ctx *tctx, - struct kpp_request *req) -{ - struct ocs_ecc_dev *ecc_dev = tctx->ecc_dev; - const struct ecc_curve *curve = tctx->curve; - u64 shared_secret[KMB_ECC_VLI_MAX_DIGITS]; - u64 pubk_buf[KMB_ECC_VLI_MAX_DIGITS * 2]; - size_t copied, nbytes, pubk_len; - struct ecc_point *pk, *result; - int rc; - - nbytes = digits_to_bytes(curve->g.ndigits); - - /* Public key is a point, thus it has two coordinates */ - pubk_len = 2 * nbytes; - - /* Copy public key from SG list to pubk_buf. */ - copied = sg_copy_to_buffer(req->src, - sg_nents_for_len(req->src, pubk_len), - pubk_buf, pubk_len); - if (copied != pubk_len) - return -EINVAL; - - /* Allocate and initialize public key point. */ - pk = ecc_alloc_point(curve->g.ndigits); - if (!pk) - return -ENOMEM; - - ecc_swap_digits(pubk_buf, pk->x, curve->g.ndigits); - ecc_swap_digits(&pubk_buf[curve->g.ndigits], pk->y, curve->g.ndigits); - - /* - * Check the public key for following - * Check 1: Verify key is not the zero point. - * Check 2: Verify key is in the range [1, p-1]. - * Check 3: Verify that y^2 == (x^3 + a·x + b) mod p - */ - rc = kmb_ocs_ecc_is_pubkey_valid_partial(ecc_dev, curve, pk); - if (rc) - goto exit_free_pk; - - /* Allocate point for storing computed shared secret. */ - result = ecc_alloc_point(pk->ndigits); - if (!result) { - rc = -ENOMEM; - goto exit_free_pk; - } - - /* Calculate the shared secret.*/ - rc = kmb_ecc_point_mult(ecc_dev, result, pk, tctx->private_key, curve); - if (rc) - goto exit_free_result; - - if (ecc_point_is_zero(result)) { - rc = -EFAULT; - goto exit_free_result; - } - - /* Copy shared secret from point to buffer. */ - ecc_swap_digits(result->x, shared_secret, result->ndigits); - - /* Request might ask for less bytes than what we have. */ - nbytes = min_t(size_t, nbytes, req->dst_len); - - copied = sg_copy_from_buffer(req->dst, - sg_nents_for_len(req->dst, nbytes), - shared_secret, nbytes); - - if (copied != nbytes) - rc = -EINVAL; - - memzero_explicit(shared_secret, sizeof(shared_secret)); - -exit_free_result: - ecc_free_point(result); - -exit_free_pk: - ecc_free_point(pk); - - return rc; -} - -/* Compute public key. */ -static int kmb_ecc_do_public_key(struct ocs_ecc_ctx *tctx, - struct kpp_request *req) -{ - const struct ecc_curve *curve = tctx->curve; - u64 pubk_buf[KMB_ECC_VLI_MAX_DIGITS * 2]; - struct ecc_point *pk; - size_t pubk_len; - size_t copied; - int rc; - - /* Public key is a point, so it has double the digits. */ - pubk_len = 2 * digits_to_bytes(curve->g.ndigits); - - pk = ecc_alloc_point(curve->g.ndigits); - if (!pk) - return -ENOMEM; - - /* Public Key(pk) = priv * G. */ - rc = kmb_ecc_point_mult(tctx->ecc_dev, pk, &curve->g, tctx->private_key, - curve); - if (rc) - goto exit; - - /* SP800-56A rev 3 5.6.2.1.3 key check */ - if (kmb_ocs_ecc_is_pubkey_valid_full(tctx->ecc_dev, curve, pk)) { - rc = -EAGAIN; - goto exit; - } - - /* Copy public key from point to buffer. */ - ecc_swap_digits(pk->x, pubk_buf, pk->ndigits); - ecc_swap_digits(pk->y, &pubk_buf[pk->ndigits], pk->ndigits); - - /* Copy public key to req->dst. */ - copied = sg_copy_from_buffer(req->dst, - sg_nents_for_len(req->dst, pubk_len), - pubk_buf, pubk_len); - - if (copied != pubk_len) - rc = -EINVAL; - -exit: - ecc_free_point(pk); - - return rc; -} - -static int kmb_ocs_ecc_do_one_request(struct crypto_engine *engine, - void *areq) -{ - struct kpp_request *req = container_of(areq, struct kpp_request, base); - struct ocs_ecc_ctx *tctx = kmb_ocs_ecc_tctx(req); - struct ocs_ecc_dev *ecc_dev = tctx->ecc_dev; - int rc; - - if (req->src) - rc = kmb_ecc_do_shared_secret(tctx, req); - else - rc = kmb_ecc_do_public_key(tctx, req); - - crypto_finalize_kpp_request(ecc_dev->engine, req, rc); - - return 0; -} - -static int kmb_ocs_ecdh_generate_public_key(struct kpp_request *req) -{ - struct ocs_ecc_ctx *tctx = kmb_ocs_ecc_tctx(req); - const struct ecc_curve *curve = tctx->curve; - - /* Ensure kmb_ocs_ecdh_set_secret() has been successfully called. */ - if (!tctx->curve) - return -EINVAL; - - /* Ensure dst is present. */ - if (!req->dst) - return -EINVAL; - - /* Check the request dst is big enough to hold the public key. */ - if (req->dst_len < (2 * digits_to_bytes(curve->g.ndigits))) - return -EINVAL; - - /* 'src' is not supposed to be present when generate pubk is called. */ - if (req->src) - return -EINVAL; - - return crypto_transfer_kpp_request_to_engine(tctx->ecc_dev->engine, - req); -} - -static int kmb_ocs_ecdh_compute_shared_secret(struct kpp_request *req) -{ - struct ocs_ecc_ctx *tctx = kmb_ocs_ecc_tctx(req); - const struct ecc_curve *curve = tctx->curve; - - /* Ensure kmb_ocs_ecdh_set_secret() has been successfully called. */ - if (!tctx->curve) - return -EINVAL; - - /* Ensure dst is present. */ - if (!req->dst) - return -EINVAL; - - /* Ensure src is present. */ - if (!req->src) - return -EINVAL; - - /* - * req->src is expected to the (other-side) public key, so its length - * must be 2 * coordinate size (in bytes). - */ - if (req->src_len != 2 * digits_to_bytes(curve->g.ndigits)) - return -EINVAL; - - return crypto_transfer_kpp_request_to_engine(tctx->ecc_dev->engine, - req); -} - -static int kmb_ecc_tctx_init(struct ocs_ecc_ctx *tctx, unsigned int curve_id) -{ - memset(tctx, 0, sizeof(*tctx)); - - tctx->ecc_dev = kmb_ocs_ecc_find_dev(tctx); - - if (IS_ERR(tctx->ecc_dev)) { - pr_err("Failed to find the device : %ld\n", - PTR_ERR(tctx->ecc_dev)); - return PTR_ERR(tctx->ecc_dev); - } - - tctx->curve = ecc_get_curve(curve_id); - if (!tctx->curve) - return -EOPNOTSUPP; - - tctx->engine_ctx.op.prepare_request = NULL; - tctx->engine_ctx.op.do_one_request = kmb_ocs_ecc_do_one_request; - tctx->engine_ctx.op.unprepare_request = NULL; - - return 0; -} - -static int kmb_ocs_ecdh_nist_p256_init_tfm(struct crypto_kpp *tfm) -{ - struct ocs_ecc_ctx *tctx = kpp_tfm_ctx(tfm); - - return kmb_ecc_tctx_init(tctx, ECC_CURVE_NIST_P256); -} - -static int kmb_ocs_ecdh_nist_p384_init_tfm(struct crypto_kpp *tfm) -{ - struct ocs_ecc_ctx *tctx = kpp_tfm_ctx(tfm); - - return kmb_ecc_tctx_init(tctx, ECC_CURVE_NIST_P384); -} - -static void kmb_ocs_ecdh_exit_tfm(struct crypto_kpp *tfm) -{ - struct ocs_ecc_ctx *tctx = kpp_tfm_ctx(tfm); - - memzero_explicit(tctx->private_key, sizeof(*tctx->private_key)); -} - -static unsigned int kmb_ocs_ecdh_max_size(struct crypto_kpp *tfm) -{ - struct ocs_ecc_ctx *tctx = kpp_tfm_ctx(tfm); - - /* Public key is made of two coordinates, so double the digits. */ - return digits_to_bytes(tctx->curve->g.ndigits) * 2; -} - -static struct kpp_alg ocs_ecdh_p256 = { - .set_secret = kmb_ocs_ecdh_set_secret, - .generate_public_key = kmb_ocs_ecdh_generate_public_key, - .compute_shared_secret = kmb_ocs_ecdh_compute_shared_secret, - .init = kmb_ocs_ecdh_nist_p256_init_tfm, - .exit = kmb_ocs_ecdh_exit_tfm, - .max_size = kmb_ocs_ecdh_max_size, - .base = { - .cra_name = "ecdh-nist-p256", - .cra_driver_name = "ecdh-nist-p256-keembay-ocs", - .cra_priority = KMB_OCS_ECC_PRIORITY, - .cra_module = THIS_MODULE, - .cra_ctxsize = sizeof(struct ocs_ecc_ctx), - }, -}; - -static struct kpp_alg ocs_ecdh_p384 = { - .set_secret = kmb_ocs_ecdh_set_secret, - .generate_public_key = kmb_ocs_ecdh_generate_public_key, - .compute_shared_secret = kmb_ocs_ecdh_compute_shared_secret, - .init = kmb_ocs_ecdh_nist_p384_init_tfm, - .exit = kmb_ocs_ecdh_exit_tfm, - .max_size = kmb_ocs_ecdh_max_size, - .base = { - .cra_name = "ecdh-nist-p384", - .cra_driver_name = "ecdh-nist-p384-keembay-ocs", - .cra_priority = KMB_OCS_ECC_PRIORITY, - .cra_module = THIS_MODULE, - .cra_ctxsize = sizeof(struct ocs_ecc_ctx), - }, -}; - -static irqreturn_t ocs_ecc_irq_handler(int irq, void *dev_id) -{ - struct ocs_ecc_dev *ecc_dev = dev_id; - u32 status; - - /* - * Read the status register and write it back to clear the - * DONE_INT_STATUS bit. - */ - status = ioread32(ecc_dev->base_reg + HW_OFFS_OCS_ECC_ISR); - iowrite32(status, ecc_dev->base_reg + HW_OFFS_OCS_ECC_ISR); - - if (!(status & HW_OCS_ECC_ISR_INT_STATUS_DONE)) - return IRQ_NONE; - - complete(&ecc_dev->irq_done); - - return IRQ_HANDLED; -} - -static int kmb_ocs_ecc_probe(struct platform_device *pdev) -{ - struct device *dev = &pdev->dev; - struct ocs_ecc_dev *ecc_dev; - int rc; - - ecc_dev = devm_kzalloc(dev, sizeof(*ecc_dev), GFP_KERNEL); - if (!ecc_dev) - return -ENOMEM; - - ecc_dev->dev = dev; - - platform_set_drvdata(pdev, ecc_dev); - - INIT_LIST_HEAD(&ecc_dev->list); - init_completion(&ecc_dev->irq_done); - - /* Get base register address. */ - ecc_dev->base_reg = devm_platform_ioremap_resource(pdev, 0); - if (IS_ERR(ecc_dev->base_reg)) { - dev_err(dev, "Failed to get base address\n"); - rc = PTR_ERR(ecc_dev->base_reg); - goto list_del; - } - - /* Get and request IRQ */ - ecc_dev->irq = platform_get_irq(pdev, 0); - if (ecc_dev->irq < 0) { - rc = ecc_dev->irq; - goto list_del; - } - - rc = devm_request_threaded_irq(dev, ecc_dev->irq, ocs_ecc_irq_handler, - NULL, 0, "keembay-ocs-ecc", ecc_dev); - if (rc < 0) { - dev_err(dev, "Could not request IRQ\n"); - goto list_del; - } - - /* Add device to the list of OCS ECC devices. */ - spin_lock(&ocs_ecc.lock); - list_add_tail(&ecc_dev->list, &ocs_ecc.dev_list); - spin_unlock(&ocs_ecc.lock); - - /* Initialize crypto engine. */ - ecc_dev->engine = crypto_engine_alloc_init(dev, 1); - if (!ecc_dev->engine) { - dev_err(dev, "Could not allocate crypto engine\n"); - rc = -ENOMEM; - goto list_del; - } - - rc = crypto_engine_start(ecc_dev->engine); - if (rc) { - dev_err(dev, "Could not start crypto engine\n"); - goto cleanup; - } - - /* Register the KPP algo. */ - rc = crypto_register_kpp(&ocs_ecdh_p256); - if (rc) { - dev_err(dev, - "Could not register OCS algorithms with Crypto API\n"); - goto cleanup; - } - - rc = crypto_register_kpp(&ocs_ecdh_p384); - if (rc) { - dev_err(dev, - "Could not register OCS algorithms with Crypto API\n"); - goto ocs_ecdh_p384_error; - } - - return 0; - -ocs_ecdh_p384_error: - crypto_unregister_kpp(&ocs_ecdh_p256); - -cleanup: - crypto_engine_exit(ecc_dev->engine); - -list_del: - spin_lock(&ocs_ecc.lock); - list_del(&ecc_dev->list); - spin_unlock(&ocs_ecc.lock); - - return rc; -} - -static int kmb_ocs_ecc_remove(struct platform_device *pdev) -{ - struct ocs_ecc_dev *ecc_dev; - - ecc_dev = platform_get_drvdata(pdev); - - crypto_unregister_kpp(&ocs_ecdh_p384); - crypto_unregister_kpp(&ocs_ecdh_p256); - - spin_lock(&ocs_ecc.lock); - list_del(&ecc_dev->list); - spin_unlock(&ocs_ecc.lock); - - crypto_engine_exit(ecc_dev->engine); - - return 0; -} - -/* Device tree driver match. */ -static const struct of_device_id kmb_ocs_ecc_of_match[] = { - { - .compatible = "intel,keembay-ocs-ecc", - }, - {} -}; - -/* The OCS driver is a platform device. */ -static struct platform_driver kmb_ocs_ecc_driver = { - .probe = kmb_ocs_ecc_probe, - .remove = kmb_ocs_ecc_remove, - .driver = { - .name = DRV_NAME, - .of_match_table = kmb_ocs_ecc_of_match, - }, -}; -module_platform_driver(kmb_ocs_ecc_driver); - -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("Intel Keem Bay OCS ECC Driver"); -MODULE_ALIAS_CRYPTO("ecdh-nist-p256"); -MODULE_ALIAS_CRYPTO("ecdh-nist-p384"); -MODULE_ALIAS_CRYPTO("ecdh-nist-p256-keembay-ocs"); -MODULE_ALIAS_CRYPTO("ecdh-nist-p384-keembay-ocs"); diff --git a/drivers/crypto/keembay/keembay-ocs-hcu-core.c b/drivers/crypto/keembay/keembay-ocs-hcu-core.c deleted file mode 100644 index d4bcbed1f546..000000000000 --- a/drivers/crypto/keembay/keembay-ocs-hcu-core.c +++ /dev/null @@ -1,1264 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Intel Keem Bay OCS HCU Crypto Driver. - * - * Copyright (C) 2018-2020 Intel Corporation - */ - -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -#include "ocs-hcu.h" - -#define DRV_NAME "keembay-ocs-hcu" - -/* Flag marking a final request. */ -#define REQ_FINAL BIT(0) -/* Flag marking a HMAC request. */ -#define REQ_FLAGS_HMAC BIT(1) -/* Flag set when HW HMAC is being used. */ -#define REQ_FLAGS_HMAC_HW BIT(2) -/* Flag set when SW HMAC is being used. */ -#define REQ_FLAGS_HMAC_SW BIT(3) - -/** - * struct ocs_hcu_ctx: OCS HCU Transform context. - * @engine_ctx: Crypto Engine context. - * @hcu_dev: The OCS HCU device used by the transformation. - * @key: The key (used only for HMAC transformations). - * @key_len: The length of the key. - * @is_sm3_tfm: Whether or not this is an SM3 transformation. - * @is_hmac_tfm: Whether or not this is a HMAC transformation. - */ -struct ocs_hcu_ctx { - struct crypto_engine_ctx engine_ctx; - struct ocs_hcu_dev *hcu_dev; - u8 key[SHA512_BLOCK_SIZE]; - size_t key_len; - bool is_sm3_tfm; - bool is_hmac_tfm; -}; - -/** - * struct ocs_hcu_rctx - Context for the request. - * @hcu_dev: OCS HCU device to be used to service the request. - * @flags: Flags tracking request status. - * @algo: Algorithm to use for the request. - * @blk_sz: Block size of the transformation / request. - * @dig_sz: Digest size of the transformation / request. - * @dma_list: OCS DMA linked list. - * @hash_ctx: OCS HCU hashing context. - * @buffer: Buffer to store: partial block of data and SW HMAC - * artifacts (ipad, opad, etc.). - * @buf_cnt: Number of bytes currently stored in the buffer. - * @buf_dma_addr: The DMA address of @buffer (when mapped). - * @buf_dma_count: The number of bytes in @buffer currently DMA-mapped. - * @sg: Head of the scatterlist entries containing data. - * @sg_data_total: Total data in the SG list at any time. - * @sg_data_offset: Offset into the data of the current individual SG node. - * @sg_dma_nents: Number of sg entries mapped in dma_list. - */ -struct ocs_hcu_rctx { - struct ocs_hcu_dev *hcu_dev; - u32 flags; - enum ocs_hcu_algo algo; - size_t blk_sz; - size_t dig_sz; - struct ocs_hcu_dma_list *dma_list; - struct ocs_hcu_hash_ctx hash_ctx; - /* - * Buffer is double the block size because we need space for SW HMAC - * artifacts, i.e: - * - ipad (1 block) + a possible partial block of data. - * - opad (1 block) + digest of H(k ^ ipad || m) - */ - u8 buffer[2 * SHA512_BLOCK_SIZE]; - size_t buf_cnt; - dma_addr_t buf_dma_addr; - size_t buf_dma_count; - struct scatterlist *sg; - unsigned int sg_data_total; - unsigned int sg_data_offset; - unsigned int sg_dma_nents; -}; - -/** - * struct ocs_hcu_drv - Driver data - * @dev_list: The list of HCU devices. - * @lock: The lock protecting dev_list. - */ -struct ocs_hcu_drv { - struct list_head dev_list; - spinlock_t lock; /* Protects dev_list. */ -}; - -static struct ocs_hcu_drv ocs_hcu = { - .dev_list = LIST_HEAD_INIT(ocs_hcu.dev_list), - .lock = __SPIN_LOCK_UNLOCKED(ocs_hcu.lock), -}; - -/* - * Return the total amount of data in the request; that is: the data in the - * request buffer + the data in the sg list. - */ -static inline unsigned int kmb_get_total_data(struct ocs_hcu_rctx *rctx) -{ - return rctx->sg_data_total + rctx->buf_cnt; -} - -/* Move remaining content of scatter-gather list to context buffer. */ -static int flush_sg_to_ocs_buffer(struct ocs_hcu_rctx *rctx) -{ - size_t count; - - if (rctx->sg_data_total > (sizeof(rctx->buffer) - rctx->buf_cnt)) { - WARN(1, "%s: sg data does not fit in buffer\n", __func__); - return -EINVAL; - } - - while (rctx->sg_data_total) { - if (!rctx->sg) { - WARN(1, "%s: unexpected NULL sg\n", __func__); - return -EINVAL; - } - /* - * If current sg has been fully processed, skip to the next - * one. - */ - if (rctx->sg_data_offset == rctx->sg->length) { - rctx->sg = sg_next(rctx->sg); - rctx->sg_data_offset = 0; - continue; - } - /* - * Determine the maximum data available to copy from the node. - * Minimum of the length left in the sg node, or the total data - * in the request. - */ - count = min(rctx->sg->length - rctx->sg_data_offset, - rctx->sg_data_total); - /* Copy from scatter-list entry to context buffer. */ - scatterwalk_map_and_copy(&rctx->buffer[rctx->buf_cnt], - rctx->sg, rctx->sg_data_offset, - count, 0); - - rctx->sg_data_offset += count; - rctx->sg_data_total -= count; - rctx->buf_cnt += count; - } - - return 0; -} - -static struct ocs_hcu_dev *kmb_ocs_hcu_find_dev(struct ahash_request *req) -{ - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - struct ocs_hcu_ctx *tctx = crypto_ahash_ctx(tfm); - - /* If the HCU device for the request was previously set, return it. */ - if (tctx->hcu_dev) - return tctx->hcu_dev; - - /* - * Otherwise, get the first HCU device available (there should be one - * and only one device). - */ - spin_lock_bh(&ocs_hcu.lock); - tctx->hcu_dev = list_first_entry_or_null(&ocs_hcu.dev_list, - struct ocs_hcu_dev, - list); - spin_unlock_bh(&ocs_hcu.lock); - - return tctx->hcu_dev; -} - -/* Free OCS DMA linked list and DMA-able context buffer. */ -static void kmb_ocs_hcu_dma_cleanup(struct ahash_request *req, - struct ocs_hcu_rctx *rctx) -{ - struct ocs_hcu_dev *hcu_dev = rctx->hcu_dev; - struct device *dev = hcu_dev->dev; - - /* Unmap rctx->buffer (if mapped). */ - if (rctx->buf_dma_count) { - dma_unmap_single(dev, rctx->buf_dma_addr, rctx->buf_dma_count, - DMA_TO_DEVICE); - rctx->buf_dma_count = 0; - } - - /* Unmap req->src (if mapped). */ - if (rctx->sg_dma_nents) { - dma_unmap_sg(dev, req->src, rctx->sg_dma_nents, DMA_TO_DEVICE); - rctx->sg_dma_nents = 0; - } - - /* Free dma_list (if allocated). */ - if (rctx->dma_list) { - ocs_hcu_dma_list_free(hcu_dev, rctx->dma_list); - rctx->dma_list = NULL; - } -} - -/* - * Prepare for DMA operation: - * - DMA-map request context buffer (if needed) - * - DMA-map SG list (only the entries to be processed, see note below) - * - Allocate OCS HCU DMA linked list (number of elements = SG entries to - * process + context buffer (if not empty)). - * - Add DMA-mapped request context buffer to OCS HCU DMA list. - * - Add SG entries to DMA list. - * - * Note: if this is a final request, we process all the data in the SG list, - * otherwise we can only process up to the maximum amount of block-aligned data - * (the remainder will be put into the context buffer and processed in the next - * request). - */ -static int kmb_ocs_dma_prepare(struct ahash_request *req) -{ - struct ocs_hcu_rctx *rctx = ahash_request_ctx_dma(req); - struct device *dev = rctx->hcu_dev->dev; - unsigned int remainder = 0; - unsigned int total; - size_t nents; - size_t count; - int rc; - int i; - - /* This function should be called only when there is data to process. */ - total = kmb_get_total_data(rctx); - if (!total) - return -EINVAL; - - /* - * If this is not a final DMA (terminated DMA), the data passed to the - * HCU must be aligned to the block size; compute the remainder data to - * be processed in the next request. - */ - if (!(rctx->flags & REQ_FINAL)) - remainder = total % rctx->blk_sz; - - /* Determine the number of scatter gather list entries to process. */ - nents = sg_nents_for_len(req->src, rctx->sg_data_total - remainder); - - /* If there are entries to process, map them. */ - if (nents) { - rctx->sg_dma_nents = dma_map_sg(dev, req->src, nents, - DMA_TO_DEVICE); - if (!rctx->sg_dma_nents) { - dev_err(dev, "Failed to MAP SG\n"); - rc = -ENOMEM; - goto cleanup; - } - /* - * The value returned by dma_map_sg() can be < nents; so update - * nents accordingly. - */ - nents = rctx->sg_dma_nents; - } - - /* - * If context buffer is not empty, map it and add extra DMA entry for - * it. - */ - if (rctx->buf_cnt) { - rctx->buf_dma_addr = dma_map_single(dev, rctx->buffer, - rctx->buf_cnt, - DMA_TO_DEVICE); - if (dma_mapping_error(dev, rctx->buf_dma_addr)) { - dev_err(dev, "Failed to map request context buffer\n"); - rc = -ENOMEM; - goto cleanup; - } - rctx->buf_dma_count = rctx->buf_cnt; - /* Increase number of dma entries. */ - nents++; - } - - /* Allocate OCS HCU DMA list. */ - rctx->dma_list = ocs_hcu_dma_list_alloc(rctx->hcu_dev, nents); - if (!rctx->dma_list) { - rc = -ENOMEM; - goto cleanup; - } - - /* Add request context buffer (if previously DMA-mapped) */ - if (rctx->buf_dma_count) { - rc = ocs_hcu_dma_list_add_tail(rctx->hcu_dev, rctx->dma_list, - rctx->buf_dma_addr, - rctx->buf_dma_count); - if (rc) - goto cleanup; - } - - /* Add the SG nodes to be processed to the DMA linked list. */ - for_each_sg(req->src, rctx->sg, rctx->sg_dma_nents, i) { - /* - * The number of bytes to add to the list entry is the minimum - * between: - * - The DMA length of the SG entry. - * - The data left to be processed. - */ - count = min(rctx->sg_data_total - remainder, - sg_dma_len(rctx->sg) - rctx->sg_data_offset); - /* - * Do not create a zero length DMA descriptor. Check in case of - * zero length SG node. - */ - if (count == 0) - continue; - /* Add sg to HCU DMA list. */ - rc = ocs_hcu_dma_list_add_tail(rctx->hcu_dev, - rctx->dma_list, - rctx->sg->dma_address, - count); - if (rc) - goto cleanup; - - /* Update amount of data remaining in SG list. */ - rctx->sg_data_total -= count; - - /* - * If remaining data is equal to remainder (note: 'less than' - * case should never happen in practice), we are done: update - * offset and exit the loop. - */ - if (rctx->sg_data_total <= remainder) { - WARN_ON(rctx->sg_data_total < remainder); - rctx->sg_data_offset += count; - break; - } - - /* - * If we get here is because we need to process the next sg in - * the list; set offset within the sg to 0. - */ - rctx->sg_data_offset = 0; - } - - return 0; -cleanup: - dev_err(dev, "Failed to prepare DMA.\n"); - kmb_ocs_hcu_dma_cleanup(req, rctx); - - return rc; -} - -static void kmb_ocs_hcu_secure_cleanup(struct ahash_request *req) -{ - struct ocs_hcu_rctx *rctx = ahash_request_ctx_dma(req); - - /* Clear buffer of any data. */ - memzero_explicit(rctx->buffer, sizeof(rctx->buffer)); -} - -static int kmb_ocs_hcu_handle_queue(struct ahash_request *req) -{ - struct ocs_hcu_dev *hcu_dev = kmb_ocs_hcu_find_dev(req); - - if (!hcu_dev) - return -ENOENT; - - return crypto_transfer_hash_request_to_engine(hcu_dev->engine, req); -} - -static int prepare_ipad(struct ahash_request *req) -{ - struct ocs_hcu_rctx *rctx = ahash_request_ctx_dma(req); - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - struct ocs_hcu_ctx *ctx = crypto_ahash_ctx(tfm); - int i; - - WARN(rctx->buf_cnt, "%s: Context buffer is not empty\n", __func__); - WARN(!(rctx->flags & REQ_FLAGS_HMAC_SW), - "%s: HMAC_SW flag is not set\n", __func__); - /* - * Key length must be equal to block size. If key is shorter, - * we pad it with zero (note: key cannot be longer, since - * longer keys are hashed by kmb_ocs_hcu_setkey()). - */ - if (ctx->key_len > rctx->blk_sz) { - WARN(1, "%s: Invalid key length in tfm context\n", __func__); - return -EINVAL; - } - memzero_explicit(&ctx->key[ctx->key_len], - rctx->blk_sz - ctx->key_len); - ctx->key_len = rctx->blk_sz; - /* - * Prepare IPAD for HMAC. Only done for first block. - * HMAC(k,m) = H(k ^ opad || H(k ^ ipad || m)) - * k ^ ipad will be first hashed block. - * k ^ opad will be calculated in the final request. - * Only needed if not using HW HMAC. - */ - for (i = 0; i < rctx->blk_sz; i++) - rctx->buffer[i] = ctx->key[i] ^ HMAC_IPAD_VALUE; - rctx->buf_cnt = rctx->blk_sz; - - return 0; -} - -static int kmb_ocs_hcu_do_one_request(struct crypto_engine *engine, void *areq) -{ - struct ahash_request *req = container_of(areq, struct ahash_request, - base); - struct ocs_hcu_dev *hcu_dev = kmb_ocs_hcu_find_dev(req); - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - struct ocs_hcu_rctx *rctx = ahash_request_ctx_dma(req); - struct ocs_hcu_ctx *tctx = crypto_ahash_ctx(tfm); - int rc; - int i; - - if (!hcu_dev) { - rc = -ENOENT; - goto error; - } - - /* - * If hardware HMAC flag is set, perform HMAC in hardware. - * - * NOTE: this flag implies REQ_FINAL && kmb_get_total_data(rctx) - */ - if (rctx->flags & REQ_FLAGS_HMAC_HW) { - /* Map input data into the HCU DMA linked list. */ - rc = kmb_ocs_dma_prepare(req); - if (rc) - goto error; - - rc = ocs_hcu_hmac(hcu_dev, rctx->algo, tctx->key, tctx->key_len, - rctx->dma_list, req->result, rctx->dig_sz); - - /* Unmap data and free DMA list regardless of return code. */ - kmb_ocs_hcu_dma_cleanup(req, rctx); - - /* Process previous return code. */ - if (rc) - goto error; - - goto done; - } - - /* Handle update request case. */ - if (!(rctx->flags & REQ_FINAL)) { - /* Update should always have input data. */ - if (!kmb_get_total_data(rctx)) - return -EINVAL; - - /* Map input data into the HCU DMA linked list. */ - rc = kmb_ocs_dma_prepare(req); - if (rc) - goto error; - - /* Do hashing step. */ - rc = ocs_hcu_hash_update(hcu_dev, &rctx->hash_ctx, - rctx->dma_list); - - /* Unmap data and free DMA list regardless of return code. */ - kmb_ocs_hcu_dma_cleanup(req, rctx); - - /* Process previous return code. */ - if (rc) - goto error; - - /* - * Reset request buffer count (data in the buffer was just - * processed). - */ - rctx->buf_cnt = 0; - /* - * Move remaining sg data into the request buffer, so that it - * will be processed during the next request. - * - * NOTE: we have remaining data if kmb_get_total_data() was not - * a multiple of block size. - */ - rc = flush_sg_to_ocs_buffer(rctx); - if (rc) - goto error; - - goto done; - } - - /* If we get here, this is a final request. */ - - /* If there is data to process, use finup. */ - if (kmb_get_total_data(rctx)) { - /* Map input data into the HCU DMA linked list. */ - rc = kmb_ocs_dma_prepare(req); - if (rc) - goto error; - - /* Do hashing step. */ - rc = ocs_hcu_hash_finup(hcu_dev, &rctx->hash_ctx, - rctx->dma_list, - req->result, rctx->dig_sz); - /* Free DMA list regardless of return code. */ - kmb_ocs_hcu_dma_cleanup(req, rctx); - - /* Process previous return code. */ - if (rc) - goto error; - - } else { /* Otherwise (if we have no data), use final. */ - rc = ocs_hcu_hash_final(hcu_dev, &rctx->hash_ctx, req->result, - rctx->dig_sz); - if (rc) - goto error; - } - - /* - * If we are finalizing a SW HMAC request, we just computed the result - * of: H(k ^ ipad || m). - * - * We now need to complete the HMAC calculation with the OPAD step, - * that is, we need to compute H(k ^ opad || digest), where digest is - * the digest we just obtained, i.e., H(k ^ ipad || m). - */ - if (rctx->flags & REQ_FLAGS_HMAC_SW) { - /* - * Compute k ^ opad and store it in the request buffer (which - * is not used anymore at this point). - * Note: key has been padded / hashed already (so keylen == - * blksz) . - */ - WARN_ON(tctx->key_len != rctx->blk_sz); - for (i = 0; i < rctx->blk_sz; i++) - rctx->buffer[i] = tctx->key[i] ^ HMAC_OPAD_VALUE; - /* Now append the digest to the rest of the buffer. */ - for (i = 0; (i < rctx->dig_sz); i++) - rctx->buffer[rctx->blk_sz + i] = req->result[i]; - - /* Now hash the buffer to obtain the final HMAC. */ - rc = ocs_hcu_digest(hcu_dev, rctx->algo, rctx->buffer, - rctx->blk_sz + rctx->dig_sz, req->result, - rctx->dig_sz); - if (rc) - goto error; - } - - /* Perform secure clean-up. */ - kmb_ocs_hcu_secure_cleanup(req); -done: - crypto_finalize_hash_request(hcu_dev->engine, req, 0); - - return 0; - -error: - kmb_ocs_hcu_secure_cleanup(req); - return rc; -} - -static int kmb_ocs_hcu_init(struct ahash_request *req) -{ - struct ocs_hcu_dev *hcu_dev = kmb_ocs_hcu_find_dev(req); - struct ocs_hcu_rctx *rctx = ahash_request_ctx_dma(req); - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - struct ocs_hcu_ctx *ctx = crypto_ahash_ctx(tfm); - - if (!hcu_dev) - return -ENOENT; - - /* Initialize entire request context to zero. */ - memset(rctx, 0, sizeof(*rctx)); - - rctx->hcu_dev = hcu_dev; - rctx->dig_sz = crypto_ahash_digestsize(tfm); - - switch (rctx->dig_sz) { -#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_HCU_HMAC_SHA224 - case SHA224_DIGEST_SIZE: - rctx->blk_sz = SHA224_BLOCK_SIZE; - rctx->algo = OCS_HCU_ALGO_SHA224; - break; -#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_HCU_HMAC_SHA224 */ - case SHA256_DIGEST_SIZE: - rctx->blk_sz = SHA256_BLOCK_SIZE; - /* - * SHA256 and SM3 have the same digest size: use info from tfm - * context to find out which one we should use. - */ - rctx->algo = ctx->is_sm3_tfm ? OCS_HCU_ALGO_SM3 : - OCS_HCU_ALGO_SHA256; - break; - case SHA384_DIGEST_SIZE: - rctx->blk_sz = SHA384_BLOCK_SIZE; - rctx->algo = OCS_HCU_ALGO_SHA384; - break; - case SHA512_DIGEST_SIZE: - rctx->blk_sz = SHA512_BLOCK_SIZE; - rctx->algo = OCS_HCU_ALGO_SHA512; - break; - default: - return -EINVAL; - } - - /* Initialize intermediate data. */ - ocs_hcu_hash_init(&rctx->hash_ctx, rctx->algo); - - /* If this a HMAC request, set HMAC flag. */ - if (ctx->is_hmac_tfm) - rctx->flags |= REQ_FLAGS_HMAC; - - return 0; -} - -static int kmb_ocs_hcu_update(struct ahash_request *req) -{ - struct ocs_hcu_rctx *rctx = ahash_request_ctx_dma(req); - int rc; - - if (!req->nbytes) - return 0; - - rctx->sg_data_total = req->nbytes; - rctx->sg_data_offset = 0; - rctx->sg = req->src; - - /* - * If we are doing HMAC, then we must use SW-assisted HMAC, since HW - * HMAC does not support context switching (there it can only be used - * with finup() or digest()). - */ - if (rctx->flags & REQ_FLAGS_HMAC && - !(rctx->flags & REQ_FLAGS_HMAC_SW)) { - rctx->flags |= REQ_FLAGS_HMAC_SW; - rc = prepare_ipad(req); - if (rc) - return rc; - } - - /* - * If remaining sg_data fits into ctx buffer, just copy it there; we'll - * process it at the next update() or final(). - */ - if (rctx->sg_data_total <= (sizeof(rctx->buffer) - rctx->buf_cnt)) - return flush_sg_to_ocs_buffer(rctx); - - return kmb_ocs_hcu_handle_queue(req); -} - -/* Common logic for kmb_ocs_hcu_final() and kmb_ocs_hcu_finup(). */ -static int kmb_ocs_hcu_fin_common(struct ahash_request *req) -{ - struct ocs_hcu_rctx *rctx = ahash_request_ctx_dma(req); - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - struct ocs_hcu_ctx *ctx = crypto_ahash_ctx(tfm); - int rc; - - rctx->flags |= REQ_FINAL; - - /* - * If this is a HMAC request and, so far, we didn't have to switch to - * SW HMAC, check if we can use HW HMAC. - */ - if (rctx->flags & REQ_FLAGS_HMAC && - !(rctx->flags & REQ_FLAGS_HMAC_SW)) { - /* - * If we are here, it means we never processed any data so far, - * so we can use HW HMAC, but only if there is some data to - * process (since OCS HW MAC does not support zero-length - * messages) and the key length is supported by the hardware - * (OCS HCU HW only supports length <= 64); if HW HMAC cannot - * be used, fall back to SW-assisted HMAC. - */ - if (kmb_get_total_data(rctx) && - ctx->key_len <= OCS_HCU_HW_KEY_LEN) { - rctx->flags |= REQ_FLAGS_HMAC_HW; - } else { - rctx->flags |= REQ_FLAGS_HMAC_SW; - rc = prepare_ipad(req); - if (rc) - return rc; - } - } - - return kmb_ocs_hcu_handle_queue(req); -} - -static int kmb_ocs_hcu_final(struct ahash_request *req) -{ - struct ocs_hcu_rctx *rctx = ahash_request_ctx_dma(req); - - rctx->sg_data_total = 0; - rctx->sg_data_offset = 0; - rctx->sg = NULL; - - return kmb_ocs_hcu_fin_common(req); -} - -static int kmb_ocs_hcu_finup(struct ahash_request *req) -{ - struct ocs_hcu_rctx *rctx = ahash_request_ctx_dma(req); - - rctx->sg_data_total = req->nbytes; - rctx->sg_data_offset = 0; - rctx->sg = req->src; - - return kmb_ocs_hcu_fin_common(req); -} - -static int kmb_ocs_hcu_digest(struct ahash_request *req) -{ - int rc = 0; - struct ocs_hcu_dev *hcu_dev = kmb_ocs_hcu_find_dev(req); - - if (!hcu_dev) - return -ENOENT; - - rc = kmb_ocs_hcu_init(req); - if (rc) - return rc; - - rc = kmb_ocs_hcu_finup(req); - - return rc; -} - -static int kmb_ocs_hcu_export(struct ahash_request *req, void *out) -{ - struct ocs_hcu_rctx *rctx = ahash_request_ctx_dma(req); - - /* Intermediate data is always stored and applied per request. */ - memcpy(out, rctx, sizeof(*rctx)); - - return 0; -} - -static int kmb_ocs_hcu_import(struct ahash_request *req, const void *in) -{ - struct ocs_hcu_rctx *rctx = ahash_request_ctx_dma(req); - - /* Intermediate data is always stored and applied per request. */ - memcpy(rctx, in, sizeof(*rctx)); - - return 0; -} - -static int kmb_ocs_hcu_setkey(struct crypto_ahash *tfm, const u8 *key, - unsigned int keylen) -{ - unsigned int digestsize = crypto_ahash_digestsize(tfm); - struct ocs_hcu_ctx *ctx = crypto_ahash_ctx(tfm); - size_t blk_sz = crypto_ahash_blocksize(tfm); - struct crypto_ahash *ahash_tfm; - struct ahash_request *req; - struct crypto_wait wait; - struct scatterlist sg; - const char *alg_name; - int rc; - - /* - * Key length must be equal to block size: - * - If key is shorter, we are done for now (the key will be padded - * later on); this is to maximize the use of HW HMAC (which works - * only for keys <= 64 bytes). - * - If key is longer, we hash it. - */ - if (keylen <= blk_sz) { - memcpy(ctx->key, key, keylen); - ctx->key_len = keylen; - return 0; - } - - switch (digestsize) { -#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_HCU_HMAC_SHA224 - case SHA224_DIGEST_SIZE: - alg_name = "sha224-keembay-ocs"; - break; -#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_HCU_HMAC_SHA224 */ - case SHA256_DIGEST_SIZE: - alg_name = ctx->is_sm3_tfm ? "sm3-keembay-ocs" : - "sha256-keembay-ocs"; - break; - case SHA384_DIGEST_SIZE: - alg_name = "sha384-keembay-ocs"; - break; - case SHA512_DIGEST_SIZE: - alg_name = "sha512-keembay-ocs"; - break; - default: - return -EINVAL; - } - - ahash_tfm = crypto_alloc_ahash(alg_name, 0, 0); - if (IS_ERR(ahash_tfm)) - return PTR_ERR(ahash_tfm); - - req = ahash_request_alloc(ahash_tfm, GFP_KERNEL); - if (!req) { - rc = -ENOMEM; - goto err_free_ahash; - } - - crypto_init_wait(&wait); - ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, - crypto_req_done, &wait); - crypto_ahash_clear_flags(ahash_tfm, ~0); - - sg_init_one(&sg, key, keylen); - ahash_request_set_crypt(req, &sg, ctx->key, keylen); - - rc = crypto_wait_req(crypto_ahash_digest(req), &wait); - if (rc == 0) - ctx->key_len = digestsize; - - ahash_request_free(req); -err_free_ahash: - crypto_free_ahash(ahash_tfm); - - return rc; -} - -/* Set request size and initialize tfm context. */ -static void __cra_init(struct crypto_tfm *tfm, struct ocs_hcu_ctx *ctx) -{ - crypto_ahash_set_reqsize_dma(__crypto_ahash_cast(tfm), - sizeof(struct ocs_hcu_rctx)); - - /* Init context to 0. */ - memzero_explicit(ctx, sizeof(*ctx)); - /* Set engine ops. */ - ctx->engine_ctx.op.do_one_request = kmb_ocs_hcu_do_one_request; -} - -static int kmb_ocs_hcu_sha_cra_init(struct crypto_tfm *tfm) -{ - struct ocs_hcu_ctx *ctx = crypto_tfm_ctx(tfm); - - __cra_init(tfm, ctx); - - return 0; -} - -static int kmb_ocs_hcu_sm3_cra_init(struct crypto_tfm *tfm) -{ - struct ocs_hcu_ctx *ctx = crypto_tfm_ctx(tfm); - - __cra_init(tfm, ctx); - - ctx->is_sm3_tfm = true; - - return 0; -} - -static int kmb_ocs_hcu_hmac_sm3_cra_init(struct crypto_tfm *tfm) -{ - struct ocs_hcu_ctx *ctx = crypto_tfm_ctx(tfm); - - __cra_init(tfm, ctx); - - ctx->is_sm3_tfm = true; - ctx->is_hmac_tfm = true; - - return 0; -} - -static int kmb_ocs_hcu_hmac_cra_init(struct crypto_tfm *tfm) -{ - struct ocs_hcu_ctx *ctx = crypto_tfm_ctx(tfm); - - __cra_init(tfm, ctx); - - ctx->is_hmac_tfm = true; - - return 0; -} - -/* Function called when 'tfm' is de-initialized. */ -static void kmb_ocs_hcu_hmac_cra_exit(struct crypto_tfm *tfm) -{ - struct ocs_hcu_ctx *ctx = crypto_tfm_ctx(tfm); - - /* Clear the key. */ - memzero_explicit(ctx->key, sizeof(ctx->key)); -} - -static struct ahash_alg ocs_hcu_algs[] = { -#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_HCU_HMAC_SHA224 -{ - .init = kmb_ocs_hcu_init, - .update = kmb_ocs_hcu_update, - .final = kmb_ocs_hcu_final, - .finup = kmb_ocs_hcu_finup, - .digest = kmb_ocs_hcu_digest, - .export = kmb_ocs_hcu_export, - .import = kmb_ocs_hcu_import, - .halg = { - .digestsize = SHA224_DIGEST_SIZE, - .statesize = sizeof(struct ocs_hcu_rctx), - .base = { - .cra_name = "sha224", - .cra_driver_name = "sha224-keembay-ocs", - .cra_priority = 255, - .cra_flags = CRYPTO_ALG_ASYNC, - .cra_blocksize = SHA224_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct ocs_hcu_ctx), - .cra_alignmask = 0, - .cra_module = THIS_MODULE, - .cra_init = kmb_ocs_hcu_sha_cra_init, - } - } -}, -{ - .init = kmb_ocs_hcu_init, - .update = kmb_ocs_hcu_update, - .final = kmb_ocs_hcu_final, - .finup = kmb_ocs_hcu_finup, - .digest = kmb_ocs_hcu_digest, - .export = kmb_ocs_hcu_export, - .import = kmb_ocs_hcu_import, - .setkey = kmb_ocs_hcu_setkey, - .halg = { - .digestsize = SHA224_DIGEST_SIZE, - .statesize = sizeof(struct ocs_hcu_rctx), - .base = { - .cra_name = "hmac(sha224)", - .cra_driver_name = "hmac-sha224-keembay-ocs", - .cra_priority = 255, - .cra_flags = CRYPTO_ALG_ASYNC, - .cra_blocksize = SHA224_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct ocs_hcu_ctx), - .cra_alignmask = 0, - .cra_module = THIS_MODULE, - .cra_init = kmb_ocs_hcu_hmac_cra_init, - .cra_exit = kmb_ocs_hcu_hmac_cra_exit, - } - } -}, -#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_HCU_HMAC_SHA224 */ -{ - .init = kmb_ocs_hcu_init, - .update = kmb_ocs_hcu_update, - .final = kmb_ocs_hcu_final, - .finup = kmb_ocs_hcu_finup, - .digest = kmb_ocs_hcu_digest, - .export = kmb_ocs_hcu_export, - .import = kmb_ocs_hcu_import, - .halg = { - .digestsize = SHA256_DIGEST_SIZE, - .statesize = sizeof(struct ocs_hcu_rctx), - .base = { - .cra_name = "sha256", - .cra_driver_name = "sha256-keembay-ocs", - .cra_priority = 255, - .cra_flags = CRYPTO_ALG_ASYNC, - .cra_blocksize = SHA256_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct ocs_hcu_ctx), - .cra_alignmask = 0, - .cra_module = THIS_MODULE, - .cra_init = kmb_ocs_hcu_sha_cra_init, - } - } -}, -{ - .init = kmb_ocs_hcu_init, - .update = kmb_ocs_hcu_update, - .final = kmb_ocs_hcu_final, - .finup = kmb_ocs_hcu_finup, - .digest = kmb_ocs_hcu_digest, - .export = kmb_ocs_hcu_export, - .import = kmb_ocs_hcu_import, - .setkey = kmb_ocs_hcu_setkey, - .halg = { - .digestsize = SHA256_DIGEST_SIZE, - .statesize = sizeof(struct ocs_hcu_rctx), - .base = { - .cra_name = "hmac(sha256)", - .cra_driver_name = "hmac-sha256-keembay-ocs", - .cra_priority = 255, - .cra_flags = CRYPTO_ALG_ASYNC, - .cra_blocksize = SHA256_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct ocs_hcu_ctx), - .cra_alignmask = 0, - .cra_module = THIS_MODULE, - .cra_init = kmb_ocs_hcu_hmac_cra_init, - .cra_exit = kmb_ocs_hcu_hmac_cra_exit, - } - } -}, -{ - .init = kmb_ocs_hcu_init, - .update = kmb_ocs_hcu_update, - .final = kmb_ocs_hcu_final, - .finup = kmb_ocs_hcu_finup, - .digest = kmb_ocs_hcu_digest, - .export = kmb_ocs_hcu_export, - .import = kmb_ocs_hcu_import, - .halg = { - .digestsize = SM3_DIGEST_SIZE, - .statesize = sizeof(struct ocs_hcu_rctx), - .base = { - .cra_name = "sm3", - .cra_driver_name = "sm3-keembay-ocs", - .cra_priority = 255, - .cra_flags = CRYPTO_ALG_ASYNC, - .cra_blocksize = SM3_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct ocs_hcu_ctx), - .cra_alignmask = 0, - .cra_module = THIS_MODULE, - .cra_init = kmb_ocs_hcu_sm3_cra_init, - } - } -}, -{ - .init = kmb_ocs_hcu_init, - .update = kmb_ocs_hcu_update, - .final = kmb_ocs_hcu_final, - .finup = kmb_ocs_hcu_finup, - .digest = kmb_ocs_hcu_digest, - .export = kmb_ocs_hcu_export, - .import = kmb_ocs_hcu_import, - .setkey = kmb_ocs_hcu_setkey, - .halg = { - .digestsize = SM3_DIGEST_SIZE, - .statesize = sizeof(struct ocs_hcu_rctx), - .base = { - .cra_name = "hmac(sm3)", - .cra_driver_name = "hmac-sm3-keembay-ocs", - .cra_priority = 255, - .cra_flags = CRYPTO_ALG_ASYNC, - .cra_blocksize = SM3_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct ocs_hcu_ctx), - .cra_alignmask = 0, - .cra_module = THIS_MODULE, - .cra_init = kmb_ocs_hcu_hmac_sm3_cra_init, - .cra_exit = kmb_ocs_hcu_hmac_cra_exit, - } - } -}, -{ - .init = kmb_ocs_hcu_init, - .update = kmb_ocs_hcu_update, - .final = kmb_ocs_hcu_final, - .finup = kmb_ocs_hcu_finup, - .digest = kmb_ocs_hcu_digest, - .export = kmb_ocs_hcu_export, - .import = kmb_ocs_hcu_import, - .halg = { - .digestsize = SHA384_DIGEST_SIZE, - .statesize = sizeof(struct ocs_hcu_rctx), - .base = { - .cra_name = "sha384", - .cra_driver_name = "sha384-keembay-ocs", - .cra_priority = 255, - .cra_flags = CRYPTO_ALG_ASYNC, - .cra_blocksize = SHA384_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct ocs_hcu_ctx), - .cra_alignmask = 0, - .cra_module = THIS_MODULE, - .cra_init = kmb_ocs_hcu_sha_cra_init, - } - } -}, -{ - .init = kmb_ocs_hcu_init, - .update = kmb_ocs_hcu_update, - .final = kmb_ocs_hcu_final, - .finup = kmb_ocs_hcu_finup, - .digest = kmb_ocs_hcu_digest, - .export = kmb_ocs_hcu_export, - .import = kmb_ocs_hcu_import, - .setkey = kmb_ocs_hcu_setkey, - .halg = { - .digestsize = SHA384_DIGEST_SIZE, - .statesize = sizeof(struct ocs_hcu_rctx), - .base = { - .cra_name = "hmac(sha384)", - .cra_driver_name = "hmac-sha384-keembay-ocs", - .cra_priority = 255, - .cra_flags = CRYPTO_ALG_ASYNC, - .cra_blocksize = SHA384_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct ocs_hcu_ctx), - .cra_alignmask = 0, - .cra_module = THIS_MODULE, - .cra_init = kmb_ocs_hcu_hmac_cra_init, - .cra_exit = kmb_ocs_hcu_hmac_cra_exit, - } - } -}, -{ - .init = kmb_ocs_hcu_init, - .update = kmb_ocs_hcu_update, - .final = kmb_ocs_hcu_final, - .finup = kmb_ocs_hcu_finup, - .digest = kmb_ocs_hcu_digest, - .export = kmb_ocs_hcu_export, - .import = kmb_ocs_hcu_import, - .halg = { - .digestsize = SHA512_DIGEST_SIZE, - .statesize = sizeof(struct ocs_hcu_rctx), - .base = { - .cra_name = "sha512", - .cra_driver_name = "sha512-keembay-ocs", - .cra_priority = 255, - .cra_flags = CRYPTO_ALG_ASYNC, - .cra_blocksize = SHA512_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct ocs_hcu_ctx), - .cra_alignmask = 0, - .cra_module = THIS_MODULE, - .cra_init = kmb_ocs_hcu_sha_cra_init, - } - } -}, -{ - .init = kmb_ocs_hcu_init, - .update = kmb_ocs_hcu_update, - .final = kmb_ocs_hcu_final, - .finup = kmb_ocs_hcu_finup, - .digest = kmb_ocs_hcu_digest, - .export = kmb_ocs_hcu_export, - .import = kmb_ocs_hcu_import, - .setkey = kmb_ocs_hcu_setkey, - .halg = { - .digestsize = SHA512_DIGEST_SIZE, - .statesize = sizeof(struct ocs_hcu_rctx), - .base = { - .cra_name = "hmac(sha512)", - .cra_driver_name = "hmac-sha512-keembay-ocs", - .cra_priority = 255, - .cra_flags = CRYPTO_ALG_ASYNC, - .cra_blocksize = SHA512_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct ocs_hcu_ctx), - .cra_alignmask = 0, - .cra_module = THIS_MODULE, - .cra_init = kmb_ocs_hcu_hmac_cra_init, - .cra_exit = kmb_ocs_hcu_hmac_cra_exit, - } - } -}, -}; - -/* Device tree driver match. */ -static const struct of_device_id kmb_ocs_hcu_of_match[] = { - { - .compatible = "intel,keembay-ocs-hcu", - }, - {} -}; - -static int kmb_ocs_hcu_remove(struct platform_device *pdev) -{ - struct ocs_hcu_dev *hcu_dev; - int rc; - - hcu_dev = platform_get_drvdata(pdev); - if (!hcu_dev) - return -ENODEV; - - crypto_unregister_ahashes(ocs_hcu_algs, ARRAY_SIZE(ocs_hcu_algs)); - - rc = crypto_engine_exit(hcu_dev->engine); - - spin_lock_bh(&ocs_hcu.lock); - list_del(&hcu_dev->list); - spin_unlock_bh(&ocs_hcu.lock); - - return rc; -} - -static int kmb_ocs_hcu_probe(struct platform_device *pdev) -{ - struct device *dev = &pdev->dev; - struct ocs_hcu_dev *hcu_dev; - struct resource *hcu_mem; - int rc; - - hcu_dev = devm_kzalloc(dev, sizeof(*hcu_dev), GFP_KERNEL); - if (!hcu_dev) - return -ENOMEM; - - hcu_dev->dev = dev; - - platform_set_drvdata(pdev, hcu_dev); - rc = dma_set_mask_and_coherent(&pdev->dev, OCS_HCU_DMA_BIT_MASK); - if (rc) - return rc; - - /* Get the memory address and remap. */ - hcu_mem = platform_get_resource(pdev, IORESOURCE_MEM, 0); - if (!hcu_mem) { - dev_err(dev, "Could not retrieve io mem resource.\n"); - return -ENODEV; - } - - hcu_dev->io_base = devm_ioremap_resource(dev, hcu_mem); - if (IS_ERR(hcu_dev->io_base)) - return PTR_ERR(hcu_dev->io_base); - - init_completion(&hcu_dev->irq_done); - - /* Get and request IRQ. */ - hcu_dev->irq = platform_get_irq(pdev, 0); - if (hcu_dev->irq < 0) - return hcu_dev->irq; - - rc = devm_request_threaded_irq(&pdev->dev, hcu_dev->irq, - ocs_hcu_irq_handler, NULL, 0, - "keembay-ocs-hcu", hcu_dev); - if (rc < 0) { - dev_err(dev, "Could not request IRQ.\n"); - return rc; - } - - INIT_LIST_HEAD(&hcu_dev->list); - - spin_lock_bh(&ocs_hcu.lock); - list_add_tail(&hcu_dev->list, &ocs_hcu.dev_list); - spin_unlock_bh(&ocs_hcu.lock); - - /* Initialize crypto engine */ - hcu_dev->engine = crypto_engine_alloc_init(dev, 1); - if (!hcu_dev->engine) { - rc = -ENOMEM; - goto list_del; - } - - rc = crypto_engine_start(hcu_dev->engine); - if (rc) { - dev_err(dev, "Could not start engine.\n"); - goto cleanup; - } - - /* Security infrastructure guarantees OCS clock is enabled. */ - - rc = crypto_register_ahashes(ocs_hcu_algs, ARRAY_SIZE(ocs_hcu_algs)); - if (rc) { - dev_err(dev, "Could not register algorithms.\n"); - goto cleanup; - } - - return 0; - -cleanup: - crypto_engine_exit(hcu_dev->engine); -list_del: - spin_lock_bh(&ocs_hcu.lock); - list_del(&hcu_dev->list); - spin_unlock_bh(&ocs_hcu.lock); - - return rc; -} - -/* The OCS driver is a platform device. */ -static struct platform_driver kmb_ocs_hcu_driver = { - .probe = kmb_ocs_hcu_probe, - .remove = kmb_ocs_hcu_remove, - .driver = { - .name = DRV_NAME, - .of_match_table = kmb_ocs_hcu_of_match, - }, -}; - -module_platform_driver(kmb_ocs_hcu_driver); - -MODULE_LICENSE("GPL"); diff --git a/drivers/crypto/keembay/ocs-aes.c b/drivers/crypto/keembay/ocs-aes.c deleted file mode 100644 index be9f32fc8f42..000000000000 --- a/drivers/crypto/keembay/ocs-aes.c +++ /dev/null @@ -1,1489 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Intel Keem Bay OCS AES Crypto Driver. - * - * Copyright (C) 2018-2020 Intel Corporation - */ - -#include -#include -#include -#include -#include - -#include -#include - -#include -#include - -#include "ocs-aes.h" - -#define AES_COMMAND_OFFSET 0x0000 -#define AES_KEY_0_OFFSET 0x0004 -#define AES_KEY_1_OFFSET 0x0008 -#define AES_KEY_2_OFFSET 0x000C -#define AES_KEY_3_OFFSET 0x0010 -#define AES_KEY_4_OFFSET 0x0014 -#define AES_KEY_5_OFFSET 0x0018 -#define AES_KEY_6_OFFSET 0x001C -#define AES_KEY_7_OFFSET 0x0020 -#define AES_IV_0_OFFSET 0x0024 -#define AES_IV_1_OFFSET 0x0028 -#define AES_IV_2_OFFSET 0x002C -#define AES_IV_3_OFFSET 0x0030 -#define AES_ACTIVE_OFFSET 0x0034 -#define AES_STATUS_OFFSET 0x0038 -#define AES_KEY_SIZE_OFFSET 0x0044 -#define AES_IER_OFFSET 0x0048 -#define AES_ISR_OFFSET 0x005C -#define AES_MULTIPURPOSE1_0_OFFSET 0x0200 -#define AES_MULTIPURPOSE1_1_OFFSET 0x0204 -#define AES_MULTIPURPOSE1_2_OFFSET 0x0208 -#define AES_MULTIPURPOSE1_3_OFFSET 0x020C -#define AES_MULTIPURPOSE2_0_OFFSET 0x0220 -#define AES_MULTIPURPOSE2_1_OFFSET 0x0224 -#define AES_MULTIPURPOSE2_2_OFFSET 0x0228 -#define AES_MULTIPURPOSE2_3_OFFSET 0x022C -#define AES_BYTE_ORDER_CFG_OFFSET 0x02C0 -#define AES_TLEN_OFFSET 0x0300 -#define AES_T_MAC_0_OFFSET 0x0304 -#define AES_T_MAC_1_OFFSET 0x0308 -#define AES_T_MAC_2_OFFSET 0x030C -#define AES_T_MAC_3_OFFSET 0x0310 -#define AES_PLEN_OFFSET 0x0314 -#define AES_A_DMA_SRC_ADDR_OFFSET 0x0400 -#define AES_A_DMA_DST_ADDR_OFFSET 0x0404 -#define AES_A_DMA_SRC_SIZE_OFFSET 0x0408 -#define AES_A_DMA_DST_SIZE_OFFSET 0x040C -#define AES_A_DMA_DMA_MODE_OFFSET 0x0410 -#define AES_A_DMA_NEXT_SRC_DESCR_OFFSET 0x0418 -#define AES_A_DMA_NEXT_DST_DESCR_OFFSET 0x041C -#define AES_A_DMA_WHILE_ACTIVE_MODE_OFFSET 0x0420 -#define AES_A_DMA_LOG_OFFSET 0x0424 -#define AES_A_DMA_STATUS_OFFSET 0x0428 -#define AES_A_DMA_PERF_CNTR_OFFSET 0x042C -#define AES_A_DMA_MSI_ISR_OFFSET 0x0480 -#define AES_A_DMA_MSI_IER_OFFSET 0x0484 -#define AES_A_DMA_MSI_MASK_OFFSET 0x0488 -#define AES_A_DMA_INBUFFER_WRITE_FIFO_OFFSET 0x0600 -#define AES_A_DMA_OUTBUFFER_READ_FIFO_OFFSET 0x0700 - -/* - * AES_A_DMA_DMA_MODE register. - * Default: 0x00000000. - * bit[31] ACTIVE - * This bit activates the DMA. When the DMA finishes, it resets - * this bit to zero. - * bit[30:26] Unused by this driver. - * bit[25] SRC_LINK_LIST_EN - * Source link list enable bit. When the linked list is terminated - * this bit is reset by the DMA. - * bit[24] DST_LINK_LIST_EN - * Destination link list enable bit. When the linked list is - * terminated this bit is reset by the DMA. - * bit[23:0] Unused by this driver. - */ -#define AES_A_DMA_DMA_MODE_ACTIVE BIT(31) -#define AES_A_DMA_DMA_MODE_SRC_LINK_LIST_EN BIT(25) -#define AES_A_DMA_DMA_MODE_DST_LINK_LIST_EN BIT(24) - -/* - * AES_ACTIVE register - * default 0x00000000 - * bit[31:10] Reserved - * bit[9] LAST_ADATA - * bit[8] LAST_GCX - * bit[7:2] Reserved - * bit[1] TERMINATION - * bit[0] TRIGGER - */ -#define AES_ACTIVE_LAST_ADATA BIT(9) -#define AES_ACTIVE_LAST_CCM_GCM BIT(8) -#define AES_ACTIVE_TERMINATION BIT(1) -#define AES_ACTIVE_TRIGGER BIT(0) - -#define AES_DISABLE_INT 0x00000000 -#define AES_DMA_CPD_ERR_INT BIT(8) -#define AES_DMA_OUTBUF_RD_ERR_INT BIT(7) -#define AES_DMA_OUTBUF_WR_ERR_INT BIT(6) -#define AES_DMA_INBUF_RD_ERR_INT BIT(5) -#define AES_DMA_INBUF_WR_ERR_INT BIT(4) -#define AES_DMA_BAD_COMP_INT BIT(3) -#define AES_DMA_SAI_INT BIT(2) -#define AES_DMA_SRC_DONE_INT BIT(0) -#define AES_COMPLETE_INT BIT(1) - -#define AES_DMA_MSI_MASK_CLEAR BIT(0) - -#define AES_128_BIT_KEY 0x00000000 -#define AES_256_BIT_KEY BIT(0) - -#define AES_DEACTIVATE_PERF_CNTR 0x00000000 -#define AES_ACTIVATE_PERF_CNTR BIT(0) - -#define AES_MAX_TAG_SIZE_U32 4 - -#define OCS_LL_DMA_FLAG_TERMINATE BIT(31) - -/* - * There is an inconsistency in the documentation. This is documented as a - * 11-bit value, but it is actually 10-bits. - */ -#define AES_DMA_STATUS_INPUT_BUFFER_OCCUPANCY_MASK 0x3FF - -/* - * During CCM decrypt, the OCS block needs to finish processing the ciphertext - * before the tag is written. For 128-bit mode this required delay is 28 OCS - * clock cycles. For 256-bit mode it is 36 OCS clock cycles. - */ -#define CCM_DECRYPT_DELAY_TAG_CLK_COUNT 36UL - -/* - * During CCM decrypt there must be a delay of at least 42 OCS clock cycles - * between setting the TRIGGER bit in AES_ACTIVE and setting the LAST_CCM_GCM - * bit in the same register (as stated in the OCS databook) - */ -#define CCM_DECRYPT_DELAY_LAST_GCX_CLK_COUNT 42UL - -/* See RFC3610 section 2.2 */ -#define L_PRIME_MIN (1) -#define L_PRIME_MAX (7) -/* - * CCM IV format from RFC 3610 section 2.3 - * - * Octet Number Contents - * ------------ --------- - * 0 Flags - * 1 ... 15-L Nonce N - * 16-L ... 15 Counter i - * - * Flags = L' = L - 1 - */ -#define L_PRIME_IDX 0 -#define COUNTER_START(lprime) (16 - ((lprime) + 1)) -#define COUNTER_LEN(lprime) ((lprime) + 1) - -enum aes_counter_mode { - AES_CTR_M_NO_INC = 0, - AES_CTR_M_32_INC = 1, - AES_CTR_M_64_INC = 2, - AES_CTR_M_128_INC = 3, -}; - -/** - * struct ocs_dma_linked_list - OCS DMA linked list entry. - * @src_addr: Source address of the data. - * @src_len: Length of data to be fetched. - * @next: Next dma_list to fetch. - * @ll_flags: Flags (Freeze @ terminate) for the DMA engine. - */ -struct ocs_dma_linked_list { - u32 src_addr; - u32 src_len; - u32 next; - u32 ll_flags; -} __packed; - -/* - * Set endianness of inputs and outputs - * AES_BYTE_ORDER_CFG - * default 0x00000000 - * bit [10] - KEY_HI_LO_SWAP - * bit [9] - KEY_HI_SWAP_DWORDS_IN_OCTWORD - * bit [8] - KEY_HI_SWAP_BYTES_IN_DWORD - * bit [7] - KEY_LO_SWAP_DWORDS_IN_OCTWORD - * bit [6] - KEY_LO_SWAP_BYTES_IN_DWORD - * bit [5] - IV_SWAP_DWORDS_IN_OCTWORD - * bit [4] - IV_SWAP_BYTES_IN_DWORD - * bit [3] - DOUT_SWAP_DWORDS_IN_OCTWORD - * bit [2] - DOUT_SWAP_BYTES_IN_DWORD - * bit [1] - DOUT_SWAP_DWORDS_IN_OCTWORD - * bit [0] - DOUT_SWAP_BYTES_IN_DWORD - */ -static inline void aes_a_set_endianness(const struct ocs_aes_dev *aes_dev) -{ - iowrite32(0x7FF, aes_dev->base_reg + AES_BYTE_ORDER_CFG_OFFSET); -} - -/* Trigger AES process start. */ -static inline void aes_a_op_trigger(const struct ocs_aes_dev *aes_dev) -{ - iowrite32(AES_ACTIVE_TRIGGER, aes_dev->base_reg + AES_ACTIVE_OFFSET); -} - -/* Indicate last bulk of data. */ -static inline void aes_a_op_termination(const struct ocs_aes_dev *aes_dev) -{ - iowrite32(AES_ACTIVE_TERMINATION, - aes_dev->base_reg + AES_ACTIVE_OFFSET); -} - -/* - * Set LAST_CCM_GCM in AES_ACTIVE register and clear all other bits. - * - * Called when DMA is programmed to fetch the last batch of data. - * - For AES-CCM it is called for the last batch of Payload data and Ciphertext - * data. - * - For AES-GCM, it is called for the last batch of Plaintext data and - * Ciphertext data. - */ -static inline void aes_a_set_last_gcx(const struct ocs_aes_dev *aes_dev) -{ - iowrite32(AES_ACTIVE_LAST_CCM_GCM, - aes_dev->base_reg + AES_ACTIVE_OFFSET); -} - -/* Wait for LAST_CCM_GCM bit to be unset. */ -static inline void aes_a_wait_last_gcx(const struct ocs_aes_dev *aes_dev) -{ - u32 aes_active_reg; - - do { - aes_active_reg = ioread32(aes_dev->base_reg + - AES_ACTIVE_OFFSET); - } while (aes_active_reg & AES_ACTIVE_LAST_CCM_GCM); -} - -/* Wait for 10 bits of input occupancy. */ -static void aes_a_dma_wait_input_buffer_occupancy(const struct ocs_aes_dev *aes_dev) -{ - u32 reg; - - do { - reg = ioread32(aes_dev->base_reg + AES_A_DMA_STATUS_OFFSET); - } while (reg & AES_DMA_STATUS_INPUT_BUFFER_OCCUPANCY_MASK); -} - - /* - * Set LAST_CCM_GCM and LAST_ADATA bits in AES_ACTIVE register (and clear all - * other bits). - * - * Called when DMA is programmed to fetch the last batch of Associated Data - * (CCM case) or Additional Authenticated Data (GCM case). - */ -static inline void aes_a_set_last_gcx_and_adata(const struct ocs_aes_dev *aes_dev) -{ - iowrite32(AES_ACTIVE_LAST_ADATA | AES_ACTIVE_LAST_CCM_GCM, - aes_dev->base_reg + AES_ACTIVE_OFFSET); -} - -/* Set DMA src and dst transfer size to 0 */ -static inline void aes_a_dma_set_xfer_size_zero(const struct ocs_aes_dev *aes_dev) -{ - iowrite32(0, aes_dev->base_reg + AES_A_DMA_SRC_SIZE_OFFSET); - iowrite32(0, aes_dev->base_reg + AES_A_DMA_DST_SIZE_OFFSET); -} - -/* Activate DMA for zero-byte transfer case. */ -static inline void aes_a_dma_active(const struct ocs_aes_dev *aes_dev) -{ - iowrite32(AES_A_DMA_DMA_MODE_ACTIVE, - aes_dev->base_reg + AES_A_DMA_DMA_MODE_OFFSET); -} - -/* Activate DMA and enable src linked list */ -static inline void aes_a_dma_active_src_ll_en(const struct ocs_aes_dev *aes_dev) -{ - iowrite32(AES_A_DMA_DMA_MODE_ACTIVE | - AES_A_DMA_DMA_MODE_SRC_LINK_LIST_EN, - aes_dev->base_reg + AES_A_DMA_DMA_MODE_OFFSET); -} - -/* Activate DMA and enable dst linked list */ -static inline void aes_a_dma_active_dst_ll_en(const struct ocs_aes_dev *aes_dev) -{ - iowrite32(AES_A_DMA_DMA_MODE_ACTIVE | - AES_A_DMA_DMA_MODE_DST_LINK_LIST_EN, - aes_dev->base_reg + AES_A_DMA_DMA_MODE_OFFSET); -} - -/* Activate DMA and enable src and dst linked lists */ -static inline void aes_a_dma_active_src_dst_ll_en(const struct ocs_aes_dev *aes_dev) -{ - iowrite32(AES_A_DMA_DMA_MODE_ACTIVE | - AES_A_DMA_DMA_MODE_SRC_LINK_LIST_EN | - AES_A_DMA_DMA_MODE_DST_LINK_LIST_EN, - aes_dev->base_reg + AES_A_DMA_DMA_MODE_OFFSET); -} - -/* Reset PERF_CNTR to 0 and activate it */ -static inline void aes_a_dma_reset_and_activate_perf_cntr(const struct ocs_aes_dev *aes_dev) -{ - iowrite32(0x00000000, aes_dev->base_reg + AES_A_DMA_PERF_CNTR_OFFSET); - iowrite32(AES_ACTIVATE_PERF_CNTR, - aes_dev->base_reg + AES_A_DMA_WHILE_ACTIVE_MODE_OFFSET); -} - -/* Wait until PERF_CNTR is > delay, then deactivate it */ -static inline void aes_a_dma_wait_and_deactivate_perf_cntr(const struct ocs_aes_dev *aes_dev, - int delay) -{ - while (ioread32(aes_dev->base_reg + AES_A_DMA_PERF_CNTR_OFFSET) < delay) - ; - iowrite32(AES_DEACTIVATE_PERF_CNTR, - aes_dev->base_reg + AES_A_DMA_WHILE_ACTIVE_MODE_OFFSET); -} - -/* Disable AES and DMA IRQ. */ -static void aes_irq_disable(struct ocs_aes_dev *aes_dev) -{ - u32 isr_val = 0; - - /* Disable interrupts */ - iowrite32(AES_DISABLE_INT, - aes_dev->base_reg + AES_A_DMA_MSI_IER_OFFSET); - iowrite32(AES_DISABLE_INT, aes_dev->base_reg + AES_IER_OFFSET); - - /* Clear any pending interrupt */ - isr_val = ioread32(aes_dev->base_reg + AES_A_DMA_MSI_ISR_OFFSET); - if (isr_val) - iowrite32(isr_val, - aes_dev->base_reg + AES_A_DMA_MSI_ISR_OFFSET); - - isr_val = ioread32(aes_dev->base_reg + AES_A_DMA_MSI_MASK_OFFSET); - if (isr_val) - iowrite32(isr_val, - aes_dev->base_reg + AES_A_DMA_MSI_MASK_OFFSET); - - isr_val = ioread32(aes_dev->base_reg + AES_ISR_OFFSET); - if (isr_val) - iowrite32(isr_val, aes_dev->base_reg + AES_ISR_OFFSET); -} - -/* Enable AES or DMA IRQ. IRQ is disabled once fired. */ -static void aes_irq_enable(struct ocs_aes_dev *aes_dev, u8 irq) -{ - if (irq == AES_COMPLETE_INT) { - /* Ensure DMA error interrupts are enabled */ - iowrite32(AES_DMA_CPD_ERR_INT | - AES_DMA_OUTBUF_RD_ERR_INT | - AES_DMA_OUTBUF_WR_ERR_INT | - AES_DMA_INBUF_RD_ERR_INT | - AES_DMA_INBUF_WR_ERR_INT | - AES_DMA_BAD_COMP_INT | - AES_DMA_SAI_INT, - aes_dev->base_reg + AES_A_DMA_MSI_IER_OFFSET); - /* - * AES_IER - * default 0x00000000 - * bits [31:3] - reserved - * bit [2] - EN_SKS_ERR - * bit [1] - EN_AES_COMPLETE - * bit [0] - reserved - */ - iowrite32(AES_COMPLETE_INT, aes_dev->base_reg + AES_IER_OFFSET); - return; - } - if (irq == AES_DMA_SRC_DONE_INT) { - /* Ensure AES interrupts are disabled */ - iowrite32(AES_DISABLE_INT, aes_dev->base_reg + AES_IER_OFFSET); - /* - * DMA_MSI_IER - * default 0x00000000 - * bits [31:9] - reserved - * bit [8] - CPD_ERR_INT_EN - * bit [7] - OUTBUF_RD_ERR_INT_EN - * bit [6] - OUTBUF_WR_ERR_INT_EN - * bit [5] - INBUF_RD_ERR_INT_EN - * bit [4] - INBUF_WR_ERR_INT_EN - * bit [3] - BAD_COMP_INT_EN - * bit [2] - SAI_INT_EN - * bit [1] - DST_DONE_INT_EN - * bit [0] - SRC_DONE_INT_EN - */ - iowrite32(AES_DMA_CPD_ERR_INT | - AES_DMA_OUTBUF_RD_ERR_INT | - AES_DMA_OUTBUF_WR_ERR_INT | - AES_DMA_INBUF_RD_ERR_INT | - AES_DMA_INBUF_WR_ERR_INT | - AES_DMA_BAD_COMP_INT | - AES_DMA_SAI_INT | - AES_DMA_SRC_DONE_INT, - aes_dev->base_reg + AES_A_DMA_MSI_IER_OFFSET); - } -} - -/* Enable and wait for IRQ (either from OCS AES engine or DMA) */ -static int ocs_aes_irq_enable_and_wait(struct ocs_aes_dev *aes_dev, u8 irq) -{ - int rc; - - reinit_completion(&aes_dev->irq_completion); - aes_irq_enable(aes_dev, irq); - rc = wait_for_completion_interruptible(&aes_dev->irq_completion); - if (rc) - return rc; - - return aes_dev->dma_err_mask ? -EIO : 0; -} - -/* Configure DMA to OCS, linked list mode */ -static inline void dma_to_ocs_aes_ll(struct ocs_aes_dev *aes_dev, - dma_addr_t dma_list) -{ - iowrite32(0, aes_dev->base_reg + AES_A_DMA_SRC_SIZE_OFFSET); - iowrite32(dma_list, - aes_dev->base_reg + AES_A_DMA_NEXT_SRC_DESCR_OFFSET); -} - -/* Configure DMA from OCS, linked list mode */ -static inline void dma_from_ocs_aes_ll(struct ocs_aes_dev *aes_dev, - dma_addr_t dma_list) -{ - iowrite32(0, aes_dev->base_reg + AES_A_DMA_DST_SIZE_OFFSET); - iowrite32(dma_list, - aes_dev->base_reg + AES_A_DMA_NEXT_DST_DESCR_OFFSET); -} - -irqreturn_t ocs_aes_irq_handler(int irq, void *dev_id) -{ - struct ocs_aes_dev *aes_dev = dev_id; - u32 aes_dma_isr; - - /* Read DMA ISR status. */ - aes_dma_isr = ioread32(aes_dev->base_reg + AES_A_DMA_MSI_ISR_OFFSET); - - /* Disable and clear interrupts. */ - aes_irq_disable(aes_dev); - - /* Save DMA error status. */ - aes_dev->dma_err_mask = aes_dma_isr & - (AES_DMA_CPD_ERR_INT | - AES_DMA_OUTBUF_RD_ERR_INT | - AES_DMA_OUTBUF_WR_ERR_INT | - AES_DMA_INBUF_RD_ERR_INT | - AES_DMA_INBUF_WR_ERR_INT | - AES_DMA_BAD_COMP_INT | - AES_DMA_SAI_INT); - - /* Signal IRQ completion. */ - complete(&aes_dev->irq_completion); - - return IRQ_HANDLED; -} - -/** - * ocs_aes_set_key() - Write key into OCS AES hardware. - * @aes_dev: The OCS AES device to write the key to. - * @key_size: The size of the key (in bytes). - * @key: The key to write. - * @cipher: The cipher the key is for. - * - * For AES @key_size must be either 16 or 32. For SM4 @key_size must be 16. - * - * Return: 0 on success, negative error code otherwise. - */ -int ocs_aes_set_key(struct ocs_aes_dev *aes_dev, u32 key_size, const u8 *key, - enum ocs_cipher cipher) -{ - const u32 *key_u32; - u32 val; - int i; - - /* OCS AES supports 128-bit and 256-bit keys only. */ - if (cipher == OCS_AES && !(key_size == 32 || key_size == 16)) { - dev_err(aes_dev->dev, - "%d-bit keys not supported by AES cipher\n", - key_size * 8); - return -EINVAL; - } - /* OCS SM4 supports 128-bit keys only. */ - if (cipher == OCS_SM4 && key_size != 16) { - dev_err(aes_dev->dev, - "%d-bit keys not supported for SM4 cipher\n", - key_size * 8); - return -EINVAL; - } - - if (!key) - return -EINVAL; - - key_u32 = (const u32 *)key; - - /* Write key to AES_KEY[0-7] registers */ - for (i = 0; i < (key_size / sizeof(u32)); i++) { - iowrite32(key_u32[i], - aes_dev->base_reg + AES_KEY_0_OFFSET + - (i * sizeof(u32))); - } - /* - * Write key size - * bits [31:1] - reserved - * bit [0] - AES_KEY_SIZE - * 0 - 128 bit key - * 1 - 256 bit key - */ - val = (key_size == 16) ? AES_128_BIT_KEY : AES_256_BIT_KEY; - iowrite32(val, aes_dev->base_reg + AES_KEY_SIZE_OFFSET); - - return 0; -} - -/* Write AES_COMMAND */ -static inline void set_ocs_aes_command(struct ocs_aes_dev *aes_dev, - enum ocs_cipher cipher, - enum ocs_mode mode, - enum ocs_instruction instruction) -{ - u32 val; - - /* AES_COMMAND - * default 0x000000CC - * bit [14] - CIPHER_SELECT - * 0 - AES - * 1 - SM4 - * bits [11:8] - OCS_AES_MODE - * 0000 - ECB - * 0001 - CBC - * 0010 - CTR - * 0110 - CCM - * 0111 - GCM - * 1001 - CTS - * bits [7:6] - AES_INSTRUCTION - * 00 - ENCRYPT - * 01 - DECRYPT - * 10 - EXPAND - * 11 - BYPASS - * bits [3:2] - CTR_M_BITS - * 00 - No increment - * 01 - Least significant 32 bits are incremented - * 10 - Least significant 64 bits are incremented - * 11 - Full 128 bits are incremented - */ - val = (cipher << 14) | (mode << 8) | (instruction << 6) | - (AES_CTR_M_128_INC << 2); - iowrite32(val, aes_dev->base_reg + AES_COMMAND_OFFSET); -} - -static void ocs_aes_init(struct ocs_aes_dev *aes_dev, - enum ocs_mode mode, - enum ocs_cipher cipher, - enum ocs_instruction instruction) -{ - /* Ensure interrupts are disabled and pending interrupts cleared. */ - aes_irq_disable(aes_dev); - - /* Set endianness recommended by data-sheet. */ - aes_a_set_endianness(aes_dev); - - /* Set AES_COMMAND register. */ - set_ocs_aes_command(aes_dev, cipher, mode, instruction); -} - -/* - * Write the byte length of the last AES/SM4 block of Payload data (without - * zero padding and without the length of the MAC) in register AES_PLEN. - */ -static inline void ocs_aes_write_last_data_blk_len(struct ocs_aes_dev *aes_dev, - u32 size) -{ - u32 val; - - if (size == 0) { - val = 0; - goto exit; - } - - val = size % AES_BLOCK_SIZE; - if (val == 0) - val = AES_BLOCK_SIZE; - -exit: - iowrite32(val, aes_dev->base_reg + AES_PLEN_OFFSET); -} - -/* - * Validate inputs according to mode. - * If OK return 0; else return -EINVAL. - */ -static int ocs_aes_validate_inputs(dma_addr_t src_dma_list, u32 src_size, - const u8 *iv, u32 iv_size, - dma_addr_t aad_dma_list, u32 aad_size, - const u8 *tag, u32 tag_size, - enum ocs_cipher cipher, enum ocs_mode mode, - enum ocs_instruction instruction, - dma_addr_t dst_dma_list) -{ - /* Ensure cipher, mode and instruction are valid. */ - if (!(cipher == OCS_AES || cipher == OCS_SM4)) - return -EINVAL; - - if (mode != OCS_MODE_ECB && mode != OCS_MODE_CBC && - mode != OCS_MODE_CTR && mode != OCS_MODE_CCM && - mode != OCS_MODE_GCM && mode != OCS_MODE_CTS) - return -EINVAL; - - if (instruction != OCS_ENCRYPT && instruction != OCS_DECRYPT && - instruction != OCS_EXPAND && instruction != OCS_BYPASS) - return -EINVAL; - - /* - * When instruction is OCS_BYPASS, OCS simply copies data from source - * to destination using DMA. - * - * AES mode is irrelevant, but both source and destination DMA - * linked-list must be defined. - */ - if (instruction == OCS_BYPASS) { - if (src_dma_list == DMA_MAPPING_ERROR || - dst_dma_list == DMA_MAPPING_ERROR) - return -EINVAL; - - return 0; - } - - /* - * For performance reasons switch based on mode to limit unnecessary - * conditionals for each mode - */ - switch (mode) { - case OCS_MODE_ECB: - /* Ensure input length is multiple of block size */ - if (src_size % AES_BLOCK_SIZE != 0) - return -EINVAL; - - /* Ensure source and destination linked lists are created */ - if (src_dma_list == DMA_MAPPING_ERROR || - dst_dma_list == DMA_MAPPING_ERROR) - return -EINVAL; - - return 0; - - case OCS_MODE_CBC: - /* Ensure input length is multiple of block size */ - if (src_size % AES_BLOCK_SIZE != 0) - return -EINVAL; - - /* Ensure source and destination linked lists are created */ - if (src_dma_list == DMA_MAPPING_ERROR || - dst_dma_list == DMA_MAPPING_ERROR) - return -EINVAL; - - /* Ensure IV is present and block size in length */ - if (!iv || iv_size != AES_BLOCK_SIZE) - return -EINVAL; - - return 0; - - case OCS_MODE_CTR: - /* Ensure input length of 1 byte or greater */ - if (src_size == 0) - return -EINVAL; - - /* Ensure source and destination linked lists are created */ - if (src_dma_list == DMA_MAPPING_ERROR || - dst_dma_list == DMA_MAPPING_ERROR) - return -EINVAL; - - /* Ensure IV is present and block size in length */ - if (!iv || iv_size != AES_BLOCK_SIZE) - return -EINVAL; - - return 0; - - case OCS_MODE_CTS: - /* Ensure input length >= block size */ - if (src_size < AES_BLOCK_SIZE) - return -EINVAL; - - /* Ensure source and destination linked lists are created */ - if (src_dma_list == DMA_MAPPING_ERROR || - dst_dma_list == DMA_MAPPING_ERROR) - return -EINVAL; - - /* Ensure IV is present and block size in length */ - if (!iv || iv_size != AES_BLOCK_SIZE) - return -EINVAL; - - return 0; - - case OCS_MODE_GCM: - /* Ensure IV is present and GCM_AES_IV_SIZE in length */ - if (!iv || iv_size != GCM_AES_IV_SIZE) - return -EINVAL; - - /* - * If input data present ensure source and destination linked - * lists are created - */ - if (src_size && (src_dma_list == DMA_MAPPING_ERROR || - dst_dma_list == DMA_MAPPING_ERROR)) - return -EINVAL; - - /* If aad present ensure aad linked list is created */ - if (aad_size && aad_dma_list == DMA_MAPPING_ERROR) - return -EINVAL; - - /* Ensure tag destination is set */ - if (!tag) - return -EINVAL; - - /* Just ensure that tag_size doesn't cause overflows. */ - if (tag_size > (AES_MAX_TAG_SIZE_U32 * sizeof(u32))) - return -EINVAL; - - return 0; - - case OCS_MODE_CCM: - /* Ensure IV is present and block size in length */ - if (!iv || iv_size != AES_BLOCK_SIZE) - return -EINVAL; - - /* 2 <= L <= 8, so 1 <= L' <= 7 */ - if (iv[L_PRIME_IDX] < L_PRIME_MIN || - iv[L_PRIME_IDX] > L_PRIME_MAX) - return -EINVAL; - - /* If aad present ensure aad linked list is created */ - if (aad_size && aad_dma_list == DMA_MAPPING_ERROR) - return -EINVAL; - - /* Just ensure that tag_size doesn't cause overflows. */ - if (tag_size > (AES_MAX_TAG_SIZE_U32 * sizeof(u32))) - return -EINVAL; - - if (instruction == OCS_DECRYPT) { - /* - * If input data present ensure source and destination - * linked lists are created - */ - if (src_size && (src_dma_list == DMA_MAPPING_ERROR || - dst_dma_list == DMA_MAPPING_ERROR)) - return -EINVAL; - - /* Ensure input tag is present */ - if (!tag) - return -EINVAL; - - return 0; - } - - /* Instruction == OCS_ENCRYPT */ - - /* - * Destination linked list always required (for tag even if no - * input data) - */ - if (dst_dma_list == DMA_MAPPING_ERROR) - return -EINVAL; - - /* If input data present ensure src linked list is created */ - if (src_size && src_dma_list == DMA_MAPPING_ERROR) - return -EINVAL; - - return 0; - - default: - return -EINVAL; - } -} - -/** - * ocs_aes_op() - Perform AES/SM4 operation. - * @aes_dev: The OCS AES device to use. - * @mode: The mode to use (ECB, CBC, CTR, or CTS). - * @cipher: The cipher to use (AES or SM4). - * @instruction: The instruction to perform (encrypt or decrypt). - * @dst_dma_list: The OCS DMA list mapping output memory. - * @src_dma_list: The OCS DMA list mapping input payload data. - * @src_size: The amount of data mapped by @src_dma_list. - * @iv: The IV vector. - * @iv_size: The size (in bytes) of @iv. - * - * Return: 0 on success, negative error code otherwise. - */ -int ocs_aes_op(struct ocs_aes_dev *aes_dev, - enum ocs_mode mode, - enum ocs_cipher cipher, - enum ocs_instruction instruction, - dma_addr_t dst_dma_list, - dma_addr_t src_dma_list, - u32 src_size, - u8 *iv, - u32 iv_size) -{ - u32 *iv32; - int rc; - - rc = ocs_aes_validate_inputs(src_dma_list, src_size, iv, iv_size, 0, 0, - NULL, 0, cipher, mode, instruction, - dst_dma_list); - if (rc) - return rc; - /* - * ocs_aes_validate_inputs() is a generic check, now ensure mode is not - * GCM or CCM. - */ - if (mode == OCS_MODE_GCM || mode == OCS_MODE_CCM) - return -EINVAL; - - /* Cast IV to u32 array. */ - iv32 = (u32 *)iv; - - ocs_aes_init(aes_dev, mode, cipher, instruction); - - if (mode == OCS_MODE_CTS) { - /* Write the byte length of the last data block to engine. */ - ocs_aes_write_last_data_blk_len(aes_dev, src_size); - } - - /* ECB is the only mode that doesn't use IV. */ - if (mode != OCS_MODE_ECB) { - iowrite32(iv32[0], aes_dev->base_reg + AES_IV_0_OFFSET); - iowrite32(iv32[1], aes_dev->base_reg + AES_IV_1_OFFSET); - iowrite32(iv32[2], aes_dev->base_reg + AES_IV_2_OFFSET); - iowrite32(iv32[3], aes_dev->base_reg + AES_IV_3_OFFSET); - } - - /* Set AES_ACTIVE.TRIGGER to start the operation. */ - aes_a_op_trigger(aes_dev); - - /* Configure and activate input / output DMA. */ - dma_to_ocs_aes_ll(aes_dev, src_dma_list); - dma_from_ocs_aes_ll(aes_dev, dst_dma_list); - aes_a_dma_active_src_dst_ll_en(aes_dev); - - if (mode == OCS_MODE_CTS) { - /* - * For CTS mode, instruct engine to activate ciphertext - * stealing if last block of data is incomplete. - */ - aes_a_set_last_gcx(aes_dev); - } else { - /* For all other modes, just write the 'termination' bit. */ - aes_a_op_termination(aes_dev); - } - - /* Wait for engine to complete processing. */ - rc = ocs_aes_irq_enable_and_wait(aes_dev, AES_COMPLETE_INT); - if (rc) - return rc; - - if (mode == OCS_MODE_CTR) { - /* Read back IV for streaming mode */ - iv32[0] = ioread32(aes_dev->base_reg + AES_IV_0_OFFSET); - iv32[1] = ioread32(aes_dev->base_reg + AES_IV_1_OFFSET); - iv32[2] = ioread32(aes_dev->base_reg + AES_IV_2_OFFSET); - iv32[3] = ioread32(aes_dev->base_reg + AES_IV_3_OFFSET); - } - - return 0; -} - -/* Compute and write J0 to engine registers. */ -static void ocs_aes_gcm_write_j0(const struct ocs_aes_dev *aes_dev, - const u8 *iv) -{ - const u32 *j0 = (u32 *)iv; - - /* - * IV must be 12 bytes; Other sizes not supported as Linux crypto API - * does only expects/allows 12 byte IV for GCM - */ - iowrite32(0x00000001, aes_dev->base_reg + AES_IV_0_OFFSET); - iowrite32(__swab32(j0[2]), aes_dev->base_reg + AES_IV_1_OFFSET); - iowrite32(__swab32(j0[1]), aes_dev->base_reg + AES_IV_2_OFFSET); - iowrite32(__swab32(j0[0]), aes_dev->base_reg + AES_IV_3_OFFSET); -} - -/* Read GCM tag from engine registers. */ -static inline void ocs_aes_gcm_read_tag(struct ocs_aes_dev *aes_dev, - u8 *tag, u32 tag_size) -{ - u32 tag_u32[AES_MAX_TAG_SIZE_U32]; - - /* - * The Authentication Tag T is stored in Little Endian order in the - * registers with the most significant bytes stored from AES_T_MAC[3] - * downward. - */ - tag_u32[0] = __swab32(ioread32(aes_dev->base_reg + AES_T_MAC_3_OFFSET)); - tag_u32[1] = __swab32(ioread32(aes_dev->base_reg + AES_T_MAC_2_OFFSET)); - tag_u32[2] = __swab32(ioread32(aes_dev->base_reg + AES_T_MAC_1_OFFSET)); - tag_u32[3] = __swab32(ioread32(aes_dev->base_reg + AES_T_MAC_0_OFFSET)); - - memcpy(tag, tag_u32, tag_size); -} - -/** - * ocs_aes_gcm_op() - Perform GCM operation. - * @aes_dev: The OCS AES device to use. - * @cipher: The Cipher to use (AES or SM4). - * @instruction: The instruction to perform (encrypt or decrypt). - * @dst_dma_list: The OCS DMA list mapping output memory. - * @src_dma_list: The OCS DMA list mapping input payload data. - * @src_size: The amount of data mapped by @src_dma_list. - * @iv: The input IV vector. - * @aad_dma_list: The OCS DMA list mapping input AAD data. - * @aad_size: The amount of data mapped by @aad_dma_list. - * @out_tag: Where to store computed tag. - * @tag_size: The size (in bytes) of @out_tag. - * - * Return: 0 on success, negative error code otherwise. - */ -int ocs_aes_gcm_op(struct ocs_aes_dev *aes_dev, - enum ocs_cipher cipher, - enum ocs_instruction instruction, - dma_addr_t dst_dma_list, - dma_addr_t src_dma_list, - u32 src_size, - const u8 *iv, - dma_addr_t aad_dma_list, - u32 aad_size, - u8 *out_tag, - u32 tag_size) -{ - u64 bit_len; - u32 val; - int rc; - - rc = ocs_aes_validate_inputs(src_dma_list, src_size, iv, - GCM_AES_IV_SIZE, aad_dma_list, - aad_size, out_tag, tag_size, cipher, - OCS_MODE_GCM, instruction, - dst_dma_list); - if (rc) - return rc; - - ocs_aes_init(aes_dev, OCS_MODE_GCM, cipher, instruction); - - /* Compute and write J0 to OCS HW. */ - ocs_aes_gcm_write_j0(aes_dev, iv); - - /* Write out_tag byte length */ - iowrite32(tag_size, aes_dev->base_reg + AES_TLEN_OFFSET); - - /* Write the byte length of the last plaintext / ciphertext block. */ - ocs_aes_write_last_data_blk_len(aes_dev, src_size); - - /* Write ciphertext bit length */ - bit_len = (u64)src_size * 8; - val = bit_len & 0xFFFFFFFF; - iowrite32(val, aes_dev->base_reg + AES_MULTIPURPOSE2_0_OFFSET); - val = bit_len >> 32; - iowrite32(val, aes_dev->base_reg + AES_MULTIPURPOSE2_1_OFFSET); - - /* Write aad bit length */ - bit_len = (u64)aad_size * 8; - val = bit_len & 0xFFFFFFFF; - iowrite32(val, aes_dev->base_reg + AES_MULTIPURPOSE2_2_OFFSET); - val = bit_len >> 32; - iowrite32(val, aes_dev->base_reg + AES_MULTIPURPOSE2_3_OFFSET); - - /* Set AES_ACTIVE.TRIGGER to start the operation. */ - aes_a_op_trigger(aes_dev); - - /* Process AAD. */ - if (aad_size) { - /* If aad present, configure DMA to feed it to the engine. */ - dma_to_ocs_aes_ll(aes_dev, aad_dma_list); - aes_a_dma_active_src_ll_en(aes_dev); - - /* Instructs engine to pad last block of aad, if needed. */ - aes_a_set_last_gcx_and_adata(aes_dev); - - /* Wait for DMA transfer to complete. */ - rc = ocs_aes_irq_enable_and_wait(aes_dev, AES_DMA_SRC_DONE_INT); - if (rc) - return rc; - } else { - aes_a_set_last_gcx_and_adata(aes_dev); - } - - /* Wait until adata (if present) has been processed. */ - aes_a_wait_last_gcx(aes_dev); - aes_a_dma_wait_input_buffer_occupancy(aes_dev); - - /* Now process payload. */ - if (src_size) { - /* Configure and activate DMA for both input and output data. */ - dma_to_ocs_aes_ll(aes_dev, src_dma_list); - dma_from_ocs_aes_ll(aes_dev, dst_dma_list); - aes_a_dma_active_src_dst_ll_en(aes_dev); - } else { - aes_a_dma_set_xfer_size_zero(aes_dev); - aes_a_dma_active(aes_dev); - } - - /* Instruct AES/SMA4 engine payload processing is over. */ - aes_a_set_last_gcx(aes_dev); - - /* Wait for OCS AES engine to complete processing. */ - rc = ocs_aes_irq_enable_and_wait(aes_dev, AES_COMPLETE_INT); - if (rc) - return rc; - - ocs_aes_gcm_read_tag(aes_dev, out_tag, tag_size); - - return 0; -} - -/* Write encrypted tag to AES/SM4 engine. */ -static void ocs_aes_ccm_write_encrypted_tag(struct ocs_aes_dev *aes_dev, - const u8 *in_tag, u32 tag_size) -{ - int i; - - /* Ensure DMA input buffer is empty */ - aes_a_dma_wait_input_buffer_occupancy(aes_dev); - - /* - * During CCM decrypt, the OCS block needs to finish processing the - * ciphertext before the tag is written. So delay needed after DMA has - * completed writing the ciphertext - */ - aes_a_dma_reset_and_activate_perf_cntr(aes_dev); - aes_a_dma_wait_and_deactivate_perf_cntr(aes_dev, - CCM_DECRYPT_DELAY_TAG_CLK_COUNT); - - /* Write encrypted tag to AES/SM4 engine. */ - for (i = 0; i < tag_size; i++) { - iowrite8(in_tag[i], aes_dev->base_reg + - AES_A_DMA_INBUFFER_WRITE_FIFO_OFFSET); - } -} - -/* - * Write B0 CCM block to OCS AES HW. - * - * Note: B0 format is documented in NIST Special Publication 800-38C - * https://nvlpubs.nist.gov/nistpubs/Legacy/SP/nistspecialpublication800-38c.pdf - * (see Section A.2.1) - */ -static int ocs_aes_ccm_write_b0(const struct ocs_aes_dev *aes_dev, - const u8 *iv, u32 adata_size, u32 tag_size, - u32 cryptlen) -{ - u8 b0[16]; /* CCM B0 block is 16 bytes long. */ - int i, q; - - /* Initialize B0 to 0. */ - memset(b0, 0, sizeof(b0)); - - /* - * B0[0] is the 'Flags Octet' and has the following structure: - * bit 7: Reserved - * bit 6: Adata flag - * bit 5-3: t value encoded as (t-2)/2 - * bit 2-0: q value encoded as q - 1 - */ - /* If there is AAD data, set the Adata flag. */ - if (adata_size) - b0[0] |= BIT(6); - /* - * t denotes the octet length of T. - * t can only be an element of { 4, 6, 8, 10, 12, 14, 16} and is - * encoded as (t - 2) / 2 - */ - b0[0] |= (((tag_size - 2) / 2) & 0x7) << 3; - /* - * q is the octet length of Q. - * q can only be an element of {2, 3, 4, 5, 6, 7, 8} and is encoded as - * q - 1 == iv[0] & 0x7; - */ - b0[0] |= iv[0] & 0x7; - /* - * Copy the Nonce N from IV to B0; N is located in iv[1]..iv[15 - q] - * and must be copied to b0[1]..b0[15-q]. - * q == (iv[0] & 0x7) + 1 - */ - q = (iv[0] & 0x7) + 1; - for (i = 1; i <= 15 - q; i++) - b0[i] = iv[i]; - /* - * The rest of B0 must contain Q, i.e., the message length. - * Q is encoded in q octets, in big-endian order, so to write it, we - * start from the end of B0 and we move backward. - */ - i = sizeof(b0) - 1; - while (q) { - b0[i] = cryptlen & 0xff; - cryptlen >>= 8; - i--; - q--; - } - /* - * If cryptlen is not zero at this point, it means that its original - * value was too big. - */ - if (cryptlen) - return -EOVERFLOW; - /* Now write B0 to OCS AES input buffer. */ - for (i = 0; i < sizeof(b0); i++) - iowrite8(b0[i], aes_dev->base_reg + - AES_A_DMA_INBUFFER_WRITE_FIFO_OFFSET); - return 0; -} - -/* - * Write adata length to OCS AES HW. - * - * Note: adata len encoding is documented in NIST Special Publication 800-38C - * https://nvlpubs.nist.gov/nistpubs/Legacy/SP/nistspecialpublication800-38c.pdf - * (see Section A.2.2) - */ -static void ocs_aes_ccm_write_adata_len(const struct ocs_aes_dev *aes_dev, - u64 adata_len) -{ - u8 enc_a[10]; /* Maximum encoded size: 10 octets. */ - int i, len; - - /* - * adata_len ('a') is encoded as follows: - * If 0 < a < 2^16 - 2^8 ==> 'a' encoded as [a]16, i.e., two octets - * (big endian). - * If 2^16 - 2^8 ≤ a < 2^32 ==> 'a' encoded as 0xff || 0xfe || [a]32, - * i.e., six octets (big endian). - * If 2^32 ≤ a < 2^64 ==> 'a' encoded as 0xff || 0xff || [a]64, - * i.e., ten octets (big endian). - */ - if (adata_len < 65280) { - len = 2; - *(__be16 *)enc_a = cpu_to_be16(adata_len); - } else if (adata_len <= 0xFFFFFFFF) { - len = 6; - *(__be16 *)enc_a = cpu_to_be16(0xfffe); - *(__be32 *)&enc_a[2] = cpu_to_be32(adata_len); - } else { /* adata_len >= 2^32 */ - len = 10; - *(__be16 *)enc_a = cpu_to_be16(0xffff); - *(__be64 *)&enc_a[2] = cpu_to_be64(adata_len); - } - for (i = 0; i < len; i++) - iowrite8(enc_a[i], - aes_dev->base_reg + - AES_A_DMA_INBUFFER_WRITE_FIFO_OFFSET); -} - -static int ocs_aes_ccm_do_adata(struct ocs_aes_dev *aes_dev, - dma_addr_t adata_dma_list, u32 adata_size) -{ - int rc; - - if (!adata_size) { - /* Since no aad the LAST_GCX bit can be set now */ - aes_a_set_last_gcx_and_adata(aes_dev); - goto exit; - } - - /* Adata case. */ - - /* - * Form the encoding of the Associated data length and write it - * to the AES/SM4 input buffer. - */ - ocs_aes_ccm_write_adata_len(aes_dev, adata_size); - - /* Configure the AES/SM4 DMA to fetch the Associated Data */ - dma_to_ocs_aes_ll(aes_dev, adata_dma_list); - - /* Activate DMA to fetch Associated data. */ - aes_a_dma_active_src_ll_en(aes_dev); - - /* Set LAST_GCX and LAST_ADATA in AES ACTIVE register. */ - aes_a_set_last_gcx_and_adata(aes_dev); - - /* Wait for DMA transfer to complete. */ - rc = ocs_aes_irq_enable_and_wait(aes_dev, AES_DMA_SRC_DONE_INT); - if (rc) - return rc; - -exit: - /* Wait until adata (if present) has been processed. */ - aes_a_wait_last_gcx(aes_dev); - aes_a_dma_wait_input_buffer_occupancy(aes_dev); - - return 0; -} - -static int ocs_aes_ccm_encrypt_do_payload(struct ocs_aes_dev *aes_dev, - dma_addr_t dst_dma_list, - dma_addr_t src_dma_list, - u32 src_size) -{ - if (src_size) { - /* - * Configure and activate DMA for both input and output - * data. - */ - dma_to_ocs_aes_ll(aes_dev, src_dma_list); - dma_from_ocs_aes_ll(aes_dev, dst_dma_list); - aes_a_dma_active_src_dst_ll_en(aes_dev); - } else { - /* Configure and activate DMA for output data only. */ - dma_from_ocs_aes_ll(aes_dev, dst_dma_list); - aes_a_dma_active_dst_ll_en(aes_dev); - } - - /* - * Set the LAST GCX bit in AES_ACTIVE Register to instruct - * AES/SM4 engine to pad the last block of data. - */ - aes_a_set_last_gcx(aes_dev); - - /* We are done, wait for IRQ and return. */ - return ocs_aes_irq_enable_and_wait(aes_dev, AES_COMPLETE_INT); -} - -static int ocs_aes_ccm_decrypt_do_payload(struct ocs_aes_dev *aes_dev, - dma_addr_t dst_dma_list, - dma_addr_t src_dma_list, - u32 src_size) -{ - if (!src_size) { - /* Let engine process 0-length input. */ - aes_a_dma_set_xfer_size_zero(aes_dev); - aes_a_dma_active(aes_dev); - aes_a_set_last_gcx(aes_dev); - - return 0; - } - - /* - * Configure and activate DMA for both input and output - * data. - */ - dma_to_ocs_aes_ll(aes_dev, src_dma_list); - dma_from_ocs_aes_ll(aes_dev, dst_dma_list); - aes_a_dma_active_src_dst_ll_en(aes_dev); - /* - * Set the LAST GCX bit in AES_ACTIVE Register; this allows the - * AES/SM4 engine to differentiate between encrypted data and - * encrypted MAC. - */ - aes_a_set_last_gcx(aes_dev); - /* - * Enable DMA DONE interrupt; once DMA transfer is over, - * interrupt handler will process the MAC/tag. - */ - return ocs_aes_irq_enable_and_wait(aes_dev, AES_DMA_SRC_DONE_INT); -} - -/* - * Compare Tag to Yr. - * - * Only used at the end of CCM decrypt. If tag == yr, message authentication - * has succeeded. - */ -static inline int ccm_compare_tag_to_yr(struct ocs_aes_dev *aes_dev, - u8 tag_size_bytes) -{ - u32 tag[AES_MAX_TAG_SIZE_U32]; - u32 yr[AES_MAX_TAG_SIZE_U32]; - u8 i; - - /* Read Tag and Yr from AES registers. */ - for (i = 0; i < AES_MAX_TAG_SIZE_U32; i++) { - tag[i] = ioread32(aes_dev->base_reg + - AES_T_MAC_0_OFFSET + (i * sizeof(u32))); - yr[i] = ioread32(aes_dev->base_reg + - AES_MULTIPURPOSE2_0_OFFSET + - (i * sizeof(u32))); - } - - return memcmp(tag, yr, tag_size_bytes) ? -EBADMSG : 0; -} - -/** - * ocs_aes_ccm_op() - Perform CCM operation. - * @aes_dev: The OCS AES device to use. - * @cipher: The Cipher to use (AES or SM4). - * @instruction: The instruction to perform (encrypt or decrypt). - * @dst_dma_list: The OCS DMA list mapping output memory. - * @src_dma_list: The OCS DMA list mapping input payload data. - * @src_size: The amount of data mapped by @src_dma_list. - * @iv: The input IV vector. - * @adata_dma_list: The OCS DMA list mapping input A-data. - * @adata_size: The amount of data mapped by @adata_dma_list. - * @in_tag: Input tag. - * @tag_size: The size (in bytes) of @in_tag. - * - * Note: for encrypt the tag is appended to the ciphertext (in the memory - * mapped by @dst_dma_list). - * - * Return: 0 on success, negative error code otherwise. - */ -int ocs_aes_ccm_op(struct ocs_aes_dev *aes_dev, - enum ocs_cipher cipher, - enum ocs_instruction instruction, - dma_addr_t dst_dma_list, - dma_addr_t src_dma_list, - u32 src_size, - u8 *iv, - dma_addr_t adata_dma_list, - u32 adata_size, - u8 *in_tag, - u32 tag_size) -{ - u32 *iv_32; - u8 lprime; - int rc; - - rc = ocs_aes_validate_inputs(src_dma_list, src_size, iv, - AES_BLOCK_SIZE, adata_dma_list, adata_size, - in_tag, tag_size, cipher, OCS_MODE_CCM, - instruction, dst_dma_list); - if (rc) - return rc; - - ocs_aes_init(aes_dev, OCS_MODE_CCM, cipher, instruction); - - /* - * Note: rfc 3610 and NIST 800-38C require counter of zero to encrypt - * auth tag so ensure this is the case - */ - lprime = iv[L_PRIME_IDX]; - memset(&iv[COUNTER_START(lprime)], 0, COUNTER_LEN(lprime)); - - /* - * Nonce is already converted to ctr0 before being passed into this - * function as iv. - */ - iv_32 = (u32 *)iv; - iowrite32(__swab32(iv_32[0]), - aes_dev->base_reg + AES_MULTIPURPOSE1_3_OFFSET); - iowrite32(__swab32(iv_32[1]), - aes_dev->base_reg + AES_MULTIPURPOSE1_2_OFFSET); - iowrite32(__swab32(iv_32[2]), - aes_dev->base_reg + AES_MULTIPURPOSE1_1_OFFSET); - iowrite32(__swab32(iv_32[3]), - aes_dev->base_reg + AES_MULTIPURPOSE1_0_OFFSET); - - /* Write MAC/tag length in register AES_TLEN */ - iowrite32(tag_size, aes_dev->base_reg + AES_TLEN_OFFSET); - /* - * Write the byte length of the last AES/SM4 block of Payload data - * (without zero padding and without the length of the MAC) in register - * AES_PLEN. - */ - ocs_aes_write_last_data_blk_len(aes_dev, src_size); - - /* Set AES_ACTIVE.TRIGGER to start the operation. */ - aes_a_op_trigger(aes_dev); - - aes_a_dma_reset_and_activate_perf_cntr(aes_dev); - - /* Form block B0 and write it to the AES/SM4 input buffer. */ - rc = ocs_aes_ccm_write_b0(aes_dev, iv, adata_size, tag_size, src_size); - if (rc) - return rc; - /* - * Ensure there has been at least CCM_DECRYPT_DELAY_LAST_GCX_CLK_COUNT - * clock cycles since TRIGGER bit was set - */ - aes_a_dma_wait_and_deactivate_perf_cntr(aes_dev, - CCM_DECRYPT_DELAY_LAST_GCX_CLK_COUNT); - - /* Process Adata. */ - ocs_aes_ccm_do_adata(aes_dev, adata_dma_list, adata_size); - - /* For Encrypt case we just process the payload and return. */ - if (instruction == OCS_ENCRYPT) { - return ocs_aes_ccm_encrypt_do_payload(aes_dev, dst_dma_list, - src_dma_list, src_size); - } - /* For Decypt we need to process the payload and then the tag. */ - rc = ocs_aes_ccm_decrypt_do_payload(aes_dev, dst_dma_list, - src_dma_list, src_size); - if (rc) - return rc; - - /* Process MAC/tag directly: feed tag to engine and wait for IRQ. */ - ocs_aes_ccm_write_encrypted_tag(aes_dev, in_tag, tag_size); - rc = ocs_aes_irq_enable_and_wait(aes_dev, AES_COMPLETE_INT); - if (rc) - return rc; - - return ccm_compare_tag_to_yr(aes_dev, tag_size); -} - -/** - * ocs_create_linked_list_from_sg() - Create OCS DMA linked list from SG list. - * @aes_dev: The OCS AES device the list will be created for. - * @sg: The SG list OCS DMA linked list will be created from. When - * passed to this function, @sg must have been already mapped - * with dma_map_sg(). - * @sg_dma_count: The number of DMA-mapped entries in @sg. This must be the - * value returned by dma_map_sg() when @sg was mapped. - * @dll_desc: The OCS DMA dma_list to use to store information about the - * created linked list. - * @data_size: The size of the data (from the SG list) to be mapped into the - * OCS DMA linked list. - * @data_offset: The offset (within the SG list) of the data to be mapped. - * - * Return: 0 on success, negative error code otherwise. - */ -int ocs_create_linked_list_from_sg(const struct ocs_aes_dev *aes_dev, - struct scatterlist *sg, - int sg_dma_count, - struct ocs_dll_desc *dll_desc, - size_t data_size, size_t data_offset) -{ - struct ocs_dma_linked_list *ll = NULL; - struct scatterlist *sg_tmp; - unsigned int tmp; - int dma_nents; - int i; - - if (!dll_desc || !sg || !aes_dev) - return -EINVAL; - - /* Default values for when no ddl_desc is created. */ - dll_desc->vaddr = NULL; - dll_desc->dma_addr = DMA_MAPPING_ERROR; - dll_desc->size = 0; - - if (data_size == 0) - return 0; - - /* Loop over sg_list until we reach entry at specified offset. */ - while (data_offset >= sg_dma_len(sg)) { - data_offset -= sg_dma_len(sg); - sg_dma_count--; - sg = sg_next(sg); - /* If we reach the end of the list, offset was invalid. */ - if (!sg || sg_dma_count == 0) - return -EINVAL; - } - - /* Compute number of DMA-mapped SG entries to add into OCS DMA list. */ - dma_nents = 0; - tmp = 0; - sg_tmp = sg; - while (tmp < data_offset + data_size) { - /* If we reach the end of the list, data_size was invalid. */ - if (!sg_tmp) - return -EINVAL; - tmp += sg_dma_len(sg_tmp); - dma_nents++; - sg_tmp = sg_next(sg_tmp); - } - if (dma_nents > sg_dma_count) - return -EINVAL; - - /* Allocate the DMA list, one entry for each SG entry. */ - dll_desc->size = sizeof(struct ocs_dma_linked_list) * dma_nents; - dll_desc->vaddr = dma_alloc_coherent(aes_dev->dev, dll_desc->size, - &dll_desc->dma_addr, GFP_KERNEL); - if (!dll_desc->vaddr) - return -ENOMEM; - - /* Populate DMA linked list entries. */ - ll = dll_desc->vaddr; - for (i = 0; i < dma_nents; i++, sg = sg_next(sg)) { - ll[i].src_addr = sg_dma_address(sg) + data_offset; - ll[i].src_len = (sg_dma_len(sg) - data_offset) < data_size ? - (sg_dma_len(sg) - data_offset) : data_size; - data_offset = 0; - data_size -= ll[i].src_len; - /* Current element points to the DMA address of the next one. */ - ll[i].next = dll_desc->dma_addr + (sizeof(*ll) * (i + 1)); - ll[i].ll_flags = 0; - } - /* Terminate last element. */ - ll[i - 1].next = 0; - ll[i - 1].ll_flags = OCS_LL_DMA_FLAG_TERMINATE; - - return 0; -} diff --git a/drivers/crypto/keembay/ocs-aes.h b/drivers/crypto/keembay/ocs-aes.h deleted file mode 100644 index c035fc48b7ed..000000000000 --- a/drivers/crypto/keembay/ocs-aes.h +++ /dev/null @@ -1,129 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Intel Keem Bay OCS AES Crypto Driver. - * - * Copyright (C) 2018-2020 Intel Corporation - */ - -#ifndef _CRYPTO_OCS_AES_H -#define _CRYPTO_OCS_AES_H - -#include - -enum ocs_cipher { - OCS_AES = 0, - OCS_SM4 = 1, -}; - -enum ocs_mode { - OCS_MODE_ECB = 0, - OCS_MODE_CBC = 1, - OCS_MODE_CTR = 2, - OCS_MODE_CCM = 6, - OCS_MODE_GCM = 7, - OCS_MODE_CTS = 9, -}; - -enum ocs_instruction { - OCS_ENCRYPT = 0, - OCS_DECRYPT = 1, - OCS_EXPAND = 2, - OCS_BYPASS = 3, -}; - -/** - * struct ocs_aes_dev - AES device context. - * @list: List head for insertion into device list hold - * by driver. - * @dev: OCS AES device. - * @irq: IRQ number. - * @base_reg: IO base address of OCS AES. - * @irq_copy_completion: Completion to indicate IRQ has been triggered. - * @dma_err_mask: Error reported by OCS DMA interrupts. - * @engine: Crypto engine for the device. - */ -struct ocs_aes_dev { - struct list_head list; - struct device *dev; - int irq; - void __iomem *base_reg; - struct completion irq_completion; - u32 dma_err_mask; - struct crypto_engine *engine; -}; - -/** - * struct ocs_dll_desc - Descriptor of an OCS DMA Linked List. - * @vaddr: Virtual address of the linked list head. - * @dma_addr: DMA address of the linked list head. - * @size: Size (in bytes) of the linked list. - */ -struct ocs_dll_desc { - void *vaddr; - dma_addr_t dma_addr; - size_t size; -}; - -int ocs_aes_set_key(struct ocs_aes_dev *aes_dev, const u32 key_size, - const u8 *key, const enum ocs_cipher cipher); - -int ocs_aes_op(struct ocs_aes_dev *aes_dev, - enum ocs_mode mode, - enum ocs_cipher cipher, - enum ocs_instruction instruction, - dma_addr_t dst_dma_list, - dma_addr_t src_dma_list, - u32 src_size, - u8 *iv, - u32 iv_size); - -/** - * ocs_aes_bypass_op() - Use OCS DMA to copy data. - * @aes_dev: The OCS AES device to use. - * @dst_dma_list: The OCS DMA list mapping the memory where input data - * will be copied to. - * @src_dma_list: The OCS DMA list mapping input data. - * @src_size: The amount of data to copy. - */ -static inline int ocs_aes_bypass_op(struct ocs_aes_dev *aes_dev, - dma_addr_t dst_dma_list, - dma_addr_t src_dma_list, u32 src_size) -{ - return ocs_aes_op(aes_dev, OCS_MODE_ECB, OCS_AES, OCS_BYPASS, - dst_dma_list, src_dma_list, src_size, NULL, 0); -} - -int ocs_aes_gcm_op(struct ocs_aes_dev *aes_dev, - enum ocs_cipher cipher, - enum ocs_instruction instruction, - dma_addr_t dst_dma_list, - dma_addr_t src_dma_list, - u32 src_size, - const u8 *iv, - dma_addr_t aad_dma_list, - u32 aad_size, - u8 *out_tag, - u32 tag_size); - -int ocs_aes_ccm_op(struct ocs_aes_dev *aes_dev, - enum ocs_cipher cipher, - enum ocs_instruction instruction, - dma_addr_t dst_dma_list, - dma_addr_t src_dma_list, - u32 src_size, - u8 *iv, - dma_addr_t adata_dma_list, - u32 adata_size, - u8 *in_tag, - u32 tag_size); - -int ocs_create_linked_list_from_sg(const struct ocs_aes_dev *aes_dev, - struct scatterlist *sg, - int sg_dma_count, - struct ocs_dll_desc *dll_desc, - size_t data_size, - size_t data_offset); - -irqreturn_t ocs_aes_irq_handler(int irq, void *dev_id); - -#endif diff --git a/drivers/crypto/keembay/ocs-hcu.c b/drivers/crypto/keembay/ocs-hcu.c deleted file mode 100644 index deb9bd460ee6..000000000000 --- a/drivers/crypto/keembay/ocs-hcu.c +++ /dev/null @@ -1,840 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Intel Keem Bay OCS HCU Crypto Driver. - * - * Copyright (C) 2018-2020 Intel Corporation - */ - -#include -#include -#include -#include -#include - -#include - -#include "ocs-hcu.h" - -/* Registers. */ -#define OCS_HCU_MODE 0x00 -#define OCS_HCU_CHAIN 0x04 -#define OCS_HCU_OPERATION 0x08 -#define OCS_HCU_KEY_0 0x0C -#define OCS_HCU_ISR 0x50 -#define OCS_HCU_IER 0x54 -#define OCS_HCU_STATUS 0x58 -#define OCS_HCU_MSG_LEN_LO 0x60 -#define OCS_HCU_MSG_LEN_HI 0x64 -#define OCS_HCU_KEY_BYTE_ORDER_CFG 0x80 -#define OCS_HCU_DMA_SRC_ADDR 0x400 -#define OCS_HCU_DMA_SRC_SIZE 0x408 -#define OCS_HCU_DMA_DST_SIZE 0x40C -#define OCS_HCU_DMA_DMA_MODE 0x410 -#define OCS_HCU_DMA_NEXT_SRC_DESCR 0x418 -#define OCS_HCU_DMA_MSI_ISR 0x480 -#define OCS_HCU_DMA_MSI_IER 0x484 -#define OCS_HCU_DMA_MSI_MASK 0x488 - -/* Register bit definitions. */ -#define HCU_MODE_ALGO_SHIFT 16 -#define HCU_MODE_HMAC_SHIFT 22 - -#define HCU_STATUS_BUSY BIT(0) - -#define HCU_BYTE_ORDER_SWAP BIT(0) - -#define HCU_IRQ_HASH_DONE BIT(2) -#define HCU_IRQ_HASH_ERR_MASK (BIT(3) | BIT(1) | BIT(0)) - -#define HCU_DMA_IRQ_SRC_DONE BIT(0) -#define HCU_DMA_IRQ_SAI_ERR BIT(2) -#define HCU_DMA_IRQ_BAD_COMP_ERR BIT(3) -#define HCU_DMA_IRQ_INBUF_RD_ERR BIT(4) -#define HCU_DMA_IRQ_INBUF_WD_ERR BIT(5) -#define HCU_DMA_IRQ_OUTBUF_WR_ERR BIT(6) -#define HCU_DMA_IRQ_OUTBUF_RD_ERR BIT(7) -#define HCU_DMA_IRQ_CRD_ERR BIT(8) -#define HCU_DMA_IRQ_ERR_MASK (HCU_DMA_IRQ_SAI_ERR | \ - HCU_DMA_IRQ_BAD_COMP_ERR | \ - HCU_DMA_IRQ_INBUF_RD_ERR | \ - HCU_DMA_IRQ_INBUF_WD_ERR | \ - HCU_DMA_IRQ_OUTBUF_WR_ERR | \ - HCU_DMA_IRQ_OUTBUF_RD_ERR | \ - HCU_DMA_IRQ_CRD_ERR) - -#define HCU_DMA_SNOOP_MASK (0x7 << 28) -#define HCU_DMA_SRC_LL_EN BIT(25) -#define HCU_DMA_EN BIT(31) - -#define OCS_HCU_ENDIANNESS_VALUE 0x2A - -#define HCU_DMA_MSI_UNMASK BIT(0) -#define HCU_DMA_MSI_DISABLE 0 -#define HCU_IRQ_DISABLE 0 - -#define OCS_HCU_START BIT(0) -#define OCS_HCU_TERMINATE BIT(1) - -#define OCS_LL_DMA_FLAG_TERMINATE BIT(31) - -#define OCS_HCU_HW_KEY_LEN_U32 (OCS_HCU_HW_KEY_LEN / sizeof(u32)) - -#define HCU_DATA_WRITE_ENDIANNESS_OFFSET 26 - -#define OCS_HCU_NUM_CHAINS_SHA256_224_SM3 (SHA256_DIGEST_SIZE / sizeof(u32)) -#define OCS_HCU_NUM_CHAINS_SHA384_512 (SHA512_DIGEST_SIZE / sizeof(u32)) - -/* - * While polling on a busy HCU, wait maximum 200us between one check and the - * other. - */ -#define OCS_HCU_WAIT_BUSY_RETRY_DELAY_US 200 -/* Wait on a busy HCU for maximum 1 second. */ -#define OCS_HCU_WAIT_BUSY_TIMEOUT_US 1000000 - -/** - * struct ocs_hcu_dma_entry - An entry in an OCS DMA linked list. - * @src_addr: Source address of the data. - * @src_len: Length of data to be fetched. - * @nxt_desc: Next descriptor to fetch. - * @ll_flags: Flags (Freeze @ terminate) for the DMA engine. - */ -struct ocs_hcu_dma_entry { - u32 src_addr; - u32 src_len; - u32 nxt_desc; - u32 ll_flags; -}; - -/** - * struct ocs_hcu_dma_list - OCS-specific DMA linked list. - * @head: The head of the list (points to the array backing the list). - * @tail: The current tail of the list; NULL if the list is empty. - * @dma_addr: The DMA address of @head (i.e., the DMA address of the backing - * array). - * @max_nents: Maximum number of entries in the list (i.e., number of elements - * in the backing array). - * - * The OCS DMA list is an array-backed list of OCS DMA descriptors. The array - * backing the list is allocated with dma_alloc_coherent() and pointed by - * @head. - */ -struct ocs_hcu_dma_list { - struct ocs_hcu_dma_entry *head; - struct ocs_hcu_dma_entry *tail; - dma_addr_t dma_addr; - size_t max_nents; -}; - -static inline u32 ocs_hcu_num_chains(enum ocs_hcu_algo algo) -{ - switch (algo) { - case OCS_HCU_ALGO_SHA224: - case OCS_HCU_ALGO_SHA256: - case OCS_HCU_ALGO_SM3: - return OCS_HCU_NUM_CHAINS_SHA256_224_SM3; - case OCS_HCU_ALGO_SHA384: - case OCS_HCU_ALGO_SHA512: - return OCS_HCU_NUM_CHAINS_SHA384_512; - default: - return 0; - }; -} - -static inline u32 ocs_hcu_digest_size(enum ocs_hcu_algo algo) -{ - switch (algo) { - case OCS_HCU_ALGO_SHA224: - return SHA224_DIGEST_SIZE; - case OCS_HCU_ALGO_SHA256: - case OCS_HCU_ALGO_SM3: - /* SM3 shares the same block size. */ - return SHA256_DIGEST_SIZE; - case OCS_HCU_ALGO_SHA384: - return SHA384_DIGEST_SIZE; - case OCS_HCU_ALGO_SHA512: - return SHA512_DIGEST_SIZE; - default: - return 0; - } -} - -/** - * ocs_hcu_wait_busy() - Wait for HCU OCS hardware to became usable. - * @hcu_dev: OCS HCU device to wait for. - * - * Return: 0 if device free, -ETIMEOUT if device busy and internal timeout has - * expired. - */ -static int ocs_hcu_wait_busy(struct ocs_hcu_dev *hcu_dev) -{ - long val; - - return readl_poll_timeout(hcu_dev->io_base + OCS_HCU_STATUS, val, - !(val & HCU_STATUS_BUSY), - OCS_HCU_WAIT_BUSY_RETRY_DELAY_US, - OCS_HCU_WAIT_BUSY_TIMEOUT_US); -} - -static void ocs_hcu_done_irq_en(struct ocs_hcu_dev *hcu_dev) -{ - /* Clear any pending interrupts. */ - writel(0xFFFFFFFF, hcu_dev->io_base + OCS_HCU_ISR); - hcu_dev->irq_err = false; - /* Enable error and HCU done interrupts. */ - writel(HCU_IRQ_HASH_DONE | HCU_IRQ_HASH_ERR_MASK, - hcu_dev->io_base + OCS_HCU_IER); -} - -static void ocs_hcu_dma_irq_en(struct ocs_hcu_dev *hcu_dev) -{ - /* Clear any pending interrupts. */ - writel(0xFFFFFFFF, hcu_dev->io_base + OCS_HCU_DMA_MSI_ISR); - hcu_dev->irq_err = false; - /* Only operating on DMA source completion and error interrupts. */ - writel(HCU_DMA_IRQ_ERR_MASK | HCU_DMA_IRQ_SRC_DONE, - hcu_dev->io_base + OCS_HCU_DMA_MSI_IER); - /* Unmask */ - writel(HCU_DMA_MSI_UNMASK, hcu_dev->io_base + OCS_HCU_DMA_MSI_MASK); -} - -static void ocs_hcu_irq_dis(struct ocs_hcu_dev *hcu_dev) -{ - writel(HCU_IRQ_DISABLE, hcu_dev->io_base + OCS_HCU_IER); - writel(HCU_DMA_MSI_DISABLE, hcu_dev->io_base + OCS_HCU_DMA_MSI_IER); -} - -static int ocs_hcu_wait_and_disable_irq(struct ocs_hcu_dev *hcu_dev) -{ - int rc; - - rc = wait_for_completion_interruptible(&hcu_dev->irq_done); - if (rc) - goto exit; - - if (hcu_dev->irq_err) { - /* Unset flag and return error. */ - hcu_dev->irq_err = false; - rc = -EIO; - goto exit; - } - -exit: - ocs_hcu_irq_dis(hcu_dev); - - return rc; -} - -/** - * ocs_hcu_get_intermediate_data() - Get intermediate data. - * @hcu_dev: The target HCU device. - * @data: Where to store the intermediate. - * @algo: The algorithm being used. - * - * This function is used to save the current hashing process state in order to - * continue it in the future. - * - * Note: once all data has been processed, the intermediate data actually - * contains the hashing result. So this function is also used to retrieve the - * final result of a hashing process. - * - * Return: 0 on success, negative error code otherwise. - */ -static int ocs_hcu_get_intermediate_data(struct ocs_hcu_dev *hcu_dev, - struct ocs_hcu_idata *data, - enum ocs_hcu_algo algo) -{ - const int n = ocs_hcu_num_chains(algo); - u32 *chain; - int rc; - int i; - - /* Data not requested. */ - if (!data) - return -EINVAL; - - chain = (u32 *)data->digest; - - /* Ensure that the OCS is no longer busy before reading the chains. */ - rc = ocs_hcu_wait_busy(hcu_dev); - if (rc) - return rc; - - /* - * This loops is safe because data->digest is an array of - * SHA512_DIGEST_SIZE bytes and the maximum value returned by - * ocs_hcu_num_chains() is OCS_HCU_NUM_CHAINS_SHA384_512 which is equal - * to SHA512_DIGEST_SIZE / sizeof(u32). - */ - for (i = 0; i < n; i++) - chain[i] = readl(hcu_dev->io_base + OCS_HCU_CHAIN); - - data->msg_len_lo = readl(hcu_dev->io_base + OCS_HCU_MSG_LEN_LO); - data->msg_len_hi = readl(hcu_dev->io_base + OCS_HCU_MSG_LEN_HI); - - return 0; -} - -/** - * ocs_hcu_set_intermediate_data() - Set intermediate data. - * @hcu_dev: The target HCU device. - * @data: The intermediate data to be set. - * @algo: The algorithm being used. - * - * This function is used to continue a previous hashing process. - */ -static void ocs_hcu_set_intermediate_data(struct ocs_hcu_dev *hcu_dev, - const struct ocs_hcu_idata *data, - enum ocs_hcu_algo algo) -{ - const int n = ocs_hcu_num_chains(algo); - u32 *chain = (u32 *)data->digest; - int i; - - /* - * This loops is safe because data->digest is an array of - * SHA512_DIGEST_SIZE bytes and the maximum value returned by - * ocs_hcu_num_chains() is OCS_HCU_NUM_CHAINS_SHA384_512 which is equal - * to SHA512_DIGEST_SIZE / sizeof(u32). - */ - for (i = 0; i < n; i++) - writel(chain[i], hcu_dev->io_base + OCS_HCU_CHAIN); - - writel(data->msg_len_lo, hcu_dev->io_base + OCS_HCU_MSG_LEN_LO); - writel(data->msg_len_hi, hcu_dev->io_base + OCS_HCU_MSG_LEN_HI); -} - -static int ocs_hcu_get_digest(struct ocs_hcu_dev *hcu_dev, - enum ocs_hcu_algo algo, u8 *dgst, size_t dgst_len) -{ - u32 *chain; - int rc; - int i; - - if (!dgst) - return -EINVAL; - - /* Length of the output buffer must match the algo digest size. */ - if (dgst_len != ocs_hcu_digest_size(algo)) - return -EINVAL; - - /* Ensure that the OCS is no longer busy before reading the chains. */ - rc = ocs_hcu_wait_busy(hcu_dev); - if (rc) - return rc; - - chain = (u32 *)dgst; - for (i = 0; i < dgst_len / sizeof(u32); i++) - chain[i] = readl(hcu_dev->io_base + OCS_HCU_CHAIN); - - return 0; -} - -/** - * ocs_hcu_hw_cfg() - Configure the HCU hardware. - * @hcu_dev: The HCU device to configure. - * @algo: The algorithm to be used by the HCU device. - * @use_hmac: Whether or not HW HMAC should be used. - * - * Return: 0 on success, negative error code otherwise. - */ -static int ocs_hcu_hw_cfg(struct ocs_hcu_dev *hcu_dev, enum ocs_hcu_algo algo, - bool use_hmac) -{ - u32 cfg; - int rc; - - if (algo != OCS_HCU_ALGO_SHA256 && algo != OCS_HCU_ALGO_SHA224 && - algo != OCS_HCU_ALGO_SHA384 && algo != OCS_HCU_ALGO_SHA512 && - algo != OCS_HCU_ALGO_SM3) - return -EINVAL; - - rc = ocs_hcu_wait_busy(hcu_dev); - if (rc) - return rc; - - /* Ensure interrupts are disabled. */ - ocs_hcu_irq_dis(hcu_dev); - - /* Configure endianness, hashing algorithm and HW HMAC (if needed) */ - cfg = OCS_HCU_ENDIANNESS_VALUE << HCU_DATA_WRITE_ENDIANNESS_OFFSET; - cfg |= algo << HCU_MODE_ALGO_SHIFT; - if (use_hmac) - cfg |= BIT(HCU_MODE_HMAC_SHIFT); - - writel(cfg, hcu_dev->io_base + OCS_HCU_MODE); - - return 0; -} - -/** - * ocs_hcu_clear_key() - Clear key stored in OCS HMAC KEY registers. - * @hcu_dev: The OCS HCU device whose key registers should be cleared. - */ -static void ocs_hcu_clear_key(struct ocs_hcu_dev *hcu_dev) -{ - int reg_off; - - /* Clear OCS_HCU_KEY_[0..15] */ - for (reg_off = 0; reg_off < OCS_HCU_HW_KEY_LEN; reg_off += sizeof(u32)) - writel(0, hcu_dev->io_base + OCS_HCU_KEY_0 + reg_off); -} - -/** - * ocs_hcu_write_key() - Write key to OCS HMAC KEY registers. - * @hcu_dev: The OCS HCU device the key should be written to. - * @key: The key to be written. - * @len: The size of the key to write. It must be OCS_HCU_HW_KEY_LEN. - * - * Return: 0 on success, negative error code otherwise. - */ -static int ocs_hcu_write_key(struct ocs_hcu_dev *hcu_dev, const u8 *key, size_t len) -{ - u32 key_u32[OCS_HCU_HW_KEY_LEN_U32]; - int i; - - if (len > OCS_HCU_HW_KEY_LEN) - return -EINVAL; - - /* Copy key into temporary u32 array. */ - memcpy(key_u32, key, len); - - /* - * Hardware requires all the bytes of the HW Key vector to be - * written. So pad with zero until we reach OCS_HCU_HW_KEY_LEN. - */ - memzero_explicit((u8 *)key_u32 + len, OCS_HCU_HW_KEY_LEN - len); - - /* - * OCS hardware expects the MSB of the key to be written at the highest - * address of the HCU Key vector; in other word, the key must be - * written in reverse order. - * - * Therefore, we first enable byte swapping for the HCU key vector; - * so that bytes of 32-bit word written to OCS_HCU_KEY_[0..15] will be - * swapped: - * 3 <---> 0, 2 <---> 1. - */ - writel(HCU_BYTE_ORDER_SWAP, - hcu_dev->io_base + OCS_HCU_KEY_BYTE_ORDER_CFG); - /* - * And then we write the 32-bit words composing the key starting from - * the end of the key. - */ - for (i = 0; i < OCS_HCU_HW_KEY_LEN_U32; i++) - writel(key_u32[OCS_HCU_HW_KEY_LEN_U32 - 1 - i], - hcu_dev->io_base + OCS_HCU_KEY_0 + (sizeof(u32) * i)); - - memzero_explicit(key_u32, OCS_HCU_HW_KEY_LEN); - - return 0; -} - -/** - * ocs_hcu_ll_dma_start() - Start OCS HCU hashing via DMA - * @hcu_dev: The OCS HCU device to use. - * @dma_list: The OCS DMA list mapping the data to hash. - * @finalize: Whether or not this is the last hashing operation and therefore - * the final hash should be compute even if data is not - * block-aligned. - * - * Return: 0 on success, negative error code otherwise. - */ -static int ocs_hcu_ll_dma_start(struct ocs_hcu_dev *hcu_dev, - const struct ocs_hcu_dma_list *dma_list, - bool finalize) -{ - u32 cfg = HCU_DMA_SNOOP_MASK | HCU_DMA_SRC_LL_EN | HCU_DMA_EN; - int rc; - - if (!dma_list) - return -EINVAL; - - /* - * For final requests we use HCU_DONE IRQ to be notified when all input - * data has been processed by the HCU; however, we cannot do so for - * non-final requests, because we don't get a HCU_DONE IRQ when we - * don't terminate the operation. - * - * Therefore, for non-final requests, we use the DMA IRQ, which - * triggers when DMA has finishing feeding all the input data to the - * HCU, but the HCU may still be processing it. This is fine, since we - * will wait for the HCU processing to be completed when we try to read - * intermediate results, in ocs_hcu_get_intermediate_data(). - */ - if (finalize) - ocs_hcu_done_irq_en(hcu_dev); - else - ocs_hcu_dma_irq_en(hcu_dev); - - reinit_completion(&hcu_dev->irq_done); - writel(dma_list->dma_addr, hcu_dev->io_base + OCS_HCU_DMA_NEXT_SRC_DESCR); - writel(0, hcu_dev->io_base + OCS_HCU_DMA_SRC_SIZE); - writel(0, hcu_dev->io_base + OCS_HCU_DMA_DST_SIZE); - - writel(OCS_HCU_START, hcu_dev->io_base + OCS_HCU_OPERATION); - - writel(cfg, hcu_dev->io_base + OCS_HCU_DMA_DMA_MODE); - - if (finalize) - writel(OCS_HCU_TERMINATE, hcu_dev->io_base + OCS_HCU_OPERATION); - - rc = ocs_hcu_wait_and_disable_irq(hcu_dev); - if (rc) - return rc; - - return 0; -} - -struct ocs_hcu_dma_list *ocs_hcu_dma_list_alloc(struct ocs_hcu_dev *hcu_dev, - int max_nents) -{ - struct ocs_hcu_dma_list *dma_list; - - dma_list = kmalloc(sizeof(*dma_list), GFP_KERNEL); - if (!dma_list) - return NULL; - - /* Total size of the DMA list to allocate. */ - dma_list->head = dma_alloc_coherent(hcu_dev->dev, - sizeof(*dma_list->head) * max_nents, - &dma_list->dma_addr, GFP_KERNEL); - if (!dma_list->head) { - kfree(dma_list); - return NULL; - } - dma_list->max_nents = max_nents; - dma_list->tail = NULL; - - return dma_list; -} - -void ocs_hcu_dma_list_free(struct ocs_hcu_dev *hcu_dev, - struct ocs_hcu_dma_list *dma_list) -{ - if (!dma_list) - return; - - dma_free_coherent(hcu_dev->dev, - sizeof(*dma_list->head) * dma_list->max_nents, - dma_list->head, dma_list->dma_addr); - - kfree(dma_list); -} - -/* Add a new DMA entry at the end of the OCS DMA list. */ -int ocs_hcu_dma_list_add_tail(struct ocs_hcu_dev *hcu_dev, - struct ocs_hcu_dma_list *dma_list, - dma_addr_t addr, u32 len) -{ - struct device *dev = hcu_dev->dev; - struct ocs_hcu_dma_entry *old_tail; - struct ocs_hcu_dma_entry *new_tail; - - if (!len) - return 0; - - if (!dma_list) - return -EINVAL; - - if (addr & ~OCS_HCU_DMA_BIT_MASK) { - dev_err(dev, - "Unexpected error: Invalid DMA address for OCS HCU\n"); - return -EINVAL; - } - - old_tail = dma_list->tail; - new_tail = old_tail ? old_tail + 1 : dma_list->head; - - /* Check if list is full. */ - if (new_tail - dma_list->head >= dma_list->max_nents) - return -ENOMEM; - - /* - * If there was an old tail (i.e., this is not the first element we are - * adding), un-terminate the old tail and make it point to the new one. - */ - if (old_tail) { - old_tail->ll_flags &= ~OCS_LL_DMA_FLAG_TERMINATE; - /* - * The old tail 'nxt_desc' must point to the DMA address of the - * new tail. - */ - old_tail->nxt_desc = dma_list->dma_addr + - sizeof(*dma_list->tail) * (new_tail - - dma_list->head); - } - - new_tail->src_addr = (u32)addr; - new_tail->src_len = (u32)len; - new_tail->ll_flags = OCS_LL_DMA_FLAG_TERMINATE; - new_tail->nxt_desc = 0; - - /* Update list tail with new tail. */ - dma_list->tail = new_tail; - - return 0; -} - -/** - * ocs_hcu_hash_init() - Initialize hash operation context. - * @ctx: The context to initialize. - * @algo: The hashing algorithm to use. - * - * Return: 0 on success, negative error code otherwise. - */ -int ocs_hcu_hash_init(struct ocs_hcu_hash_ctx *ctx, enum ocs_hcu_algo algo) -{ - if (!ctx) - return -EINVAL; - - ctx->algo = algo; - ctx->idata.msg_len_lo = 0; - ctx->idata.msg_len_hi = 0; - /* No need to set idata.digest to 0. */ - - return 0; -} - -/** - * ocs_hcu_hash_update() - Perform a hashing iteration. - * @hcu_dev: The OCS HCU device to use. - * @ctx: The OCS HCU hashing context. - * @dma_list: The OCS DMA list mapping the input data to process. - * - * Return: 0 on success; negative error code otherwise. - */ -int ocs_hcu_hash_update(struct ocs_hcu_dev *hcu_dev, - struct ocs_hcu_hash_ctx *ctx, - const struct ocs_hcu_dma_list *dma_list) -{ - int rc; - - if (!hcu_dev || !ctx) - return -EINVAL; - - /* Configure the hardware for the current request. */ - rc = ocs_hcu_hw_cfg(hcu_dev, ctx->algo, false); - if (rc) - return rc; - - /* If we already processed some data, idata needs to be set. */ - if (ctx->idata.msg_len_lo || ctx->idata.msg_len_hi) - ocs_hcu_set_intermediate_data(hcu_dev, &ctx->idata, ctx->algo); - - /* Start linked-list DMA hashing. */ - rc = ocs_hcu_ll_dma_start(hcu_dev, dma_list, false); - if (rc) - return rc; - - /* Update idata and return. */ - return ocs_hcu_get_intermediate_data(hcu_dev, &ctx->idata, ctx->algo); -} - -/** - * ocs_hcu_hash_finup() - Update and finalize hash computation. - * @hcu_dev: The OCS HCU device to use. - * @ctx: The OCS HCU hashing context. - * @dma_list: The OCS DMA list mapping the input data to process. - * @dgst: The buffer where to save the computed digest. - * @dgst_len: The length of @dgst. - * - * Return: 0 on success; negative error code otherwise. - */ -int ocs_hcu_hash_finup(struct ocs_hcu_dev *hcu_dev, - const struct ocs_hcu_hash_ctx *ctx, - const struct ocs_hcu_dma_list *dma_list, - u8 *dgst, size_t dgst_len) -{ - int rc; - - if (!hcu_dev || !ctx) - return -EINVAL; - - /* Configure the hardware for the current request. */ - rc = ocs_hcu_hw_cfg(hcu_dev, ctx->algo, false); - if (rc) - return rc; - - /* If we already processed some data, idata needs to be set. */ - if (ctx->idata.msg_len_lo || ctx->idata.msg_len_hi) - ocs_hcu_set_intermediate_data(hcu_dev, &ctx->idata, ctx->algo); - - /* Start linked-list DMA hashing. */ - rc = ocs_hcu_ll_dma_start(hcu_dev, dma_list, true); - if (rc) - return rc; - - /* Get digest and return. */ - return ocs_hcu_get_digest(hcu_dev, ctx->algo, dgst, dgst_len); -} - -/** - * ocs_hcu_hash_final() - Finalize hash computation. - * @hcu_dev: The OCS HCU device to use. - * @ctx: The OCS HCU hashing context. - * @dgst: The buffer where to save the computed digest. - * @dgst_len: The length of @dgst. - * - * Return: 0 on success; negative error code otherwise. - */ -int ocs_hcu_hash_final(struct ocs_hcu_dev *hcu_dev, - const struct ocs_hcu_hash_ctx *ctx, u8 *dgst, - size_t dgst_len) -{ - int rc; - - if (!hcu_dev || !ctx) - return -EINVAL; - - /* Configure the hardware for the current request. */ - rc = ocs_hcu_hw_cfg(hcu_dev, ctx->algo, false); - if (rc) - return rc; - - /* If we already processed some data, idata needs to be set. */ - if (ctx->idata.msg_len_lo || ctx->idata.msg_len_hi) - ocs_hcu_set_intermediate_data(hcu_dev, &ctx->idata, ctx->algo); - - /* - * Enable HCU interrupts, so that HCU_DONE will be triggered once the - * final hash is computed. - */ - ocs_hcu_done_irq_en(hcu_dev); - reinit_completion(&hcu_dev->irq_done); - writel(OCS_HCU_TERMINATE, hcu_dev->io_base + OCS_HCU_OPERATION); - - rc = ocs_hcu_wait_and_disable_irq(hcu_dev); - if (rc) - return rc; - - /* Get digest and return. */ - return ocs_hcu_get_digest(hcu_dev, ctx->algo, dgst, dgst_len); -} - -/** - * ocs_hcu_digest() - Compute hash digest. - * @hcu_dev: The OCS HCU device to use. - * @algo: The hash algorithm to use. - * @data: The input data to process. - * @data_len: The length of @data. - * @dgst: The buffer where to save the computed digest. - * @dgst_len: The length of @dgst. - * - * Return: 0 on success; negative error code otherwise. - */ -int ocs_hcu_digest(struct ocs_hcu_dev *hcu_dev, enum ocs_hcu_algo algo, - void *data, size_t data_len, u8 *dgst, size_t dgst_len) -{ - struct device *dev = hcu_dev->dev; - dma_addr_t dma_handle; - u32 reg; - int rc; - - /* Configure the hardware for the current request. */ - rc = ocs_hcu_hw_cfg(hcu_dev, algo, false); - if (rc) - return rc; - - dma_handle = dma_map_single(dev, data, data_len, DMA_TO_DEVICE); - if (dma_mapping_error(dev, dma_handle)) - return -EIO; - - reg = HCU_DMA_SNOOP_MASK | HCU_DMA_EN; - - ocs_hcu_done_irq_en(hcu_dev); - - reinit_completion(&hcu_dev->irq_done); - - writel(dma_handle, hcu_dev->io_base + OCS_HCU_DMA_SRC_ADDR); - writel(data_len, hcu_dev->io_base + OCS_HCU_DMA_SRC_SIZE); - writel(OCS_HCU_START, hcu_dev->io_base + OCS_HCU_OPERATION); - writel(reg, hcu_dev->io_base + OCS_HCU_DMA_DMA_MODE); - - writel(OCS_HCU_TERMINATE, hcu_dev->io_base + OCS_HCU_OPERATION); - - rc = ocs_hcu_wait_and_disable_irq(hcu_dev); - if (rc) - return rc; - - dma_unmap_single(dev, dma_handle, data_len, DMA_TO_DEVICE); - - return ocs_hcu_get_digest(hcu_dev, algo, dgst, dgst_len); -} - -/** - * ocs_hcu_hmac() - Compute HMAC. - * @hcu_dev: The OCS HCU device to use. - * @algo: The hash algorithm to use with HMAC. - * @key: The key to use. - * @dma_list: The OCS DMA list mapping the input data to process. - * @key_len: The length of @key. - * @dgst: The buffer where to save the computed HMAC. - * @dgst_len: The length of @dgst. - * - * Return: 0 on success; negative error code otherwise. - */ -int ocs_hcu_hmac(struct ocs_hcu_dev *hcu_dev, enum ocs_hcu_algo algo, - const u8 *key, size_t key_len, - const struct ocs_hcu_dma_list *dma_list, - u8 *dgst, size_t dgst_len) -{ - int rc; - - /* Ensure 'key' is not NULL. */ - if (!key || key_len == 0) - return -EINVAL; - - /* Configure the hardware for the current request. */ - rc = ocs_hcu_hw_cfg(hcu_dev, algo, true); - if (rc) - return rc; - - rc = ocs_hcu_write_key(hcu_dev, key, key_len); - if (rc) - return rc; - - rc = ocs_hcu_ll_dma_start(hcu_dev, dma_list, true); - - /* Clear HW key before processing return code. */ - ocs_hcu_clear_key(hcu_dev); - - if (rc) - return rc; - - return ocs_hcu_get_digest(hcu_dev, algo, dgst, dgst_len); -} - -irqreturn_t ocs_hcu_irq_handler(int irq, void *dev_id) -{ - struct ocs_hcu_dev *hcu_dev = dev_id; - u32 hcu_irq; - u32 dma_irq; - - /* Read and clear the HCU interrupt. */ - hcu_irq = readl(hcu_dev->io_base + OCS_HCU_ISR); - writel(hcu_irq, hcu_dev->io_base + OCS_HCU_ISR); - - /* Read and clear the HCU DMA interrupt. */ - dma_irq = readl(hcu_dev->io_base + OCS_HCU_DMA_MSI_ISR); - writel(dma_irq, hcu_dev->io_base + OCS_HCU_DMA_MSI_ISR); - - /* Check for errors. */ - if (hcu_irq & HCU_IRQ_HASH_ERR_MASK || dma_irq & HCU_DMA_IRQ_ERR_MASK) { - hcu_dev->irq_err = true; - goto complete; - } - - /* Check for DONE IRQs. */ - if (hcu_irq & HCU_IRQ_HASH_DONE || dma_irq & HCU_DMA_IRQ_SRC_DONE) - goto complete; - - return IRQ_NONE; - -complete: - complete(&hcu_dev->irq_done); - - return IRQ_HANDLED; -} - -MODULE_LICENSE("GPL"); diff --git a/drivers/crypto/keembay/ocs-hcu.h b/drivers/crypto/keembay/ocs-hcu.h deleted file mode 100644 index fbbbb92a0592..000000000000 --- a/drivers/crypto/keembay/ocs-hcu.h +++ /dev/null @@ -1,106 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Intel Keem Bay OCS HCU Crypto Driver. - * - * Copyright (C) 2018-2020 Intel Corporation - */ - -#include - -#ifndef _CRYPTO_OCS_HCU_H -#define _CRYPTO_OCS_HCU_H - -#define OCS_HCU_DMA_BIT_MASK DMA_BIT_MASK(32) - -#define OCS_HCU_HW_KEY_LEN 64 - -struct ocs_hcu_dma_list; - -enum ocs_hcu_algo { - OCS_HCU_ALGO_SHA256 = 2, - OCS_HCU_ALGO_SHA224 = 3, - OCS_HCU_ALGO_SHA384 = 4, - OCS_HCU_ALGO_SHA512 = 5, - OCS_HCU_ALGO_SM3 = 6, -}; - -/** - * struct ocs_hcu_dev - OCS HCU device context. - * @list: List of device contexts. - * @dev: OCS HCU device. - * @io_base: Base address of OCS HCU registers. - * @engine: Crypto engine for the device. - * @irq: IRQ number. - * @irq_done: Completion for IRQ. - * @irq_err: Flag indicating an IRQ error has happened. - */ -struct ocs_hcu_dev { - struct list_head list; - struct device *dev; - void __iomem *io_base; - struct crypto_engine *engine; - int irq; - struct completion irq_done; - bool irq_err; -}; - -/** - * struct ocs_hcu_idata - Intermediate data generated by the HCU. - * @msg_len_lo: Length of data the HCU has operated on in bits, low 32b. - * @msg_len_hi: Length of data the HCU has operated on in bits, high 32b. - * @digest: The digest read from the HCU. If the HCU is terminated, it will - * contain the actual hash digest. Otherwise it is the intermediate - * state. - */ -struct ocs_hcu_idata { - u32 msg_len_lo; - u32 msg_len_hi; - u8 digest[SHA512_DIGEST_SIZE]; -}; - -/** - * struct ocs_hcu_hash_ctx - Context for OCS HCU hashing operation. - * @algo: The hashing algorithm being used. - * @idata: The current intermediate data. - */ -struct ocs_hcu_hash_ctx { - enum ocs_hcu_algo algo; - struct ocs_hcu_idata idata; -}; - -irqreturn_t ocs_hcu_irq_handler(int irq, void *dev_id); - -struct ocs_hcu_dma_list *ocs_hcu_dma_list_alloc(struct ocs_hcu_dev *hcu_dev, - int max_nents); - -void ocs_hcu_dma_list_free(struct ocs_hcu_dev *hcu_dev, - struct ocs_hcu_dma_list *dma_list); - -int ocs_hcu_dma_list_add_tail(struct ocs_hcu_dev *hcu_dev, - struct ocs_hcu_dma_list *dma_list, - dma_addr_t addr, u32 len); - -int ocs_hcu_hash_init(struct ocs_hcu_hash_ctx *ctx, enum ocs_hcu_algo algo); - -int ocs_hcu_hash_update(struct ocs_hcu_dev *hcu_dev, - struct ocs_hcu_hash_ctx *ctx, - const struct ocs_hcu_dma_list *dma_list); - -int ocs_hcu_hash_finup(struct ocs_hcu_dev *hcu_dev, - const struct ocs_hcu_hash_ctx *ctx, - const struct ocs_hcu_dma_list *dma_list, - u8 *dgst, size_t dgst_len); - -int ocs_hcu_hash_final(struct ocs_hcu_dev *hcu_dev, - const struct ocs_hcu_hash_ctx *ctx, u8 *dgst, - size_t dgst_len); - -int ocs_hcu_digest(struct ocs_hcu_dev *hcu_dev, enum ocs_hcu_algo algo, - void *data, size_t data_len, u8 *dgst, size_t dgst_len); - -int ocs_hcu_hmac(struct ocs_hcu_dev *hcu_dev, enum ocs_hcu_algo algo, - const u8 *key, size_t key_len, - const struct ocs_hcu_dma_list *dma_list, - u8 *dgst, size_t dgst_len); - -#endif /* _CRYPTO_OCS_HCU_H */ -- cgit v1.2.1