diff --git a/arch/arm64/configs/hulk_defconfig b/arch/arm64/configs/hulk_defconfig index 92afa4e0b9a07f1f93a654e9c02ea5ecb2a8ca02..333a4e7c187f559fedb92bbea2e70460b166cea7 100644 --- a/arch/arm64/configs/hulk_defconfig +++ b/arch/arm64/configs/hulk_defconfig @@ -5271,6 +5271,7 @@ CONFIG_CRYPTO_DEV_HISI_QM=m CONFIG_CRYPTO_QM_UACCE=y CONFIG_CRYPTO_DEV_HISI_ZIP=m CONFIG_CRYPTO_DEV_HISI_HPRE=m +CONFIG_CRYPTO_HISI_SGL=m CONFIG_CRYPTO_DEV_HISI_SEC2=m CONFIG_CRYPTO_DEV_HISI_RDE=m CONFIG_ASYMMETRIC_KEY_TYPE=y diff --git a/arch/arm64/configs/syzkaller_defconfig b/arch/arm64/configs/syzkaller_defconfig index d885b3901d4f93d0beb2ade16c569dc40b9509f5..ce2fc26cb3d53278edca954a2a9751735984b1a4 100644 --- a/arch/arm64/configs/syzkaller_defconfig +++ b/arch/arm64/configs/syzkaller_defconfig @@ -5248,6 +5248,7 @@ CONFIG_CRYPTO_DEV_HISI_QM=m CONFIG_CRYPTO_QM_UACCE=y CONFIG_CRYPTO_DEV_HISI_ZIP=m CONFIG_CRYPTO_DEV_HISI_HPRE=m +CONFIG_CRYPTO_HISI_SGL=m CONFIG_CRYPTO_DEV_HISI_SEC2=m CONFIG_CRYPTO_DEV_HISI_RDE=m CONFIG_ASYMMETRIC_KEY_TYPE=y diff --git a/drivers/crypto/hisilicon/Kconfig b/drivers/crypto/hisilicon/Kconfig index 841eea7b7e5bfe7d1605afc5b9bee8c52a97943f..faa57a5220f7fe89cffb68301ae67e1a57077876 100644 --- a/drivers/crypto/hisilicon/Kconfig +++ b/drivers/crypto/hisilicon/Kconfig @@ -17,6 +17,10 @@ config CRYPTO_DEV_HISI_QM tristate depends on ARM64 && PCI +config CRYPTO_HISI_SGL + tristate + depends on ARM64 + config CRYPTO_QM_UACCE bool "enable UACCE support for all acceleartor with Hisi QM" depends on CRYPTO_DEV_HISI_QM @@ -42,6 +46,7 @@ config CRYPTO_DEV_HISI_SEC2 tristate "Support for HISI SEC Driver" depends on ARM64 select CRYPTO_DEV_HISI_QM + select CRYPTO_HISI_SGL select CRYPTO_BLKCIPHER select CRYPTO_ALGAPI help diff --git a/drivers/crypto/hisilicon/Makefile b/drivers/crypto/hisilicon/Makefile index adf4bfbe4bae6343fc5ca55665d9a57df8abb939..7952ab2605cd946aa31401fc32326bf9927666c4 100644 --- a/drivers/crypto/hisilicon/Makefile +++ b/drivers/crypto/hisilicon/Makefile @@ -1,6 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_CRYPTO_DEV_HISI_SEC) += sec/ obj-$(CONFIG_CRYPTO_DEV_HISI_QM) += qm.o +obj-$(CONFIG_CRYPTO_HISI_SGL) += sgl.o obj-$(CONFIG_CRYPTO_DEV_HISI_ZIP) += zip/ obj-$(CONFIG_CRYPTO_DEV_HISI_HPRE) += hpre/ obj-$(CONFIG_CRYPTO_DEV_HISI_SEC2) += sec2/ diff --git a/drivers/crypto/hisilicon/sec2/Makefile b/drivers/crypto/hisilicon/sec2/Makefile index 04190157d41bf06b41cb88607f878331f5f9a0e4..b4f6cf14be3a4f9e76e3d057ac833f7f3e7c6128 100644 --- a/drivers/crypto/hisilicon/sec2/Makefile +++ b/drivers/crypto/hisilicon/sec2/Makefile @@ -1,2 +1,2 @@ -obj-$(CONFIG_CRYPTO_DEV_HISI_SEC) += hisi_sec.o -hisi_sec-objs = sec_main.o sec_crypto.o +obj-$(CONFIG_CRYPTO_DEV_HISI_SEC2) += hisi_sec2.o +hisi_sec2-objs = sec_main.o sec_crypto.o diff --git a/drivers/crypto/hisilicon/sgl.c b/drivers/crypto/hisilicon/sgl.c new file mode 100644 index 0000000000000000000000000000000000000000..328877e03777dc69f19429bf1c676178bdfcac2d --- /dev/null +++ b/drivers/crypto/hisilicon/sgl.c @@ -0,0 +1,230 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2019 HiSilicon Limited. */ +#include <linux/dma-mapping.h> +#include <linux/dmapool.h> +#include <linux/module.h> +#include <linux/delay.h> +#include "./sgl.h" + +#define ACC_SGL_SGE_NR_MIN 1 +#define ACC_SGL_SGE_NR_MAX 255 +#define ACC_SGL_SGE_NR 64 +#define ACC_SGL_NR_MAX 256 +#define ACC_SGL_ALIGN_SIZE 64 + +struct acc_hw_sge { + dma_addr_t buf; + void *page_ctrl; + __le32 len; + __le32 pad; + __le32 pad0; + __le32 pad1; +}; + +/* use default sgl head size 64B */ +struct acc_hw_sgl { + dma_addr_t next_dma; + __le16 entry_sum_in_chain; + __le16 entry_sum_in_sgl; + __le16 entry_length_in_sgl; + __le16 pad0; + __le64 pad1[5]; + struct acc_hw_sgl *next; + struct acc_hw_sge sge_entries[ACC_SGL_SGE_NR]; +} __aligned(1); + +/** + * acc_create_sgl_pool() - Create a hw sgl pool. + * @dev: The device which hw sgl pool belongs to. + * @name: Name of this pool. + * + * This function creates a hw sgl pool for a device, after this user can + * allocate hw sgl memory from it. + */ +struct dma_pool *acc_create_sgl_pool(struct device *dev, const char *name) +{ + if (!dev || !name) + return ERR_PTR(-EINVAL); + + return dmam_pool_create(name, dev, sizeof(struct acc_hw_sgl), + ACC_SGL_ALIGN_SIZE, 0); +} +EXPORT_SYMBOL_GPL(acc_create_sgl_pool); + +void acc_free_sgl_pool(struct dma_pool *pool) {} + +struct acc_hw_sgl *acc_allocate_sgl(struct dma_pool *pool, + dma_addr_t *hw_sgl_dma) +{ + if (!pool || !hw_sgl_dma) + return ERR_PTR(-EINVAL); + + return dma_pool_zalloc(pool, GFP_KERNEL, hw_sgl_dma); +} + +void acc_free_sgl(struct dma_pool *pool, struct acc_hw_sgl *hw_sgl, + dma_addr_t hw_sgl_dma) +{ + if (!pool || !hw_sgl_dma) + return; + + return dma_pool_free(pool, hw_sgl, hw_sgl_dma); +} + +/* return true if sgl has empty sge */ +static bool has_empty_sge(struct acc_hw_sgl *hw_sgl) +{ + return (hw_sgl->entry_sum_in_sgl < hw_sgl->entry_length_in_sgl); +} + +static void add_to_sgl(struct acc_hw_sgl *last_hw_sgl, + struct acc_hw_sgl *new_hw_sgl, dma_addr_t new_sgl_dma) +{ + last_hw_sgl->next = new_hw_sgl; + last_hw_sgl->next_dma = new_sgl_dma; +} + +static void __sg_map_to_hw_sg(struct scatterlist *sgl, + struct acc_hw_sge *hw_sge) +{ + hw_sge->buf = sgl->dma_address; + hw_sge->len = sgl->dma_length; +} + +static void inc_hw_sgl_sge(struct acc_hw_sgl *hw_sgl) +{ + hw_sgl->entry_sum_in_sgl++; +} + +static void update_hw_sgl_sum_sge(struct acc_hw_sgl *hw_sgl, u16 sum) +{ + hw_sgl->entry_sum_in_chain = sum; +} + +static void acc_free_sgls(struct dma_pool *pool, struct acc_hw_sgl *hw_sgl, + dma_addr_t hw_sgl_dma) +{ + if (!pool || !hw_sgl) + return; + + if (!hw_sgl->next) { + acc_free_sgl(pool, hw_sgl, hw_sgl_dma); + } else { + acc_free_sgls(pool, hw_sgl->next, hw_sgl->next_dma); + acc_free_sgl(pool, hw_sgl, hw_sgl_dma); + } +} + +/** + * acc_sg_buf_map_to_hw_sgl - Map a scatterlist to a hw sgl. + * @dev: The device which hw sgl belongs to. + * @sgl: Scatterlist which will be mapped to hw sgl. + * @pool: Pool which hw sgl memory will be allocated in. + * @hw_sgl_dma: The dma address of allocated hw sgl. + * + * This function builds hw sgl according input sgl, user can use hw_sgl_dma + * as src/dst in its BD. + */ +struct acc_hw_sgl *acc_sg_buf_map_to_hw_sgl(struct device *dev, + struct scatterlist *sgl, + struct dma_pool *pool, + dma_addr_t *hw_sgl_dma) +{ + struct acc_hw_sgl *head_hw_sgl, *curr_hw_sgl, *last_hw_sgl; + dma_addr_t curr_sgl_dma, head_sgl_dma; + struct acc_hw_sge *curr_hw_sge; + struct scatterlist *sg; + int sg_n = sg_nents(sgl); + int i, ret, sgl_n = 0; + + if (!dev || !sgl || !pool || !hw_sgl_dma) + return ERR_PTR(-EINVAL); + + ret = dma_map_sg(dev, sgl, sg_n, DMA_BIDIRECTIONAL); + if (!ret) + return ERR_PTR(-EINVAL); + + curr_hw_sgl = acc_allocate_sgl(pool, &curr_sgl_dma); + if (!curr_hw_sgl) { + ret = -ENOMEM; + goto err_unmap_sg; + } + curr_hw_sgl->entry_length_in_sgl = ACC_SGL_SGE_NR; + sgl_n++; + + head_sgl_dma = curr_sgl_dma; + head_hw_sgl = last_hw_sgl = curr_hw_sgl; + curr_hw_sge = curr_hw_sgl->sge_entries; + + for_each_sg(sgl, sg, sg_n, i) { + /* if there is no sge in this hw sgl, allocate a new hw sgl */ + if (unlikely(!has_empty_sge(curr_hw_sgl))) { + if (sgl_n == ACC_SGL_NR_MAX) { + ret = -ENOSPC; + dev_err(dev, "sgl number should not be more than %d\n", + ACC_SGL_NR_MAX); + goto err_free_sgls; + } + + curr_hw_sgl = acc_allocate_sgl(pool, &curr_sgl_dma); + if (!curr_hw_sgl) { + ret = -ENOMEM; + goto err_free_sgls; + } + curr_hw_sgl->entry_length_in_sgl = ACC_SGL_SGE_NR; + sgl_n++; + + add_to_sgl(last_hw_sgl, curr_hw_sgl, curr_sgl_dma); + + last_hw_sgl = curr_hw_sgl; + curr_hw_sge = curr_hw_sgl->sge_entries; + } + + __sg_map_to_hw_sg(sg, curr_hw_sge); + + inc_hw_sgl_sge(curr_hw_sgl); + curr_hw_sge++; + usleep_range(1, 2); + } + + update_hw_sgl_sum_sge(head_hw_sgl, ACC_SGL_SGE_NR * sgl_n); + *hw_sgl_dma = head_sgl_dma; + + return head_hw_sgl; + +err_free_sgls: + acc_free_sgls(pool, head_hw_sgl, head_sgl_dma); +err_unmap_sg: + dma_unmap_sg(dev, sgl, sg_n, DMA_BIDIRECTIONAL); + return ERR_PTR(ret); +} +EXPORT_SYMBOL_GPL(acc_sg_buf_map_to_hw_sgl); + +/** + * acc_sg_buf_unmap() - Destroy and unmap allocated hw sgl. + * @dev: The device which hw sgl belongs to. + * @sgl: Related scatterlist. + * @hw_sgl: Virtual address of hw sgl. + * @hw_sgl_dma: DMA address of hw sgl. + * @pool: Pool which hw sgl is allocated in. + * + * This function destroies and unmap allocated hw sgl. + */ +void acc_sg_buf_unmap(struct device *dev, struct scatterlist *sgl, + struct acc_hw_sgl *hw_sgl, dma_addr_t hw_sgl_dma, + struct dma_pool *pool) +{ + int sg_n = sg_nents(sgl); + + if (!dev || !sgl || !hw_sgl || !hw_sgl_dma || !pool) + return; + + acc_free_sgls(pool, hw_sgl, hw_sgl_dma); + + dma_unmap_sg(dev, sgl, sg_n, DMA_BIDIRECTIONAL); +} +EXPORT_SYMBOL_GPL(acc_sg_buf_unmap); + +MODULE_LICENSE("GPL v2"); +MODULE_AUTHOR("Zhou Wang <wangzhou1@hisilicon.com>"); +MODULE_DESCRIPTION("HiSilicon Accelerator SGL support"); diff --git a/drivers/crypto/hisilicon/sgl.h b/drivers/crypto/hisilicon/sgl.h new file mode 100644 index 0000000000000000000000000000000000000000..badb1d7889e7abf1efc73ee7d70ac15eef70579b --- /dev/null +++ b/drivers/crypto/hisilicon/sgl.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* Copyright (c) 2019 HiSilicon Limited. */ +#ifndef HISI_ACC_SGL_H +#define HISI_ACC_SGL_H + +struct acc_hw_sgl *acc_sg_buf_map_to_hw_sgl(struct device *dev, + struct scatterlist *sgl, + struct dma_pool *pool, + dma_addr_t *hw_sgl_dma); +void acc_sg_buf_unmap(struct device *dev, struct scatterlist *sgl, + struct acc_hw_sgl *hw_sgl, dma_addr_t hw_sgl_dma, + struct dma_pool *pool); +struct dma_pool *acc_create_sgl_pool(struct device *dev, const char *name); +#endif