From 8ccdaf56abdcf33a35cb905db73216467adea17a Mon Sep 17 00:00:00 2001 From: Jason Lee Date: Wed, 21 Jun 2023 16:14:54 +0000 Subject: [PATCH] Intel QAT Z.I.A. provider for ZFS only This provider takes the original Intel QAT code found in ZFS and moves it into a Z.I.A. provider. Because the original code uses symbols from ZFS, this provider should be treated as only usable by ZFS. The original QAT code in ZFS did not use the new operation chaining capability that was added to the QAT API in March 2019. This provider does not update the code to use it as using chaining would cause operations to be performed out of sync with ZFS. The Intel QAT does not maintain memory locally and instead always reads from in-memory buffers and write results to in-memory buffers. This results in a lot of memory traffic. Additionally, because the DPUSM (intentionally) does not have an interface for passing raw pointers from the caller to the accelerator, this provider would have allocated another in-memory buffer to copy each offloaded ZFS buffer into, doubling memory usage, but a layer violation was introduced to prevent these unnecessary memcpys. The vdev_raidz, vdev_file, and vdev_disk code was copied from the example software provider. Encryption is not supported by the DPUSM. The original code was moved along with the rest of the QAT code, but is now not used by ZFS. Signed-off-by: Jason Lee --- include/Makefile.am | 1 - include/sys/qat.h | 207 ------ module/Kbuild.in | 37 +- module/Makefile.in | 6 +- module/os/linux/zfs/qat.c | 105 --- module/os/linux/zfs/zio_crypt.c | 37 +- module/zfs-qat-provider/provider.c | 640 ++++++++++++++++++ module/zfs-qat-provider/qat.c | 41 ++ module/zfs-qat-provider/qat.h | 85 +++ .../zfs => zfs-qat-provider}/qat_compress.c | 50 +- .../zfs => zfs-qat-provider}/qat_crypt.c | 94 +-- module/zfs/gzip.c | 26 - module/zfs/sha2_zfs.c | 13 - module/zfs/spa_misc.c | 3 - 14 files changed, 807 insertions(+), 538 deletions(-) delete mode 100644 include/sys/qat.h delete mode 100644 module/os/linux/zfs/qat.c create mode 100644 module/zfs-qat-provider/provider.c create mode 100644 module/zfs-qat-provider/qat.c create mode 100644 module/zfs-qat-provider/qat.h rename module/{os/linux/zfs => zfs-qat-provider}/qat_compress.c (92%) rename module/{os/linux/zfs => zfs-qat-provider}/qat_crypt.c (86%) diff --git a/include/Makefile.am b/include/Makefile.am index fa16093d2003..bb08e6765d87 100644 --- a/include/Makefile.am +++ b/include/Makefile.am @@ -74,7 +74,6 @@ COMMON_H = \ sys/nvpair_impl.h \ sys/objlist.h \ sys/pathname.h \ - sys/qat.h \ sys/range_tree.h \ sys/rrwlock.h \ sys/sa.h \ diff --git a/include/sys/qat.h b/include/sys/qat.h deleted file mode 100644 index 76360ba99042..000000000000 --- a/include/sys/qat.h +++ /dev/null @@ -1,207 +0,0 @@ -/* - * CDDL HEADER START - * - * The contents of this file are subject to the terms of the - * Common Development and Distribution License (the "License"). - * You may not use this file except in compliance with the License. - * - * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE - * or https://opensource.org/licenses/CDDL-1.0. - * See the License for the specific language governing permissions - * and limitations under the License. - * - * When distributing Covered Code, include this CDDL HEADER in each - * file and include the License file at usr/src/OPENSOLARIS.LICENSE. - * If applicable, add the following below this CDDL HEADER, with the - * fields enclosed by brackets "[]" replaced with your own identifying - * information: Portions Copyright [yyyy] [name of copyright owner] - * - * CDDL HEADER END - */ - -#ifndef _SYS_QAT_H -#define _SYS_QAT_H - -typedef enum qat_compress_dir { - QAT_DECOMPRESS = 0, - QAT_COMPRESS = 1, -} qat_compress_dir_t; - -typedef enum qat_encrypt_dir { - QAT_DECRYPT = 0, - QAT_ENCRYPT = 1, -} qat_encrypt_dir_t; - - -#if defined(_KERNEL) && defined(HAVE_QAT) -#include -#include -#include "cpa.h" -#include "dc/cpa_dc.h" -#include "lac/cpa_cy_sym.h" - -/* - * The minimal and maximal buffer size which are not restricted - * in the QAT hardware, but with the input buffer size between 4KB - * and 128KB the hardware can provide the optimal performance. - */ -#define QAT_MIN_BUF_SIZE (4*1024) -#define QAT_MAX_BUF_SIZE (128*1024) - -/* - * Used for QAT kstat. - */ -typedef struct qat_stats { - /* - * Number of jobs submitted to QAT compression engine. - */ - kstat_named_t comp_requests; - /* - * Total bytes sent to QAT compression engine. - */ - kstat_named_t comp_total_in_bytes; - /* - * Total bytes output from QAT compression engine. - */ - kstat_named_t comp_total_out_bytes; - /* - * Number of jobs submitted to QAT de-compression engine. - */ - kstat_named_t decomp_requests; - /* - * Total bytes sent to QAT de-compression engine. - */ - kstat_named_t decomp_total_in_bytes; - /* - * Total bytes output from QAT de-compression engine. - */ - kstat_named_t decomp_total_out_bytes; - /* - * Number of fails in the QAT compression / decompression engine. - * Note: when a QAT error happens, it doesn't necessarily indicate a - * critical hardware issue. Sometimes it is because the output buffer - * is not big enough. The compression job will be transferred to the - * gzip software implementation so the functionality of ZFS is not - * impacted. - */ - kstat_named_t dc_fails; - - /* - * Number of jobs submitted to QAT encryption engine. - */ - kstat_named_t encrypt_requests; - /* - * Total bytes sent to QAT encryption engine. - */ - kstat_named_t encrypt_total_in_bytes; - /* - * Total bytes output from QAT encryption engine. - */ - kstat_named_t encrypt_total_out_bytes; - /* - * Number of jobs submitted to QAT decryption engine. - */ - kstat_named_t decrypt_requests; - /* - * Total bytes sent to QAT decryption engine. - */ - kstat_named_t decrypt_total_in_bytes; - /* - * Total bytes output from QAT decryption engine. - */ - kstat_named_t decrypt_total_out_bytes; - /* - * Number of fails in the QAT encryption / decryption engine. - * Note: when a QAT error happens, it doesn't necessarily indicate a - * critical hardware issue. The encryption job will be transferred - * to the software implementation so the functionality of ZFS is - * not impacted. - */ - kstat_named_t crypt_fails; - - /* - * Number of jobs submitted to QAT checksum engine. - */ - kstat_named_t cksum_requests; - /* - * Total bytes sent to QAT checksum engine. - */ - kstat_named_t cksum_total_in_bytes; - /* - * Number of fails in the QAT checksum engine. - * Note: when a QAT error happens, it doesn't necessarily indicate a - * critical hardware issue. The checksum job will be transferred to the - * software implementation so the functionality of ZFS is not impacted. - */ - kstat_named_t cksum_fails; -} qat_stats_t; - -#define QAT_STAT_INCR(stat, val) \ - atomic_add_64(&qat_stats.stat.value.ui64, (val)) -#define QAT_STAT_BUMP(stat) \ - QAT_STAT_INCR(stat, 1) - -extern qat_stats_t qat_stats; -extern int zfs_qat_compress_disable; -extern int zfs_qat_checksum_disable; -extern int zfs_qat_encrypt_disable; - -/* inlined for performance */ -static inline struct page * -qat_mem_to_page(void *addr) -{ - if (!is_vmalloc_addr(addr)) - return (virt_to_page(addr)); - - return (vmalloc_to_page(addr)); -} - -CpaStatus qat_mem_alloc_contig(void **pp_mem_addr, Cpa32U size_bytes); -void qat_mem_free_contig(void **pp_mem_addr); -#define QAT_PHYS_CONTIG_ALLOC(pp_mem_addr, size_bytes) \ - qat_mem_alloc_contig((void *)(pp_mem_addr), (size_bytes)) -#define QAT_PHYS_CONTIG_FREE(p_mem_addr) \ - qat_mem_free_contig((void *)&(p_mem_addr)) - -extern int qat_dc_init(void); -extern void qat_dc_fini(void); -extern int qat_cy_init(void); -extern void qat_cy_fini(void); -extern int qat_init(void); -extern void qat_fini(void); - -/* fake CpaStatus used to indicate data was not compressible */ -#define CPA_STATUS_INCOMPRESSIBLE (-127) - -extern boolean_t qat_dc_use_accel(size_t s_len); -extern boolean_t qat_crypt_use_accel(size_t s_len); -extern boolean_t qat_checksum_use_accel(size_t s_len); -extern int qat_compress(qat_compress_dir_t dir, char *src, int src_len, - char *dst, int dst_len, size_t *c_len); -extern int qat_crypt(qat_encrypt_dir_t dir, uint8_t *src_buf, uint8_t *dst_buf, - uint8_t *aad_buf, uint32_t aad_len, uint8_t *iv_buf, uint8_t *digest_buf, - crypto_key_t *key, uint64_t crypt, uint32_t enc_len); -extern int qat_checksum(uint64_t cksum, uint8_t *buf, uint64_t size, - zio_cksum_t *zcp); -#else -#define CPA_STATUS_SUCCESS 0 -#define CPA_STATUS_INCOMPRESSIBLE (-127) -#define qat_init() -#define qat_fini() -#define qat_dc_use_accel(s_len) ((void) sizeof (s_len), 0) -#define qat_crypt_use_accel(s_len) ((void) sizeof (s_len), 0) -#define qat_checksum_use_accel(s_len) ((void) sizeof (s_len), 0) -#define qat_compress(dir, s, sl, d, dl, cl) \ - ((void) sizeof (dir), (void) sizeof (s), (void) sizeof (sl), \ - (void) sizeof (d), (void) sizeof (dl), (void) sizeof (cl), 0) -#define qat_crypt(dir, s, d, a, al, i, db, k, c, el) \ - ((void) sizeof (dir), (void) sizeof (s), (void) sizeof (d), \ - (void) sizeof (a), (void) sizeof (al), (void) sizeof (i), \ - (void) sizeof (db), (void) sizeof (k), (void) sizeof (c), \ - (void) sizeof (el), 0) -#define qat_checksum(c, buf, s, z) \ - ((void) sizeof (c), (void) sizeof (buf), (void) sizeof (s), \ - (void) sizeof (z), 0) -#endif - -#endif /* _SYS_QAT_H */ diff --git a/module/Kbuild.in b/module/Kbuild.in index bd8c9d50262c..d7a32ed83e11 100644 --- a/module/Kbuild.in +++ b/module/Kbuild.in @@ -49,11 +49,6 @@ CFLAGS_zfs/zap_micro.o += -mllvm -x86-cmov-converter=false endif endif -ifneq ($(KBUILD_EXTMOD),) -@CONFIG_QAT_TRUE@ZFS_MODULE_CFLAGS += -I@QAT_SRC@/include -@CONFIG_QAT_TRUE@KBUILD_EXTRA_SYMBOLS += @QAT_SYMBOLS@ -endif - asflags-y := $(ZFS_MODULE_CFLAGS) $(ZFS_MODULE_CPPFLAGS) ccflags-y := $(ZFS_MODULE_CFLAGS) $(ZFS_MODULE_CPPFLAGS) @@ -439,9 +434,6 @@ ZFS_OBJS_OS := \ arc_os.o \ mmp_os.o \ policy.o \ - qat.o \ - qat_compress.o \ - qat_crypt.o \ spa_misc_os.o \ trace.o \ vdev_disk.o \ @@ -506,17 +498,34 @@ $(obj)/zfs/vdev_raidz_math_powerpc_altivec.o : c_flags += -maltivec endif ifneq ("@DPUSM_SYMBOLS@","") -obj-$(CONFIG_ZFS) += zia-software-provider.o +@ZIA_ENABLED_TRUE@KBUILD_EXTRA_SYMBOLS += @DPUSM_SYMBOLS@ + +@ZIA_ENABLED_TRUE@obj-$(CONFIG_ZFS) += zia-software-provider.o ZIA_SOFTWARE_PROVIDER_OBJS := \ provider.o \ kernel_offloader.o -zia-software-provider-objs += $(addprefix zia-software-provider/,$(ZIA_SOFTWARE_PROVIDER_OBJS)) +@ZIA_ENABLED_TRUE@zia-software-provider-objs += $(addprefix zia-software-provider/,$(ZIA_SOFTWARE_PROVIDER_OBJS)) # zfs_file_os does not have any dependencies, so just link to it directly -zia-software-provider-objs += os/linux/zfs/zfs_file_os.o +@ZIA_ENABLED_TRUE@zia-software-provider-objs += os/linux/zfs/zfs_file_os.o -$(addprefix $(obj)/zia-software-provider/,$(ZIA_SOFTWARE_PROVIDER_OBJS)) : ccflags-y += -I@abs_top_builddir@ $(ZFS_MODULE_CFLAGS) -I@abs_srcdir@/zia-software-provider/ -I@DPUSM_ROOT@/include +@ZIA_ENABLED_TRUE@$(addprefix $(obj)/zia-software-provider/,$(ZIA_SOFTWARE_PROVIDER_OBJS)) : ccflags-y += -I@abs_top_builddir@ $(ZFS_MODULE_CFLAGS) -I@abs_srcdir@/zia-software-provider/ -I@DPUSM_ROOT@/include/ -@ZIA_ENABLED_TRUE@KBUILD_EXTRA_SYMBOLS += @DPUSM_SYMBOLS@ -endif \ No newline at end of file +@ZIA_ENABLED_TRUE@@CONFIG_QAT_TRUE@obj-$(CONFIG_ZFS) += zfs-qat-provider.o + +ZFS_QAT_PROVIDER_OBJS := \ + qat.o \ + qat_compress.o \ + qat_crypt.o \ + provider.o + +@ZIA_ENABLED_TRUE@@CONFIG_QAT_TRUE@zfs-qat-provider-objs += $(addprefix zfs-qat-provider/,$(ZFS_QAT_PROVIDER_OBJS)) +# zfs_file_os does not have any dependencies, so just link to it directly +@ZIA_ENABLED_TRUE@@CONFIG_QAT_TRUE@zfs-qat-provider-objs += os/linux/zfs/zfs_file_os.o + +@ZIA_ENABLED_TRUE@@CONFIG_QAT_TRUE@$(addprefix $(obj)/zfs-qat-provider/,$(ZFS_QAT_PROVIDER_OBJS)) : ccflags-y += -I@abs_top_builddir@ $(ZFS_MODULE_CFLAGS) -I@abs_srcdir@/zfs-qat-provider/ -I@DPUSM_ROOT@/include/ -I@QAT_SRC@/include/ -Wframe-larger-than=20480 + +@ZIA_ENABLED_TRUE@KBUILD_EXTRA_SYMBOLS += @QAT_SYMBOLS@ + +endif diff --git a/module/Makefile.in b/module/Makefile.in index c780c491de39..c3f435e24504 100644 --- a/module/Makefile.in +++ b/module/Makefile.in @@ -80,7 +80,7 @@ clean: clean-@ac_system@ .PHONY: modules_uninstall-Linux-legacy modules_uninstall-Linux-legacy: - $(RM) -r $(addprefix $(KMODDIR)/$(INSTALL_MOD_DIR)/,spl/ avl/ icp/ lua/ nvpair/ unicode/ zcommon/ zfs/ zstd/ zia-software-provider) + $(RM) -r $(addprefix $(KMODDIR)/$(INSTALL_MOD_DIR)/,spl/ avl/ icp/ lua/ nvpair/ unicode/ zcommon/ zfs/ zstd/ zia-software-provider zfs-qat-provider) KMODDIR := $(INSTALL_MOD_PATH)/lib/modules/@LINUX_VERSION@ modules_install-Linux: modules_uninstall-Linux-legacy @@ -123,7 +123,7 @@ data_install: data_install-@ac_system@ modules_uninstall-Linux: modules_uninstall-Linux-legacy @# Uninstall the kernel modules - $(RM) $(addprefix $(KMODDIR)/$(INSTALL_MOD_DIR)/,zfs.ko spl.ko zia-software-provider) + $(RM) $(addprefix $(KMODDIR)/$(INSTALL_MOD_DIR)/,zfs.ko spl.ko zia-software-provider zfs-qat-provider) modules_uninstall-FreeBSD: @false @@ -153,7 +153,7 @@ cppcheck-Linux: -I @top_srcdir@/include/os/linux/spl \ -I @top_srcdir@/include/os/linux/zfs \ -I @top_srcdir@/include \ - avl icp lua nvpair unicode zcommon zfs zstd os/linux zia-software-provider + avl icp lua nvpair unicode zcommon zfs zstd os/linux zia-software-provider zfs-qat-provider cppcheck-FreeBSD: @true diff --git a/module/os/linux/zfs/qat.c b/module/os/linux/zfs/qat.c deleted file mode 100644 index 07e0cafabb0e..000000000000 --- a/module/os/linux/zfs/qat.c +++ /dev/null @@ -1,105 +0,0 @@ -/* - * CDDL HEADER START - * - * The contents of this file are subject to the terms of the - * Common Development and Distribution License (the "License"). - * You may not use this file except in compliance with the License. - * - * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE - * or https://opensource.org/licenses/CDDL-1.0. - * See the License for the specific language governing permissions - * and limitations under the License. - * - * When distributing Covered Code, include this CDDL HEADER in each - * file and include the License file at usr/src/OPENSOLARIS.LICENSE. - * If applicable, add the following below this CDDL HEADER, with the - * fields enclosed by brackets "[]" replaced with your own identifying - * information: Portions Copyright [yyyy] [name of copyright owner] - * - * CDDL HEADER END - */ - -#if defined(_KERNEL) && defined(HAVE_QAT) -#include -#include - -qat_stats_t qat_stats = { - { "comp_requests", KSTAT_DATA_UINT64 }, - { "comp_total_in_bytes", KSTAT_DATA_UINT64 }, - { "comp_total_out_bytes", KSTAT_DATA_UINT64 }, - { "decomp_requests", KSTAT_DATA_UINT64 }, - { "decomp_total_in_bytes", KSTAT_DATA_UINT64 }, - { "decomp_total_out_bytes", KSTAT_DATA_UINT64 }, - { "dc_fails", KSTAT_DATA_UINT64 }, - { "encrypt_requests", KSTAT_DATA_UINT64 }, - { "encrypt_total_in_bytes", KSTAT_DATA_UINT64 }, - { "encrypt_total_out_bytes", KSTAT_DATA_UINT64 }, - { "decrypt_requests", KSTAT_DATA_UINT64 }, - { "decrypt_total_in_bytes", KSTAT_DATA_UINT64 }, - { "decrypt_total_out_bytes", KSTAT_DATA_UINT64 }, - { "crypt_fails", KSTAT_DATA_UINT64 }, - { "cksum_requests", KSTAT_DATA_UINT64 }, - { "cksum_total_in_bytes", KSTAT_DATA_UINT64 }, - { "cksum_fails", KSTAT_DATA_UINT64 }, -}; - -static kstat_t *qat_ksp = NULL; - -CpaStatus -qat_mem_alloc_contig(void **pp_mem_addr, Cpa32U size_bytes) -{ - *pp_mem_addr = kmalloc(size_bytes, GFP_KERNEL); - if (*pp_mem_addr == NULL) - return (CPA_STATUS_RESOURCE); - return (CPA_STATUS_SUCCESS); -} - -void -qat_mem_free_contig(void **pp_mem_addr) -{ - if (*pp_mem_addr != NULL) { - kfree(*pp_mem_addr); - *pp_mem_addr = NULL; - } -} - -int -qat_init(void) -{ - qat_ksp = kstat_create("zfs", 0, "qat", "misc", - KSTAT_TYPE_NAMED, sizeof (qat_stats) / sizeof (kstat_named_t), - KSTAT_FLAG_VIRTUAL); - if (qat_ksp != NULL) { - qat_ksp->ks_data = &qat_stats; - kstat_install(qat_ksp); - } - - /* - * Just set the disable flag when qat init failed, qat can be - * turned on again in post-process after zfs module is loaded, e.g.: - * echo 0 > /sys/module/zfs/parameters/zfs_qat_compress_disable - */ - if (qat_dc_init() != 0) - zfs_qat_compress_disable = 1; - - if (qat_cy_init() != 0) { - zfs_qat_checksum_disable = 1; - zfs_qat_encrypt_disable = 1; - } - - return (0); -} - -void -qat_fini(void) -{ - if (qat_ksp != NULL) { - kstat_delete(qat_ksp); - qat_ksp = NULL; - } - - qat_cy_fini(); - qat_dc_fini(); -} - -#endif diff --git a/module/os/linux/zfs/zio_crypt.c b/module/os/linux/zfs/zio_crypt.c index 21f3740f6fe6..ff8666511351 100644 --- a/module/os/linux/zfs/zio_crypt.c +++ b/module/os/linux/zfs/zio_crypt.c @@ -26,7 +26,6 @@ #include #include #include -#include /* * This file is responsible for handling all of the details of generating @@ -1955,37 +1954,6 @@ zio_do_crypt_data(boolean_t encrypt, zio_crypt_key_t *key, tmpl = NULL; } - /* - * Attempt to use QAT acceleration if we can. We currently don't - * do this for metadnode and ZIL blocks, since they have a much - * more involved buffer layout and the qat_crypt() function only - * works in-place. - */ - if (qat_crypt_use_accel(datalen) && - ot != DMU_OT_INTENT_LOG && ot != DMU_OT_DNODE) { - uint8_t *srcbuf, *dstbuf; - - if (encrypt) { - srcbuf = plainbuf; - dstbuf = cipherbuf; - } else { - srcbuf = cipherbuf; - dstbuf = plainbuf; - } - - ret = qat_crypt((encrypt) ? QAT_ENCRYPT : QAT_DECRYPT, srcbuf, - dstbuf, NULL, 0, iv, mac, ckey, key->zk_crypt, datalen); - if (ret == CPA_STATUS_SUCCESS) { - if (locked) { - rw_exit(&key->zk_salt_lock); - locked = B_FALSE; - } - - return (0); - } - /* If the hardware implementation fails fall back to software */ - } - /* create uios for encryption */ ret = zio_crypt_init_uios(encrypt, key->zk_version, ot, plainbuf, cipherbuf, datalen, byteswap, mac, &puio, &cuio, &enc_len, @@ -2077,4 +2045,9 @@ zio_do_crypt_abd(boolean_t encrypt, zio_crypt_key_t *key, dmu_object_type_t ot, module_param(zfs_key_max_salt_uses, ulong, 0644); MODULE_PARM_DESC(zfs_key_max_salt_uses, "Max number of times a salt value " "can be used for generating encryption keys before it is rotated"); + +#if defined(ZIA) && defined(HAVE_QAT) +EXPORT_SYMBOL(zio_crypt_table); +#endif + #endif diff --git a/module/zfs-qat-provider/provider.c b/module/zfs-qat-provider/provider.c new file mode 100644 index 000000000000..571d3e658b4e --- /dev/null +++ b/module/zfs-qat-provider/provider.c @@ -0,0 +1,640 @@ +/* + * © 2021. Triad National Security, LLC. All rights reserved. + * + * This program was produced under U.S. Government contract + * 89233218CNA000001 for Los Alamos National Laboratory (LANL), which + * is operated by Triad National Security, LLC for the U.S. + * Department of Energy/National Nuclear Security Administration. All + * rights in the program are reserved by Triad National Security, LLC, + * and the U.S. Department of Energy/National Nuclear Security + * Administration. The Government is granted for itself and others + * acting on its behalf a nonexclusive, paid-up, irrevocable worldwide + * license in this material to reproduce, prepare derivative works, + * distribute copies to the public, perform publicly and display + * publicly, and to permit others to do so. + * + * ---- + * + * This program is open source under the BSD-3 License. + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from this + * software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include +#include +#include + +#include /* the DPUSM provider API */ +#include /* QAT wrapper from ZFS */ + +/* should not be here */ +#include +#include +#include +#include +#include +#include +#include + +typedef enum zfs_qat_handle_type { + ZQH_REAL, + ZQH_REF, +} zqh_type_t; + +typedef struct zfs_qat_provider_handle { + zqh_type_t type; + void *ptr; + size_t size; +} zqh_t; + +static void * +ptr_start(zqh_t *zqh, const size_t offset) +{ + return (((char *)zqh->ptr) + offset); +} + +static int +zfs_qat_provider_algorithms(int *compress, int *decompress, + int *checksum, int *checksum_byteorder, int *raid) +{ + *compress = + DPUSM_COMPRESS_GZIP_1 | + DPUSM_COMPRESS_GZIP_2 | + DPUSM_COMPRESS_GZIP_3 | + DPUSM_COMPRESS_GZIP_4 | + DPUSM_COMPRESS_GZIP_5 | + DPUSM_COMPRESS_GZIP_6 | + DPUSM_COMPRESS_GZIP_7 | + DPUSM_COMPRESS_GZIP_8 | + DPUSM_COMPRESS_GZIP_9; + + *decompress = *compress; + + *checksum = DPUSM_CHECKSUM_SHA256 | DPUSM_CHECKSUM_SHA512; + + *checksum_byteorder = DPUSM_BYTEORDER_NATIVE | DPUSM_BYTEORDER_BYTESWAP; + + *raid = + DPUSM_RAID_1_GEN | + DPUSM_RAID_2_GEN | + DPUSM_RAID_3_GEN | + DPUSM_RAID_1_REC | + DPUSM_RAID_2_REC | + DPUSM_RAID_3_REC; + + return (DPUSM_OK); +} + +static void * +zfs_qat_provider_alloc(size_t size) +{ + zqh_t *buf = kmalloc(sizeof (zqh_t), GFP_KERNEL); + buf->type = ZQH_REAL; + buf->ptr = kmalloc(size, GFP_KERNEL); + buf->size = size; + return (buf); +} + +static void * +zfs_qat_provider_alloc_ref(void *src_handle, size_t offset, size_t size) +{ + zqh_t *src = (zqh_t *)src_handle; + + zqh_t *ref = kmalloc(sizeof (zqh_t), GFP_KERNEL); + ref->type = ZQH_REF; + ref->ptr = ptr_start(src, offset); + ref->size = size; + + return (ref); +} + +static int +zfs_qat_provider_get_size(void *handle, size_t *size, size_t *actual) +{ + zqh_t *buf = (zqh_t *)handle; + if (size) { + *size = buf->size; + } + + if (actual) { + *actual = buf->size; + } + + return (DPUSM_OK); +} + +static int +zfs_qat_provider_free(void *handle) +{ + zqh_t *buf = (zqh_t *)handle; + if (buf->type == ZQH_REAL) { + kfree(buf->ptr); + } + kfree(buf); + + return (DPUSM_OK); +} + +static int +zfs_qat_provider_copy_from_generic(dpusm_mv_t *mv, const void *buf, size_t size) +{ + zqh_t *dst = (zqh_t *)mv->handle; + /* + * If the handle points to a real buffer, free it, + * and convert the handle to a reference to the + * user's pointer. + * + * This is a layer violation, but is done to prevent + * unnecessary copies of the data due to the QAT using + * in-memory addresses. + */ + if ((dst->type == ZQH_REAL) && + (mv->offset == 0)) { + kfree(dst->ptr); + + dst->type = ZQH_REF; + dst->ptr = (void *) buf; + dst->size = size; + } + else + { + memcpy(ptr_start(dst, mv->offset), buf, size); + } + return (DPUSM_OK); +} + +static int +zfs_qat_provider_copy_to_generic(dpusm_mv_t *mv, void *buf, size_t size) +{ + zqh_t *src = (zqh_t *)mv->handle; + /* + * if the handle is still a real buffer, it is not + * the same as buf, so the contents have to be copied + * into buf. + */ + void *start = ptr_start(src, mv->offset); + if (start != buf) { + memcpy(buf, start, size); + } + return (DPUSM_OK); +} + +static int +zfs_qat_provider_zero_fill(void *handle, size_t offset, size_t size) +{ + memset(ptr_start(handle, offset), 0, size); + return (DPUSM_OK); +} + +static int +zfs_qat_provider_all_zeros(void *handle, size_t offset, size_t size) +{ + zqh_t *zqh = (zqh_t *)handle; + for (size_t i = 0; i < size; i++) { + if (((char *)zqh->ptr)[offset + i]) { + return (DPUSM_ERROR); + } + } + return (DPUSM_OK); +} + +static int +zfs_qat_provider_compress(dpusm_compress_t alg, int level, + void *src, size_t s_len, void *dst, size_t *d_len) +{ + (void) alg; /* unused */ + (void) level; /* unused */ + + /* check if hardware accelerator can be used */ + if (!qat_dc_use_accel(s_len)) { + return (DPUSM_ERROR); + } + + zqh_t *s = (zqh_t *)src; + zqh_t *d = (zqh_t *)dst; + + if ((s_len > s->size) || + (*d_len > d->size)) { + return (DPUSM_ERROR); + } + + void *s_start = ptr_start(s, 0); + void *d_start = ptr_start(d, 0); + + const int ret = qat_compress(QAT_COMPRESS, + s_start, s_len, d_start, *d_len, d_len); + + if (ret == CPA_STATUS_SUCCESS) { + return (DPUSM_OK); + } else if (ret == CPA_STATUS_INCOMPRESSIBLE) { + *d_len = s_len; + return (DPUSM_OK); + } + return (DPUSM_ERROR); +} + +static int +zfs_qat_provider_decompress(dpusm_compress_t alg, int *level, + void *src, size_t s_len, void *dst, size_t *d_len) +{ + (void) alg; /* unused */ + (void) level; /* unused */ + + /* check if hardware accelerator can be used */ + if (!qat_dc_use_accel(*d_len)) { + return (DPUSM_ERROR); + } + + zqh_t *s = (zqh_t *)src; + zqh_t *d = (zqh_t *)dst; + + if ((s_len > s->size) || + (*d_len > d->size)) { + return (DPUSM_ERROR); + } + + void *s_start = ptr_start(s, 0); + void *d_start = ptr_start(d, 0); + + if (qat_compress(QAT_DECOMPRESS, s_start, s_len, + d_start, *d_len, d_len) == CPA_STATUS_SUCCESS) { + return (DPUSM_OK); + } + + return (DPUSM_ERROR); +} + +static int +zfs_qat_provider_checksum(dpusm_checksum_t alg, + dpusm_checksum_byteorder_t order, void *data, size_t size, + void *cksum, size_t cksum_size) +{ + (void) order; /* might have to handle here */ + + if (!qat_checksum_use_accel(size)) { + return (DPUSM_ERROR); + } + + if (alg != DPUSM_CHECKSUM_SHA256) { + return (DPUSM_NOT_IMPLEMENTED); + } + + if (cksum_size < sizeof (zio_cksum_t)) { + return (DPUSM_ERROR); + } + + zqh_t *src = (zqh_t *)data; + if (size > src->size) { + return (DPUSM_ERROR); + } + + const int ret = qat_checksum(ZIO_CHECKSUM_SHA256, src->ptr, size, + cksum); + return ((ret == CPA_STATUS_SUCCESS)?DPUSM_OK:DPUSM_ERROR); +} + +static int +zfs_qat_provider_raidz_can_compute(size_t nparity, size_t ndata, + size_t *col_sizes, int rec) +{ + if ((nparity < 1) || (nparity > 3)) { + return (DPUSM_NOT_SUPPORTED); + } + + return (DPUSM_OK); +} + +static void * +zfs_qat_provider_raidz_alloc(size_t nparity, size_t ndata) +{ + const size_t ncols = nparity + ndata; + + const size_t rr_size = offsetof(raidz_row_t, rr_col[ncols]); + raidz_row_t *rr = kzalloc(rr_size, GFP_KERNEL); + rr->rr_cols = ncols; + rr->rr_firstdatacol = nparity; + + return (rr); +} + +/* attaches a column to the raidz struct */ +static int +zfs_qat_provider_raidz_set_column(void *raidz, uint64_t c, + void *col, size_t size) +{ + raidz_row_t *rr = (raidz_row_t *)raidz; + zqh_t *zqh = (zqh_t *)col; + + if (!rr || !zqh) { + return (DPUSM_ERROR); + } + + /* c is too big */ + if (c >= rr->rr_cols) { + return (DPUSM_ERROR); + } + + /* "active" size is larger than allocated size */ + if (size > zqh->size) { + return (DPUSM_ERROR); + } + + raidz_col_t *rc = &rr->rr_col[c]; + + /* clean up old column */ + abd_free(rc->rc_abd); + + /* + * rc->rc_abd does not take ownership of zqh->ptr, + * so don't need to release ownership + */ + rc->rc_abd = abd_get_from_buf(zqh->ptr, size); + rc->rc_size = size; + + return (DPUSM_OK); +} + +static int +zfs_qat_provider_raidz_free(void *raidz) +{ + raidz_row_t *rr = (raidz_row_t *)raidz; + for (int c = 0; c < rr->rr_cols; c++) { + raidz_col_t *rc = &rr->rr_col[c]; + abd_free(rc->rc_abd); + } + kfree(rr); + + return (DPUSM_OK); +} + +static int +zfs_qat_provider_raidz_gen(void *raidz) +{ + raidz_row_t *rr = (raidz_row_t *)raidz; + switch (rr->rr_firstdatacol) { + case 1: + vdev_raidz_generate_parity_p(rr); + break; + case 2: + vdev_raidz_generate_parity_pq(rr); + break; + case 3: + vdev_raidz_generate_parity_pqr(rr); + break; + } + + return (DPUSM_OK); +} + +static int +zfs_qat_provider_raidz_rec(void *raidz, int *tgts, int ntgts) +{ + raidz_row_t *rr = (raidz_row_t *)raidz; + vdev_raidz_reconstruct_general(rr, tgts, ntgts); + + return (DPUSM_OK); +} + +static int +zfs_qat_provider_raidz_cmp(void *lhs_handle, void *rhs_handle, int *diff) +{ + zqh_t *lhs = (zqh_t *)lhs_handle; + zqh_t *rhs = (zqh_t *)rhs_handle; + + if (!diff) { + return (DPUSM_ERROR); + } + + size_t len = rhs->size; + if (lhs->size != rhs->size) { + len = + (lhs->size < rhs->size)?lhs->size:rhs->size; + } + + *diff = memcmp(ptr_start(lhs, 0), + ptr_start(rhs, 0), len); + + return (DPUSM_OK); +} + +static void * +zfs_qat_provider_file_open(const char *path, int flags, int mode) +{ + zfs_file_t *fp = NULL; + /* on error, fp should still be NULL */ + zfs_file_open(path, flags, mode, &fp); + return (fp); +} + +static int +zfs_qat_provider_file_write(void *fp_handle, void *handle, size_t count, + size_t trailing_zeros, loff_t offset, ssize_t *resid, int *err) +{ + zfs_file_t *fp = (zfs_file_t *)fp_handle; + zqh_t *zqh = (zqh_t *)handle; + + if (!err) { + return (EIO); + } + + *err = zfs_file_pwrite(fp, ptr_start(zqh, 0), + count, offset, resid); + + if (*err == 0) { + void *zeros = kzalloc(trailing_zeros, GFP_KERNEL); + *err = zfs_file_pwrite(fp, zeros, + trailing_zeros, offset + count, resid); + kfree(zeros); + } + + return (*err); +} + +static void +zfs_qat_provider_file_close(void *fp_handle) +{ + zfs_file_close(fp_handle); +} + +static void * +zfs_qat_provider_disk_open(dpusm_dd_t *disk_data) +{ + return (disk_data->bdev); +} + +static int +zfs_qat_provider_disk_invalidate(void *disk_handle) +{ + struct block_device *bdev = + (struct block_device *)disk_handle; + invalidate_bdev(bdev); + return (DPUSM_OK); +} + +static int +zfs_qat_provider_disk_write(void *disk_handle, void *handle, size_t data_size, + size_t trailing_zeros, uint64_t io_offset, int flags, + dpusm_disk_write_completion_t write_completion, void *wc_args) +{ + struct block_device *bdev = + (struct block_device *)disk_handle; + zqh_t *zqh = (zqh_t *)handle; + + const size_t io_size = data_size + trailing_zeros; + + if (trailing_zeros) { + /* create a copy of the data with the trailing zeros attached */ + void *copy = kzalloc(io_size, GFP_KERNEL); + memcpy(copy, ptr_start(zqh, 0), data_size); + + /* need to keep copy alive, so replace zqh->ptr */ + if (zqh->type == ZQH_REAL) { + kfree(zqh->ptr); + } + + zqh->type = ZQH_REAL; + zqh->ptr = copy; + zqh->size = io_size; + } + + abd_t *abd = abd_get_from_buf(zqh->ptr, io_size); + zio_push_transform(wc_args, abd, io_size, io_size, NULL); + + /* __vdev_disk_physio already adds write_completion */ + (void) write_completion; + + return (__vdev_classic_physio(bdev, wc_args, + io_size, io_offset, WRITE, flags)); +} + +static int +zfs_qat_provider_disk_flush(void *disk_handle, + dpusm_disk_flush_completion_t flush_completion, void *fc_args) +{ + struct block_device *bdev = + (struct block_device *)disk_handle; + + /* vdev_disk_io_flush already adds flush completion */ + (void) flush_completion; + + return (vdev_disk_io_flush(bdev, fc_args)); +} + +static void +zfs_qat_provider_disk_close(void *disk_handle) +{} + +/* + * "zfs-qat-provider" instead of "qat-provider" + * because this provider links with ZFS symbols + */ +/* BEGIN CSTYLED */ +static const dpusm_pf_t zfs_qat_provider_functions = { + .algorithms = zfs_qat_provider_algorithms, + .alloc = zfs_qat_provider_alloc, + .alloc_ref = zfs_qat_provider_alloc_ref, + .get_size = zfs_qat_provider_get_size, + .free = zfs_qat_provider_free, + .copy = { + .from = { + .generic = zfs_qat_provider_copy_from_generic, + .ptr = NULL, + .scatterlist = NULL, + }, + .to = { + .generic = zfs_qat_provider_copy_to_generic, + .ptr = NULL, + .scatterlist = NULL, + }, + }, + .at_connect = NULL, + .at_disconnect = NULL, + .mem_stats = NULL, + .zero_fill = zfs_qat_provider_zero_fill, + .all_zeros = zfs_qat_provider_all_zeros, + .compress = zfs_qat_provider_compress, + .decompress = zfs_qat_provider_decompress, + .checksum = zfs_qat_provider_checksum, + .raid = { + .can_compute = zfs_qat_provider_raidz_can_compute, + .alloc = zfs_qat_provider_raidz_alloc, + .set_column = zfs_qat_provider_raidz_set_column, + .free = zfs_qat_provider_raidz_free, + .gen = zfs_qat_provider_raidz_gen, + .cmp = zfs_qat_provider_raidz_cmp, + .rec = zfs_qat_provider_raidz_rec, + }, + .file = { + .open = zfs_qat_provider_file_open, + .write = zfs_qat_provider_file_write, + .close = zfs_qat_provider_file_close, + }, + .disk = { + .open = zfs_qat_provider_disk_open, + .invalidate = zfs_qat_provider_disk_invalidate, + .write = zfs_qat_provider_disk_write, + .flush = zfs_qat_provider_disk_flush, + .close = zfs_qat_provider_disk_close, + }, +}; +/* END CSTYLED */ + +static int __init +zfs_qat_provider_init(void) +{ + Cpa16U dev_count = 0; + if (qat_dc_init(&dev_count) != 0) { + qat_dc_fini(); + printk("%s: Error: Could not initialize provider\n", + module_name(THIS_MODULE)); + return (-ENODEV); + } + + if (dev_count == 0) { + printk("%s: Error: No devices found\n", + module_name(THIS_MODULE)); + return (-ENODEV); + } + + printk("%s: Initialized\n", module_name(THIS_MODULE)); + + return (dpusm_register_bsd(THIS_MODULE, &zfs_qat_provider_functions)); +} + +static void __exit +zfs_qat_provider_exit(void) +{ + dpusm_unregister_bsd(THIS_MODULE); + + qat_dc_fini(); + printk("%s: Uninitialized\n", module_name(THIS_MODULE)); +} + +module_init(zfs_qat_provider_init); +module_exit(zfs_qat_provider_exit); + +MODULE_LICENSE("CDDL"); diff --git a/module/zfs-qat-provider/qat.c b/module/zfs-qat-provider/qat.c new file mode 100644 index 000000000000..0c6abc573135 --- /dev/null +++ b/module/zfs-qat-provider/qat.c @@ -0,0 +1,41 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or https://opensource.org/licenses/CDDL-1.0. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +#include +#include + +CpaStatus +qat_mem_alloc_contig(void **pp_mem_addr, Cpa32U size_bytes) +{ + *pp_mem_addr = kmalloc(size_bytes, GFP_KERNEL); + if (*pp_mem_addr == NULL) + return (CPA_STATUS_RESOURCE); + return (CPA_STATUS_SUCCESS); +} + +void +qat_mem_free_contig(void **pp_mem_addr) +{ + if (*pp_mem_addr != NULL) { + kfree(*pp_mem_addr); + *pp_mem_addr = NULL; + } +} diff --git a/module/zfs-qat-provider/qat.h b/module/zfs-qat-provider/qat.h new file mode 100644 index 000000000000..cbade8a891b3 --- /dev/null +++ b/module/zfs-qat-provider/qat.h @@ -0,0 +1,85 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or https://opensource.org/licenses/CDDL-1.0. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +#ifndef _QAT_H +#define _QAT_H + +typedef enum qat_compress_dir { + QAT_DECOMPRESS = 0, + QAT_COMPRESS = 1, +} qat_compress_dir_t; + +typedef enum qat_encrypt_dir { + QAT_DECRYPT = 0, + QAT_ENCRYPT = 1, +} qat_encrypt_dir_t; + +#include +#include +#include "cpa.h" +#include "dc/cpa_dc.h" +#include "lac/cpa_cy_sym.h" + +/* + * The minimal and maximal buffer size which are not restricted + * in the QAT hardware, but with the input buffer size between 4KB + * and 128KB the hardware can provide the optimal performance. + */ +#define QAT_MIN_BUF_SIZE (4*1024) +#define QAT_MAX_BUF_SIZE (128*1024) + +/* inlined for performance */ +static inline struct page * +qat_mem_to_page(void *addr) +{ + if (!is_vmalloc_addr(addr)) + return (virt_to_page(addr)); + + return (vmalloc_to_page(addr)); +} + +CpaStatus qat_mem_alloc_contig(void **pp_mem_addr, Cpa32U size_bytes); +void qat_mem_free_contig(void **pp_mem_addr); +#define QAT_PHYS_CONTIG_ALLOC(pp_mem_addr, size_bytes) \ + qat_mem_alloc_contig((void *)(pp_mem_addr), (size_bytes)) +#define QAT_PHYS_CONTIG_FREE(p_mem_addr) \ + qat_mem_free_contig((void *)&(p_mem_addr)) + +extern int qat_dc_init(Cpa16U *dev_count); +extern void qat_dc_fini(void); +extern int qat_cy_init(void); +extern void qat_cy_fini(void); + +/* fake CpaStatus used to indicate data was not compressible */ +#define CPA_STATUS_INCOMPRESSIBLE (-127) + +extern boolean_t qat_dc_use_accel(size_t s_len); +extern boolean_t qat_crypt_use_accel(size_t s_len); +extern boolean_t qat_checksum_use_accel(size_t s_len); +extern int qat_compress(qat_compress_dir_t dir, char *src, int src_len, + char *dst, int dst_len, size_t *c_len); +extern int qat_crypt(qat_encrypt_dir_t dir, uint8_t *src_buf, uint8_t *dst_buf, + uint8_t *aad_buf, uint32_t aad_len, uint8_t *iv_buf, uint8_t *digest_buf, + crypto_key_t *key, uint64_t crypt, uint32_t enc_len); +extern int qat_checksum(uint64_t cksum, uint8_t *buf, uint64_t size, + zio_cksum_t *zcp); + +#endif /* _QAT_H */ diff --git a/module/os/linux/zfs/qat_compress.c b/module/zfs-qat-provider/qat_compress.c similarity index 92% rename from module/os/linux/zfs/qat_compress.c rename to module/zfs-qat-provider/qat_compress.c index 6d0595dd5f76..afeeed268c9a 100644 --- a/module/os/linux/zfs/qat_compress.c +++ b/module/zfs-qat-provider/qat_compress.c @@ -19,7 +19,6 @@ * CDDL HEADER END */ -#if defined(_KERNEL) && defined(HAVE_QAT) #include #include #include @@ -27,7 +26,7 @@ #include #include #include -#include +#include /* * Max instances in a QAT device, each instance is a channel to submit @@ -49,13 +48,11 @@ static CpaBufferList **buffer_array[QAT_DC_MAX_INSTANCES]; static Cpa16U num_inst = 0; static Cpa32U inst_num = 0; static boolean_t qat_dc_init_done = B_FALSE; -int zfs_qat_compress_disable = 0; boolean_t qat_dc_use_accel(size_t s_len) { - return (!zfs_qat_compress_disable && - qat_dc_init_done && + return (qat_dc_init_done && s_len >= QAT_MIN_BUF_SIZE && s_len <= QAT_MAX_BUF_SIZE); } @@ -102,7 +99,7 @@ qat_dc_clean(void) } int -qat_dc_init(void) +qat_dc_init(Cpa16U *dev_count) { CpaStatus status = CPA_STATUS_SUCCESS; Cpa32U sess_size = 0; @@ -214,6 +211,9 @@ qat_dc_init(void) } qat_dc_init_done = B_TRUE; + if (dev_count) { + *dev_count = num_inst; + } return (0); fail: qat_dc_clean(); @@ -391,9 +391,6 @@ qat_compress_impl(qat_compress_dir_t dir, char *src, int src_len, init_completion(&complete); if (dir == QAT_COMPRESS) { - QAT_STAT_BUMP(comp_requests); - QAT_STAT_INCR(comp_total_in_bytes, src_len); - cpaDcGenerateHeader(session_handle, buf_list_dst->pBuffers, &hdr_sz); buf_list_dst->pBuffers->pData += hdr_sz; @@ -426,11 +423,8 @@ qat_compress_impl(qat_compress_dir_t dir, char *src, int src_len, BSWAP_32(dc_results.checksum); *c_len = hdr_sz + compressed_sz + ZLIB_FOOT_SZ; - QAT_STAT_INCR(comp_total_out_bytes, *c_len); } else { ASSERT3U(dir, ==, QAT_DECOMPRESS); - QAT_STAT_BUMP(decomp_requests); - QAT_STAT_INCR(decomp_total_in_bytes, src_len); buf_list_src->pBuffers->pData += ZLIB_HEAD_SZ; buf_list_src->pBuffers->dataLenInBytes -= ZLIB_HEAD_SZ; @@ -458,13 +452,9 @@ qat_compress_impl(qat_compress_dir_t dir, char *src, int src_len, goto fail; } *c_len = dc_results.produced; - QAT_STAT_INCR(decomp_total_out_bytes, *c_len); } fail: - if (status != CPA_STATUS_SUCCESS && status != CPA_STATUS_INCOMPRESSIBLE) - QAT_STAT_BUMP(dc_fails); - if (in_pages) { for (page_num = 0; page_num < buf_list_src->numBuffers; @@ -522,31 +512,3 @@ qat_compress(qat_compress_dir_t dir, char *src, int src_len, return (ret); } - -static int -param_set_qat_compress(const char *val, zfs_kernel_param_t *kp) -{ - int ret; - int *pvalue = kp->arg; - ret = param_set_int(val, kp); - if (ret) - return (ret); - /* - * zfs_qat_compress_disable = 0: enable qat compress - * try to initialize qat instance if it has not been done - */ - if (*pvalue == 0 && !qat_dc_init_done) { - ret = qat_dc_init(); - if (ret != 0) { - zfs_qat_compress_disable = 1; - return (ret); - } - } - return (ret); -} - -module_param_call(zfs_qat_compress_disable, param_set_qat_compress, - param_get_int, &zfs_qat_compress_disable, 0644); -MODULE_PARM_DESC(zfs_qat_compress_disable, "Enable/Disable QAT compression"); - -#endif diff --git a/module/os/linux/zfs/qat_crypt.c b/module/zfs-qat-provider/qat_crypt.c similarity index 86% rename from module/os/linux/zfs/qat_crypt.c rename to module/zfs-qat-provider/qat_crypt.c index 0523a23c61e1..3a9ce58d5b0b 100644 --- a/module/os/linux/zfs/qat_crypt.c +++ b/module/zfs-qat-provider/qat_crypt.c @@ -26,7 +26,6 @@ * compression instances, so that code is separated into qat_compress.c */ -#if defined(_KERNEL) && defined(HAVE_QAT) #include #include #include @@ -35,7 +34,7 @@ #include #include "lac/cpa_cy_im.h" #include "lac/cpa_cy_common.h" -#include +#include /* * Max instances in a QAT device, each instance is a channel to submit @@ -51,8 +50,6 @@ static Cpa32U inst_num = 0; static Cpa16U num_inst = 0; static CpaInstanceHandle cy_inst_handles[QAT_CRYPT_MAX_INSTANCES]; static boolean_t qat_cy_init_done = B_FALSE; -int zfs_qat_encrypt_disable = 0; -int zfs_qat_checksum_disable = 0; typedef struct cy_callback { CpaBoolean verify_result; @@ -75,8 +72,7 @@ symcallback(void *p_callback, CpaStatus status, const CpaCySymOp operation, boolean_t qat_crypt_use_accel(size_t s_len) { - return (!zfs_qat_encrypt_disable && - qat_cy_init_done && + return (qat_cy_init_done && s_len >= QAT_MIN_BUF_SIZE && s_len <= QAT_MAX_BUF_SIZE); } @@ -84,13 +80,12 @@ qat_crypt_use_accel(size_t s_len) boolean_t qat_checksum_use_accel(size_t s_len) { - return (!zfs_qat_checksum_disable && - qat_cy_init_done && + return (qat_cy_init_done && s_len >= QAT_MIN_BUF_SIZE && s_len <= QAT_MAX_BUF_SIZE); } -void +static void qat_cy_clean(void) { for (Cpa16U i = 0; i < num_inst; i++) @@ -316,23 +311,12 @@ qat_crypt(qat_encrypt_dir_t dir, uint8_t *src_buf, uint8_t *dst_buf, Cpa32U in_page_off = 0; Cpa32U out_page_off = 0; - if (dir == QAT_ENCRYPT) { - QAT_STAT_BUMP(encrypt_requests); - QAT_STAT_INCR(encrypt_total_in_bytes, enc_len); - } else { - QAT_STAT_BUMP(decrypt_requests); - QAT_STAT_INCR(decrypt_total_in_bytes, enc_len); - } - i = (Cpa32U)atomic_inc_32_nv(&inst_num) % num_inst; cy_inst_handle = cy_inst_handles[i]; status = qat_init_crypt_session_ctx(dir, cy_inst_handle, &cy_session_ctx, key, crypt, aad_len); if (status != CPA_STATUS_SUCCESS) { - /* don't count CCM as a failure since it's not supported */ - if (zio_crypt_table[crypt].ci_crypt_type == ZC_TYPE_GCM) - QAT_STAT_BUMP(crypt_fails); return (status); } @@ -436,15 +420,9 @@ qat_crypt(qat_encrypt_dir_t dir, uint8_t *src_buf, uint8_t *dst_buf, if (dir == QAT_ENCRYPT) { /* if dir is QAT_ENCRYPT, save pDigestResult to digest_buf */ memcpy(digest_buf, op_data.pDigestResult, ZIO_DATA_MAC_LEN); - QAT_STAT_INCR(encrypt_total_out_bytes, enc_len); - } else { - QAT_STAT_INCR(decrypt_total_out_bytes, enc_len); } fail: - if (status != CPA_STATUS_SUCCESS) - QAT_STAT_BUMP(crypt_fails); - for (i = 0; i < in_page_num; i++) kunmap(in_pages[i]); for (i = 0; i < out_page_num; i++) @@ -484,19 +462,12 @@ qat_checksum(uint64_t cksum, uint8_t *buf, uint64_t size, zio_cksum_t *zcp) Cpa32U page_num = 0; Cpa32U page_off = 0; - QAT_STAT_BUMP(cksum_requests); - QAT_STAT_INCR(cksum_total_in_bytes, size); - i = (Cpa32U)atomic_inc_32_nv(&inst_num) % num_inst; cy_inst_handle = cy_inst_handles[i]; status = qat_init_checksum_session_ctx(cy_inst_handle, &cy_session_ctx, cksum); if (status != CPA_STATUS_SUCCESS) { - /* don't count unsupported checksums as a failure */ - if (cksum == ZIO_CHECKSUM_SHA256 || - cksum == ZIO_CHECKSUM_SHA512) - QAT_STAT_BUMP(cksum_fails); return (status); } @@ -560,9 +531,6 @@ qat_checksum(uint64_t cksum, uint8_t *buf, uint64_t size, zio_cksum_t *zcp) memcpy(zcp, digest_buffer, sizeof (zio_cksum_t)); fail: - if (status != CPA_STATUS_SUCCESS) - QAT_STAT_BUMP(cksum_fails); - for (i = 0; i < page_num; i++) kunmap(in_pages[i]); @@ -574,57 +542,3 @@ qat_checksum(uint64_t cksum, uint8_t *buf, uint64_t size, zio_cksum_t *zcp) return (status); } - -static int -param_set_qat_encrypt(const char *val, zfs_kernel_param_t *kp) -{ - int ret; - int *pvalue = kp->arg; - ret = param_set_int(val, kp); - if (ret) - return (ret); - /* - * zfs_qat_encrypt_disable = 0: enable qat encrypt - * try to initialize qat instance if it has not been done - */ - if (*pvalue == 0 && !qat_cy_init_done) { - ret = qat_cy_init(); - if (ret != 0) { - zfs_qat_encrypt_disable = 1; - return (ret); - } - } - return (ret); -} - -static int -param_set_qat_checksum(const char *val, zfs_kernel_param_t *kp) -{ - int ret; - int *pvalue = kp->arg; - ret = param_set_int(val, kp); - if (ret) - return (ret); - /* - * set_checksum_param_ops = 0: enable qat checksum - * try to initialize qat instance if it has not been done - */ - if (*pvalue == 0 && !qat_cy_init_done) { - ret = qat_cy_init(); - if (ret != 0) { - zfs_qat_checksum_disable = 1; - return (ret); - } - } - return (ret); -} - -module_param_call(zfs_qat_encrypt_disable, param_set_qat_encrypt, - param_get_int, &zfs_qat_encrypt_disable, 0644); -MODULE_PARM_DESC(zfs_qat_encrypt_disable, "Enable/Disable QAT encryption"); - -module_param_call(zfs_qat_checksum_disable, param_set_qat_checksum, - param_get_int, &zfs_qat_checksum_disable, 0644); -MODULE_PARM_DESC(zfs_qat_checksum_disable, "Enable/Disable QAT checksumming"); - -#endif diff --git a/module/zfs/gzip.c b/module/zfs/gzip.c index f3b19446352a..c2a325ee9ff6 100644 --- a/module/zfs/gzip.c +++ b/module/zfs/gzip.c @@ -28,7 +28,6 @@ #include #include -#include #include #ifdef _KERNEL @@ -50,27 +49,10 @@ typedef uLongf zlen_t; size_t gzip_compress(void *s_start, void *d_start, size_t s_len, size_t d_len, int n) { - int ret; zlen_t dstlen = d_len; ASSERT(d_len <= s_len); - /* check if hardware accelerator can be used */ - if (qat_dc_use_accel(s_len)) { - ret = qat_compress(QAT_COMPRESS, s_start, s_len, d_start, - d_len, &dstlen); - if (ret == CPA_STATUS_SUCCESS) { - return ((size_t)dstlen); - } else if (ret == CPA_STATUS_INCOMPRESSIBLE) { - if (d_len != s_len) - return (s_len); - - memcpy(d_start, s_start, s_len); - return (s_len); - } - /* if hardware compression fails, do it again with software */ - } - if (compress_func(d_start, &dstlen, s_start, s_len, n) != Z_OK) { if (d_len != s_len) return (s_len); @@ -90,14 +72,6 @@ gzip_decompress(void *s_start, void *d_start, size_t s_len, size_t d_len, int n) ASSERT(d_len >= s_len); - /* check if hardware accelerator can be used */ - if (qat_dc_use_accel(d_len)) { - if (qat_compress(QAT_DECOMPRESS, s_start, s_len, - d_start, d_len, &dstlen) == CPA_STATUS_SUCCESS) - return (0); - /* if hardware de-compress fail, do it again with software */ - } - if (uncompress_func(d_start, &dstlen, s_start, s_len) != Z_OK) return (-1); diff --git a/module/zfs/sha2_zfs.c b/module/zfs/sha2_zfs.c index 872b1e53ee66..9621a69bf377 100644 --- a/module/zfs/sha2_zfs.c +++ b/module/zfs/sha2_zfs.c @@ -29,7 +29,6 @@ #include #include #include -#include static int sha_incremental(void *buf, size_t size, void *arg) @@ -44,25 +43,13 @@ abd_checksum_sha256(abd_t *abd, uint64_t size, const void *ctx_template, zio_cksum_t *zcp) { (void) ctx_template; - int ret; SHA2_CTX ctx; zio_cksum_t tmp; - if (qat_checksum_use_accel(size)) { - uint8_t *buf = abd_borrow_buf_copy(abd, size); - ret = qat_checksum(ZIO_CHECKSUM_SHA256, buf, size, &tmp); - abd_return_buf(abd, buf, size); - if (ret == CPA_STATUS_SUCCESS) - goto bswap; - - /* If the hardware implementation fails fall back to software */ - } - SHA2Init(SHA256, &ctx); (void) abd_iterate_func(abd, 0, size, sha_incremental, &ctx); SHA2Final(&tmp, &ctx); -bswap: /* * A prior implementation of this function had a * private SHA256 implementation always wrote things out in diff --git a/module/zfs/spa_misc.c b/module/zfs/spa_misc.c index 97191e768549..5f87fb9cc067 100644 --- a/module/zfs/spa_misc.c +++ b/module/zfs/spa_misc.c @@ -64,7 +64,6 @@ #include "zfs_prop.h" #include #include -#include #include /* @@ -2588,7 +2587,6 @@ spa_init(spa_mode_t mode) vdev_prop_init(); l2arc_start(); scan_init(); - qat_init(); spa_import_progress_init(); } @@ -2614,7 +2612,6 @@ spa_fini(void) zfs_refcount_fini(); fm_fini(); scan_fini(); - qat_fini(); spa_import_progress_destroy(); avl_destroy(&spa_namespace_avl);