diff --git a/include/Makefile.am b/include/Makefile.am index fa16093d2003..bb08e6765d87 100644 --- a/include/Makefile.am +++ b/include/Makefile.am @@ -74,7 +74,6 @@ COMMON_H = \ sys/nvpair_impl.h \ sys/objlist.h \ sys/pathname.h \ - sys/qat.h \ sys/range_tree.h \ sys/rrwlock.h \ sys/sa.h \ diff --git a/include/sys/qat.h b/include/sys/qat.h deleted file mode 100644 index 76360ba99042..000000000000 --- a/include/sys/qat.h +++ /dev/null @@ -1,207 +0,0 @@ -/* - * CDDL HEADER START - * - * The contents of this file are subject to the terms of the - * Common Development and Distribution License (the "License"). - * You may not use this file except in compliance with the License. - * - * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE - * or https://opensource.org/licenses/CDDL-1.0. - * See the License for the specific language governing permissions - * and limitations under the License. - * - * When distributing Covered Code, include this CDDL HEADER in each - * file and include the License file at usr/src/OPENSOLARIS.LICENSE. - * If applicable, add the following below this CDDL HEADER, with the - * fields enclosed by brackets "[]" replaced with your own identifying - * information: Portions Copyright [yyyy] [name of copyright owner] - * - * CDDL HEADER END - */ - -#ifndef _SYS_QAT_H -#define _SYS_QAT_H - -typedef enum qat_compress_dir { - QAT_DECOMPRESS = 0, - QAT_COMPRESS = 1, -} qat_compress_dir_t; - -typedef enum qat_encrypt_dir { - QAT_DECRYPT = 0, - QAT_ENCRYPT = 1, -} qat_encrypt_dir_t; - - -#if defined(_KERNEL) && defined(HAVE_QAT) -#include -#include -#include "cpa.h" -#include "dc/cpa_dc.h" -#include "lac/cpa_cy_sym.h" - -/* - * The minimal and maximal buffer size which are not restricted - * in the QAT hardware, but with the input buffer size between 4KB - * and 128KB the hardware can provide the optimal performance. - */ -#define QAT_MIN_BUF_SIZE (4*1024) -#define QAT_MAX_BUF_SIZE (128*1024) - -/* - * Used for QAT kstat. - */ -typedef struct qat_stats { - /* - * Number of jobs submitted to QAT compression engine. - */ - kstat_named_t comp_requests; - /* - * Total bytes sent to QAT compression engine. - */ - kstat_named_t comp_total_in_bytes; - /* - * Total bytes output from QAT compression engine. - */ - kstat_named_t comp_total_out_bytes; - /* - * Number of jobs submitted to QAT de-compression engine. - */ - kstat_named_t decomp_requests; - /* - * Total bytes sent to QAT de-compression engine. - */ - kstat_named_t decomp_total_in_bytes; - /* - * Total bytes output from QAT de-compression engine. - */ - kstat_named_t decomp_total_out_bytes; - /* - * Number of fails in the QAT compression / decompression engine. - * Note: when a QAT error happens, it doesn't necessarily indicate a - * critical hardware issue. Sometimes it is because the output buffer - * is not big enough. The compression job will be transferred to the - * gzip software implementation so the functionality of ZFS is not - * impacted. - */ - kstat_named_t dc_fails; - - /* - * Number of jobs submitted to QAT encryption engine. - */ - kstat_named_t encrypt_requests; - /* - * Total bytes sent to QAT encryption engine. - */ - kstat_named_t encrypt_total_in_bytes; - /* - * Total bytes output from QAT encryption engine. - */ - kstat_named_t encrypt_total_out_bytes; - /* - * Number of jobs submitted to QAT decryption engine. - */ - kstat_named_t decrypt_requests; - /* - * Total bytes sent to QAT decryption engine. - */ - kstat_named_t decrypt_total_in_bytes; - /* - * Total bytes output from QAT decryption engine. - */ - kstat_named_t decrypt_total_out_bytes; - /* - * Number of fails in the QAT encryption / decryption engine. - * Note: when a QAT error happens, it doesn't necessarily indicate a - * critical hardware issue. The encryption job will be transferred - * to the software implementation so the functionality of ZFS is - * not impacted. - */ - kstat_named_t crypt_fails; - - /* - * Number of jobs submitted to QAT checksum engine. - */ - kstat_named_t cksum_requests; - /* - * Total bytes sent to QAT checksum engine. - */ - kstat_named_t cksum_total_in_bytes; - /* - * Number of fails in the QAT checksum engine. - * Note: when a QAT error happens, it doesn't necessarily indicate a - * critical hardware issue. The checksum job will be transferred to the - * software implementation so the functionality of ZFS is not impacted. - */ - kstat_named_t cksum_fails; -} qat_stats_t; - -#define QAT_STAT_INCR(stat, val) \ - atomic_add_64(&qat_stats.stat.value.ui64, (val)) -#define QAT_STAT_BUMP(stat) \ - QAT_STAT_INCR(stat, 1) - -extern qat_stats_t qat_stats; -extern int zfs_qat_compress_disable; -extern int zfs_qat_checksum_disable; -extern int zfs_qat_encrypt_disable; - -/* inlined for performance */ -static inline struct page * -qat_mem_to_page(void *addr) -{ - if (!is_vmalloc_addr(addr)) - return (virt_to_page(addr)); - - return (vmalloc_to_page(addr)); -} - -CpaStatus qat_mem_alloc_contig(void **pp_mem_addr, Cpa32U size_bytes); -void qat_mem_free_contig(void **pp_mem_addr); -#define QAT_PHYS_CONTIG_ALLOC(pp_mem_addr, size_bytes) \ - qat_mem_alloc_contig((void *)(pp_mem_addr), (size_bytes)) -#define QAT_PHYS_CONTIG_FREE(p_mem_addr) \ - qat_mem_free_contig((void *)&(p_mem_addr)) - -extern int qat_dc_init(void); -extern void qat_dc_fini(void); -extern int qat_cy_init(void); -extern void qat_cy_fini(void); -extern int qat_init(void); -extern void qat_fini(void); - -/* fake CpaStatus used to indicate data was not compressible */ -#define CPA_STATUS_INCOMPRESSIBLE (-127) - -extern boolean_t qat_dc_use_accel(size_t s_len); -extern boolean_t qat_crypt_use_accel(size_t s_len); -extern boolean_t qat_checksum_use_accel(size_t s_len); -extern int qat_compress(qat_compress_dir_t dir, char *src, int src_len, - char *dst, int dst_len, size_t *c_len); -extern int qat_crypt(qat_encrypt_dir_t dir, uint8_t *src_buf, uint8_t *dst_buf, - uint8_t *aad_buf, uint32_t aad_len, uint8_t *iv_buf, uint8_t *digest_buf, - crypto_key_t *key, uint64_t crypt, uint32_t enc_len); -extern int qat_checksum(uint64_t cksum, uint8_t *buf, uint64_t size, - zio_cksum_t *zcp); -#else -#define CPA_STATUS_SUCCESS 0 -#define CPA_STATUS_INCOMPRESSIBLE (-127) -#define qat_init() -#define qat_fini() -#define qat_dc_use_accel(s_len) ((void) sizeof (s_len), 0) -#define qat_crypt_use_accel(s_len) ((void) sizeof (s_len), 0) -#define qat_checksum_use_accel(s_len) ((void) sizeof (s_len), 0) -#define qat_compress(dir, s, sl, d, dl, cl) \ - ((void) sizeof (dir), (void) sizeof (s), (void) sizeof (sl), \ - (void) sizeof (d), (void) sizeof (dl), (void) sizeof (cl), 0) -#define qat_crypt(dir, s, d, a, al, i, db, k, c, el) \ - ((void) sizeof (dir), (void) sizeof (s), (void) sizeof (d), \ - (void) sizeof (a), (void) sizeof (al), (void) sizeof (i), \ - (void) sizeof (db), (void) sizeof (k), (void) sizeof (c), \ - (void) sizeof (el), 0) -#define qat_checksum(c, buf, s, z) \ - ((void) sizeof (c), (void) sizeof (buf), (void) sizeof (s), \ - (void) sizeof (z), 0) -#endif - -#endif /* _SYS_QAT_H */ diff --git a/module/Kbuild.in b/module/Kbuild.in index bd8c9d50262c..d7a32ed83e11 100644 --- a/module/Kbuild.in +++ b/module/Kbuild.in @@ -49,11 +49,6 @@ CFLAGS_zfs/zap_micro.o += -mllvm -x86-cmov-converter=false endif endif -ifneq ($(KBUILD_EXTMOD),) -@CONFIG_QAT_TRUE@ZFS_MODULE_CFLAGS += -I@QAT_SRC@/include -@CONFIG_QAT_TRUE@KBUILD_EXTRA_SYMBOLS += @QAT_SYMBOLS@ -endif - asflags-y := $(ZFS_MODULE_CFLAGS) $(ZFS_MODULE_CPPFLAGS) ccflags-y := $(ZFS_MODULE_CFLAGS) $(ZFS_MODULE_CPPFLAGS) @@ -439,9 +434,6 @@ ZFS_OBJS_OS := \ arc_os.o \ mmp_os.o \ policy.o \ - qat.o \ - qat_compress.o \ - qat_crypt.o \ spa_misc_os.o \ trace.o \ vdev_disk.o \ @@ -506,17 +498,34 @@ $(obj)/zfs/vdev_raidz_math_powerpc_altivec.o : c_flags += -maltivec endif ifneq ("@DPUSM_SYMBOLS@","") -obj-$(CONFIG_ZFS) += zia-software-provider.o +@ZIA_ENABLED_TRUE@KBUILD_EXTRA_SYMBOLS += @DPUSM_SYMBOLS@ + +@ZIA_ENABLED_TRUE@obj-$(CONFIG_ZFS) += zia-software-provider.o ZIA_SOFTWARE_PROVIDER_OBJS := \ provider.o \ kernel_offloader.o -zia-software-provider-objs += $(addprefix zia-software-provider/,$(ZIA_SOFTWARE_PROVIDER_OBJS)) +@ZIA_ENABLED_TRUE@zia-software-provider-objs += $(addprefix zia-software-provider/,$(ZIA_SOFTWARE_PROVIDER_OBJS)) # zfs_file_os does not have any dependencies, so just link to it directly -zia-software-provider-objs += os/linux/zfs/zfs_file_os.o +@ZIA_ENABLED_TRUE@zia-software-provider-objs += os/linux/zfs/zfs_file_os.o -$(addprefix $(obj)/zia-software-provider/,$(ZIA_SOFTWARE_PROVIDER_OBJS)) : ccflags-y += -I@abs_top_builddir@ $(ZFS_MODULE_CFLAGS) -I@abs_srcdir@/zia-software-provider/ -I@DPUSM_ROOT@/include +@ZIA_ENABLED_TRUE@$(addprefix $(obj)/zia-software-provider/,$(ZIA_SOFTWARE_PROVIDER_OBJS)) : ccflags-y += -I@abs_top_builddir@ $(ZFS_MODULE_CFLAGS) -I@abs_srcdir@/zia-software-provider/ -I@DPUSM_ROOT@/include/ -@ZIA_ENABLED_TRUE@KBUILD_EXTRA_SYMBOLS += @DPUSM_SYMBOLS@ -endif \ No newline at end of file +@ZIA_ENABLED_TRUE@@CONFIG_QAT_TRUE@obj-$(CONFIG_ZFS) += zfs-qat-provider.o + +ZFS_QAT_PROVIDER_OBJS := \ + qat.o \ + qat_compress.o \ + qat_crypt.o \ + provider.o + +@ZIA_ENABLED_TRUE@@CONFIG_QAT_TRUE@zfs-qat-provider-objs += $(addprefix zfs-qat-provider/,$(ZFS_QAT_PROVIDER_OBJS)) +# zfs_file_os does not have any dependencies, so just link to it directly +@ZIA_ENABLED_TRUE@@CONFIG_QAT_TRUE@zfs-qat-provider-objs += os/linux/zfs/zfs_file_os.o + +@ZIA_ENABLED_TRUE@@CONFIG_QAT_TRUE@$(addprefix $(obj)/zfs-qat-provider/,$(ZFS_QAT_PROVIDER_OBJS)) : ccflags-y += -I@abs_top_builddir@ $(ZFS_MODULE_CFLAGS) -I@abs_srcdir@/zfs-qat-provider/ -I@DPUSM_ROOT@/include/ -I@QAT_SRC@/include/ -Wframe-larger-than=20480 + +@ZIA_ENABLED_TRUE@KBUILD_EXTRA_SYMBOLS += @QAT_SYMBOLS@ + +endif diff --git a/module/Makefile.in b/module/Makefile.in index c780c491de39..c3f435e24504 100644 --- a/module/Makefile.in +++ b/module/Makefile.in @@ -80,7 +80,7 @@ clean: clean-@ac_system@ .PHONY: modules_uninstall-Linux-legacy modules_uninstall-Linux-legacy: - $(RM) -r $(addprefix $(KMODDIR)/$(INSTALL_MOD_DIR)/,spl/ avl/ icp/ lua/ nvpair/ unicode/ zcommon/ zfs/ zstd/ zia-software-provider) + $(RM) -r $(addprefix $(KMODDIR)/$(INSTALL_MOD_DIR)/,spl/ avl/ icp/ lua/ nvpair/ unicode/ zcommon/ zfs/ zstd/ zia-software-provider zfs-qat-provider) KMODDIR := $(INSTALL_MOD_PATH)/lib/modules/@LINUX_VERSION@ modules_install-Linux: modules_uninstall-Linux-legacy @@ -123,7 +123,7 @@ data_install: data_install-@ac_system@ modules_uninstall-Linux: modules_uninstall-Linux-legacy @# Uninstall the kernel modules - $(RM) $(addprefix $(KMODDIR)/$(INSTALL_MOD_DIR)/,zfs.ko spl.ko zia-software-provider) + $(RM) $(addprefix $(KMODDIR)/$(INSTALL_MOD_DIR)/,zfs.ko spl.ko zia-software-provider zfs-qat-provider) modules_uninstall-FreeBSD: @false @@ -153,7 +153,7 @@ cppcheck-Linux: -I @top_srcdir@/include/os/linux/spl \ -I @top_srcdir@/include/os/linux/zfs \ -I @top_srcdir@/include \ - avl icp lua nvpair unicode zcommon zfs zstd os/linux zia-software-provider + avl icp lua nvpair unicode zcommon zfs zstd os/linux zia-software-provider zfs-qat-provider cppcheck-FreeBSD: @true diff --git a/module/os/linux/zfs/qat.c b/module/os/linux/zfs/qat.c deleted file mode 100644 index 07e0cafabb0e..000000000000 --- a/module/os/linux/zfs/qat.c +++ /dev/null @@ -1,105 +0,0 @@ -/* - * CDDL HEADER START - * - * The contents of this file are subject to the terms of the - * Common Development and Distribution License (the "License"). - * You may not use this file except in compliance with the License. - * - * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE - * or https://opensource.org/licenses/CDDL-1.0. - * See the License for the specific language governing permissions - * and limitations under the License. - * - * When distributing Covered Code, include this CDDL HEADER in each - * file and include the License file at usr/src/OPENSOLARIS.LICENSE. - * If applicable, add the following below this CDDL HEADER, with the - * fields enclosed by brackets "[]" replaced with your own identifying - * information: Portions Copyright [yyyy] [name of copyright owner] - * - * CDDL HEADER END - */ - -#if defined(_KERNEL) && defined(HAVE_QAT) -#include -#include - -qat_stats_t qat_stats = { - { "comp_requests", KSTAT_DATA_UINT64 }, - { "comp_total_in_bytes", KSTAT_DATA_UINT64 }, - { "comp_total_out_bytes", KSTAT_DATA_UINT64 }, - { "decomp_requests", KSTAT_DATA_UINT64 }, - { "decomp_total_in_bytes", KSTAT_DATA_UINT64 }, - { "decomp_total_out_bytes", KSTAT_DATA_UINT64 }, - { "dc_fails", KSTAT_DATA_UINT64 }, - { "encrypt_requests", KSTAT_DATA_UINT64 }, - { "encrypt_total_in_bytes", KSTAT_DATA_UINT64 }, - { "encrypt_total_out_bytes", KSTAT_DATA_UINT64 }, - { "decrypt_requests", KSTAT_DATA_UINT64 }, - { "decrypt_total_in_bytes", KSTAT_DATA_UINT64 }, - { "decrypt_total_out_bytes", KSTAT_DATA_UINT64 }, - { "crypt_fails", KSTAT_DATA_UINT64 }, - { "cksum_requests", KSTAT_DATA_UINT64 }, - { "cksum_total_in_bytes", KSTAT_DATA_UINT64 }, - { "cksum_fails", KSTAT_DATA_UINT64 }, -}; - -static kstat_t *qat_ksp = NULL; - -CpaStatus -qat_mem_alloc_contig(void **pp_mem_addr, Cpa32U size_bytes) -{ - *pp_mem_addr = kmalloc(size_bytes, GFP_KERNEL); - if (*pp_mem_addr == NULL) - return (CPA_STATUS_RESOURCE); - return (CPA_STATUS_SUCCESS); -} - -void -qat_mem_free_contig(void **pp_mem_addr) -{ - if (*pp_mem_addr != NULL) { - kfree(*pp_mem_addr); - *pp_mem_addr = NULL; - } -} - -int -qat_init(void) -{ - qat_ksp = kstat_create("zfs", 0, "qat", "misc", - KSTAT_TYPE_NAMED, sizeof (qat_stats) / sizeof (kstat_named_t), - KSTAT_FLAG_VIRTUAL); - if (qat_ksp != NULL) { - qat_ksp->ks_data = &qat_stats; - kstat_install(qat_ksp); - } - - /* - * Just set the disable flag when qat init failed, qat can be - * turned on again in post-process after zfs module is loaded, e.g.: - * echo 0 > /sys/module/zfs/parameters/zfs_qat_compress_disable - */ - if (qat_dc_init() != 0) - zfs_qat_compress_disable = 1; - - if (qat_cy_init() != 0) { - zfs_qat_checksum_disable = 1; - zfs_qat_encrypt_disable = 1; - } - - return (0); -} - -void -qat_fini(void) -{ - if (qat_ksp != NULL) { - kstat_delete(qat_ksp); - qat_ksp = NULL; - } - - qat_cy_fini(); - qat_dc_fini(); -} - -#endif diff --git a/module/os/linux/zfs/zio_crypt.c b/module/os/linux/zfs/zio_crypt.c index 21f3740f6fe6..ff8666511351 100644 --- a/module/os/linux/zfs/zio_crypt.c +++ b/module/os/linux/zfs/zio_crypt.c @@ -26,7 +26,6 @@ #include #include #include -#include /* * This file is responsible for handling all of the details of generating @@ -1955,37 +1954,6 @@ zio_do_crypt_data(boolean_t encrypt, zio_crypt_key_t *key, tmpl = NULL; } - /* - * Attempt to use QAT acceleration if we can. We currently don't - * do this for metadnode and ZIL blocks, since they have a much - * more involved buffer layout and the qat_crypt() function only - * works in-place. - */ - if (qat_crypt_use_accel(datalen) && - ot != DMU_OT_INTENT_LOG && ot != DMU_OT_DNODE) { - uint8_t *srcbuf, *dstbuf; - - if (encrypt) { - srcbuf = plainbuf; - dstbuf = cipherbuf; - } else { - srcbuf = cipherbuf; - dstbuf = plainbuf; - } - - ret = qat_crypt((encrypt) ? QAT_ENCRYPT : QAT_DECRYPT, srcbuf, - dstbuf, NULL, 0, iv, mac, ckey, key->zk_crypt, datalen); - if (ret == CPA_STATUS_SUCCESS) { - if (locked) { - rw_exit(&key->zk_salt_lock); - locked = B_FALSE; - } - - return (0); - } - /* If the hardware implementation fails fall back to software */ - } - /* create uios for encryption */ ret = zio_crypt_init_uios(encrypt, key->zk_version, ot, plainbuf, cipherbuf, datalen, byteswap, mac, &puio, &cuio, &enc_len, @@ -2077,4 +2045,9 @@ zio_do_crypt_abd(boolean_t encrypt, zio_crypt_key_t *key, dmu_object_type_t ot, module_param(zfs_key_max_salt_uses, ulong, 0644); MODULE_PARM_DESC(zfs_key_max_salt_uses, "Max number of times a salt value " "can be used for generating encryption keys before it is rotated"); + +#if defined(ZIA) && defined(HAVE_QAT) +EXPORT_SYMBOL(zio_crypt_table); +#endif + #endif diff --git a/module/zfs-qat-provider/provider.c b/module/zfs-qat-provider/provider.c new file mode 100644 index 000000000000..571d3e658b4e --- /dev/null +++ b/module/zfs-qat-provider/provider.c @@ -0,0 +1,640 @@ +/* + * © 2021. Triad National Security, LLC. All rights reserved. + * + * This program was produced under U.S. Government contract + * 89233218CNA000001 for Los Alamos National Laboratory (LANL), which + * is operated by Triad National Security, LLC for the U.S. + * Department of Energy/National Nuclear Security Administration. All + * rights in the program are reserved by Triad National Security, LLC, + * and the U.S. Department of Energy/National Nuclear Security + * Administration. The Government is granted for itself and others + * acting on its behalf a nonexclusive, paid-up, irrevocable worldwide + * license in this material to reproduce, prepare derivative works, + * distribute copies to the public, perform publicly and display + * publicly, and to permit others to do so. + * + * ---- + * + * This program is open source under the BSD-3 License. + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from this + * software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include +#include +#include + +#include /* the DPUSM provider API */ +#include /* QAT wrapper from ZFS */ + +/* should not be here */ +#include +#include +#include +#include +#include +#include +#include + +typedef enum zfs_qat_handle_type { + ZQH_REAL, + ZQH_REF, +} zqh_type_t; + +typedef struct zfs_qat_provider_handle { + zqh_type_t type; + void *ptr; + size_t size; +} zqh_t; + +static void * +ptr_start(zqh_t *zqh, const size_t offset) +{ + return (((char *)zqh->ptr) + offset); +} + +static int +zfs_qat_provider_algorithms(int *compress, int *decompress, + int *checksum, int *checksum_byteorder, int *raid) +{ + *compress = + DPUSM_COMPRESS_GZIP_1 | + DPUSM_COMPRESS_GZIP_2 | + DPUSM_COMPRESS_GZIP_3 | + DPUSM_COMPRESS_GZIP_4 | + DPUSM_COMPRESS_GZIP_5 | + DPUSM_COMPRESS_GZIP_6 | + DPUSM_COMPRESS_GZIP_7 | + DPUSM_COMPRESS_GZIP_8 | + DPUSM_COMPRESS_GZIP_9; + + *decompress = *compress; + + *checksum = DPUSM_CHECKSUM_SHA256 | DPUSM_CHECKSUM_SHA512; + + *checksum_byteorder = DPUSM_BYTEORDER_NATIVE | DPUSM_BYTEORDER_BYTESWAP; + + *raid = + DPUSM_RAID_1_GEN | + DPUSM_RAID_2_GEN | + DPUSM_RAID_3_GEN | + DPUSM_RAID_1_REC | + DPUSM_RAID_2_REC | + DPUSM_RAID_3_REC; + + return (DPUSM_OK); +} + +static void * +zfs_qat_provider_alloc(size_t size) +{ + zqh_t *buf = kmalloc(sizeof (zqh_t), GFP_KERNEL); + buf->type = ZQH_REAL; + buf->ptr = kmalloc(size, GFP_KERNEL); + buf->size = size; + return (buf); +} + +static void * +zfs_qat_provider_alloc_ref(void *src_handle, size_t offset, size_t size) +{ + zqh_t *src = (zqh_t *)src_handle; + + zqh_t *ref = kmalloc(sizeof (zqh_t), GFP_KERNEL); + ref->type = ZQH_REF; + ref->ptr = ptr_start(src, offset); + ref->size = size; + + return (ref); +} + +static int +zfs_qat_provider_get_size(void *handle, size_t *size, size_t *actual) +{ + zqh_t *buf = (zqh_t *)handle; + if (size) { + *size = buf->size; + } + + if (actual) { + *actual = buf->size; + } + + return (DPUSM_OK); +} + +static int +zfs_qat_provider_free(void *handle) +{ + zqh_t *buf = (zqh_t *)handle; + if (buf->type == ZQH_REAL) { + kfree(buf->ptr); + } + kfree(buf); + + return (DPUSM_OK); +} + +static int +zfs_qat_provider_copy_from_generic(dpusm_mv_t *mv, const void *buf, size_t size) +{ + zqh_t *dst = (zqh_t *)mv->handle; + /* + * If the handle points to a real buffer, free it, + * and convert the handle to a reference to the + * user's pointer. + * + * This is a layer violation, but is done to prevent + * unnecessary copies of the data due to the QAT using + * in-memory addresses. + */ + if ((dst->type == ZQH_REAL) && + (mv->offset == 0)) { + kfree(dst->ptr); + + dst->type = ZQH_REF; + dst->ptr = (void *) buf; + dst->size = size; + } + else + { + memcpy(ptr_start(dst, mv->offset), buf, size); + } + return (DPUSM_OK); +} + +static int +zfs_qat_provider_copy_to_generic(dpusm_mv_t *mv, void *buf, size_t size) +{ + zqh_t *src = (zqh_t *)mv->handle; + /* + * if the handle is still a real buffer, it is not + * the same as buf, so the contents have to be copied + * into buf. + */ + void *start = ptr_start(src, mv->offset); + if (start != buf) { + memcpy(buf, start, size); + } + return (DPUSM_OK); +} + +static int +zfs_qat_provider_zero_fill(void *handle, size_t offset, size_t size) +{ + memset(ptr_start(handle, offset), 0, size); + return (DPUSM_OK); +} + +static int +zfs_qat_provider_all_zeros(void *handle, size_t offset, size_t size) +{ + zqh_t *zqh = (zqh_t *)handle; + for (size_t i = 0; i < size; i++) { + if (((char *)zqh->ptr)[offset + i]) { + return (DPUSM_ERROR); + } + } + return (DPUSM_OK); +} + +static int +zfs_qat_provider_compress(dpusm_compress_t alg, int level, + void *src, size_t s_len, void *dst, size_t *d_len) +{ + (void) alg; /* unused */ + (void) level; /* unused */ + + /* check if hardware accelerator can be used */ + if (!qat_dc_use_accel(s_len)) { + return (DPUSM_ERROR); + } + + zqh_t *s = (zqh_t *)src; + zqh_t *d = (zqh_t *)dst; + + if ((s_len > s->size) || + (*d_len > d->size)) { + return (DPUSM_ERROR); + } + + void *s_start = ptr_start(s, 0); + void *d_start = ptr_start(d, 0); + + const int ret = qat_compress(QAT_COMPRESS, + s_start, s_len, d_start, *d_len, d_len); + + if (ret == CPA_STATUS_SUCCESS) { + return (DPUSM_OK); + } else if (ret == CPA_STATUS_INCOMPRESSIBLE) { + *d_len = s_len; + return (DPUSM_OK); + } + return (DPUSM_ERROR); +} + +static int +zfs_qat_provider_decompress(dpusm_compress_t alg, int *level, + void *src, size_t s_len, void *dst, size_t *d_len) +{ + (void) alg; /* unused */ + (void) level; /* unused */ + + /* check if hardware accelerator can be used */ + if (!qat_dc_use_accel(*d_len)) { + return (DPUSM_ERROR); + } + + zqh_t *s = (zqh_t *)src; + zqh_t *d = (zqh_t *)dst; + + if ((s_len > s->size) || + (*d_len > d->size)) { + return (DPUSM_ERROR); + } + + void *s_start = ptr_start(s, 0); + void *d_start = ptr_start(d, 0); + + if (qat_compress(QAT_DECOMPRESS, s_start, s_len, + d_start, *d_len, d_len) == CPA_STATUS_SUCCESS) { + return (DPUSM_OK); + } + + return (DPUSM_ERROR); +} + +static int +zfs_qat_provider_checksum(dpusm_checksum_t alg, + dpusm_checksum_byteorder_t order, void *data, size_t size, + void *cksum, size_t cksum_size) +{ + (void) order; /* might have to handle here */ + + if (!qat_checksum_use_accel(size)) { + return (DPUSM_ERROR); + } + + if (alg != DPUSM_CHECKSUM_SHA256) { + return (DPUSM_NOT_IMPLEMENTED); + } + + if (cksum_size < sizeof (zio_cksum_t)) { + return (DPUSM_ERROR); + } + + zqh_t *src = (zqh_t *)data; + if (size > src->size) { + return (DPUSM_ERROR); + } + + const int ret = qat_checksum(ZIO_CHECKSUM_SHA256, src->ptr, size, + cksum); + return ((ret == CPA_STATUS_SUCCESS)?DPUSM_OK:DPUSM_ERROR); +} + +static int +zfs_qat_provider_raidz_can_compute(size_t nparity, size_t ndata, + size_t *col_sizes, int rec) +{ + if ((nparity < 1) || (nparity > 3)) { + return (DPUSM_NOT_SUPPORTED); + } + + return (DPUSM_OK); +} + +static void * +zfs_qat_provider_raidz_alloc(size_t nparity, size_t ndata) +{ + const size_t ncols = nparity + ndata; + + const size_t rr_size = offsetof(raidz_row_t, rr_col[ncols]); + raidz_row_t *rr = kzalloc(rr_size, GFP_KERNEL); + rr->rr_cols = ncols; + rr->rr_firstdatacol = nparity; + + return (rr); +} + +/* attaches a column to the raidz struct */ +static int +zfs_qat_provider_raidz_set_column(void *raidz, uint64_t c, + void *col, size_t size) +{ + raidz_row_t *rr = (raidz_row_t *)raidz; + zqh_t *zqh = (zqh_t *)col; + + if (!rr || !zqh) { + return (DPUSM_ERROR); + } + + /* c is too big */ + if (c >= rr->rr_cols) { + return (DPUSM_ERROR); + } + + /* "active" size is larger than allocated size */ + if (size > zqh->size) { + return (DPUSM_ERROR); + } + + raidz_col_t *rc = &rr->rr_col[c]; + + /* clean up old column */ + abd_free(rc->rc_abd); + + /* + * rc->rc_abd does not take ownership of zqh->ptr, + * so don't need to release ownership + */ + rc->rc_abd = abd_get_from_buf(zqh->ptr, size); + rc->rc_size = size; + + return (DPUSM_OK); +} + +static int +zfs_qat_provider_raidz_free(void *raidz) +{ + raidz_row_t *rr = (raidz_row_t *)raidz; + for (int c = 0; c < rr->rr_cols; c++) { + raidz_col_t *rc = &rr->rr_col[c]; + abd_free(rc->rc_abd); + } + kfree(rr); + + return (DPUSM_OK); +} + +static int +zfs_qat_provider_raidz_gen(void *raidz) +{ + raidz_row_t *rr = (raidz_row_t *)raidz; + switch (rr->rr_firstdatacol) { + case 1: + vdev_raidz_generate_parity_p(rr); + break; + case 2: + vdev_raidz_generate_parity_pq(rr); + break; + case 3: + vdev_raidz_generate_parity_pqr(rr); + break; + } + + return (DPUSM_OK); +} + +static int +zfs_qat_provider_raidz_rec(void *raidz, int *tgts, int ntgts) +{ + raidz_row_t *rr = (raidz_row_t *)raidz; + vdev_raidz_reconstruct_general(rr, tgts, ntgts); + + return (DPUSM_OK); +} + +static int +zfs_qat_provider_raidz_cmp(void *lhs_handle, void *rhs_handle, int *diff) +{ + zqh_t *lhs = (zqh_t *)lhs_handle; + zqh_t *rhs = (zqh_t *)rhs_handle; + + if (!diff) { + return (DPUSM_ERROR); + } + + size_t len = rhs->size; + if (lhs->size != rhs->size) { + len = + (lhs->size < rhs->size)?lhs->size:rhs->size; + } + + *diff = memcmp(ptr_start(lhs, 0), + ptr_start(rhs, 0), len); + + return (DPUSM_OK); +} + +static void * +zfs_qat_provider_file_open(const char *path, int flags, int mode) +{ + zfs_file_t *fp = NULL; + /* on error, fp should still be NULL */ + zfs_file_open(path, flags, mode, &fp); + return (fp); +} + +static int +zfs_qat_provider_file_write(void *fp_handle, void *handle, size_t count, + size_t trailing_zeros, loff_t offset, ssize_t *resid, int *err) +{ + zfs_file_t *fp = (zfs_file_t *)fp_handle; + zqh_t *zqh = (zqh_t *)handle; + + if (!err) { + return (EIO); + } + + *err = zfs_file_pwrite(fp, ptr_start(zqh, 0), + count, offset, resid); + + if (*err == 0) { + void *zeros = kzalloc(trailing_zeros, GFP_KERNEL); + *err = zfs_file_pwrite(fp, zeros, + trailing_zeros, offset + count, resid); + kfree(zeros); + } + + return (*err); +} + +static void +zfs_qat_provider_file_close(void *fp_handle) +{ + zfs_file_close(fp_handle); +} + +static void * +zfs_qat_provider_disk_open(dpusm_dd_t *disk_data) +{ + return (disk_data->bdev); +} + +static int +zfs_qat_provider_disk_invalidate(void *disk_handle) +{ + struct block_device *bdev = + (struct block_device *)disk_handle; + invalidate_bdev(bdev); + return (DPUSM_OK); +} + +static int +zfs_qat_provider_disk_write(void *disk_handle, void *handle, size_t data_size, + size_t trailing_zeros, uint64_t io_offset, int flags, + dpusm_disk_write_completion_t write_completion, void *wc_args) +{ + struct block_device *bdev = + (struct block_device *)disk_handle; + zqh_t *zqh = (zqh_t *)handle; + + const size_t io_size = data_size + trailing_zeros; + + if (trailing_zeros) { + /* create a copy of the data with the trailing zeros attached */ + void *copy = kzalloc(io_size, GFP_KERNEL); + memcpy(copy, ptr_start(zqh, 0), data_size); + + /* need to keep copy alive, so replace zqh->ptr */ + if (zqh->type == ZQH_REAL) { + kfree(zqh->ptr); + } + + zqh->type = ZQH_REAL; + zqh->ptr = copy; + zqh->size = io_size; + } + + abd_t *abd = abd_get_from_buf(zqh->ptr, io_size); + zio_push_transform(wc_args, abd, io_size, io_size, NULL); + + /* __vdev_disk_physio already adds write_completion */ + (void) write_completion; + + return (__vdev_classic_physio(bdev, wc_args, + io_size, io_offset, WRITE, flags)); +} + +static int +zfs_qat_provider_disk_flush(void *disk_handle, + dpusm_disk_flush_completion_t flush_completion, void *fc_args) +{ + struct block_device *bdev = + (struct block_device *)disk_handle; + + /* vdev_disk_io_flush already adds flush completion */ + (void) flush_completion; + + return (vdev_disk_io_flush(bdev, fc_args)); +} + +static void +zfs_qat_provider_disk_close(void *disk_handle) +{} + +/* + * "zfs-qat-provider" instead of "qat-provider" + * because this provider links with ZFS symbols + */ +/* BEGIN CSTYLED */ +static const dpusm_pf_t zfs_qat_provider_functions = { + .algorithms = zfs_qat_provider_algorithms, + .alloc = zfs_qat_provider_alloc, + .alloc_ref = zfs_qat_provider_alloc_ref, + .get_size = zfs_qat_provider_get_size, + .free = zfs_qat_provider_free, + .copy = { + .from = { + .generic = zfs_qat_provider_copy_from_generic, + .ptr = NULL, + .scatterlist = NULL, + }, + .to = { + .generic = zfs_qat_provider_copy_to_generic, + .ptr = NULL, + .scatterlist = NULL, + }, + }, + .at_connect = NULL, + .at_disconnect = NULL, + .mem_stats = NULL, + .zero_fill = zfs_qat_provider_zero_fill, + .all_zeros = zfs_qat_provider_all_zeros, + .compress = zfs_qat_provider_compress, + .decompress = zfs_qat_provider_decompress, + .checksum = zfs_qat_provider_checksum, + .raid = { + .can_compute = zfs_qat_provider_raidz_can_compute, + .alloc = zfs_qat_provider_raidz_alloc, + .set_column = zfs_qat_provider_raidz_set_column, + .free = zfs_qat_provider_raidz_free, + .gen = zfs_qat_provider_raidz_gen, + .cmp = zfs_qat_provider_raidz_cmp, + .rec = zfs_qat_provider_raidz_rec, + }, + .file = { + .open = zfs_qat_provider_file_open, + .write = zfs_qat_provider_file_write, + .close = zfs_qat_provider_file_close, + }, + .disk = { + .open = zfs_qat_provider_disk_open, + .invalidate = zfs_qat_provider_disk_invalidate, + .write = zfs_qat_provider_disk_write, + .flush = zfs_qat_provider_disk_flush, + .close = zfs_qat_provider_disk_close, + }, +}; +/* END CSTYLED */ + +static int __init +zfs_qat_provider_init(void) +{ + Cpa16U dev_count = 0; + if (qat_dc_init(&dev_count) != 0) { + qat_dc_fini(); + printk("%s: Error: Could not initialize provider\n", + module_name(THIS_MODULE)); + return (-ENODEV); + } + + if (dev_count == 0) { + printk("%s: Error: No devices found\n", + module_name(THIS_MODULE)); + return (-ENODEV); + } + + printk("%s: Initialized\n", module_name(THIS_MODULE)); + + return (dpusm_register_bsd(THIS_MODULE, &zfs_qat_provider_functions)); +} + +static void __exit +zfs_qat_provider_exit(void) +{ + dpusm_unregister_bsd(THIS_MODULE); + + qat_dc_fini(); + printk("%s: Uninitialized\n", module_name(THIS_MODULE)); +} + +module_init(zfs_qat_provider_init); +module_exit(zfs_qat_provider_exit); + +MODULE_LICENSE("CDDL"); diff --git a/module/zfs-qat-provider/qat.c b/module/zfs-qat-provider/qat.c new file mode 100644 index 000000000000..0c6abc573135 --- /dev/null +++ b/module/zfs-qat-provider/qat.c @@ -0,0 +1,41 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or https://opensource.org/licenses/CDDL-1.0. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +#include +#include + +CpaStatus +qat_mem_alloc_contig(void **pp_mem_addr, Cpa32U size_bytes) +{ + *pp_mem_addr = kmalloc(size_bytes, GFP_KERNEL); + if (*pp_mem_addr == NULL) + return (CPA_STATUS_RESOURCE); + return (CPA_STATUS_SUCCESS); +} + +void +qat_mem_free_contig(void **pp_mem_addr) +{ + if (*pp_mem_addr != NULL) { + kfree(*pp_mem_addr); + *pp_mem_addr = NULL; + } +} diff --git a/module/zfs-qat-provider/qat.h b/module/zfs-qat-provider/qat.h new file mode 100644 index 000000000000..cbade8a891b3 --- /dev/null +++ b/module/zfs-qat-provider/qat.h @@ -0,0 +1,85 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or https://opensource.org/licenses/CDDL-1.0. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +#ifndef _QAT_H +#define _QAT_H + +typedef enum qat_compress_dir { + QAT_DECOMPRESS = 0, + QAT_COMPRESS = 1, +} qat_compress_dir_t; + +typedef enum qat_encrypt_dir { + QAT_DECRYPT = 0, + QAT_ENCRYPT = 1, +} qat_encrypt_dir_t; + +#include +#include +#include "cpa.h" +#include "dc/cpa_dc.h" +#include "lac/cpa_cy_sym.h" + +/* + * The minimal and maximal buffer size which are not restricted + * in the QAT hardware, but with the input buffer size between 4KB + * and 128KB the hardware can provide the optimal performance. + */ +#define QAT_MIN_BUF_SIZE (4*1024) +#define QAT_MAX_BUF_SIZE (128*1024) + +/* inlined for performance */ +static inline struct page * +qat_mem_to_page(void *addr) +{ + if (!is_vmalloc_addr(addr)) + return (virt_to_page(addr)); + + return (vmalloc_to_page(addr)); +} + +CpaStatus qat_mem_alloc_contig(void **pp_mem_addr, Cpa32U size_bytes); +void qat_mem_free_contig(void **pp_mem_addr); +#define QAT_PHYS_CONTIG_ALLOC(pp_mem_addr, size_bytes) \ + qat_mem_alloc_contig((void *)(pp_mem_addr), (size_bytes)) +#define QAT_PHYS_CONTIG_FREE(p_mem_addr) \ + qat_mem_free_contig((void *)&(p_mem_addr)) + +extern int qat_dc_init(Cpa16U *dev_count); +extern void qat_dc_fini(void); +extern int qat_cy_init(void); +extern void qat_cy_fini(void); + +/* fake CpaStatus used to indicate data was not compressible */ +#define CPA_STATUS_INCOMPRESSIBLE (-127) + +extern boolean_t qat_dc_use_accel(size_t s_len); +extern boolean_t qat_crypt_use_accel(size_t s_len); +extern boolean_t qat_checksum_use_accel(size_t s_len); +extern int qat_compress(qat_compress_dir_t dir, char *src, int src_len, + char *dst, int dst_len, size_t *c_len); +extern int qat_crypt(qat_encrypt_dir_t dir, uint8_t *src_buf, uint8_t *dst_buf, + uint8_t *aad_buf, uint32_t aad_len, uint8_t *iv_buf, uint8_t *digest_buf, + crypto_key_t *key, uint64_t crypt, uint32_t enc_len); +extern int qat_checksum(uint64_t cksum, uint8_t *buf, uint64_t size, + zio_cksum_t *zcp); + +#endif /* _QAT_H */ diff --git a/module/os/linux/zfs/qat_compress.c b/module/zfs-qat-provider/qat_compress.c similarity index 92% rename from module/os/linux/zfs/qat_compress.c rename to module/zfs-qat-provider/qat_compress.c index 6d0595dd5f76..afeeed268c9a 100644 --- a/module/os/linux/zfs/qat_compress.c +++ b/module/zfs-qat-provider/qat_compress.c @@ -19,7 +19,6 @@ * CDDL HEADER END */ -#if defined(_KERNEL) && defined(HAVE_QAT) #include #include #include @@ -27,7 +26,7 @@ #include #include #include -#include +#include /* * Max instances in a QAT device, each instance is a channel to submit @@ -49,13 +48,11 @@ static CpaBufferList **buffer_array[QAT_DC_MAX_INSTANCES]; static Cpa16U num_inst = 0; static Cpa32U inst_num = 0; static boolean_t qat_dc_init_done = B_FALSE; -int zfs_qat_compress_disable = 0; boolean_t qat_dc_use_accel(size_t s_len) { - return (!zfs_qat_compress_disable && - qat_dc_init_done && + return (qat_dc_init_done && s_len >= QAT_MIN_BUF_SIZE && s_len <= QAT_MAX_BUF_SIZE); } @@ -102,7 +99,7 @@ qat_dc_clean(void) } int -qat_dc_init(void) +qat_dc_init(Cpa16U *dev_count) { CpaStatus status = CPA_STATUS_SUCCESS; Cpa32U sess_size = 0; @@ -214,6 +211,9 @@ qat_dc_init(void) } qat_dc_init_done = B_TRUE; + if (dev_count) { + *dev_count = num_inst; + } return (0); fail: qat_dc_clean(); @@ -391,9 +391,6 @@ qat_compress_impl(qat_compress_dir_t dir, char *src, int src_len, init_completion(&complete); if (dir == QAT_COMPRESS) { - QAT_STAT_BUMP(comp_requests); - QAT_STAT_INCR(comp_total_in_bytes, src_len); - cpaDcGenerateHeader(session_handle, buf_list_dst->pBuffers, &hdr_sz); buf_list_dst->pBuffers->pData += hdr_sz; @@ -426,11 +423,8 @@ qat_compress_impl(qat_compress_dir_t dir, char *src, int src_len, BSWAP_32(dc_results.checksum); *c_len = hdr_sz + compressed_sz + ZLIB_FOOT_SZ; - QAT_STAT_INCR(comp_total_out_bytes, *c_len); } else { ASSERT3U(dir, ==, QAT_DECOMPRESS); - QAT_STAT_BUMP(decomp_requests); - QAT_STAT_INCR(decomp_total_in_bytes, src_len); buf_list_src->pBuffers->pData += ZLIB_HEAD_SZ; buf_list_src->pBuffers->dataLenInBytes -= ZLIB_HEAD_SZ; @@ -458,13 +452,9 @@ qat_compress_impl(qat_compress_dir_t dir, char *src, int src_len, goto fail; } *c_len = dc_results.produced; - QAT_STAT_INCR(decomp_total_out_bytes, *c_len); } fail: - if (status != CPA_STATUS_SUCCESS && status != CPA_STATUS_INCOMPRESSIBLE) - QAT_STAT_BUMP(dc_fails); - if (in_pages) { for (page_num = 0; page_num < buf_list_src->numBuffers; @@ -522,31 +512,3 @@ qat_compress(qat_compress_dir_t dir, char *src, int src_len, return (ret); } - -static int -param_set_qat_compress(const char *val, zfs_kernel_param_t *kp) -{ - int ret; - int *pvalue = kp->arg; - ret = param_set_int(val, kp); - if (ret) - return (ret); - /* - * zfs_qat_compress_disable = 0: enable qat compress - * try to initialize qat instance if it has not been done - */ - if (*pvalue == 0 && !qat_dc_init_done) { - ret = qat_dc_init(); - if (ret != 0) { - zfs_qat_compress_disable = 1; - return (ret); - } - } - return (ret); -} - -module_param_call(zfs_qat_compress_disable, param_set_qat_compress, - param_get_int, &zfs_qat_compress_disable, 0644); -MODULE_PARM_DESC(zfs_qat_compress_disable, "Enable/Disable QAT compression"); - -#endif diff --git a/module/os/linux/zfs/qat_crypt.c b/module/zfs-qat-provider/qat_crypt.c similarity index 86% rename from module/os/linux/zfs/qat_crypt.c rename to module/zfs-qat-provider/qat_crypt.c index 0523a23c61e1..3a9ce58d5b0b 100644 --- a/module/os/linux/zfs/qat_crypt.c +++ b/module/zfs-qat-provider/qat_crypt.c @@ -26,7 +26,6 @@ * compression instances, so that code is separated into qat_compress.c */ -#if defined(_KERNEL) && defined(HAVE_QAT) #include #include #include @@ -35,7 +34,7 @@ #include #include "lac/cpa_cy_im.h" #include "lac/cpa_cy_common.h" -#include +#include /* * Max instances in a QAT device, each instance is a channel to submit @@ -51,8 +50,6 @@ static Cpa32U inst_num = 0; static Cpa16U num_inst = 0; static CpaInstanceHandle cy_inst_handles[QAT_CRYPT_MAX_INSTANCES]; static boolean_t qat_cy_init_done = B_FALSE; -int zfs_qat_encrypt_disable = 0; -int zfs_qat_checksum_disable = 0; typedef struct cy_callback { CpaBoolean verify_result; @@ -75,8 +72,7 @@ symcallback(void *p_callback, CpaStatus status, const CpaCySymOp operation, boolean_t qat_crypt_use_accel(size_t s_len) { - return (!zfs_qat_encrypt_disable && - qat_cy_init_done && + return (qat_cy_init_done && s_len >= QAT_MIN_BUF_SIZE && s_len <= QAT_MAX_BUF_SIZE); } @@ -84,13 +80,12 @@ qat_crypt_use_accel(size_t s_len) boolean_t qat_checksum_use_accel(size_t s_len) { - return (!zfs_qat_checksum_disable && - qat_cy_init_done && + return (qat_cy_init_done && s_len >= QAT_MIN_BUF_SIZE && s_len <= QAT_MAX_BUF_SIZE); } -void +static void qat_cy_clean(void) { for (Cpa16U i = 0; i < num_inst; i++) @@ -316,23 +311,12 @@ qat_crypt(qat_encrypt_dir_t dir, uint8_t *src_buf, uint8_t *dst_buf, Cpa32U in_page_off = 0; Cpa32U out_page_off = 0; - if (dir == QAT_ENCRYPT) { - QAT_STAT_BUMP(encrypt_requests); - QAT_STAT_INCR(encrypt_total_in_bytes, enc_len); - } else { - QAT_STAT_BUMP(decrypt_requests); - QAT_STAT_INCR(decrypt_total_in_bytes, enc_len); - } - i = (Cpa32U)atomic_inc_32_nv(&inst_num) % num_inst; cy_inst_handle = cy_inst_handles[i]; status = qat_init_crypt_session_ctx(dir, cy_inst_handle, &cy_session_ctx, key, crypt, aad_len); if (status != CPA_STATUS_SUCCESS) { - /* don't count CCM as a failure since it's not supported */ - if (zio_crypt_table[crypt].ci_crypt_type == ZC_TYPE_GCM) - QAT_STAT_BUMP(crypt_fails); return (status); } @@ -436,15 +420,9 @@ qat_crypt(qat_encrypt_dir_t dir, uint8_t *src_buf, uint8_t *dst_buf, if (dir == QAT_ENCRYPT) { /* if dir is QAT_ENCRYPT, save pDigestResult to digest_buf */ memcpy(digest_buf, op_data.pDigestResult, ZIO_DATA_MAC_LEN); - QAT_STAT_INCR(encrypt_total_out_bytes, enc_len); - } else { - QAT_STAT_INCR(decrypt_total_out_bytes, enc_len); } fail: - if (status != CPA_STATUS_SUCCESS) - QAT_STAT_BUMP(crypt_fails); - for (i = 0; i < in_page_num; i++) kunmap(in_pages[i]); for (i = 0; i < out_page_num; i++) @@ -484,19 +462,12 @@ qat_checksum(uint64_t cksum, uint8_t *buf, uint64_t size, zio_cksum_t *zcp) Cpa32U page_num = 0; Cpa32U page_off = 0; - QAT_STAT_BUMP(cksum_requests); - QAT_STAT_INCR(cksum_total_in_bytes, size); - i = (Cpa32U)atomic_inc_32_nv(&inst_num) % num_inst; cy_inst_handle = cy_inst_handles[i]; status = qat_init_checksum_session_ctx(cy_inst_handle, &cy_session_ctx, cksum); if (status != CPA_STATUS_SUCCESS) { - /* don't count unsupported checksums as a failure */ - if (cksum == ZIO_CHECKSUM_SHA256 || - cksum == ZIO_CHECKSUM_SHA512) - QAT_STAT_BUMP(cksum_fails); return (status); } @@ -560,9 +531,6 @@ qat_checksum(uint64_t cksum, uint8_t *buf, uint64_t size, zio_cksum_t *zcp) memcpy(zcp, digest_buffer, sizeof (zio_cksum_t)); fail: - if (status != CPA_STATUS_SUCCESS) - QAT_STAT_BUMP(cksum_fails); - for (i = 0; i < page_num; i++) kunmap(in_pages[i]); @@ -574,57 +542,3 @@ qat_checksum(uint64_t cksum, uint8_t *buf, uint64_t size, zio_cksum_t *zcp) return (status); } - -static int -param_set_qat_encrypt(const char *val, zfs_kernel_param_t *kp) -{ - int ret; - int *pvalue = kp->arg; - ret = param_set_int(val, kp); - if (ret) - return (ret); - /* - * zfs_qat_encrypt_disable = 0: enable qat encrypt - * try to initialize qat instance if it has not been done - */ - if (*pvalue == 0 && !qat_cy_init_done) { - ret = qat_cy_init(); - if (ret != 0) { - zfs_qat_encrypt_disable = 1; - return (ret); - } - } - return (ret); -} - -static int -param_set_qat_checksum(const char *val, zfs_kernel_param_t *kp) -{ - int ret; - int *pvalue = kp->arg; - ret = param_set_int(val, kp); - if (ret) - return (ret); - /* - * set_checksum_param_ops = 0: enable qat checksum - * try to initialize qat instance if it has not been done - */ - if (*pvalue == 0 && !qat_cy_init_done) { - ret = qat_cy_init(); - if (ret != 0) { - zfs_qat_checksum_disable = 1; - return (ret); - } - } - return (ret); -} - -module_param_call(zfs_qat_encrypt_disable, param_set_qat_encrypt, - param_get_int, &zfs_qat_encrypt_disable, 0644); -MODULE_PARM_DESC(zfs_qat_encrypt_disable, "Enable/Disable QAT encryption"); - -module_param_call(zfs_qat_checksum_disable, param_set_qat_checksum, - param_get_int, &zfs_qat_checksum_disable, 0644); -MODULE_PARM_DESC(zfs_qat_checksum_disable, "Enable/Disable QAT checksumming"); - -#endif diff --git a/module/zfs/gzip.c b/module/zfs/gzip.c index f3b19446352a..c2a325ee9ff6 100644 --- a/module/zfs/gzip.c +++ b/module/zfs/gzip.c @@ -28,7 +28,6 @@ #include #include -#include #include #ifdef _KERNEL @@ -50,27 +49,10 @@ typedef uLongf zlen_t; size_t gzip_compress(void *s_start, void *d_start, size_t s_len, size_t d_len, int n) { - int ret; zlen_t dstlen = d_len; ASSERT(d_len <= s_len); - /* check if hardware accelerator can be used */ - if (qat_dc_use_accel(s_len)) { - ret = qat_compress(QAT_COMPRESS, s_start, s_len, d_start, - d_len, &dstlen); - if (ret == CPA_STATUS_SUCCESS) { - return ((size_t)dstlen); - } else if (ret == CPA_STATUS_INCOMPRESSIBLE) { - if (d_len != s_len) - return (s_len); - - memcpy(d_start, s_start, s_len); - return (s_len); - } - /* if hardware compression fails, do it again with software */ - } - if (compress_func(d_start, &dstlen, s_start, s_len, n) != Z_OK) { if (d_len != s_len) return (s_len); @@ -90,14 +72,6 @@ gzip_decompress(void *s_start, void *d_start, size_t s_len, size_t d_len, int n) ASSERT(d_len >= s_len); - /* check if hardware accelerator can be used */ - if (qat_dc_use_accel(d_len)) { - if (qat_compress(QAT_DECOMPRESS, s_start, s_len, - d_start, d_len, &dstlen) == CPA_STATUS_SUCCESS) - return (0); - /* if hardware de-compress fail, do it again with software */ - } - if (uncompress_func(d_start, &dstlen, s_start, s_len) != Z_OK) return (-1); diff --git a/module/zfs/sha2_zfs.c b/module/zfs/sha2_zfs.c index 872b1e53ee66..9621a69bf377 100644 --- a/module/zfs/sha2_zfs.c +++ b/module/zfs/sha2_zfs.c @@ -29,7 +29,6 @@ #include #include #include -#include static int sha_incremental(void *buf, size_t size, void *arg) @@ -44,25 +43,13 @@ abd_checksum_sha256(abd_t *abd, uint64_t size, const void *ctx_template, zio_cksum_t *zcp) { (void) ctx_template; - int ret; SHA2_CTX ctx; zio_cksum_t tmp; - if (qat_checksum_use_accel(size)) { - uint8_t *buf = abd_borrow_buf_copy(abd, size); - ret = qat_checksum(ZIO_CHECKSUM_SHA256, buf, size, &tmp); - abd_return_buf(abd, buf, size); - if (ret == CPA_STATUS_SUCCESS) - goto bswap; - - /* If the hardware implementation fails fall back to software */ - } - SHA2Init(SHA256, &ctx); (void) abd_iterate_func(abd, 0, size, sha_incremental, &ctx); SHA2Final(&tmp, &ctx); -bswap: /* * A prior implementation of this function had a * private SHA256 implementation always wrote things out in diff --git a/module/zfs/spa_misc.c b/module/zfs/spa_misc.c index 97191e768549..5f87fb9cc067 100644 --- a/module/zfs/spa_misc.c +++ b/module/zfs/spa_misc.c @@ -64,7 +64,6 @@ #include "zfs_prop.h" #include #include -#include #include /* @@ -2588,7 +2587,6 @@ spa_init(spa_mode_t mode) vdev_prop_init(); l2arc_start(); scan_init(); - qat_init(); spa_import_progress_init(); } @@ -2614,7 +2612,6 @@ spa_fini(void) zfs_refcount_fini(); fm_fini(); scan_fini(); - qat_fini(); spa_import_progress_destroy(); avl_destroy(&spa_namespace_avl);