Intel QAT Z.I.A. provider for ZFS only

This provider takes the original Intel QAT code found in ZFS
and moves it into a Z.I.A. provider. Because the original
code uses symbols from ZFS, this provider should be treated
as only usable by ZFS.

The original QAT code in ZFS did not use the new operation
chaining capability that was added to the QAT API in March
2019. This provider does not update the code to use it as
using chaining would cause operations to be performed out of
sync with ZFS.

The Intel QAT does not maintain memory locally and instead
always reads from in-memory buffers and write results to
in-memory buffers. This results in a lot of memory traffic.

Additionally, because the DPUSM (intentionally) does not have
an interface for passing raw pointers from the caller to the
accelerator, this provider would have allocated another
in-memory buffer to copy each offloaded ZFS buffer into,
doubling memory usage, but a layer violation was introduced
to prevent these unnecessary memcpys.

The vdev_raidz, vdev_file, and vdev_disk code was copied
from the example software provider.

Encryption is not supported by the DPUSM. The original
code was moved along with the rest of the QAT code, but
is now not used by ZFS.

Signed-off-by: Jason Lee <jasonlee@lanl.gov>

include/Makefile.am

@@ -74,7 +74,6 @@ COMMON_H = \
 	sys/nvpair_impl.h \
 	sys/objlist.h \
 	sys/pathname.h \
-	sys/qat.h \
 	sys/range_tree.h \
 	sys/rrwlock.h \
 	sys/sa.h \

module/Kbuild.in

@@ -49,11 +49,6 @@ CFLAGS_zfs/zap_micro.o += -mllvm -x86-cmov-converter=false
 endif
 endif
 
-ifneq ($(KBUILD_EXTMOD),)
-@CONFIG_QAT_TRUE@ZFS_MODULE_CFLAGS += -I@QAT_SRC@/include
-@CONFIG_QAT_TRUE@KBUILD_EXTRA_SYMBOLS += @QAT_SYMBOLS@
-endif
-
 asflags-y := $(ZFS_MODULE_CFLAGS) $(ZFS_MODULE_CPPFLAGS)
 ccflags-y := $(ZFS_MODULE_CFLAGS) $(ZFS_MODULE_CPPFLAGS)
 
@@ -439,9 +434,6 @@ ZFS_OBJS_OS := \
 	arc_os.o \
 	mmp_os.o \
 	policy.o \
-	qat.o \
-	qat_compress.o \
-	qat_crypt.o \
 	spa_misc_os.o \
 	trace.o \
 	vdev_disk.o \
@@ -506,17 +498,34 @@ $(obj)/zfs/vdev_raidz_math_powerpc_altivec.o : c_flags += -maltivec
 endif
 
 ifneq ("@DPUSM_SYMBOLS@","")
-obj-$(CONFIG_ZFS) += zia-software-provider.o
+@ZIA_ENABLED_TRUE@KBUILD_EXTRA_SYMBOLS += @DPUSM_SYMBOLS@
+
+@ZIA_ENABLED_TRUE@obj-$(CONFIG_ZFS) += zia-software-provider.o
 
 ZIA_SOFTWARE_PROVIDER_OBJS := \
 	provider.o \
 	kernel_offloader.o
 
-zia-software-provider-objs += $(addprefix zia-software-provider/,$(ZIA_SOFTWARE_PROVIDER_OBJS))
+@ZIA_ENABLED_TRUE@zia-software-provider-objs += $(addprefix zia-software-provider/,$(ZIA_SOFTWARE_PROVIDER_OBJS))
 # zfs_file_os does not have any dependencies, so just link to it directly
-zia-software-provider-objs += os/linux/zfs/zfs_file_os.o
+@ZIA_ENABLED_TRUE@zia-software-provider-objs += os/linux/zfs/zfs_file_os.o
 
-$(addprefix $(obj)/zia-software-provider/,$(ZIA_SOFTWARE_PROVIDER_OBJS)) : ccflags-y += -I@abs_top_builddir@ $(ZFS_MODULE_CFLAGS) -I@abs_srcdir@/zia-software-provider/ -I@DPUSM_ROOT@/include
+@ZIA_ENABLED_TRUE@$(addprefix $(obj)/zia-software-provider/,$(ZIA_SOFTWARE_PROVIDER_OBJS)) : ccflags-y += -I@abs_top_builddir@ $(ZFS_MODULE_CFLAGS) -I@abs_srcdir@/zia-software-provider/ -I@DPUSM_ROOT@/include/
 
-@ZIA_ENABLED_TRUE@KBUILD_EXTRA_SYMBOLS += @DPUSM_SYMBOLS@
-endif
+@ZIA_ENABLED_TRUE@@CONFIG_QAT_TRUE@obj-$(CONFIG_ZFS) += zfs-qat-provider.o
+
+ZFS_QAT_PROVIDER_OBJS := \
+    qat.o \
+    qat_compress.o \
+    qat_crypt.o \
+    provider.o
+
+@ZIA_ENABLED_TRUE@@CONFIG_QAT_TRUE@zfs-qat-provider-objs += $(addprefix zfs-qat-provider/,$(ZFS_QAT_PROVIDER_OBJS))
+# zfs_file_os does not have any dependencies, so just link to it directly
+@ZIA_ENABLED_TRUE@@CONFIG_QAT_TRUE@zfs-qat-provider-objs += os/linux/zfs/zfs_file_os.o
+
+@ZIA_ENABLED_TRUE@@CONFIG_QAT_TRUE@$(addprefix $(obj)/zfs-qat-provider/,$(ZFS_QAT_PROVIDER_OBJS)) : ccflags-y += -I@abs_top_builddir@ $(ZFS_MODULE_CFLAGS) -I@abs_srcdir@/zfs-qat-provider/ -I@DPUSM_ROOT@/include/ -I@QAT_SRC@/include/ -Wframe-larger-than=20480
+
+@ZIA_ENABLED_TRUE@KBUILD_EXTRA_SYMBOLS += @QAT_SYMBOLS@
+
+endif

module/Makefile.in

@@ -80,7 +80,7 @@ clean: clean-@ac_system@
 
 .PHONY: modules_uninstall-Linux-legacy
 modules_uninstall-Linux-legacy:
-	$(RM) -r $(addprefix $(KMODDIR)/$(INSTALL_MOD_DIR)/,spl/ avl/ icp/ lua/ nvpair/ unicode/ zcommon/ zfs/ zstd/ zia-software-provider)
+	$(RM) -r $(addprefix $(KMODDIR)/$(INSTALL_MOD_DIR)/,spl/ avl/ icp/ lua/ nvpair/ unicode/ zcommon/ zfs/ zstd/ zia-software-provider zfs-qat-provider)
 
 KMODDIR := $(INSTALL_MOD_PATH)/lib/modules/@LINUX_VERSION@
 modules_install-Linux: modules_uninstall-Linux-legacy
@@ -123,7 +123,7 @@ data_install: data_install-@ac_system@
 
 modules_uninstall-Linux: modules_uninstall-Linux-legacy
 	@# Uninstall the kernel modules
-	$(RM) $(addprefix $(KMODDIR)/$(INSTALL_MOD_DIR)/,zfs.ko spl.ko zia-software-provider)
+	$(RM) $(addprefix $(KMODDIR)/$(INSTALL_MOD_DIR)/,zfs.ko spl.ko zia-software-provider zfs-qat-provider)
 
 modules_uninstall-FreeBSD:
 	@false
@@ -153,7 +153,7 @@ cppcheck-Linux:
 		-I @top_srcdir@/include/os/linux/spl \
 		-I @top_srcdir@/include/os/linux/zfs \
 		-I @top_srcdir@/include \
-		avl icp lua nvpair unicode zcommon zfs zstd os/linux zia-software-provider
+		avl icp lua nvpair unicode zcommon zfs zstd os/linux zia-software-provider zfs-qat-provider
 
 cppcheck-FreeBSD:
 	@true