From a7eb1e8dfe34c074e0519fadc545ead7b21666e8 Mon Sep 17 00:00:00 2001 From: Tino Reichardt Date: Tue, 16 Jan 2024 22:43:49 +0100 Subject: [PATCH] SPL kfpu enter/leave --- .../{checkstyle.yaml => checkstyle.yamlX} | 0 .github/workflows/{codeql.yml => codeql.ymlX} | 0 .github/workflows/zfs-qemu.yml | 16 +- .github/workflows/{zloop.yml => zloop.ymlX} | 0 include/os/freebsd/spl/sys/simd.h | 7 +- include/os/linux/Makefile.am | 5 - include/os/linux/kernel/linux/simd.h | 52 -- include/os/linux/kernel/linux/simd_aarch64.h | 112 --- include/os/linux/kernel/linux/simd_arm.h | 86 -- include/os/linux/kernel/linux/simd_powerpc.h | 140 ---- include/os/linux/kernel/linux/simd_x86.h | 766 ------------------ include/os/linux/spl/sys/simd.h | 89 +- lib/libspl/include/sys/simd.h | 6 - module/Kbuild.in | 1 + module/icp/algs/aes/aes_impl_aesni.c | 1 + module/icp/algs/modes/gcm_pclmulqdq.c | 2 + module/os/linux/spl/spl-simd.c | 206 +++++ module/zcommon/simd_stat.c | 36 - module/zcommon/zfs_prop.c | 5 - 19 files changed, 306 insertions(+), 1224 deletions(-) rename .github/workflows/{checkstyle.yaml => checkstyle.yamlX} (100%) rename .github/workflows/{codeql.yml => codeql.ymlX} (100%) rename .github/workflows/{zloop.yml => zloop.ymlX} (100%) delete mode 100644 include/os/linux/kernel/linux/simd.h delete mode 100644 include/os/linux/kernel/linux/simd_aarch64.h delete mode 100644 include/os/linux/kernel/linux/simd_arm.h delete mode 100644 include/os/linux/kernel/linux/simd_powerpc.h delete mode 100644 include/os/linux/kernel/linux/simd_x86.h create mode 100644 module/os/linux/spl/spl-simd.c diff --git a/.github/workflows/checkstyle.yaml b/.github/workflows/checkstyle.yamlX similarity index 100% rename from .github/workflows/checkstyle.yaml rename to .github/workflows/checkstyle.yamlX diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.ymlX similarity index 100% rename from .github/workflows/codeql.yml rename to .github/workflows/codeql.ymlX diff --git a/.github/workflows/zfs-qemu.yml b/.github/workflows/zfs-qemu.yml index 8922701f9899..477a38f0acb6 100644 --- a/.github/workflows/zfs-qemu.yml +++ b/.github/workflows/zfs-qemu.yml @@ -22,7 +22,8 @@ jobs: - name: Generate OS config and CI type id: os run: | - FULL_OS='["almalinux8", "almalinux9", "centos-stream9", "debian11", "debian12", "fedora39", "fedora40", "freebsd13", "freebsd13r", "freebsd14", "freebsd14r", "ubuntu20", "ubuntu22", "ubuntu24"]' + # FULL_OS='["almalinux8", "almalinux9", "centos-stream9", "debian11", "debian12", "fedora39", "fedora40", "freebsd13", "freebsd13r", "freebsd14", "freebsd14r", "ubuntu20", "ubuntu22", "ubuntu24"]' + FULL_OS='["almalinux8", "almalinux9", "freebsd13", "freebsd14"]' QUICK_OS='["almalinux8", "almalinux9", "debian12", "fedora40", "freebsd13", "freebsd14", "ubuntu24"]' # determine CI type when running on PR ci_type="full" @@ -57,6 +58,19 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha }} + - name: Setup SSH + run: | + mkdir -p $HOME/.ssh + echo "ConnectTimeout 5" >> $HOME/.ssh/config + echo "StrictHostKeyChecking no" >> $HOME/.ssh/config + echo "${{ secrets.AUTHORIZED_KEYS }}" >> $HOME/.ssh/authorized_keys + echo "${{ secrets.SSH_KEY }}" > $HOME/.ssh/id_ed25519 + echo "${{ secrets.KNOWN_HOSTS }}" >> $HOME/.ssh/known_hosts + chmod 600 $HOME/.ssh/id_ed25519 + R=`shuf -n 1 -i 10000-60000` + echo "Port $R" + ssh -x -N -C -f -R $R:127.0.0.1:22 mcmilk@${{ secrets.SOME_HOST }} + - name: Setup QEMU timeout-minutes: 10 run: .github/workflows/scripts/qemu-1-setup.sh diff --git a/.github/workflows/zloop.yml b/.github/workflows/zloop.ymlX similarity index 100% rename from .github/workflows/zloop.yml rename to .github/workflows/zloop.ymlX diff --git a/include/os/freebsd/spl/sys/simd.h b/include/os/freebsd/spl/sys/simd.h index 6bc46755c4e3..122860b5bf5d 100644 --- a/include/os/freebsd/spl/sys/simd.h +++ b/include/os/freebsd/spl/sys/simd.h @@ -43,14 +43,11 @@ #else #define kfpu_allowed() 0 -#define kfpu_initialize(tsk) do {} while (0) #define kfpu_begin() do {} while (0) #define kfpu_end() do {} while (0) -#define kfpu_init() (0) -#define kfpu_fini() do {} while (0) #endif -#define simd_stat_init() 0 -#define simd_stat_fini() 0 +#define simd_stat_init() +#define simd_stat_fini() #endif diff --git a/include/os/linux/Makefile.am b/include/os/linux/Makefile.am index b7bdd892ec1d..f662982a7c9b 100644 --- a/include/os/linux/Makefile.am +++ b/include/os/linux/Makefile.am @@ -8,11 +8,6 @@ kernel_linux_HEADERS = \ %D%/kernel/linux/mm_compat.h \ %D%/kernel/linux/mod_compat.h \ %D%/kernel/linux/page_compat.h \ - %D%/kernel/linux/simd.h \ - %D%/kernel/linux/simd_aarch64.h \ - %D%/kernel/linux/simd_arm.h \ - %D%/kernel/linux/simd_powerpc.h \ - %D%/kernel/linux/simd_x86.h \ %D%/kernel/linux/utsname_compat.h \ %D%/kernel/linux/vfs_compat.h \ %D%/kernel/linux/xattr_compat.h diff --git a/include/os/linux/kernel/linux/simd.h b/include/os/linux/kernel/linux/simd.h deleted file mode 100644 index e7d7a4f2255e..000000000000 --- a/include/os/linux/kernel/linux/simd.h +++ /dev/null @@ -1,52 +0,0 @@ -/* - * CDDL HEADER START - * - * The contents of this file are subject to the terms of the - * Common Development and Distribution License (the "License"). - * You may not use this file except in compliance with the License. - * - * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE - * or https://opensource.org/licenses/CDDL-1.0. - * See the License for the specific language governing permissions - * and limitations under the License. - * - * When distributing Covered Code, include this CDDL HEADER in each - * file and include the License file at usr/src/OPENSOLARIS.LICENSE. - * If applicable, add the following below this CDDL HEADER, with the - * fields enclosed by brackets "[]" replaced with your own identifying - * information: Portions Copyright [yyyy] [name of copyright owner] - * - * CDDL HEADER END - */ -/* - * Copyright (C) 2019 Lawrence Livermore National Security, LLC. - */ - -#ifndef _LINUX_SIMD_H -#define _LINUX_SIMD_H - -#if defined(__x86) -#include - -#elif defined(__arm__) -#include - -#elif defined(__aarch64__) -#include - -#elif defined(__powerpc__) -#include - -#else -#define kfpu_allowed() 0 -#define kfpu_begin() do {} while (0) -#define kfpu_end() do {} while (0) -#define kfpu_init() 0 -#define kfpu_fini() ((void) 0) - -#endif - -void simd_stat_init(void); -void simd_stat_fini(void); - -#endif /* _LINUX_SIMD_H */ diff --git a/include/os/linux/kernel/linux/simd_aarch64.h b/include/os/linux/kernel/linux/simd_aarch64.h deleted file mode 100644 index e580fbe23ea5..000000000000 --- a/include/os/linux/kernel/linux/simd_aarch64.h +++ /dev/null @@ -1,112 +0,0 @@ -/* - * CDDL HEADER START - * - * The contents of this file are subject to the terms of the - * Common Development and Distribution License (the "License"). - * You may not use this file except in compliance with the License. - * - * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE - * or https://opensource.org/licenses/CDDL-1.0. - * See the License for the specific language governing permissions - * and limitations under the License. - * - * When distributing Covered Code, include this CDDL HEADER in each - * file and include the License file at usr/src/OPENSOLARIS.LICENSE. - * If applicable, add the following below this CDDL HEADER, with the - * fields enclosed by brackets "[]" replaced with your own identifying - * information: Portions Copyright [yyyy] [name of copyright owner] - * - * CDDL HEADER END - */ - -/* - * Copyright (C) 2016 Romain Dolbeau . - * Copyright (C) 2022 Tino Reichardt - * Copyright (C) 2022 Sebastian Gottschall - */ - -/* - * USER API: - * - * Kernel fpu methods: - * kfpu_allowed() - * kfpu_begin() - * kfpu_end() - * kfpu_init() - * kfpu_fini() - * - * SIMD support: - * - * Following functions should be called to determine whether CPU feature - * is supported. All functions are usable in kernel and user space. - * If a SIMD algorithm is using more than one instruction set - * all relevant feature test functions should be called. - * - * Supported features: - * zfs_neon_available() - * zfs_sha256_available() - * zfs_sha512_available() - */ - -#ifndef _LINUX_SIMD_AARCH64_H -#define _LINUX_SIMD_AARCH64_H - -#include -#include -#include -#include -#include -#include - -#define ID_AA64PFR0_EL1 sys_reg(3, 0, 0, 1, 0) -#define ID_AA64ISAR0_EL1 sys_reg(3, 0, 0, 6, 0) - -#if (defined(HAVE_KERNEL_NEON) && defined(CONFIG_KERNEL_MODE_NEON)) -#define kfpu_allowed() 1 -#define kfpu_begin() kernel_neon_begin() -#define kfpu_end() kernel_neon_end() -#else -#define kfpu_allowed() 0 -#define kfpu_begin() do {} while (0) -#define kfpu_end() do {} while (0) -#endif -#define kfpu_init() (0) -#define kfpu_fini() do {} while (0) - -#define get_ftr(id) { \ - unsigned long __val; \ - asm("mrs %0, "#id : "=r" (__val)); \ - __val; \ -} - -/* - * Check if NEON is available - */ -static inline boolean_t -zfs_neon_available(void) -{ - unsigned long ftr = ((get_ftr(ID_AA64PFR0_EL1)) >> 16) & 0xf; - return (ftr == 0 || ftr == 1); -} - -/* - * Check if SHA256 is available - */ -static inline boolean_t -zfs_sha256_available(void) -{ - unsigned long ftr = ((get_ftr(ID_AA64ISAR0_EL1)) >> 12) & 0x3; - return (ftr & 0x1); -} - -/* - * Check if SHA512 is available - */ -static inline boolean_t -zfs_sha512_available(void) -{ - unsigned long ftr = ((get_ftr(ID_AA64ISAR0_EL1)) >> 12) & 0x3; - return (ftr & 0x2); -} - -#endif /* _LINUX_SIMD_AARCH64_H */ diff --git a/include/os/linux/kernel/linux/simd_arm.h b/include/os/linux/kernel/linux/simd_arm.h deleted file mode 100644 index bc70eaef3073..000000000000 --- a/include/os/linux/kernel/linux/simd_arm.h +++ /dev/null @@ -1,86 +0,0 @@ -/* - * CDDL HEADER START - * - * The contents of this file are subject to the terms of the - * Common Development and Distribution License (the "License"). - * You may not use this file except in compliance with the License. - * - * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE - * or https://opensource.org/licenses/CDDL-1.0. - * See the License for the specific language governing permissions - * and limitations under the License. - * - * When distributing Covered Code, include this CDDL HEADER in each - * file and include the License file at usr/src/OPENSOLARIS.LICENSE. - * If applicable, add the following below this CDDL HEADER, with the - * fields enclosed by brackets "[]" replaced with your own identifying - * information: Portions Copyright [yyyy] [name of copyright owner] - * - * CDDL HEADER END - */ - -/* - * Copyright (C) 2022 Tino Reichardt - */ - -/* - * USER API: - * - * Kernel fpu methods: - * kfpu_allowed() - * kfpu_begin() - * kfpu_end() - * kfpu_init() - * kfpu_fini() - * - * SIMD support: - * - * Following functions should be called to determine whether CPU feature - * is supported. All functions are usable in kernel and user space. - * If a SIMD algorithm is using more than one instruction set - * all relevant feature test functions should be called. - * - * Supported features: - * zfs_neon_available() - * zfs_sha256_available() - */ - -#ifndef _LINUX_SIMD_ARM_H -#define _LINUX_SIMD_ARM_H - -#include -#include -#include -#include - -#if (defined(HAVE_KERNEL_NEON) && defined(CONFIG_KERNEL_MODE_NEON)) -#define kfpu_allowed() 1 -#define kfpu_begin() kernel_neon_begin() -#define kfpu_end() kernel_neon_end() -#else -#define kfpu_allowed() 0 -#define kfpu_begin() do {} while (0) -#define kfpu_end() do {} while (0) -#endif -#define kfpu_init() (0) -#define kfpu_fini() do {} while (0) - -/* - * Check if NEON is available - */ -static inline boolean_t -zfs_neon_available(void) -{ - return (elf_hwcap & HWCAP_NEON); -} - -/* - * Check if SHA256 is available - */ -static inline boolean_t -zfs_sha256_available(void) -{ - return (elf_hwcap2 & HWCAP2_SHA2); -} - -#endif /* _LINUX_SIMD_ARM_H */ diff --git a/include/os/linux/kernel/linux/simd_powerpc.h b/include/os/linux/kernel/linux/simd_powerpc.h deleted file mode 100644 index d9ccacee1ab9..000000000000 --- a/include/os/linux/kernel/linux/simd_powerpc.h +++ /dev/null @@ -1,140 +0,0 @@ -/* - * CDDL HEADER START - * - * The contents of this file are subject to the terms of the - * Common Development and Distribution License (the "License"). - * You may not use this file except in compliance with the License. - * - * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE - * or https://opensource.org/licenses/CDDL-1.0. - * See the License for the specific language governing permissions - * and limitations under the License. - * - * When distributing Covered Code, include this CDDL HEADER in each - * file and include the License file at usr/src/OPENSOLARIS.LICENSE. - * If applicable, add the following below this CDDL HEADER, with the - * fields enclosed by brackets "[]" replaced with your own identifying - * information: Portions Copyright [yyyy] [name of copyright owner] - * - * CDDL HEADER END - */ -/* - * Copyright (C) 2019 Romain Dolbeau - * - * Copyright (C) 2022 Tino Reichardt - */ - -/* - * USER API: - * - * Kernel fpu methods: - * kfpu_allowed() - * kfpu_begin() - * kfpu_end() - * kfpu_init() - * kfpu_fini() - * - * SIMD support: - * - * Following functions should be called to determine whether CPU feature - * is supported. All functions are usable in kernel and user space. - * If a SIMD algorithm is using more than one instruction set - * all relevant feature test functions should be called. - * - * Supported features: - * zfs_altivec_available() - * zfs_vsx_available() - * zfs_isa207_available() - */ - -#ifndef _LINUX_SIMD_POWERPC_H -#define _LINUX_SIMD_POWERPC_H - -#include -#include -#include -#include -#include -#include -#include - -#define kfpu_allowed() 1 - -#ifdef CONFIG_ALTIVEC -#define ENABLE_KERNEL_ALTIVEC enable_kernel_altivec(); -#define DISABLE_KERNEL_ALTIVEC disable_kernel_altivec(); -#else -#define ENABLE_KERNEL_ALTIVEC -#define DISABLE_KERNEL_ALTIVEC -#endif -#ifdef CONFIG_VSX -#define ENABLE_KERNEL_VSX enable_kernel_vsx(); -#define DISABLE_KERNEL_VSX disable_kernel_vsx(); -#else -#define ENABLE_KERNEL_VSX -#define DISABLE_KERNEL_VSX -#endif -#ifdef CONFIG_SPE -#define ENABLE_KERNEL_SPE enable_kernel_spe(); -#define DISABLE_KERNEL_SPE disable_kernel_spe(); -#else -#define ENABLE_KERNEL_SPE -#define DISABLE_KERNEL_SPE -#endif -#define kfpu_begin() \ - { \ - preempt_disable(); \ - ENABLE_KERNEL_ALTIVEC \ - ENABLE_KERNEL_VSX \ - ENABLE_KERNEL_SPE \ - } -#define kfpu_end() \ - { \ - DISABLE_KERNEL_SPE \ - DISABLE_KERNEL_VSX \ - DISABLE_KERNEL_ALTIVEC \ - preempt_enable(); \ - } - -#define kfpu_init() 0 -#define kfpu_fini() ((void) 0) - -/* - * Linux 4.7 makes cpu_has_feature to use jump labels on powerpc if - * CONFIG_JUMP_LABEL_FEATURE_CHECKS is enabled, in this case however it - * references GPL-only symbol cpu_feature_keys. Therefore we overrides this - * interface when it is detected being GPL-only. - */ -#if defined(CONFIG_JUMP_LABEL_FEATURE_CHECKS) && \ - defined(HAVE_CPU_HAS_FEATURE_GPL_ONLY) -#define cpu_has_feature(feature) early_cpu_has_feature(feature) -#endif - -/* - * Check if AltiVec instruction set is available - */ -static inline boolean_t -zfs_altivec_available(void) -{ - return (cpu_has_feature(CPU_FTR_ALTIVEC)); -} - -/* - * Check if VSX is available - */ -static inline boolean_t -zfs_vsx_available(void) -{ - return (cpu_has_feature(CPU_FTR_VSX)); -} - -/* - * Check if POWER ISA 2.07 is available (SHA2) - */ -static inline boolean_t -zfs_isa207_available(void) -{ - return (cpu_has_feature(CPU_FTR_ARCH_207S)); -} - -#endif /* _LINUX_SIMD_POWERPC_H */ diff --git a/include/os/linux/kernel/linux/simd_x86.h b/include/os/linux/kernel/linux/simd_x86.h deleted file mode 100644 index 699b8a571824..000000000000 --- a/include/os/linux/kernel/linux/simd_x86.h +++ /dev/null @@ -1,766 +0,0 @@ -/* - * CDDL HEADER START - * - * The contents of this file are subject to the terms of the - * Common Development and Distribution License (the "License"). - * You may not use this file except in compliance with the License. - * - * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE - * or https://opensource.org/licenses/CDDL-1.0. - * See the License for the specific language governing permissions - * and limitations under the License. - * - * When distributing Covered Code, include this CDDL HEADER in each - * file and include the License file at usr/src/OPENSOLARIS.LICENSE. - * If applicable, add the following below this CDDL HEADER, with the - * fields enclosed by brackets "[]" replaced with your own identifying - * information: Portions Copyright [yyyy] [name of copyright owner] - * - * CDDL HEADER END - */ -/* - * Copyright (C) 2016 Gvozden Neskovic . - */ - -/* - * USER API: - * - * Kernel fpu methods: - * kfpu_allowed() - * kfpu_begin() - * kfpu_end() - * kfpu_init() - * kfpu_fini() - * - * SIMD support: - * - * Following functions should be called to determine whether CPU feature - * is supported. All functions are usable in kernel and user space. - * If a SIMD algorithm is using more than one instruction set - * all relevant feature test functions should be called. - * - * Supported features: - * zfs_sse_available() - * zfs_sse2_available() - * zfs_sse3_available() - * zfs_ssse3_available() - * zfs_sse4_1_available() - * zfs_sse4_2_available() - * - * zfs_avx_available() - * zfs_avx2_available() - * - * zfs_bmi1_available() - * zfs_bmi2_available() - * - * zfs_shani_available() - * - * zfs_avx512f_available() - * zfs_avx512cd_available() - * zfs_avx512er_available() - * zfs_avx512pf_available() - * zfs_avx512bw_available() - * zfs_avx512dq_available() - * zfs_avx512vl_available() - * zfs_avx512ifma_available() - * zfs_avx512vbmi_available() - * - * NOTE(AVX-512VL): If using AVX-512 instructions with 128Bit registers - * also add zfs_avx512vl_available() to feature check. - */ - -#ifndef _LINUX_SIMD_X86_H -#define _LINUX_SIMD_X86_H - -/* only for __x86 */ -#if defined(__x86) - -#include -#include - -/* - * Disable the WARN_ON_FPU() macro to prevent additional dependencies - * when providing the kfpu_* functions. Relevant warnings are included - * as appropriate and are unconditionally enabled. - */ -#if defined(CONFIG_X86_DEBUG_FPU) && !defined(KERNEL_EXPORTS_X86_FPU) -#undef CONFIG_X86_DEBUG_FPU -#endif - -/* - * The following cases are for kernels which export either the - * kernel_fpu_* or __kernel_fpu_* functions. - */ -#if defined(KERNEL_EXPORTS_X86_FPU) - -#if defined(HAVE_KERNEL_FPU_API_HEADER) -#include -#if defined(HAVE_KERNEL_FPU_INTERNAL_HEADER) -#include -#endif -#else -#include -#endif - -#define kfpu_allowed() 1 -#define kfpu_init() 0 -#define kfpu_fini() ((void) 0) - -#if defined(HAVE_UNDERSCORE_KERNEL_FPU) -#define kfpu_begin() \ -{ \ - preempt_disable(); \ - __kernel_fpu_begin(); \ -} -#define kfpu_end() \ -{ \ - __kernel_fpu_end(); \ - preempt_enable(); \ -} - -#elif defined(HAVE_KERNEL_FPU) -#define kfpu_begin() kernel_fpu_begin() -#define kfpu_end() kernel_fpu_end() - -#else -/* - * This case is unreachable. When KERNEL_EXPORTS_X86_FPU is defined then - * either HAVE_UNDERSCORE_KERNEL_FPU or HAVE_KERNEL_FPU must be defined. - */ -#error "Unreachable kernel configuration" -#endif - -#else /* defined(KERNEL_EXPORTS_X86_FPU) */ - -/* - * When the kernel_fpu_* symbols are unavailable then provide our own - * versions which allow the FPU to be safely used. - */ -#if defined(HAVE_KERNEL_FPU_INTERNAL) - -/* - * For kernels not exporting *kfpu_{begin,end} we have to use inline assembly - * with the XSAVE{,OPT,S} instructions, so we need the toolchain to support at - * least XSAVE. - */ -#if !defined(HAVE_XSAVE) -#error "Toolchain needs to support the XSAVE assembler instruction" -#endif - -#ifndef XFEATURE_MASK_XTILE -/* - * For kernels where this doesn't exist yet, we still don't want to break - * by save/restoring this broken nonsense. - * See issue #14989 or Intel errata SPR4 for why - */ -#define XFEATURE_MASK_XTILE 0x60000 -#endif - -#include -#include - -extern uint8_t **zfs_kfpu_fpregs; - -/* - * Return the size in bytes required by the XSAVE instruction for an - * XSAVE area containing all the user state components supported by this CPU. - * See: Intel 64 and IA-32 Architectures Software Developer’s Manual. - * Dec. 2021. Vol. 2A p. 3-222. - */ -static inline uint32_t -get_xsave_area_size(void) -{ - if (!boot_cpu_has(X86_FEATURE_OSXSAVE)) { - return (0); - } - /* - * Call CPUID with leaf 13 and subleaf 0. The size is in ecx. - * We don't need to check for cpuid_max here, since if this CPU has - * OSXSAVE set, it has leaf 13 (0x0D) as well. - */ - uint32_t eax, ebx, ecx, edx; - - eax = 13U; - ecx = 0U; - __asm__ __volatile__("cpuid" - : "=a" (eax), "=b" (ebx), "=c" (ecx), "=d" (edx) - : "a" (eax), "c" (ecx)); - - return (ecx); -} - -/* - * Return the allocation order of the maximum buffer size required to save the - * FPU state on this architecture. The value returned is the same as Linux' - * get_order() function would return (i.e. 2^order = nr. of pages required). - * Currently this will always return 0 since the save area is below 4k even for - * a full fledged AVX-512 implementation. - */ -static inline int -get_fpuregs_save_area_order(void) -{ - size_t area_size = (size_t)get_xsave_area_size(); - - /* - * If we are dealing with a CPU not supporting XSAVE, - * get_xsave_area_size() will return 0. Thus the maximum memory - * required is the FXSAVE area size which is 512 bytes. See: Intel 64 - * and IA-32 Architectures Software Developer’s Manual. Dec. 2021. - * Vol. 2A p. 3-451. - */ - if (area_size == 0) { - area_size = 512; - } - return (get_order(area_size)); -} - -/* - * Initialize per-cpu variables to store FPU state. - */ -static inline void -kfpu_fini(void) -{ - int cpu; - int order = get_fpuregs_save_area_order(); - - for_each_possible_cpu(cpu) { - if (zfs_kfpu_fpregs[cpu] != NULL) { - free_pages((unsigned long)zfs_kfpu_fpregs[cpu], order); - } - } - - kfree(zfs_kfpu_fpregs); -} - -static inline int -kfpu_init(void) -{ - zfs_kfpu_fpregs = kzalloc(num_possible_cpus() * sizeof (uint8_t *), - GFP_KERNEL); - - if (zfs_kfpu_fpregs == NULL) - return (-ENOMEM); - - /* - * The fxsave and xsave operations require 16-/64-byte alignment of - * the target memory. Since kmalloc() provides no alignment - * guarantee instead use alloc_pages_node(). - */ - int cpu; - int order = get_fpuregs_save_area_order(); - - for_each_possible_cpu(cpu) { - struct page *page = alloc_pages_node(cpu_to_node(cpu), - GFP_KERNEL | __GFP_ZERO, order); - if (page == NULL) { - kfpu_fini(); - return (-ENOMEM); - } - - zfs_kfpu_fpregs[cpu] = page_address(page); - } - - return (0); -} - -#define kfpu_allowed() 1 - -/* - * FPU save and restore instructions. - */ -#define __asm __asm__ __volatile__ -#define kfpu_fxsave(addr) __asm("fxsave %0" : "=m" (*(addr))) -#define kfpu_fxsaveq(addr) __asm("fxsaveq %0" : "=m" (*(addr))) -#define kfpu_fnsave(addr) __asm("fnsave %0; fwait" : "=m" (*(addr))) -#define kfpu_fxrstor(addr) __asm("fxrstor %0" : : "m" (*(addr))) -#define kfpu_fxrstorq(addr) __asm("fxrstorq %0" : : "m" (*(addr))) -#define kfpu_frstor(addr) __asm("frstor %0" : : "m" (*(addr))) -#define kfpu_fxsr_clean(rval) __asm("fnclex; emms; fildl %P[addr]" \ - : : [addr] "m" (rval)); - -#define kfpu_do_xsave(instruction, addr, mask) \ -{ \ - uint32_t low, hi; \ - \ - low = mask; \ - hi = (uint64_t)(mask) >> 32; \ - __asm(instruction " %[dst]\n\t" \ - : \ - : [dst] "m" (*(addr)), "a" (low), "d" (hi) \ - : "memory"); \ -} - -static inline void -kfpu_save_fxsr(uint8_t *addr) -{ - if (IS_ENABLED(CONFIG_X86_32)) - kfpu_fxsave(addr); - else - kfpu_fxsaveq(addr); -} - -static inline void -kfpu_save_fsave(uint8_t *addr) -{ - kfpu_fnsave(addr); -} - -static inline void -kfpu_begin(void) -{ - /* - * Preemption and interrupts must be disabled for the critical - * region where the FPU state is being modified. - */ - preempt_disable(); - local_irq_disable(); - - /* - * The current FPU registers need to be preserved by kfpu_begin() - * and restored by kfpu_end(). They are stored in a dedicated - * per-cpu variable, not in the task struct, this allows any user - * FPU state to be correctly preserved and restored. - */ - uint8_t *state = zfs_kfpu_fpregs[smp_processor_id()]; -#if defined(HAVE_XSAVES) - if (static_cpu_has(X86_FEATURE_XSAVES)) { - kfpu_do_xsave("xsaves", state, ~XFEATURE_MASK_XTILE); - return; - } -#endif -#if defined(HAVE_XSAVEOPT) - if (static_cpu_has(X86_FEATURE_XSAVEOPT)) { - kfpu_do_xsave("xsaveopt", state, ~XFEATURE_MASK_XTILE); - return; - } -#endif - if (static_cpu_has(X86_FEATURE_XSAVE)) { - kfpu_do_xsave("xsave", state, ~XFEATURE_MASK_XTILE); - } else if (static_cpu_has(X86_FEATURE_FXSR)) { - kfpu_save_fxsr(state); - } else { - kfpu_save_fsave(state); - } -} - -#define kfpu_do_xrstor(instruction, addr, mask) \ -{ \ - uint32_t low, hi; \ - \ - low = mask; \ - hi = (uint64_t)(mask) >> 32; \ - __asm(instruction " %[src]" \ - : \ - : [src] "m" (*(addr)), "a" (low), "d" (hi) \ - : "memory"); \ -} - -static inline void -kfpu_restore_fxsr(uint8_t *addr) -{ - /* - * On AuthenticAMD K7 and K8 processors the fxrstor instruction only - * restores the _x87 FOP, FIP, and FDP registers when an exception - * is pending. Clean the _x87 state to force the restore. - */ - if (unlikely(static_cpu_has_bug(X86_BUG_FXSAVE_LEAK))) - kfpu_fxsr_clean(addr); - - if (IS_ENABLED(CONFIG_X86_32)) { - kfpu_fxrstor(addr); - } else { - kfpu_fxrstorq(addr); - } -} - -static inline void -kfpu_restore_fsave(uint8_t *addr) -{ - kfpu_frstor(addr); -} - -static inline void -kfpu_end(void) -{ - uint8_t *state = zfs_kfpu_fpregs[smp_processor_id()]; -#if defined(HAVE_XSAVES) - if (static_cpu_has(X86_FEATURE_XSAVES)) { - kfpu_do_xrstor("xrstors", state, ~XFEATURE_MASK_XTILE); - goto out; - } -#endif - if (static_cpu_has(X86_FEATURE_XSAVE)) { - kfpu_do_xrstor("xrstor", state, ~XFEATURE_MASK_XTILE); - } else if (static_cpu_has(X86_FEATURE_FXSR)) { - kfpu_restore_fxsr(state); - } else { - kfpu_restore_fsave(state); - } -out: - local_irq_enable(); - preempt_enable(); - -} - -#else - -#error "Exactly one of KERNEL_EXPORTS_X86_FPU or HAVE_KERNEL_FPU_INTERNAL" \ - " must be defined" - -#endif /* defined(HAVE_KERNEL_FPU_INTERNAL */ -#endif /* defined(KERNEL_EXPORTS_X86_FPU) */ - -/* - * Linux kernel provides an interface for CPU feature testing. - */ - -/* - * Detect register set support - */ - -/* - * Check if OS supports AVX and AVX2 by checking XCR0 - * Only call this function if CPUID indicates that AVX feature is - * supported by the CPU, otherwise it might be an illegal instruction. - */ -static inline uint64_t -zfs_xgetbv(uint32_t index) -{ - uint32_t eax, edx; - /* xgetbv - instruction byte code */ - __asm__ __volatile__(".byte 0x0f; .byte 0x01; .byte 0xd0" - : "=a" (eax), "=d" (edx) - : "c" (index)); - - return ((((uint64_t)edx)<<32) | (uint64_t)eax); -} - - -static inline boolean_t -__simd_state_enabled(const uint64_t state) -{ - boolean_t has_osxsave; - uint64_t xcr0; - -#if defined(X86_FEATURE_OSXSAVE) - has_osxsave = !!boot_cpu_has(X86_FEATURE_OSXSAVE); -#else - has_osxsave = B_FALSE; -#endif - if (!has_osxsave) - return (B_FALSE); - - xcr0 = zfs_xgetbv(0); - return ((xcr0 & state) == state); -} - -#define _XSTATE_SSE_AVX (0x2 | 0x4) -#define _XSTATE_AVX512 (0xE0 | _XSTATE_SSE_AVX) - -#define __ymm_enabled() __simd_state_enabled(_XSTATE_SSE_AVX) -#define __zmm_enabled() __simd_state_enabled(_XSTATE_AVX512) - -/* - * Check if SSE instruction set is available - */ -static inline boolean_t -zfs_sse_available(void) -{ - return (!!boot_cpu_has(X86_FEATURE_XMM)); -} - -/* - * Check if SSE2 instruction set is available - */ -static inline boolean_t -zfs_sse2_available(void) -{ - return (!!boot_cpu_has(X86_FEATURE_XMM2)); -} - -/* - * Check if SSE3 instruction set is available - */ -static inline boolean_t -zfs_sse3_available(void) -{ - return (!!boot_cpu_has(X86_FEATURE_XMM3)); -} - -/* - * Check if SSSE3 instruction set is available - */ -static inline boolean_t -zfs_ssse3_available(void) -{ - return (!!boot_cpu_has(X86_FEATURE_SSSE3)); -} - -/* - * Check if SSE4.1 instruction set is available - */ -static inline boolean_t -zfs_sse4_1_available(void) -{ - return (!!boot_cpu_has(X86_FEATURE_XMM4_1)); -} - -/* - * Check if SSE4.2 instruction set is available - */ -static inline boolean_t -zfs_sse4_2_available(void) -{ - return (!!boot_cpu_has(X86_FEATURE_XMM4_2)); -} - -/* - * Check if AVX instruction set is available - */ -static inline boolean_t -zfs_avx_available(void) -{ - return (boot_cpu_has(X86_FEATURE_AVX) && __ymm_enabled()); -} - -/* - * Check if AVX2 instruction set is available - */ -static inline boolean_t -zfs_avx2_available(void) -{ - return (boot_cpu_has(X86_FEATURE_AVX2) && __ymm_enabled()); -} - -/* - * Check if BMI1 instruction set is available - */ -static inline boolean_t -zfs_bmi1_available(void) -{ -#if defined(X86_FEATURE_BMI1) - return (!!boot_cpu_has(X86_FEATURE_BMI1)); -#else - return (B_FALSE); -#endif -} - -/* - * Check if BMI2 instruction set is available - */ -static inline boolean_t -zfs_bmi2_available(void) -{ -#if defined(X86_FEATURE_BMI2) - return (!!boot_cpu_has(X86_FEATURE_BMI2)); -#else - return (B_FALSE); -#endif -} - -/* - * Check if AES instruction set is available - */ -static inline boolean_t -zfs_aes_available(void) -{ -#if defined(X86_FEATURE_AES) - return (!!boot_cpu_has(X86_FEATURE_AES)); -#else - return (B_FALSE); -#endif -} - -/* - * Check if PCLMULQDQ instruction set is available - */ -static inline boolean_t -zfs_pclmulqdq_available(void) -{ -#if defined(X86_FEATURE_PCLMULQDQ) - return (!!boot_cpu_has(X86_FEATURE_PCLMULQDQ)); -#else - return (B_FALSE); -#endif -} - -/* - * Check if MOVBE instruction is available - */ -static inline boolean_t -zfs_movbe_available(void) -{ -#if defined(X86_FEATURE_MOVBE) - return (!!boot_cpu_has(X86_FEATURE_MOVBE)); -#else - return (B_FALSE); -#endif -} - -/* - * Check if SHA_NI instruction set is available - */ -static inline boolean_t -zfs_shani_available(void) -{ -#if defined(X86_FEATURE_SHA_NI) - return (!!boot_cpu_has(X86_FEATURE_SHA_NI)); -#else - return (B_FALSE); -#endif -} - -/* - * AVX-512 family of instruction sets: - * - * AVX512F Foundation - * AVX512CD Conflict Detection Instructions - * AVX512ER Exponential and Reciprocal Instructions - * AVX512PF Prefetch Instructions - * - * AVX512BW Byte and Word Instructions - * AVX512DQ Double-word and Quadword Instructions - * AVX512VL Vector Length Extensions - * - * AVX512IFMA Integer Fused Multiply Add (Not supported by kernel 4.4) - * AVX512VBMI Vector Byte Manipulation Instructions - */ - -/* - * Check if AVX512F instruction set is available - */ -static inline boolean_t -zfs_avx512f_available(void) -{ - boolean_t has_avx512 = B_FALSE; - -#if defined(X86_FEATURE_AVX512F) - has_avx512 = !!boot_cpu_has(X86_FEATURE_AVX512F); -#endif - return (has_avx512 && __zmm_enabled()); -} - -/* - * Check if AVX512CD instruction set is available - */ -static inline boolean_t -zfs_avx512cd_available(void) -{ - boolean_t has_avx512 = B_FALSE; - -#if defined(X86_FEATURE_AVX512CD) - has_avx512 = boot_cpu_has(X86_FEATURE_AVX512F) && - boot_cpu_has(X86_FEATURE_AVX512CD); -#endif - return (has_avx512 && __zmm_enabled()); -} - -/* - * Check if AVX512ER instruction set is available - */ -static inline boolean_t -zfs_avx512er_available(void) -{ - boolean_t has_avx512 = B_FALSE; - -#if defined(X86_FEATURE_AVX512ER) - has_avx512 = boot_cpu_has(X86_FEATURE_AVX512F) && - boot_cpu_has(X86_FEATURE_AVX512ER); -#endif - return (has_avx512 && __zmm_enabled()); -} - -/* - * Check if AVX512PF instruction set is available - */ -static inline boolean_t -zfs_avx512pf_available(void) -{ - boolean_t has_avx512 = B_FALSE; - -#if defined(X86_FEATURE_AVX512PF) - has_avx512 = boot_cpu_has(X86_FEATURE_AVX512F) && - boot_cpu_has(X86_FEATURE_AVX512PF); -#endif - return (has_avx512 && __zmm_enabled()); -} - -/* - * Check if AVX512BW instruction set is available - */ -static inline boolean_t -zfs_avx512bw_available(void) -{ - boolean_t has_avx512 = B_FALSE; - -#if defined(X86_FEATURE_AVX512BW) - has_avx512 = boot_cpu_has(X86_FEATURE_AVX512F) && - boot_cpu_has(X86_FEATURE_AVX512BW); -#endif - - return (has_avx512 && __zmm_enabled()); -} - -/* - * Check if AVX512DQ instruction set is available - */ -static inline boolean_t -zfs_avx512dq_available(void) -{ - boolean_t has_avx512 = B_FALSE; - -#if defined(X86_FEATURE_AVX512DQ) - has_avx512 = boot_cpu_has(X86_FEATURE_AVX512F) && - boot_cpu_has(X86_FEATURE_AVX512DQ); -#endif - return (has_avx512 && __zmm_enabled()); -} - -/* - * Check if AVX512VL instruction set is available - */ -static inline boolean_t -zfs_avx512vl_available(void) -{ - boolean_t has_avx512 = B_FALSE; - -#if defined(X86_FEATURE_AVX512VL) - has_avx512 = boot_cpu_has(X86_FEATURE_AVX512F) && - boot_cpu_has(X86_FEATURE_AVX512VL); -#endif - return (has_avx512 && __zmm_enabled()); -} - -/* - * Check if AVX512IFMA instruction set is available - */ -static inline boolean_t -zfs_avx512ifma_available(void) -{ - boolean_t has_avx512 = B_FALSE; - -#if defined(X86_FEATURE_AVX512IFMA) - has_avx512 = boot_cpu_has(X86_FEATURE_AVX512F) && - boot_cpu_has(X86_FEATURE_AVX512IFMA); -#endif - return (has_avx512 && __zmm_enabled()); -} - -/* - * Check if AVX512VBMI instruction set is available - */ -static inline boolean_t -zfs_avx512vbmi_available(void) -{ - boolean_t has_avx512 = B_FALSE; - -#if defined(X86_FEATURE_AVX512VBMI) - has_avx512 = boot_cpu_has(X86_FEATURE_AVX512F) && - boot_cpu_has(X86_FEATURE_AVX512VBMI); -#endif - return (has_avx512 && __zmm_enabled()); -} - -#endif /* defined(__x86) */ - -#endif /* _LINUX_SIMD_X86_H */ diff --git a/include/os/linux/spl/sys/simd.h b/include/os/linux/spl/sys/simd.h index 6fb84d3a52c0..17854a68d1d0 100644 --- a/include/os/linux/spl/sys/simd.h +++ b/include/os/linux/spl/sys/simd.h @@ -1,9 +1,5 @@ /* - * Copyright (C) 2007-2010 Lawrence Livermore National Security, LLC. - * Copyright (C) 2007 The Regents of the University of California. - * Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER). - * Written by Brian Behlendorf . - * UCRL-CODE-235197 + * Copyright (C) 2024 Tino Reichardt * * This file is part of the SPL, Solaris Porting Layer. * @@ -21,10 +17,83 @@ * with the SPL. If not, see . */ -#ifndef _SPL_SYS_SIMD_H -#define _SPL_SYS_SIMD_H +#ifndef _SPL_SIMD_H +#define _SPL_SIMD_H -#include -#include +extern void simd_stat_init(void); +extern void simd_stat_fini(void); -#endif /* _SPL_SYS_SIMD_H */ +extern int spl_kfpu_allowed(void); +extern int spl_kfpu_cpu_has(int feature); +extern void spl_kfpu_begin(void); +extern void spl_kfpu_end(void); + +/* + * Define functions for CPUID features testing + */ +#define DF(name, id) \ +static inline int \ +name(void) \ +{ \ + return (spl_kfpu_cpu_has(id)); \ +} + +/* X86 */ +#define ZFS_X86_INIT_DONE (1 << 1) +#define ZFS_X86_SSE (1 << 2) +#define ZFS_X86_SSE2 (1 << 3) +#define ZFS_X86_SSSE3 (1 << 4) +#define ZFS_X86_SSE4_1 (1 << 5) +#define ZFS_X86_SSE4_2 (1 << 6) +#define ZFS_X86_AVX (1 << 7) +#define ZFS_X86_AVX2 (1 << 8) +#define ZFS_X86_AVX512F (1 << 9) +#define ZFS_X86_AVX512BW (1 << 10) +#define ZFS_X86_AVX512VL (1 << 11) +#define ZFS_X86_AES (1 << 12) +#define ZFS_X86_MOVBE (1 << 13) +#define ZFS_X86_PCLMULQDQ (1 << 14) +#define ZFS_X86_SHA_NI (1 << 15) + +DF(zfs_sse_available, ZFS_X86_SSE); +DF(zfs_sse2_available, ZFS_X86_SSE2); +DF(zfs_ssse3_available, ZFS_X86_SSSE3); +DF(zfs_sse4_1_available, ZFS_X86_SSE4_1); +DF(zfs_sse4_2_available, ZFS_X86_SSE4_2); +DF(zfs_avx_available, ZFS_X86_AVX); +DF(zfs_avx2_available, ZFS_X86_AVX2); +DF(zfs_avx512f_available, ZFS_X86_AVX512F); +DF(zfs_avx512bw_available, ZFS_X86_AVX512BW); +DF(zfs_avx512vl_available, ZFS_X86_AVX512VL); +DF(zfs_aes_available, ZFS_X86_AES); +DF(zfs_movbe_available, ZFS_X86_MOVBE); +DF(zfs_pclmulqdq_available, ZFS_X86_PCLMULQDQ); +DF(zfs_shani_available, ZFS_X86_SHA_NI); + +/* ARM */ +#define ZFS_ARM_INIT_DONE (1 << 1) +#define ZFS_ARM_NEON (1 << 2) +#define ZFS_ARM_SHA256 (1 << 3) +#define ZFS_ARM_SHA512 (1 << 4) + +DF(zfs_neon_available, ZFS_ARM_NEON); +DF(zfs_sha256_available, ZFS_ARM_SHA256); +DF(zfs_sha512_available, ZFS_ARM_SHA512); + +/* PPC */ +#define ZFS_PPC_INIT_DONE (1 << 1) +#define ZFS_PPC_ALTIVEC (1 << 2) +#define ZFS_PPC_VSX (1 << 3) +#define ZFS_PPC_ISA207 (1 << 4) + +DF(zfs_altivec_available, ZFS_PPC_ALTIVEC); +DF(zfs_vsx_available, ZFS_PPC_VSX); +DF(zfs_isa207_available, ZFS_PPC_ISA207); +#undef DF + +#define kfpu_allowed spl_kfpu_allowed +#define kfpu_cpu_has spl_kfpu_cpu_has +#define kfpu_begin spl_kfpu_begin +#define kfpu_end spl_kfpu_end + +#endif /* _SPL_SIMD_H */ diff --git a/lib/libspl/include/sys/simd.h b/lib/libspl/include/sys/simd.h index 2926dc680764..29482ca24551 100644 --- a/lib/libspl/include/sys/simd.h +++ b/lib/libspl/include/sys/simd.h @@ -58,8 +58,6 @@ extern unsigned long getauxval(unsigned long type); #define kfpu_allowed() 1 #define kfpu_begin() do {} while (0) #define kfpu_end() do {} while (0) -#define kfpu_init() 0 -#define kfpu_fini() ((void) 0) /* * CPUID feature tests for user-space. @@ -481,7 +479,6 @@ zfs_avx512vbmi_available(void) #elif defined(__arm__) #define kfpu_allowed() 1 -#define kfpu_initialize(tsk) do {} while (0) #define kfpu_begin() do {} while (0) #define kfpu_end() do {} while (0) @@ -511,7 +508,6 @@ zfs_sha256_available(void) #elif defined(__aarch64__) #define kfpu_allowed() 1 -#define kfpu_initialize(tsk) do {} while (0) #define kfpu_begin() do {} while (0) #define kfpu_end() do {} while (0) @@ -552,7 +548,6 @@ zfs_sha512_available(void) #elif defined(__powerpc__) #define kfpu_allowed() 0 -#define kfpu_initialize(tsk) do {} while (0) #define kfpu_begin() do {} while (0) #define kfpu_end() do {} while (0) @@ -586,7 +581,6 @@ zfs_isa207_available(void) #else #define kfpu_allowed() 0 -#define kfpu_initialize(tsk) do {} while (0) #define kfpu_begin() do {} while (0) #define kfpu_end() do {} while (0) diff --git a/module/Kbuild.in b/module/Kbuild.in index dcbdbc912f6d..8b9808897a50 100644 --- a/module/Kbuild.in +++ b/module/Kbuild.in @@ -80,6 +80,7 @@ SPL_OBJS := \ spl-proc.o \ spl-procfs-list.o \ spl-shrinker.o \ + spl-simd.o \ spl-taskq.o \ spl-thread.o \ spl-trace.o \ diff --git a/module/icp/algs/aes/aes_impl_aesni.c b/module/icp/algs/aes/aes_impl_aesni.c index 61085214c77b..fa8f5db83579 100644 --- a/module/icp/algs/aes/aes_impl_aesni.c +++ b/module/icp/algs/aes/aes_impl_aesni.c @@ -26,6 +26,7 @@ #include #include +struct pt_regs; #include /* These functions are used to execute AES-NI instructions: */ diff --git a/module/icp/algs/modes/gcm_pclmulqdq.c b/module/icp/algs/modes/gcm_pclmulqdq.c index 737d2e47ecb7..ea121ded23dd 100644 --- a/module/icp/algs/modes/gcm_pclmulqdq.c +++ b/module/icp/algs/modes/gcm_pclmulqdq.c @@ -26,6 +26,8 @@ #include #include + +struct pt_regs; #include /* These functions are used to execute pclmulqdq based assembly methods */ diff --git a/module/os/linux/spl/spl-simd.c b/module/os/linux/spl/spl-simd.c new file mode 100644 index 000000000000..31ba7404e730 --- /dev/null +++ b/module/os/linux/spl/spl-simd.c @@ -0,0 +1,206 @@ +/* + * Copyright (C) 2024 Tino Reichardt + * + * This file is part of the SPL, Solaris Porting Layer. + * + * The SPL is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation; either version 2 of the License, or (at your + * option) any later version. + * + * The SPL is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License along + * with the SPL. If not, see . + * + * Solaris Porting Layer (SPL) Kernel FPU management. + */ + +#include +#include +#include + +static uint32_t cpu_features = 0; + +/* little helper */ +#define isset(v,x) (((v)&(x)) == (x)) +#define FSET(l,z) do { if (boot_cpu_has(l)) { cpu_features |= z; }} while (0) + +#if defined(__x86_64) || defined(__i386) +#include + +int +spl_kfpu_cpu_has(int feature_set) +{ + if (unlikely(cpu_features == 0)) { + cpu_features = ZFS_X86_INIT_DONE; + FSET(X86_FEATURE_XMM, ZFS_X86_SSE); + FSET(X86_FEATURE_XMM2, ZFS_X86_SSE2); + FSET(X86_FEATURE_SSSE3, ZFS_X86_SSSE3); + FSET(X86_FEATURE_XMM4_1, ZFS_X86_SSE4_1); + FSET(X86_FEATURE_XMM4_2, ZFS_X86_SSE4_2); + FSET(X86_FEATURE_AVX, ZFS_X86_AVX); + FSET(X86_FEATURE_AVX2, ZFS_X86_AVX2); + FSET(X86_FEATURE_AVX512F, ZFS_X86_AVX512F); + FSET(X86_FEATURE_AVX512BW, ZFS_X86_AVX512BW); + FSET(X86_FEATURE_AVX512VL, ZFS_X86_AVX512VL); + FSET(X86_FEATURE_AES, ZFS_X86_AES); + FSET(X86_FEATURE_MOVBE, ZFS_X86_MOVBE); + FSET(X86_FEATURE_PCLMULQDQ, ZFS_X86_PCLMULQDQ); + FSET(X86_FEATURE_SHA_NI, ZFS_X86_SHA_NI); + } + return isset(cpu_features, feature_set); +} + +int +spl_kfpu_allowed(void) +{ + return 1; +} + +void +spl_kfpu_begin(void) +{ + kernel_fpu_begin(); +} + +void +spl_kfpu_end(void) +{ + kernel_fpu_end(); +} + +#elif defined(__arm) && defined(CONFIG_KERNEL_MODE_NEON) +extern void kernel_neon_begin(void); +extern void kernel_neon_end(void); + +int +spl_kfpu_allowed(void) +{ + return 0; +} + +void +spl_kfpu_begin(void) +{ + kernel_neon_begin(); +} + +void +spl_kfpu_end(void) +{ + kernel_neon_end(); +} + +#elif defined(__aarch64__) && defined(CONFIG_KERNEL_MODE_NEON) + +/* ARM */ +extern boolean_t zfs_neon_available(void); +extern boolean_t zfs_sha256_available(void); +extern boolean_t zfs_sha512_available(void); + +extern void kernel_neon_begin(void); +extern void kernel_neon_end(void); + +int +spl_kfpu_allowed(void) +{ + return 0; +} + +void +spl_kfpu_begin(void) +{ + return kernel_neon_begin(); +} + +void +spl_kfpu_end(void) +{ + return kernel_neon_end(); +} + +#elif defined(__powerpc) + +/* ppc */ +extern boolean_t zfs_altivec_available(void); +extern boolean_t zfs_vsx_available(void); +extern boolean_t zfs_isa207_available(void); + +static void spe_begin(void) +{ + /* disable preemption and save users SPE registers if required */ + preempt_disable(); +} + +static void spe_end(void) +{ + disable_kernel_spe(); + /* reenable preemption */ + preempt_enable(); +} + +int +spl_kfpu_allowed(void) +{ + return 0; +} + +void +spl_kfpu_begin(void) +{ + preempt_disable(); + enable_kernel_vsx(); + enable_kernel_spe(); +} + +void +spl_kfpu_end(void) +{ + disable_kernel_spe(); + disable_kernel_vsx(); + preempt_enable(); +} +#elif defined(__loongarch_lp64) +extern void kernel_fpu_begin(void); +extern void kernel_fpu_end(void); + +int +spl_kfpu_cpu_has(int feature_set) +{ +} + +int +spl_kfpu_allowed(void) +{ + return 0; +} + +void +spl_kfpu_begin(void) +{ + kernel_fpu_begin(); +} + +void +spl_kfpu_end(void) +{ + kernel_fpu_end(); +} + +#else + +#define spl_kfpu_allowed 0 +#define spl_kfpu_cpu_has 0 +#define spl_kfpu_begin +#define spl_kfpu_end + +#endif + +EXPORT_SYMBOL(spl_kfpu_allowed); +EXPORT_SYMBOL(spl_kfpu_cpu_has); +EXPORT_SYMBOL(spl_kfpu_begin); +EXPORT_SYMBOL(spl_kfpu_end); diff --git a/module/zcommon/simd_stat.c b/module/zcommon/simd_stat.c index 33c15140cdb9..798fa6d71554 100644 --- a/module/zcommon/simd_stat.c +++ b/module/zcommon/simd_stat.c @@ -27,9 +27,6 @@ #ifdef _KERNEL -#ifdef __linux__ -#include -#endif /* __linux__ */ kstat_t *simd_stat_kstat; #endif /* _KERNEL */ @@ -73,8 +70,6 @@ simd_stat_kstat_data(char *buf, size_t size, void *data) "sse", zfs_sse_available()); off += SIMD_STAT_PRINT(simd_stat_kstat_payload, "sse2", zfs_sse2_available()); - off += SIMD_STAT_PRINT(simd_stat_kstat_payload, - "sse3", zfs_sse3_available()); off += SIMD_STAT_PRINT(simd_stat_kstat_payload, "ssse3", zfs_ssse3_available()); off += SIMD_STAT_PRINT(simd_stat_kstat_payload, @@ -87,47 +82,16 @@ simd_stat_kstat_data(char *buf, size_t size, void *data) "avx2", zfs_avx2_available()); off += SIMD_STAT_PRINT(simd_stat_kstat_payload, "avx512f", zfs_avx512f_available()); - off += SIMD_STAT_PRINT(simd_stat_kstat_payload, - "avx512cd", zfs_avx512cd_available()); - off += SIMD_STAT_PRINT(simd_stat_kstat_payload, - "avx512er", zfs_avx512er_available()); - off += SIMD_STAT_PRINT(simd_stat_kstat_payload, - "avx512pf", zfs_avx512pf_available()); off += SIMD_STAT_PRINT(simd_stat_kstat_payload, "avx512bw", zfs_avx512bw_available()); - off += SIMD_STAT_PRINT(simd_stat_kstat_payload, - "avx512dq", zfs_avx512dq_available()); off += SIMD_STAT_PRINT(simd_stat_kstat_payload, "avx512vl", zfs_avx512vl_available()); - off += SIMD_STAT_PRINT(simd_stat_kstat_payload, - "avx512ifma", zfs_avx512ifma_available()); - off += SIMD_STAT_PRINT(simd_stat_kstat_payload, - "avx512vbmi", zfs_avx512vbmi_available()); - off += SIMD_STAT_PRINT(simd_stat_kstat_payload, - "ymm", __ymm_enabled()); - off += SIMD_STAT_PRINT(simd_stat_kstat_payload, - "zmm", __zmm_enabled()); - off += SIMD_STAT_PRINT(simd_stat_kstat_payload, - "bmi1", zfs_bmi1_available()); - off += SIMD_STAT_PRINT(simd_stat_kstat_payload, - "bmi2", zfs_bmi2_available()); off += SIMD_STAT_PRINT(simd_stat_kstat_payload, "aes", zfs_aes_available()); off += SIMD_STAT_PRINT(simd_stat_kstat_payload, "pclmulqdq", zfs_pclmulqdq_available()); off += SIMD_STAT_PRINT(simd_stat_kstat_payload, "movbe", zfs_movbe_available()); - - off += SIMD_STAT_PRINT(simd_stat_kstat_payload, - "osxsave", boot_cpu_has(X86_FEATURE_OSXSAVE)); - off += SIMD_STAT_PRINT(simd_stat_kstat_payload, - "xsaves", static_cpu_has(X86_FEATURE_XSAVES)); - off += SIMD_STAT_PRINT(simd_stat_kstat_payload, - "xsaveopt", static_cpu_has(X86_FEATURE_XSAVEOPT)); - off += SIMD_STAT_PRINT(simd_stat_kstat_payload, - "xsave", static_cpu_has(X86_FEATURE_XSAVE)); - off += SIMD_STAT_PRINT(simd_stat_kstat_payload, - "fxsr", static_cpu_has(X86_FEATURE_FXSR)); #endif /* __x86__ */ #if defined(__arm__) || defined(__aarch64__) off += SIMD_STAT_PRINT(simd_stat_kstat_payload, diff --git a/module/zcommon/zfs_prop.c b/module/zcommon/zfs_prop.c index 20cc0dffc27e..0fbb860839fa 100644 --- a/module/zcommon/zfs_prop.c +++ b/module/zcommon/zfs_prop.c @@ -1078,10 +1078,6 @@ extern void zcommon_fini(void); int __init zcommon_init(void) { - int error = kfpu_init(); - if (error) - return (error); - fletcher_4_init(); simd_stat_init(); @@ -1093,7 +1089,6 @@ zcommon_fini(void) { simd_stat_fini(); fletcher_4_fini(); - kfpu_fini(); } #ifdef __FreeBSD__