From c284569457febd659f6afecdc74c0e26ea700fd4 Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Wed, 1 Dec 2021 12:52:23 -0800 Subject: [PATCH] [asm] Share portability macros and restrict ASM further Move portability macros to `lib/common/portability_macros.h`. This file only contains platform/feature detection (e.g. 0/1 macros). This file is shared between C and ASM code, so it cannot include any C code. Rename `HUF_` ASM macros to be `ZSTD_` prefixed, and move to the new header. Restrict `ZSTD_ASM_SUPPORTED` to `__GNUC__`, because we need the GAS assembler. Finally, only include the ASM code if we are actually going to use it. This disables it on all Windows platforms, which should resolve the problem brought up in Issue #2789. --- build/single_file_libs/zstd-in.c | 2 +- build/single_file_libs/zstddeclib-in.c | 2 +- contrib/linux-kernel/decompress_sources.h | 11 +- lib/common/compiler.h | 60 +--------- lib/common/portability_macros.h | 129 ++++++++++++++++++++++ lib/decompress/huf_decompress.c | 48 ++------ lib/decompress/huf_decompress_amd64.S | 4 +- lib/libzstd.mk | 2 +- 8 files changed, 155 insertions(+), 103 deletions(-) create mode 100644 lib/common/portability_macros.h diff --git a/build/single_file_libs/zstd-in.c b/build/single_file_libs/zstd-in.c index 733dcb75710..b694681396a 100644 --- a/build/single_file_libs/zstd-in.c +++ b/build/single_file_libs/zstd-in.c @@ -44,7 +44,7 @@ #endif #define ZSTD_TRACE 0 /* TODO: Can't amalgamate ASM function */ -#define HUF_DISABLE_ASM 1 +#define ZSTD_DISABLE_ASM 1 /* Include zstd_deps.h first with all the options we need enabled. */ #define ZSTD_DEPS_NEED_MALLOC diff --git a/build/single_file_libs/zstddeclib-in.c b/build/single_file_libs/zstddeclib-in.c index cbf70c6197d..72abe61343d 100644 --- a/build/single_file_libs/zstddeclib-in.c +++ b/build/single_file_libs/zstddeclib-in.c @@ -40,7 +40,7 @@ #define ZSTD_STRIP_ERROR_STRINGS #define ZSTD_TRACE 0 /* TODO: Can't amalgamate ASM function */ -#define HUF_DISABLE_ASM 1 +#define ZSTD_DISABLE_ASM 1 /* Include zstd_deps.h first with all the options we need enabled. */ #define ZSTD_DEPS_NEED_MALLOC diff --git a/contrib/linux-kernel/decompress_sources.h b/contrib/linux-kernel/decompress_sources.h index a2aefe8f227..a06ca187aab 100644 --- a/contrib/linux-kernel/decompress_sources.h +++ b/contrib/linux-kernel/decompress_sources.h @@ -16,16 +16,17 @@ * decompression. */ +/* + * Disable the ASM Huffman implementation because we need to + * include all the sources. + */ +#define ZSTD_DISABLE_ASM 1 + #include "common/debug.c" #include "common/entropy_common.c" #include "common/error_private.c" #include "common/fse_decompress.c" #include "common/zstd_common.c" -/* - * Disable the ASM Huffman implementation because we need to - * include all the sources. - */ -#define HUF_DISABLE_ASM 1 #include "decompress/huf_decompress.c" #include "decompress/zstd_ddict.c" #include "decompress/zstd_decompress.c" diff --git a/lib/common/compiler.h b/lib/common/compiler.h index 95ecb75e0fd..516930c01ec 100644 --- a/lib/common/compiler.h +++ b/lib/common/compiler.h @@ -11,6 +11,8 @@ #ifndef ZSTD_COMPILER_H #define ZSTD_COMPILER_H +#include "portability_macros.h" + /*-******************************************************* * Compiler specifics *********************************************************/ @@ -92,9 +94,6 @@ /* target attribute */ -#ifndef __has_attribute - #define __has_attribute(x) 0 /* Compatibility with non-clang compilers. */ -#endif #if defined(__GNUC__) || defined(__ICCARM__) # define TARGET_ATTRIBUTE(target) __attribute__((__target__(target))) #else @@ -107,22 +106,6 @@ */ #define BMI2_TARGET_ATTRIBUTE TARGET_ATTRIBUTE("lzcnt,bmi,bmi2") - -/* Enable runtime BMI2 dispatch based on the CPU. - * Enabled for clang & gcc >=4.8 on x86 when BMI2 isn't enabled by default. - */ -#ifndef DYNAMIC_BMI2 - #if ((defined(__clang__) && __has_attribute(__target__)) \ - || (defined(__GNUC__) \ - && (__GNUC__ >= 5 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)))) \ - && (defined(__x86_64__) || defined(_M_X64)) \ - && !defined(__BMI2__) - # define DYNAMIC_BMI2 1 - #else - # define DYNAMIC_BMI2 0 - #endif -#endif - /* prefetch * can be disabled, by declaring NO_PREFETCH build macro */ #if defined(NO_PREFETCH) @@ -221,16 +204,6 @@ # endif #endif -/* compat. with non-clang compilers */ -#ifndef __has_builtin -# define __has_builtin(x) 0 -#endif - -/* compat. with non-clang compilers */ -#ifndef __has_feature -# define __has_feature(x) 0 -#endif - /* C-language Attributes are added in C23. */ #if defined(__STDC_VERSION__) && (__STDC_VERSION__ > 201710L) && defined(__has_c_attribute) # define ZSTD_HAS_C_ATTRIBUTE(x) __has_c_attribute(x) @@ -267,24 +240,6 @@ # endif #endif -/* detects whether we are being compiled under msan */ -#ifndef ZSTD_MEMORY_SANITIZER -# if __has_feature(memory_sanitizer) -# define ZSTD_MEMORY_SANITIZER 1 -# else -# define ZSTD_MEMORY_SANITIZER 0 -# endif -#endif - -/* detects whether we are being compiled undef dfsan */ -#ifndef ZSTD_DATAFLOW_SANITIZER -# if __has_feature(dataflow_sanitizer) -# define ZSTD_DATAFLOW_SANITIZER 1 -# else -# define ZSTD_DATAFLOW_SANITIZER 0 -# endif -#endif - /*-************************************************************** * Alignment check *****************************************************************/ @@ -339,17 +294,6 @@ void __msan_poison(const volatile void *a, size_t size); intptr_t __msan_test_shadow(const volatile void *x, size_t size); #endif -/* detects whether we are being compiled under asan */ -#ifndef ZSTD_ADDRESS_SANITIZER -# if __has_feature(address_sanitizer) -# define ZSTD_ADDRESS_SANITIZER 1 -# elif defined(__SANITIZE_ADDRESS__) -# define ZSTD_ADDRESS_SANITIZER 1 -# else -# define ZSTD_ADDRESS_SANITIZER 0 -# endif -#endif - #if ZSTD_ADDRESS_SANITIZER /* Not all platforms that support asan provide sanitizers/asan_interface.h. * We therefore declare the functions we need ourselves, rather than trying to diff --git a/lib/common/portability_macros.h b/lib/common/portability_macros.h new file mode 100644 index 00000000000..ef7987d41c0 --- /dev/null +++ b/lib/common/portability_macros.h @@ -0,0 +1,129 @@ +/* + * Copyright (c) Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_PORTABILITY_MACROS_H +#define ZSTD_PORTABILITY_MACROS_H + +/** + * This header file contains macro defintions to support portability. + * This header is shared between C and ASM code, so it MUST only + * contain macro definitions. It MUST not contain any C code. + * + * This header ONLY defines macros to detect platforms/feature support. + * + */ + + +/* compat. with non-clang compilers */ +#ifndef __has_attribute + #define __has_attribute(x) 0 +#endif + +/* compat. with non-clang compilers */ +#ifndef __has_builtin +# define __has_builtin(x) 0 +#endif + +/* compat. with non-clang compilers */ +#ifndef __has_feature +# define __has_feature(x) 0 +#endif + +/* detects whether we are being compiled under msan */ +#ifndef ZSTD_MEMORY_SANITIZER +# if __has_feature(memory_sanitizer) +# define ZSTD_MEMORY_SANITIZER 1 +# else +# define ZSTD_MEMORY_SANITIZER 0 +# endif +#endif + +/* detects whether we are being compiled under asan */ +#ifndef ZSTD_ADDRESS_SANITIZER +# if __has_feature(address_sanitizer) +# define ZSTD_ADDRESS_SANITIZER 1 +# elif defined(__SANITIZE_ADDRESS__) +# define ZSTD_ADDRESS_SANITIZER 1 +# else +# define ZSTD_ADDRESS_SANITIZER 0 +# endif +#endif + +/* detects whether we are being compiled under dfsan */ +#ifndef ZSTD_DATAFLOW_SANITIZER +# if __has_feature(dataflow_sanitizer) +# define ZSTD_DATAFLOW_SANITIZER 1 +# else +# define ZSTD_DATAFLOW_SANITIZER 0 +# endif +#endif + + +/* Enable runtime BMI2 dispatch based on the CPU. + * Enabled for clang & gcc >=4.8 on x86 when BMI2 isn't enabled by default. + */ +#ifndef DYNAMIC_BMI2 + #if ((defined(__clang__) && __has_attribute(__target__)) \ + || (defined(__GNUC__) \ + && (__GNUC__ >= 5 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)))) \ + && (defined(__x86_64__) || defined(_M_X64)) \ + && !defined(__BMI2__) + # define DYNAMIC_BMI2 1 + #else + # define DYNAMIC_BMI2 0 + #endif +#endif + +/** + * Only enable assembly for GNUC comptabile compilers, + * because other platforms may not support GAS assembly syntax. + * + * Only enable assembly for Linux / MacOS, other platforms may + * work, but they haven't been tested. This could likely be + * extended to BSD systems. + * + * Disable assembly when MSAN is enabled, because MSAN requires + * 100% of code to be instrumented to work. + */ +#if defined(__GNUC__) +# if defined(__linux__) || defined(__linux) || defined(__APPLE__) +# if ZSTD_MEMORY_SANITIZER +# define ZSTD_ASM_SUPPORTED 0 +# else +# define ZSTD_ASM_SUPPORTED 1 +# endif +# else +# define ZSTD_ASM_SUPPORTED 0 +# endif +#else +# define ZSTD_ASM_SUPPORTED 0 +#endif + +/** + * Determines whether we should enable assembly for x86-64 + * with BMI2. + * + * Enable if all of the following conditions hold: + * - ASM hasn't been explicitly disabled by defining ZSTD_DISABLE_ASM + * - Assembly is supported + * - We are compiling for x86-64 and either: + * - DYNAMIC_BMI2 is enabled + * - BMI2 is supported at compile time + */ +#if !defined(ZSTD_DISABLE_ASM) && \ + ZSTD_ASM_SUPPORTED && \ + defined(__x86_64__) && \ + (DYNAMIC_BMI2 || defined(__BMI2__)) +# define ZSTD_ENABLE_ASM_X86_64_BMI2 1 +#else +# define ZSTD_ENABLE_ASM_X86_64_BMI2 0 +#endif + +#endif /* ZSTD_PORTABILITY_MACROS_H */ diff --git a/lib/decompress/huf_decompress.c b/lib/decompress/huf_decompress.c index dd6c466c715..a00f938ce45 100644 --- a/lib/decompress/huf_decompress.c +++ b/lib/decompress/huf_decompress.c @@ -43,31 +43,7 @@ #error "Cannot force the use of the X1 and X2 decoders at the same time!" #endif -/* Only use assembly on Linux / MacOS. - * Disable when MSAN is enabled. - */ -#if defined(__linux__) || defined(__linux) || defined(__APPLE__) -# if ZSTD_MEMORY_SANITIZER -# define HUF_ASM_SUPPORTED 0 -# elif ZSTD_DATAFLOW_SANITIZER -# define HUF_ASM_SUPPORTED 0 -# else -# define HUF_ASM_SUPPORTED 1 -# endif -#else -# define HUF_ASM_SUPPORTED 0 -#endif - -/* HUF_DISABLE_ASM: Disables all ASM implementations. */ -#if !defined(HUF_DISABLE_ASM) && \ - HUF_ASM_SUPPORTED && \ - defined(__x86_64__) && (DYNAMIC_BMI2 || defined(__BMI2__)) -# define HUF_ENABLE_ASM_X86_64_BMI2 1 -#else -# define HUF_ENABLE_ASM_X86_64_BMI2 0 -#endif - -#if HUF_ENABLE_ASM_X86_64_BMI2 && DYNAMIC_BMI2 +#if ZSTD_ENABLE_ASM_X86_64_BMI2 && DYNAMIC_BMI2 # define HUF_ASM_X86_64_BMI2_ATTRS BMI2_TARGET_ATTRIBUTE #else # define HUF_ASM_X86_64_BMI2_ATTRS @@ -80,13 +56,13 @@ #endif #define HUF_ASM_DECL HUF_EXTERN_C -#if DYNAMIC_BMI2 || (HUF_ENABLE_ASM_X86_64_BMI2 && defined(__BMI2__)) +#if DYNAMIC_BMI2 || (ZSTD_ENABLE_ASM_X86_64_BMI2 && defined(__BMI2__)) # define HUF_NEED_BMI2_FUNCTION 1 #else # define HUF_NEED_BMI2_FUNCTION 0 #endif -#if !(HUF_ENABLE_ASM_X86_64_BMI2 && defined(__BMI2__)) +#if !(ZSTD_ENABLE_ASM_X86_64_BMI2 && defined(__BMI2__)) # define HUF_NEED_DEFAULT_FUNCTION 1 #else # define HUF_NEED_DEFAULT_FUNCTION 0 @@ -162,7 +138,7 @@ static DTableDesc HUF_getDTableDesc(const HUF_DTable* table) return dtd; } -#if HUF_ENABLE_ASM_X86_64_BMI2 +#if ZSTD_ENABLE_ASM_X86_64_BMI2 static size_t HUF_initDStream(BYTE const* ip) { BYTE const lastByte = ip[7]; @@ -685,7 +661,7 @@ size_t HUF_decompress4X1_usingDTable_internal_default(void* dst, size_t dstSize, } #endif -#if HUF_ENABLE_ASM_X86_64_BMI2 +#if ZSTD_ENABLE_ASM_X86_64_BMI2 HUF_ASM_DECL void HUF_decompress4X1_usingDTable_internal_bmi2_asm_loop(HUF_DecompressAsmArgs* args); @@ -741,7 +717,7 @@ HUF_decompress4X1_usingDTable_internal_bmi2_asm( /* decoded size */ return dstSize; } -#endif /* HUF_ENABLE_ASM_X86_64_BMI2 */ +#endif /* ZSTD_ENABLE_ASM_X86_64_BMI2 */ typedef size_t (*HUF_decompress_usingDTable_t)(void *dst, size_t dstSize, const void *cSrc, @@ -755,7 +731,7 @@ static size_t HUF_decompress4X1_usingDTable_internal(void* dst, size_t dstSize, { #if DYNAMIC_BMI2 if (bmi2) { -# if HUF_ENABLE_ASM_X86_64_BMI2 +# if ZSTD_ENABLE_ASM_X86_64_BMI2 return HUF_decompress4X1_usingDTable_internal_bmi2_asm(dst, dstSize, cSrc, cSrcSize, DTable); # else return HUF_decompress4X1_usingDTable_internal_bmi2(dst, dstSize, cSrc, cSrcSize, DTable); @@ -765,7 +741,7 @@ static size_t HUF_decompress4X1_usingDTable_internal(void* dst, size_t dstSize, (void)bmi2; #endif -#if HUF_ENABLE_ASM_X86_64_BMI2 && defined(__BMI2__) +#if ZSTD_ENABLE_ASM_X86_64_BMI2 && defined(__BMI2__) return HUF_decompress4X1_usingDTable_internal_bmi2_asm(dst, dstSize, cSrc, cSrcSize, DTable); #else return HUF_decompress4X1_usingDTable_internal_default(dst, dstSize, cSrc, cSrcSize, DTable); @@ -1401,7 +1377,7 @@ size_t HUF_decompress4X2_usingDTable_internal_default(void* dst, size_t dstSize, } #endif -#if HUF_ENABLE_ASM_X86_64_BMI2 +#if ZSTD_ENABLE_ASM_X86_64_BMI2 HUF_ASM_DECL void HUF_decompress4X2_usingDTable_internal_bmi2_asm_loop(HUF_DecompressAsmArgs* args); @@ -1453,14 +1429,14 @@ HUF_decompress4X2_usingDTable_internal_bmi2_asm( /* decoded size */ return dstSize; } -#endif /* HUF_ENABLE_ASM_X86_64_BMI2 */ +#endif /* ZSTD_ENABLE_ASM_X86_64_BMI2 */ static size_t HUF_decompress4X2_usingDTable_internal(void* dst, size_t dstSize, void const* cSrc, size_t cSrcSize, HUF_DTable const* DTable, int bmi2) { #if DYNAMIC_BMI2 if (bmi2) { -# if HUF_ENABLE_ASM_X86_64_BMI2 +# if ZSTD_ENABLE_ASM_X86_64_BMI2 return HUF_decompress4X2_usingDTable_internal_bmi2_asm(dst, dstSize, cSrc, cSrcSize, DTable); # else return HUF_decompress4X2_usingDTable_internal_bmi2(dst, dstSize, cSrc, cSrcSize, DTable); @@ -1470,7 +1446,7 @@ static size_t HUF_decompress4X2_usingDTable_internal(void* dst, size_t dstSize, (void)bmi2; #endif -#if HUF_ENABLE_ASM_X86_64_BMI2 && defined(__BMI2__) +#if ZSTD_ENABLE_ASM_X86_64_BMI2 && defined(__BMI2__) return HUF_decompress4X2_usingDTable_internal_bmi2_asm(dst, dstSize, cSrc, cSrcSize, DTable); #else return HUF_decompress4X2_usingDTable_internal_default(dst, dstSize, cSrc, cSrcSize, DTable); diff --git a/lib/decompress/huf_decompress_amd64.S b/lib/decompress/huf_decompress_amd64.S index bb517d7aa43..98173cce863 100644 --- a/lib/decompress/huf_decompress_amd64.S +++ b/lib/decompress/huf_decompress_amd64.S @@ -1,4 +1,6 @@ -#if !defined(HUF_DISABLE_ASM) && defined(__x86_64__) +#include "../common/portability_macros.h" + +#if ZSTD_ENABLE_ASM_X86_64_BMI2 /* Stack marking * ref: https://wiki.gentoo.org/wiki/Hardened/GNU_stack_quickstart diff --git a/lib/libzstd.mk b/lib/libzstd.mk index 45ee1ce629a..af12daffe12 100644 --- a/lib/libzstd.mk +++ b/lib/libzstd.mk @@ -114,7 +114,7 @@ ZSTD_LEGACY_FILES := ZSTD_DECOMPRESS_AMD64_ASM_FILES := $(sort $(wildcard $(LIBZSTD)/decompress/*_amd64.S)) ifneq ($(ZSTD_NO_ASM), 0) - CPPFLAGS += -DHUF_DISABLE_ASM + CPPFLAGS += -DZSTD_DISABLE_ASM else # Unconditionally add the ASM files they are disabled by # macros in the .S file.