diff --git a/compiler-rt/lib/builtins/cpu_model.c b/compiler-rt/lib/builtins/cpu_model.c --- a/compiler-rt/lib/builtins/cpu_model.c +++ b/compiler-rt/lib/builtins/cpu_model.c @@ -838,136 +838,6 @@ return 0; } #elif defined(__aarch64__) -// LSE support detection for out-of-line atomics -// using HWCAP and Auxiliary vector -_Bool __aarch64_have_lse_atomics - __attribute__((visibility("hidden"), nocommon)); - -#if defined(__has_include) -#if __has_include() -#include -#if __has_include() -#include - -#if defined(__ANDROID__) -#include -#include -#elif defined(__Fuchsia__) -#include -#include -#endif - -// Detect Exynos 9810 CPU -#define IF_EXYNOS9810 \ - char arch[PROP_VALUE_MAX]; \ - if (__system_property_get("ro.arch", arch) > 0 && \ - strncmp(arch, "exynos9810", sizeof("exynos9810") - 1) == 0) - -static void CONSTRUCTOR_ATTRIBUTE init_have_lse_atomics(void) { -#if defined(__FreeBSD__) - unsigned long hwcap; - int result = elf_aux_info(AT_HWCAP, &hwcap, sizeof hwcap); - __aarch64_have_lse_atomics = result == 0 && (hwcap & HWCAP_ATOMICS) != 0; -#elif defined(__Fuchsia__) - // This ensures the vDSO is a direct link-time dependency of anything that - // needs this initializer code. -#pragma comment(lib, "zircon") - uint32_t features; - zx_status_t status = _zx_system_get_features(ZX_FEATURE_KIND_CPU, &features); - __aarch64_have_lse_atomics = - status == ZX_OK && (features & ZX_ARM64_FEATURE_ISA_ATOMICS) != 0; -#else - unsigned long hwcap = getauxval(AT_HWCAP); - _Bool result = (hwcap & HWCAP_ATOMICS) != 0; -#if defined(__ANDROID__) - if (result) { - // Some cores in the Exynos 9810 CPU are ARMv8.2 and others are ARMv8.0; - // only the former support LSE atomics. However, the kernel in the - // initial Android 8.0 release of Galaxy S9/S9+ devices incorrectly - // reported the feature as being supported. - // - // The kernel appears to have been corrected to mark it unsupported as of - // the Android 9.0 release on those devices, and this issue has not been - // observed anywhere else. Thus, this workaround may be removed if - // compiler-rt ever drops support for Android 8.0. - IF_EXYNOS9810 result = false; - } -#endif // defined(__ANDROID__) - __aarch64_have_lse_atomics = result; -#endif // defined(__FreeBSD__) -} - -#if !defined(DISABLE_AARCH64_FMV) -// CPUFeatures must correspond to the same AArch64 features in -// AArch64TargetParser.h -enum CPUFeatures { - FEAT_RNG, - FEAT_FLAGM, - FEAT_FLAGM2, - FEAT_FP16FML, - FEAT_DOTPROD, - FEAT_SM4, - FEAT_RDM, - FEAT_LSE, - FEAT_FP, - FEAT_SIMD, - FEAT_CRC, - FEAT_SHA1, - FEAT_SHA2, - FEAT_SHA3, - FEAT_AES, - FEAT_PMULL, - FEAT_FP16, - FEAT_DIT, - FEAT_DPB, - FEAT_DPB2, - FEAT_JSCVT, - FEAT_FCMA, - FEAT_RCPC, - FEAT_RCPC2, - FEAT_FRINTTS, - FEAT_DGH, - FEAT_I8MM, - FEAT_BF16, - FEAT_EBF16, - FEAT_RPRES, - FEAT_SVE, - FEAT_SVE_BF16, - FEAT_SVE_EBF16, - FEAT_SVE_I8MM, - FEAT_SVE_F32MM, - FEAT_SVE_F64MM, - FEAT_SVE2, - FEAT_SVE_AES, - FEAT_SVE_PMULL128, - FEAT_SVE_BITPERM, - FEAT_SVE_SHA3, - FEAT_SVE_SM4, - FEAT_SME, - FEAT_MEMTAG, - FEAT_MEMTAG2, - FEAT_MEMTAG3, - FEAT_SB, - FEAT_PREDRES, - FEAT_SSBS, - FEAT_SSBS2, - FEAT_BTI, - FEAT_LS64, - FEAT_LS64_V, - FEAT_LS64_ACCDATA, - FEAT_WFXT, - FEAT_SME_F64, - FEAT_SME_I64, - FEAT_SME2, - FEAT_MAX -}; - -// Architecture features used -// in Function Multi Versioning -struct { - unsigned long long features; - // As features grows new fields could be added -} __aarch64_cpu_features __attribute__((visibility("hidden"), nocommon)); #ifndef AT_HWCAP #define AT_HWCAP 16 @@ -1136,6 +1006,137 @@ #define HWCAP2_SVE_EBF16 (1UL << 33) #endif +// LSE support detection for out-of-line atomics +// using HWCAP and Auxiliary vector +_Bool __aarch64_have_lse_atomics + __attribute__((visibility("hidden"), nocommon)); + +#if defined(__has_include) +#if __has_include() +#include +#if __has_include() +#include + +#if defined(__ANDROID__) +#include +#include +#elif defined(__Fuchsia__) +#include +#include +#endif + +// Detect Exynos 9810 CPU +#define IF_EXYNOS9810 \ + char arch[PROP_VALUE_MAX]; \ + if (__system_property_get("ro.arch", arch) > 0 && \ + strncmp(arch, "exynos9810", sizeof("exynos9810") - 1) == 0) + +static void CONSTRUCTOR_ATTRIBUTE init_have_lse_atomics(void) { +#if defined(__FreeBSD__) + unsigned long hwcap; + int result = elf_aux_info(AT_HWCAP, &hwcap, sizeof hwcap); + __aarch64_have_lse_atomics = result == 0 && (hwcap & HWCAP_ATOMICS) != 0; +#elif defined(__Fuchsia__) + // This ensures the vDSO is a direct link-time dependency of anything that + // needs this initializer code. +#pragma comment(lib, "zircon") + uint32_t features; + zx_status_t status = _zx_system_get_features(ZX_FEATURE_KIND_CPU, &features); + __aarch64_have_lse_atomics = + status == ZX_OK && (features & ZX_ARM64_FEATURE_ISA_ATOMICS) != 0; +#else + unsigned long hwcap = getauxval(AT_HWCAP); + _Bool result = (hwcap & HWCAP_ATOMICS) != 0; +#if defined(__ANDROID__) + if (result) { + // Some cores in the Exynos 9810 CPU are ARMv8.2 and others are ARMv8.0; + // only the former support LSE atomics. However, the kernel in the + // initial Android 8.0 release of Galaxy S9/S9+ devices incorrectly + // reported the feature as being supported. + // + // The kernel appears to have been corrected to mark it unsupported as of + // the Android 9.0 release on those devices, and this issue has not been + // observed anywhere else. Thus, this workaround may be removed if + // compiler-rt ever drops support for Android 8.0. + IF_EXYNOS9810 result = false; + } +#endif // defined(__ANDROID__) + __aarch64_have_lse_atomics = result; +#endif // defined(__FreeBSD__) +} + +#if !defined(DISABLE_AARCH64_FMV) +// CPUFeatures must correspond to the same AArch64 features in +// AArch64TargetParser.h +enum CPUFeatures { + FEAT_RNG, + FEAT_FLAGM, + FEAT_FLAGM2, + FEAT_FP16FML, + FEAT_DOTPROD, + FEAT_SM4, + FEAT_RDM, + FEAT_LSE, + FEAT_FP, + FEAT_SIMD, + FEAT_CRC, + FEAT_SHA1, + FEAT_SHA2, + FEAT_SHA3, + FEAT_AES, + FEAT_PMULL, + FEAT_FP16, + FEAT_DIT, + FEAT_DPB, + FEAT_DPB2, + FEAT_JSCVT, + FEAT_FCMA, + FEAT_RCPC, + FEAT_RCPC2, + FEAT_FRINTTS, + FEAT_DGH, + FEAT_I8MM, + FEAT_BF16, + FEAT_EBF16, + FEAT_RPRES, + FEAT_SVE, + FEAT_SVE_BF16, + FEAT_SVE_EBF16, + FEAT_SVE_I8MM, + FEAT_SVE_F32MM, + FEAT_SVE_F64MM, + FEAT_SVE2, + FEAT_SVE_AES, + FEAT_SVE_PMULL128, + FEAT_SVE_BITPERM, + FEAT_SVE_SHA3, + FEAT_SVE_SM4, + FEAT_SME, + FEAT_MEMTAG, + FEAT_MEMTAG2, + FEAT_MEMTAG3, + FEAT_SB, + FEAT_PREDRES, + FEAT_SSBS, + FEAT_SSBS2, + FEAT_BTI, + FEAT_LS64, + FEAT_LS64_V, + FEAT_LS64_ACCDATA, + FEAT_WFXT, + FEAT_SME_F64, + FEAT_SME_I64, + FEAT_SME2, + FEAT_MAX +}; + +// Architecture features used +// in Function Multi Versioning +struct { + unsigned long long features; + // As features grows new fields could be added +} __aarch64_cpu_features __attribute__((visibility("hidden"), nocommon)); + void init_cpu_features_resolver(unsigned long hwcap, unsigned long hwcap2) { #define setCPUFeature(F) __aarch64_cpu_features.features |= 1ULL << F #define getCPUFeature(id, ftr) __asm__("mrs %0, " #id : "=r"(ftr))