1/* cpu_features.c -- Processor features detection. 2 * 3 * Copyright 2018 The Chromium Authors 4 * Use of this source code is governed by a BSD-style license that can be 5 * found in the Chromium source repository LICENSE file. 6 */ 7 8#include "cpu_features.h" 9#include "zutil.h" 10 11#include <stdint.h> 12#if defined(_MSC_VER) 13#include <intrin.h> 14#elif defined(ADLER32_SIMD_SSSE3) 15#include <cpuid.h> 16#endif 17 18/* TODO(cavalcantii): remove checks for x86_flags on deflate. 19 */ 20#if defined(ARMV8_OS_MACOS) 21/* Crypto extensions (crc32/pmull) are a baseline feature in ARMv8.1-A, and 22 * OSX running on arm64 is new enough that these can be assumed without 23 * runtime detection. 24 */ 25int ZLIB_INTERNAL arm_cpu_enable_crc32 = 1; 26int ZLIB_INTERNAL arm_cpu_enable_pmull = 1; 27#else 28int ZLIB_INTERNAL arm_cpu_enable_crc32 = 0; 29int ZLIB_INTERNAL arm_cpu_enable_pmull = 0; 30#endif 31int ZLIB_INTERNAL x86_cpu_enable_sse2 = 0; 32int ZLIB_INTERNAL x86_cpu_enable_ssse3 = 0; 33int ZLIB_INTERNAL x86_cpu_enable_simd = 0; 34int ZLIB_INTERNAL x86_cpu_enable_avx512 = 0; 35 36#ifndef CPU_NO_SIMD 37 38#if defined(ARMV8_OS_ANDROID) || defined(ARMV8_OS_LINUX) || defined(ARMV8_OS_FUCHSIA) || defined(ARMV8_OS_IOS) 39#include <pthread.h> 40#endif 41 42#if defined(ARMV8_OS_ANDROID) 43#include <cpu-features.h> 44#elif defined(ARMV8_OS_LINUX) 45#include <asm/hwcap.h> 46#include <sys/auxv.h> 47#elif defined(ARMV8_OS_FUCHSIA) 48#include <zircon/features.h> 49#include <zircon/syscalls.h> 50#include <zircon/types.h> 51#elif defined(ARMV8_OS_WINDOWS) || defined(X86_WINDOWS) 52#include <windows.h> 53#elif defined(ARMV8_OS_IOS) 54#include <sys/sysctl.h> 55#elif !defined(_MSC_VER) 56#include <pthread.h> 57#else 58#error cpu_features.c CPU feature detection in not defined for your platform 59#endif 60 61#if !defined(CPU_NO_SIMD) && !defined(ARMV8_OS_MACOS) 62static void _cpu_check_features(void); 63#endif 64 65#if defined(ARMV8_OS_ANDROID) || defined(ARMV8_OS_LINUX) || defined(ARMV8_OS_MACOS) || defined(ARMV8_OS_FUCHSIA) || defined(X86_NOT_WINDOWS) || defined(ARMV8_OS_IOS) 66#if !defined(ARMV8_OS_MACOS) 67// _cpu_check_features() doesn't need to do anything on mac/arm since all 68// features are known at build time, so don't call it. 69// Do provide cpu_check_features() (with a no-op implementation) so that we 70// don't have to make all callers of it check for mac/arm. 71static pthread_once_t cpu_check_inited_once = PTHREAD_ONCE_INIT; 72#endif 73void ZLIB_INTERNAL cpu_check_features(void) 74{ 75#if !defined(ARMV8_OS_MACOS) 76 pthread_once(&cpu_check_inited_once, _cpu_check_features); 77#endif 78} 79#elif defined(ARMV8_OS_WINDOWS) || defined(X86_WINDOWS) 80static INIT_ONCE cpu_check_inited_once = INIT_ONCE_STATIC_INIT; 81static BOOL CALLBACK _cpu_check_features_forwarder(PINIT_ONCE once, PVOID param, PVOID* context) 82{ 83 _cpu_check_features(); 84 return TRUE; 85} 86void ZLIB_INTERNAL cpu_check_features(void) 87{ 88 InitOnceExecuteOnce(&cpu_check_inited_once, _cpu_check_features_forwarder, 89 NULL, NULL); 90} 91#endif 92 93#if (defined(__ARM_NEON__) || defined(__ARM_NEON)) 94#if !defined(ARMV8_OS_MACOS) 95/* 96 * See http://bit.ly/2CcoEsr for run-time detection of ARM features and also 97 * crbug.com/931275 for android_getCpuFeatures() use in the Android sandbox. 98 */ 99static void _cpu_check_features(void) 100{ 101#if defined(ARMV8_OS_ANDROID) && defined(__aarch64__) 102 uint64_t features = android_getCpuFeatures(); 103 arm_cpu_enable_crc32 = !!(features & ANDROID_CPU_ARM64_FEATURE_CRC32); 104 arm_cpu_enable_pmull = !!(features & ANDROID_CPU_ARM64_FEATURE_PMULL); 105#elif defined(ARMV8_OS_ANDROID) /* aarch32 */ 106 uint64_t features = android_getCpuFeatures(); 107 arm_cpu_enable_crc32 = !!(features & ANDROID_CPU_ARM_FEATURE_CRC32); 108 arm_cpu_enable_pmull = !!(features & ANDROID_CPU_ARM_FEATURE_PMULL); 109#elif defined(ARMV8_OS_LINUX) && defined(__aarch64__) 110 unsigned long features = getauxval(AT_HWCAP); 111 arm_cpu_enable_crc32 = !!(features & HWCAP_CRC32); 112 arm_cpu_enable_pmull = !!(features & HWCAP_PMULL); 113#elif defined(ARMV8_OS_LINUX) && (defined(__ARM_NEON) || defined(__ARM_NEON__)) 114 /* Query HWCAP2 for ARMV8-A SoCs running in aarch32 mode */ 115 unsigned long features = getauxval(AT_HWCAP2); 116 arm_cpu_enable_crc32 = !!(features & HWCAP2_CRC32); 117 arm_cpu_enable_pmull = !!(features & HWCAP2_PMULL); 118#elif defined(ARMV8_OS_FUCHSIA) 119 uint32_t features; 120 zx_status_t rc = zx_system_get_features(ZX_FEATURE_KIND_CPU, &features); 121 if (rc != ZX_OK || (features & ZX_ARM64_FEATURE_ISA_ASIMD) == 0) 122 return; /* Report nothing if ASIMD(NEON) is missing */ 123 arm_cpu_enable_crc32 = !!(features & ZX_ARM64_FEATURE_ISA_CRC32); 124 arm_cpu_enable_pmull = !!(features & ZX_ARM64_FEATURE_ISA_PMULL); 125#elif defined(ARMV8_OS_WINDOWS) 126 arm_cpu_enable_crc32 = IsProcessorFeaturePresent(PF_ARM_V8_CRC32_INSTRUCTIONS_AVAILABLE); 127 arm_cpu_enable_pmull = IsProcessorFeaturePresent(PF_ARM_V8_CRYPTO_INSTRUCTIONS_AVAILABLE); 128#elif defined(ARMV8_OS_IOS) 129 // Determine what features are supported dynamically. This code is applicable to macOS 130 // as well if we wish to do that dynamically on that platform in the future. 131 // See https://developer.apple.com/documentation/kernel/1387446-sysctlbyname/determining_instruction_set_characteristics 132 int val = 0; 133 size_t len = sizeof(val); 134 arm_cpu_enable_crc32 = sysctlbyname("hw.optional.armv8_crc32", &val, &len, 0, 0) == 0 135 && val != 0; 136 val = 0; 137 len = sizeof(val); 138 arm_cpu_enable_pmull = sysctlbyname("hw.optional.arm.FEAT_PMULL", &val, &len, 0, 0) == 0 139 && val != 0; 140#endif 141} 142#endif 143#elif defined(X86_NOT_WINDOWS) || defined(X86_WINDOWS) 144/* 145 * iOS@x86 (i.e. emulator) is another special case where we disable 146 * SIMD optimizations. 147 */ 148#ifndef CPU_NO_SIMD 149/* On x86 we simply use a instruction to check the CPU features. 150 * (i.e. CPUID). 151 */ 152#ifdef CRC32_SIMD_AVX512_PCLMUL 153#include <immintrin.h> 154#include <xsaveintrin.h> 155#endif 156static void _cpu_check_features(void) 157{ 158 int x86_cpu_has_sse2; 159 int x86_cpu_has_ssse3; 160 int x86_cpu_has_sse42; 161 int x86_cpu_has_pclmulqdq; 162 int abcd[4]; 163 164#ifdef _MSC_VER 165 __cpuid(abcd, 1); 166#else 167 __cpuid(1, abcd[0], abcd[1], abcd[2], abcd[3]); 168#endif 169 170 x86_cpu_has_sse2 = abcd[3] & 0x4000000; 171 x86_cpu_has_ssse3 = abcd[2] & 0x000200; 172 x86_cpu_has_sse42 = abcd[2] & 0x100000; 173 x86_cpu_has_pclmulqdq = abcd[2] & 0x2; 174 175 x86_cpu_enable_sse2 = x86_cpu_has_sse2; 176 177 x86_cpu_enable_ssse3 = x86_cpu_has_ssse3; 178 179 x86_cpu_enable_simd = x86_cpu_has_sse2 && 180 x86_cpu_has_sse42 && 181 x86_cpu_has_pclmulqdq; 182 183#ifdef CRC32_SIMD_AVX512_PCLMUL 184 x86_cpu_enable_avx512 = _xgetbv(0) & 0x00000040; 185#endif 186} 187#endif 188#endif 189#endif 190