1 /* cpu_features.c -- Processor features detection.
2  *
3  * Copyright 2018 The Chromium Authors
4  * Use of this source code is governed by a BSD-style license that can be
5  * found in the Chromium source repository LICENSE file.
6  */
7 
8 #include "cpu_features.h"
9 #include "zutil.h"
10 
11 #include <stdint.h>
12 #if defined(_MSC_VER)
13 #include <intrin.h>
14 #elif defined(ADLER32_SIMD_SSSE3)
15 #include <cpuid.h>
16 #endif
17 
18 /* TODO(cavalcantii): remove checks for x86_flags on deflate.
19  */
20 #if defined(ARMV8_OS_MACOS)
21 /* Crypto extensions (crc32/pmull) are a baseline feature in ARMv8.1-A, and
22  * OSX running on arm64 is new enough that these can be assumed without
23  * runtime detection.
24  */
25 int ZLIB_INTERNAL arm_cpu_enable_crc32 = 1;
26 int ZLIB_INTERNAL arm_cpu_enable_pmull = 1;
27 #else
28 int ZLIB_INTERNAL arm_cpu_enable_crc32 = 0;
29 int ZLIB_INTERNAL arm_cpu_enable_pmull = 0;
30 #endif
31 int ZLIB_INTERNAL x86_cpu_enable_sse2 = 0;
32 int ZLIB_INTERNAL x86_cpu_enable_ssse3 = 0;
33 int ZLIB_INTERNAL x86_cpu_enable_simd = 0;
34 int ZLIB_INTERNAL x86_cpu_enable_avx512 = 0;
35 
36 #ifndef CPU_NO_SIMD
37 
38 #if defined(ARMV8_OS_ANDROID) || defined(ARMV8_OS_LINUX) || defined(ARMV8_OS_FUCHSIA) || defined(ARMV8_OS_IOS)
39 #include <pthread.h>
40 #endif
41 
42 #if defined(ARMV8_OS_ANDROID)
43 #include <cpu-features.h>
44 #elif defined(ARMV8_OS_LINUX)
45 #include <asm/hwcap.h>
46 #include <sys/auxv.h>
47 #elif defined(ARMV8_OS_FUCHSIA)
48 #include <zircon/features.h>
49 #include <zircon/syscalls.h>
50 #include <zircon/types.h>
51 #elif defined(ARMV8_OS_WINDOWS) || defined(X86_WINDOWS)
52 #include <windows.h>
53 #elif defined(ARMV8_OS_IOS)
54 #include <sys/sysctl.h>
55 #elif !defined(_MSC_VER)
56 #include <pthread.h>
57 #else
58 #error cpu_features.c CPU feature detection in not defined for your platform
59 #endif
60 
61 #if !defined(CPU_NO_SIMD) && !defined(ARMV8_OS_MACOS)
62 static void _cpu_check_features(void);
63 #endif
64 
65 #if defined(ARMV8_OS_ANDROID) || defined(ARMV8_OS_LINUX) || defined(ARMV8_OS_MACOS) || defined(ARMV8_OS_FUCHSIA) || defined(X86_NOT_WINDOWS) || defined(ARMV8_OS_IOS)
66 #if !defined(ARMV8_OS_MACOS)
67 // _cpu_check_features() doesn't need to do anything on mac/arm since all
68 // features are known at build time, so don't call it.
69 // Do provide cpu_check_features() (with a no-op implementation) so that we
70 // don't have to make all callers of it check for mac/arm.
71 static pthread_once_t cpu_check_inited_once = PTHREAD_ONCE_INIT;
72 #endif
cpu_check_features(void)73 void ZLIB_INTERNAL cpu_check_features(void)
74 {
75 #if !defined(ARMV8_OS_MACOS)
76     pthread_once(&cpu_check_inited_once, _cpu_check_features);
77 #endif
78 }
79 #elif defined(ARMV8_OS_WINDOWS) || defined(X86_WINDOWS)
80 static INIT_ONCE cpu_check_inited_once = INIT_ONCE_STATIC_INIT;
_cpu_check_features_forwarder(PINIT_ONCE once, PVOID param, PVOID* context)81 static BOOL CALLBACK _cpu_check_features_forwarder(PINIT_ONCE once, PVOID param, PVOID* context)
82 {
83     _cpu_check_features();
84     return TRUE;
85 }
cpu_check_features(void)86 void ZLIB_INTERNAL cpu_check_features(void)
87 {
88     InitOnceExecuteOnce(&cpu_check_inited_once, _cpu_check_features_forwarder,
89                         NULL, NULL);
90 }
91 #endif
92 
93 #if (defined(__ARM_NEON__) || defined(__ARM_NEON))
94 #if !defined(ARMV8_OS_MACOS)
95 /*
96  * See http://bit.ly/2CcoEsr for run-time detection of ARM features and also
97  * crbug.com/931275 for android_getCpuFeatures() use in the Android sandbox.
98  */
_cpu_check_features(void)99 static void _cpu_check_features(void)
100 {
101 #if defined(ARMV8_OS_ANDROID) && defined(__aarch64__)
102     uint64_t features = android_getCpuFeatures();
103     arm_cpu_enable_crc32 = !!(features & ANDROID_CPU_ARM64_FEATURE_CRC32);
104     arm_cpu_enable_pmull = !!(features & ANDROID_CPU_ARM64_FEATURE_PMULL);
105 #elif defined(ARMV8_OS_ANDROID) /* aarch32 */
106     uint64_t features = android_getCpuFeatures();
107     arm_cpu_enable_crc32 = !!(features & ANDROID_CPU_ARM_FEATURE_CRC32);
108     arm_cpu_enable_pmull = !!(features & ANDROID_CPU_ARM_FEATURE_PMULL);
109 #elif defined(ARMV8_OS_LINUX) && defined(__aarch64__)
110     unsigned long features = getauxval(AT_HWCAP);
111     arm_cpu_enable_crc32 = !!(features & HWCAP_CRC32);
112     arm_cpu_enable_pmull = !!(features & HWCAP_PMULL);
113 #elif defined(ARMV8_OS_LINUX) && (defined(__ARM_NEON) || defined(__ARM_NEON__))
114     /* Query HWCAP2 for ARMV8-A SoCs running in aarch32 mode */
115     unsigned long features = getauxval(AT_HWCAP2);
116     arm_cpu_enable_crc32 = !!(features & HWCAP2_CRC32);
117     arm_cpu_enable_pmull = !!(features & HWCAP2_PMULL);
118 #elif defined(ARMV8_OS_FUCHSIA)
119     uint32_t features;
120     zx_status_t rc = zx_system_get_features(ZX_FEATURE_KIND_CPU, &features);
121     if (rc != ZX_OK || (features & ZX_ARM64_FEATURE_ISA_ASIMD) == 0)
122         return;  /* Report nothing if ASIMD(NEON) is missing */
123     arm_cpu_enable_crc32 = !!(features & ZX_ARM64_FEATURE_ISA_CRC32);
124     arm_cpu_enable_pmull = !!(features & ZX_ARM64_FEATURE_ISA_PMULL);
125 #elif defined(ARMV8_OS_WINDOWS)
126     arm_cpu_enable_crc32 = IsProcessorFeaturePresent(PF_ARM_V8_CRC32_INSTRUCTIONS_AVAILABLE);
127     arm_cpu_enable_pmull = IsProcessorFeaturePresent(PF_ARM_V8_CRYPTO_INSTRUCTIONS_AVAILABLE);
128 #elif defined(ARMV8_OS_IOS)
129     // Determine what features are supported dynamically. This code is applicable to macOS
130     // as well if we wish to do that dynamically on that platform in the future.
131     // See https://developer.apple.com/documentation/kernel/1387446-sysctlbyname/determining_instruction_set_characteristics
132     int val = 0;
133     size_t len = sizeof(val);
134     arm_cpu_enable_crc32 = sysctlbyname("hw.optional.armv8_crc32", &val, &len, 0, 0) == 0
135                && val != 0;
136     val = 0;
137     len = sizeof(val);
138     arm_cpu_enable_pmull = sysctlbyname("hw.optional.arm.FEAT_PMULL", &val, &len, 0, 0) == 0
139                && val != 0;
140 #endif
141 }
142 #endif
143 #elif defined(X86_NOT_WINDOWS) || defined(X86_WINDOWS)
144 /*
145  * iOS@x86 (i.e. emulator) is another special case where we disable
146  * SIMD optimizations.
147  */
148 #ifndef CPU_NO_SIMD
149 /* On x86 we simply use a instruction to check the CPU features.
150  * (i.e. CPUID).
151  */
152 #ifdef CRC32_SIMD_AVX512_PCLMUL
153 #include <immintrin.h>
154 #include <xsaveintrin.h>
155 #endif
_cpu_check_features(void)156 static void _cpu_check_features(void)
157 {
158     int x86_cpu_has_sse2;
159     int x86_cpu_has_ssse3;
160     int x86_cpu_has_sse42;
161     int x86_cpu_has_pclmulqdq;
162     int abcd[4];
163 
164 #ifdef _MSC_VER
165     __cpuid(abcd, 1);
166 #else
167     __cpuid(1, abcd[0], abcd[1], abcd[2], abcd[3]);
168 #endif
169 
170     x86_cpu_has_sse2 = abcd[3] & 0x4000000;
171     x86_cpu_has_ssse3 = abcd[2] & 0x000200;
172     x86_cpu_has_sse42 = abcd[2] & 0x100000;
173     x86_cpu_has_pclmulqdq = abcd[2] & 0x2;
174 
175     x86_cpu_enable_sse2 = x86_cpu_has_sse2;
176 
177     x86_cpu_enable_ssse3 = x86_cpu_has_ssse3;
178 
179     x86_cpu_enable_simd = x86_cpu_has_sse2 &&
180                           x86_cpu_has_sse42 &&
181                           x86_cpu_has_pclmulqdq;
182 
183 #ifdef CRC32_SIMD_AVX512_PCLMUL
184     x86_cpu_enable_avx512 = _xgetbv(0) & 0x00000040;
185 #endif
186 }
187 #endif
188 #endif
189 #endif
190