1// SPDX-License-Identifier: Apache-2.0 2// ---------------------------------------------------------------------------- 3// Copyright 2019-2023 Arm Limited 4// 5// Licensed under the Apache License, Version 2.0 (the "License"); you may not 6// use this file except in compliance with the License. You may obtain a copy 7// of the License at: 8// 9// http://www.apache.org/licenses/LICENSE-2.0 10// 11// Unless required by applicable law or agreed to in writing, software 12// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 13// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 14// License for the specific language governing permissions and limitations 15// under the License. 16// ---------------------------------------------------------------------------- 17 18/** 19 * @brief 4x32-bit vectors, implemented using SSE. 20 * 21 * This module implements 4-wide 32-bit float, int, and mask vectors for x86 22 * SSE. The implementation requires at least SSE2, but higher levels of SSE can 23 * be selected at compile time to improve performance. 24 * 25 * There is a baseline level of functionality provided by all vector widths and 26 * implementations. This is implemented using identical function signatures, 27 * modulo data type, so we can use them as substitutable implementations in VLA 28 * code. 29 * 30 * The 4-wide vectors are also used as a fixed-width type, and significantly 31 * extend the functionality above that available to VLA code. 32 */ 33 34#ifndef ASTC_VECMATHLIB_SSE_4_H_INCLUDED 35#define ASTC_VECMATHLIB_SSE_4_H_INCLUDED 36 37#ifndef ASTCENC_SIMD_INLINE 38 #error "Include astcenc_vecmathlib.h, do not include directly" 39#endif 40 41#include <cstdio> 42#include <cstring> 43 44// ============================================================================ 45// vfloat4 data type 46// ============================================================================ 47 48/** 49 * @brief Data type for 4-wide floats. 50 */ 51struct vfloat4 52{ 53 /** 54 * @brief Construct from zero-initialized value. 55 */ 56 ASTCENC_SIMD_INLINE vfloat4() = default; 57 58 /** 59 * @brief Construct from 4 values loaded from an unaligned address. 60 * 61 * Consider using loada() which is better with vectors if data is aligned 62 * to vector length. 63 */ 64 ASTCENC_SIMD_INLINE explicit vfloat4(const float *p) 65 { 66 m = _mm_loadu_ps(p); 67 } 68 69 /** 70 * @brief Construct from 1 scalar value replicated across all lanes. 71 * 72 * Consider using zero() for constexpr zeros. 73 */ 74 ASTCENC_SIMD_INLINE explicit vfloat4(float a) 75 { 76 m = _mm_set1_ps(a); 77 } 78 79 /** 80 * @brief Construct from 4 scalar values. 81 * 82 * The value of @c a is stored to lane 0 (LSB) in the SIMD register. 83 */ 84 ASTCENC_SIMD_INLINE explicit vfloat4(float a, float b, float c, float d) 85 { 86 m = _mm_set_ps(d, c, b, a); 87 } 88 89 /** 90 * @brief Construct from an existing SIMD register. 91 */ 92 ASTCENC_SIMD_INLINE explicit vfloat4(__m128 a) 93 { 94 m = a; 95 } 96 97 /** 98 * @brief Get the scalar value of a single lane. 99 */ 100 template <int l> ASTCENC_SIMD_INLINE float lane() const 101 { 102 return _mm_cvtss_f32(_mm_shuffle_ps(m, m, l)); 103 } 104 105 /** 106 * @brief Set the scalar value of a single lane. 107 */ 108 template <int l> ASTCENC_SIMD_INLINE void set_lane(float a) 109 { 110#if ASTCENC_SSE >= 41 111 __m128 v = _mm_set1_ps(a); 112 m = _mm_insert_ps(m, v, l << 6 | l << 4); 113#else 114 alignas(16) float idx[4]; 115 _mm_store_ps(idx, m); 116 idx[l] = a; 117 m = _mm_load_ps(idx); 118#endif 119 } 120 121 /** 122 * @brief Factory that returns a vector of zeros. 123 */ 124 static ASTCENC_SIMD_INLINE vfloat4 zero() 125 { 126 return vfloat4(_mm_setzero_ps()); 127 } 128 129 /** 130 * @brief Factory that returns a replicated scalar loaded from memory. 131 */ 132 static ASTCENC_SIMD_INLINE vfloat4 load1(const float* p) 133 { 134 return vfloat4(_mm_load_ps1(p)); 135 } 136 137 /** 138 * @brief Factory that returns a vector loaded from 16B aligned memory. 139 */ 140 static ASTCENC_SIMD_INLINE vfloat4 loada(const float* p) 141 { 142 return vfloat4(_mm_load_ps(p)); 143 } 144 145 /** 146 * @brief Factory that returns a vector containing the lane IDs. 147 */ 148 static ASTCENC_SIMD_INLINE vfloat4 lane_id() 149 { 150 return vfloat4(_mm_set_ps(3, 2, 1, 0)); 151 } 152 153 /** 154 * @brief Return a swizzled float 2. 155 */ 156 template <int l0, int l1> ASTCENC_SIMD_INLINE vfloat4 swz() const 157 { 158 vfloat4 result(_mm_shuffle_ps(m, m, l0 | l1 << 2)); 159 result.set_lane<2>(0.0f); 160 result.set_lane<3>(0.0f); 161 return result; 162 } 163 164 /** 165 * @brief Return a swizzled float 3. 166 */ 167 template <int l0, int l1, int l2> ASTCENC_SIMD_INLINE vfloat4 swz() const 168 { 169 vfloat4 result(_mm_shuffle_ps(m, m, l0 | l1 << 2 | l2 << 4)); 170 result.set_lane<3>(0.0f); 171 return result; 172 } 173 174 /** 175 * @brief Return a swizzled float 4. 176 */ 177 template <int l0, int l1, int l2, int l3> ASTCENC_SIMD_INLINE vfloat4 swz() const 178 { 179 return vfloat4(_mm_shuffle_ps(m, m, l0 | l1 << 2 | l2 << 4 | l3 << 6)); 180 } 181 182 /** 183 * @brief The vector ... 184 */ 185 __m128 m; 186}; 187 188// ============================================================================ 189// vint4 data type 190// ============================================================================ 191 192/** 193 * @brief Data type for 4-wide ints. 194 */ 195struct vint4 196{ 197 /** 198 * @brief Construct from zero-initialized value. 199 */ 200 ASTCENC_SIMD_INLINE vint4() = default; 201 202 /** 203 * @brief Construct from 4 values loaded from an unaligned address. 204 * 205 * Consider using loada() which is better with vectors if data is aligned 206 * to vector length. 207 */ 208 ASTCENC_SIMD_INLINE explicit vint4(const int *p) 209 { 210 m = _mm_loadu_si128(reinterpret_cast<const __m128i*>(p)); 211 } 212 213 /** 214 * @brief Construct from 4 uint8_t loaded from an unaligned address. 215 */ 216 ASTCENC_SIMD_INLINE explicit vint4(const uint8_t *p) 217 { 218 // _mm_loadu_si32 would be nicer syntax, but missing on older GCC 219 __m128i t = _mm_cvtsi32_si128(*reinterpret_cast<const int*>(p)); 220 221#if ASTCENC_SSE >= 41 222 m = _mm_cvtepu8_epi32(t); 223#else 224 t = _mm_unpacklo_epi8(t, _mm_setzero_si128()); 225 m = _mm_unpacklo_epi16(t, _mm_setzero_si128()); 226#endif 227 } 228 229 /** 230 * @brief Construct from 1 scalar value replicated across all lanes. 231 * 232 * Consider using vfloat4::zero() for constexpr zeros. 233 */ 234 ASTCENC_SIMD_INLINE explicit vint4(int a) 235 { 236 m = _mm_set1_epi32(a); 237 } 238 239 /** 240 * @brief Construct from 4 scalar values. 241 * 242 * The value of @c a is stored to lane 0 (LSB) in the SIMD register. 243 */ 244 ASTCENC_SIMD_INLINE explicit vint4(int a, int b, int c, int d) 245 { 246 m = _mm_set_epi32(d, c, b, a); 247 } 248 249 /** 250 * @brief Construct from an existing SIMD register. 251 */ 252 ASTCENC_SIMD_INLINE explicit vint4(__m128i a) 253 { 254 m = a; 255 } 256 257 /** 258 * @brief Get the scalar from a single lane. 259 */ 260 template <int l> ASTCENC_SIMD_INLINE int lane() const 261 { 262 return _mm_cvtsi128_si32(_mm_shuffle_epi32(m, l)); 263 } 264 265 /** 266 * @brief Set the scalar value of a single lane. 267 */ 268 template <int l> ASTCENC_SIMD_INLINE void set_lane(int a) 269 { 270#if ASTCENC_SSE >= 41 271 m = _mm_insert_epi32(m, a, l); 272#else 273 alignas(16) int idx[4]; 274 _mm_store_si128(reinterpret_cast<__m128i*>(idx), m); 275 idx[l] = a; 276 m = _mm_load_si128(reinterpret_cast<const __m128i*>(idx)); 277#endif 278 } 279 280 /** 281 * @brief Factory that returns a vector of zeros. 282 */ 283 static ASTCENC_SIMD_INLINE vint4 zero() 284 { 285 return vint4(_mm_setzero_si128()); 286 } 287 288 /** 289 * @brief Factory that returns a replicated scalar loaded from memory. 290 */ 291 static ASTCENC_SIMD_INLINE vint4 load1(const int* p) 292 { 293 return vint4(*p); 294 } 295 296 /** 297 * @brief Factory that returns a vector loaded from unaligned memory. 298 */ 299 static ASTCENC_SIMD_INLINE vint4 load(const uint8_t* p) 300 { 301#if ASTCENC_SSE >= 41 302 return vint4(_mm_lddqu_si128(reinterpret_cast<const __m128i*>(p))); 303#else 304 return vint4(_mm_loadu_si128(reinterpret_cast<const __m128i*>(p))); 305#endif 306 } 307 308 /** 309 * @brief Factory that returns a vector loaded from 16B aligned memory. 310 */ 311 static ASTCENC_SIMD_INLINE vint4 loada(const int* p) 312 { 313 return vint4(_mm_load_si128(reinterpret_cast<const __m128i*>(p))); 314 } 315 316 /** 317 * @brief Factory that returns a vector containing the lane IDs. 318 */ 319 static ASTCENC_SIMD_INLINE vint4 lane_id() 320 { 321 return vint4(_mm_set_epi32(3, 2, 1, 0)); 322 } 323 324 /** 325 * @brief The vector ... 326 */ 327 __m128i m; 328}; 329 330// ============================================================================ 331// vmask4 data type 332// ============================================================================ 333 334/** 335 * @brief Data type for 4-wide control plane masks. 336 */ 337struct vmask4 338{ 339 /** 340 * @brief Construct from an existing SIMD register. 341 */ 342 ASTCENC_SIMD_INLINE explicit vmask4(__m128 a) 343 { 344 m = a; 345 } 346 347 /** 348 * @brief Construct from an existing SIMD register. 349 */ 350 ASTCENC_SIMD_INLINE explicit vmask4(__m128i a) 351 { 352 m = _mm_castsi128_ps(a); 353 } 354 355 /** 356 * @brief Construct from 1 scalar value. 357 */ 358 ASTCENC_SIMD_INLINE explicit vmask4(bool a) 359 { 360 vint4 mask(a == false ? 0 : -1); 361 m = _mm_castsi128_ps(mask.m); 362 } 363 364 /** 365 * @brief Construct from 4 scalar values. 366 * 367 * The value of @c a is stored to lane 0 (LSB) in the SIMD register. 368 */ 369 ASTCENC_SIMD_INLINE explicit vmask4(bool a, bool b, bool c, bool d) 370 { 371 vint4 mask(a == false ? 0 : -1, 372 b == false ? 0 : -1, 373 c == false ? 0 : -1, 374 d == false ? 0 : -1); 375 376 m = _mm_castsi128_ps(mask.m); 377 } 378 379 /** 380 * @brief Get the scalar value of a single lane. 381 */ 382 template <int l> ASTCENC_SIMD_INLINE bool lane() const 383 { 384 return _mm_cvtss_f32(_mm_shuffle_ps(m, m, l)) != 0.0f; 385 } 386 387 /** 388 * @brief The vector ... 389 */ 390 __m128 m; 391}; 392 393// ============================================================================ 394// vmask4 operators and functions 395// ============================================================================ 396 397/** 398 * @brief Overload: mask union (or). 399 */ 400ASTCENC_SIMD_INLINE vmask4 operator|(vmask4 a, vmask4 b) 401{ 402 return vmask4(_mm_or_ps(a.m, b.m)); 403} 404 405/** 406 * @brief Overload: mask intersect (and). 407 */ 408ASTCENC_SIMD_INLINE vmask4 operator&(vmask4 a, vmask4 b) 409{ 410 return vmask4(_mm_and_ps(a.m, b.m)); 411} 412 413/** 414 * @brief Overload: mask difference (xor). 415 */ 416ASTCENC_SIMD_INLINE vmask4 operator^(vmask4 a, vmask4 b) 417{ 418 return vmask4(_mm_xor_ps(a.m, b.m)); 419} 420 421/** 422 * @brief Overload: mask invert (not). 423 */ 424ASTCENC_SIMD_INLINE vmask4 operator~(vmask4 a) 425{ 426 return vmask4(_mm_xor_si128(_mm_castps_si128(a.m), _mm_set1_epi32(-1))); 427} 428 429/** 430 * @brief Return a 4-bit mask code indicating mask status. 431 * 432 * bit0 = lane 0 433 */ 434ASTCENC_SIMD_INLINE unsigned int mask(vmask4 a) 435{ 436 return static_cast<unsigned int>(_mm_movemask_ps(a.m)); 437} 438 439// ============================================================================ 440// vint4 operators and functions 441// ============================================================================ 442 443/** 444 * @brief Overload: vector by vector addition. 445 */ 446ASTCENC_SIMD_INLINE vint4 operator+(vint4 a, vint4 b) 447{ 448 return vint4(_mm_add_epi32(a.m, b.m)); 449} 450 451/** 452 * @brief Overload: vector by vector subtraction. 453 */ 454ASTCENC_SIMD_INLINE vint4 operator-(vint4 a, vint4 b) 455{ 456 return vint4(_mm_sub_epi32(a.m, b.m)); 457} 458 459/** 460 * @brief Overload: vector by vector multiplication. 461 */ 462ASTCENC_SIMD_INLINE vint4 operator*(vint4 a, vint4 b) 463{ 464#if ASTCENC_SSE >= 41 465 return vint4(_mm_mullo_epi32 (a.m, b.m)); 466#else 467 __m128i t1 = _mm_mul_epu32(a.m, b.m); 468 __m128i t2 = _mm_mul_epu32( 469 _mm_srli_si128(a.m, 4), 470 _mm_srli_si128(b.m, 4)); 471 __m128i r = _mm_unpacklo_epi32( 472 _mm_shuffle_epi32(t1, _MM_SHUFFLE (0, 0, 2, 0)), 473 _mm_shuffle_epi32(t2, _MM_SHUFFLE (0, 0, 2, 0))); 474 return vint4(r); 475#endif 476} 477 478/** 479 * @brief Overload: vector bit invert. 480 */ 481ASTCENC_SIMD_INLINE vint4 operator~(vint4 a) 482{ 483 return vint4(_mm_xor_si128(a.m, _mm_set1_epi32(-1))); 484} 485 486/** 487 * @brief Overload: vector by vector bitwise or. 488 */ 489ASTCENC_SIMD_INLINE vint4 operator|(vint4 a, vint4 b) 490{ 491 return vint4(_mm_or_si128(a.m, b.m)); 492} 493 494/** 495 * @brief Overload: vector by vector bitwise and. 496 */ 497ASTCENC_SIMD_INLINE vint4 operator&(vint4 a, vint4 b) 498{ 499 return vint4(_mm_and_si128(a.m, b.m)); 500} 501 502/** 503 * @brief Overload: vector by vector bitwise xor. 504 */ 505ASTCENC_SIMD_INLINE vint4 operator^(vint4 a, vint4 b) 506{ 507 return vint4(_mm_xor_si128(a.m, b.m)); 508} 509 510/** 511 * @brief Overload: vector by vector equality. 512 */ 513ASTCENC_SIMD_INLINE vmask4 operator==(vint4 a, vint4 b) 514{ 515 return vmask4(_mm_cmpeq_epi32(a.m, b.m)); 516} 517 518/** 519 * @brief Overload: vector by vector inequality. 520 */ 521ASTCENC_SIMD_INLINE vmask4 operator!=(vint4 a, vint4 b) 522{ 523 return ~vmask4(_mm_cmpeq_epi32(a.m, b.m)); 524} 525 526/** 527 * @brief Overload: vector by vector less than. 528 */ 529ASTCENC_SIMD_INLINE vmask4 operator<(vint4 a, vint4 b) 530{ 531 return vmask4(_mm_cmplt_epi32(a.m, b.m)); 532} 533 534/** 535 * @brief Overload: vector by vector greater than. 536 */ 537ASTCENC_SIMD_INLINE vmask4 operator>(vint4 a, vint4 b) 538{ 539 return vmask4(_mm_cmpgt_epi32(a.m, b.m)); 540} 541 542/** 543 * @brief Logical shift left. 544 */ 545template <int s> ASTCENC_SIMD_INLINE vint4 lsl(vint4 a) 546{ 547 return vint4(_mm_slli_epi32(a.m, s)); 548} 549 550/** 551 * @brief Logical shift right. 552 */ 553template <int s> ASTCENC_SIMD_INLINE vint4 lsr(vint4 a) 554{ 555 return vint4(_mm_srli_epi32(a.m, s)); 556} 557 558/** 559 * @brief Arithmetic shift right. 560 */ 561template <int s> ASTCENC_SIMD_INLINE vint4 asr(vint4 a) 562{ 563 return vint4(_mm_srai_epi32(a.m, s)); 564} 565 566/** 567 * @brief Return the min vector of two vectors. 568 */ 569ASTCENC_SIMD_INLINE vint4 min(vint4 a, vint4 b) 570{ 571#if ASTCENC_SSE >= 41 572 return vint4(_mm_min_epi32(a.m, b.m)); 573#else 574 vmask4 d = a < b; 575 __m128i ap = _mm_and_si128(_mm_castps_si128(d.m), a.m); 576 __m128i bp = _mm_andnot_si128(_mm_castps_si128(d.m), b.m); 577 return vint4(_mm_or_si128(ap,bp)); 578#endif 579} 580 581/** 582 * @brief Return the max vector of two vectors. 583 */ 584ASTCENC_SIMD_INLINE vint4 max(vint4 a, vint4 b) 585{ 586#if ASTCENC_SSE >= 41 587 return vint4(_mm_max_epi32(a.m, b.m)); 588#else 589 vmask4 d = a > b; 590 __m128i ap = _mm_and_si128(_mm_castps_si128(d.m), a.m); 591 __m128i bp = _mm_andnot_si128(_mm_castps_si128(d.m), b.m); 592 return vint4(_mm_or_si128(ap,bp)); 593#endif 594} 595 596/** 597 * @brief Return the horizontal minimum of a vector. 598 */ 599ASTCENC_SIMD_INLINE vint4 hmin(vint4 a) 600{ 601 a = min(a, vint4(_mm_shuffle_epi32(a.m, _MM_SHUFFLE(0, 0, 3, 2)))); 602 a = min(a, vint4(_mm_shuffle_epi32(a.m, _MM_SHUFFLE(0, 0, 0, 1)))); 603 return vint4(_mm_shuffle_epi32(a.m, _MM_SHUFFLE(0, 0, 0, 0))); 604} 605 606/* 607 * @brief Return the horizontal maximum of a vector. 608 */ 609ASTCENC_SIMD_INLINE vint4 hmax(vint4 a) 610{ 611 a = max(a, vint4(_mm_shuffle_epi32(a.m, _MM_SHUFFLE(0, 0, 3, 2)))); 612 a = max(a, vint4(_mm_shuffle_epi32(a.m, _MM_SHUFFLE(0, 0, 0, 1)))); 613 return vint4(_mm_shuffle_epi32(a.m, _MM_SHUFFLE(0, 0, 0, 0))); 614} 615 616/** 617 * @brief Return the horizontal sum of a vector as a scalar. 618 */ 619ASTCENC_SIMD_INLINE int hadd_s(vint4 a) 620{ 621 // Add top and bottom halves, lane 1/0 622 __m128i fold = _mm_castps_si128(_mm_movehl_ps(_mm_castsi128_ps(a.m), 623 _mm_castsi128_ps(a.m))); 624 __m128i t = _mm_add_epi32(a.m, fold); 625 626 // Add top and bottom halves, lane 0 (_mm_hadd_ps exists but slow) 627 t = _mm_add_epi32(t, _mm_shuffle_epi32(t, 0x55)); 628 629 return _mm_cvtsi128_si32(t); 630} 631 632/** 633 * @brief Store a vector to a 16B aligned memory address. 634 */ 635ASTCENC_SIMD_INLINE void storea(vint4 a, int* p) 636{ 637 _mm_store_si128(reinterpret_cast<__m128i*>(p), a.m); 638} 639 640/** 641 * @brief Store a vector to an unaligned memory address. 642 */ 643ASTCENC_SIMD_INLINE void store(vint4 a, int* p) 644{ 645 // Cast due to missing intrinsics 646 _mm_storeu_ps(reinterpret_cast<float*>(p), _mm_castsi128_ps(a.m)); 647} 648 649/** 650 * @brief Store a vector to an unaligned memory address. 651 */ 652ASTCENC_SIMD_INLINE void store(vint4 a, uint8_t* p) 653{ 654 std::memcpy(p, &a.m, sizeof(int) * 4); 655} 656 657/** 658 * @brief Store lowest N (vector width) bytes into an unaligned address. 659 */ 660ASTCENC_SIMD_INLINE void store_nbytes(vint4 a, uint8_t* p) 661{ 662 // Cast due to missing intrinsics 663 _mm_store_ss(reinterpret_cast<float*>(p), _mm_castsi128_ps(a.m)); 664} 665 666/** 667 * @brief Gather N (vector width) indices from the array. 668 */ 669ASTCENC_SIMD_INLINE vint4 gatheri(const int* base, vint4 indices) 670{ 671#if ASTCENC_AVX >= 2 672 return vint4(_mm_i32gather_epi32(base, indices.m, 4)); 673#else 674 alignas(16) int idx[4]; 675 storea(indices, idx); 676 return vint4(base[idx[0]], base[idx[1]], base[idx[2]], base[idx[3]]); 677#endif 678} 679 680/** 681 * @brief Pack low 8 bits of N (vector width) lanes into bottom of vector. 682 */ 683ASTCENC_SIMD_INLINE vint4 pack_low_bytes(vint4 a) 684{ 685#if ASTCENC_SSE >= 41 686 __m128i shuf = _mm_set_epi8(0,0,0,0, 0,0,0,0, 0,0,0,0, 12,8,4,0); 687 return vint4(_mm_shuffle_epi8(a.m, shuf)); 688#else 689 __m128i va = _mm_unpacklo_epi8(a.m, _mm_shuffle_epi32(a.m, _MM_SHUFFLE(1,1,1,1))); 690 __m128i vb = _mm_unpackhi_epi8(a.m, _mm_shuffle_epi32(a.m, _MM_SHUFFLE(3,3,3,3))); 691 return vint4(_mm_unpacklo_epi16(va, vb)); 692#endif 693} 694 695/** 696 * @brief Return lanes from @c b if @c cond is set, else @c a. 697 */ 698ASTCENC_SIMD_INLINE vint4 select(vint4 a, vint4 b, vmask4 cond) 699{ 700 __m128i condi = _mm_castps_si128(cond.m); 701 702#if ASTCENC_SSE >= 41 703 return vint4(_mm_blendv_epi8(a.m, b.m, condi)); 704#else 705 return vint4(_mm_or_si128(_mm_and_si128(condi, b.m), _mm_andnot_si128(condi, a.m))); 706#endif 707} 708 709// ============================================================================ 710// vfloat4 operators and functions 711// ============================================================================ 712 713/** 714 * @brief Overload: vector by vector addition. 715 */ 716ASTCENC_SIMD_INLINE vfloat4 operator+(vfloat4 a, vfloat4 b) 717{ 718 return vfloat4(_mm_add_ps(a.m, b.m)); 719} 720 721/** 722 * @brief Overload: vector by vector subtraction. 723 */ 724ASTCENC_SIMD_INLINE vfloat4 operator-(vfloat4 a, vfloat4 b) 725{ 726 return vfloat4(_mm_sub_ps(a.m, b.m)); 727} 728 729/** 730 * @brief Overload: vector by vector multiplication. 731 */ 732ASTCENC_SIMD_INLINE vfloat4 operator*(vfloat4 a, vfloat4 b) 733{ 734 return vfloat4(_mm_mul_ps(a.m, b.m)); 735} 736 737/** 738 * @brief Overload: vector by vector division. 739 */ 740ASTCENC_SIMD_INLINE vfloat4 operator/(vfloat4 a, vfloat4 b) 741{ 742 return vfloat4(_mm_div_ps(a.m, b.m)); 743} 744 745/** 746 * @brief Overload: vector by vector equality. 747 */ 748ASTCENC_SIMD_INLINE vmask4 operator==(vfloat4 a, vfloat4 b) 749{ 750 return vmask4(_mm_cmpeq_ps(a.m, b.m)); 751} 752 753/** 754 * @brief Overload: vector by vector inequality. 755 */ 756ASTCENC_SIMD_INLINE vmask4 operator!=(vfloat4 a, vfloat4 b) 757{ 758 return vmask4(_mm_cmpneq_ps(a.m, b.m)); 759} 760 761/** 762 * @brief Overload: vector by vector less than. 763 */ 764ASTCENC_SIMD_INLINE vmask4 operator<(vfloat4 a, vfloat4 b) 765{ 766 return vmask4(_mm_cmplt_ps(a.m, b.m)); 767} 768 769/** 770 * @brief Overload: vector by vector greater than. 771 */ 772ASTCENC_SIMD_INLINE vmask4 operator>(vfloat4 a, vfloat4 b) 773{ 774 return vmask4(_mm_cmpgt_ps(a.m, b.m)); 775} 776 777/** 778 * @brief Overload: vector by vector less than or equal. 779 */ 780ASTCENC_SIMD_INLINE vmask4 operator<=(vfloat4 a, vfloat4 b) 781{ 782 return vmask4(_mm_cmple_ps(a.m, b.m)); 783} 784 785/** 786 * @brief Overload: vector by vector greater than or equal. 787 */ 788ASTCENC_SIMD_INLINE vmask4 operator>=(vfloat4 a, vfloat4 b) 789{ 790 return vmask4(_mm_cmpge_ps(a.m, b.m)); 791} 792 793/** 794 * @brief Return the min vector of two vectors. 795 * 796 * If either lane value is NaN, @c b will be returned for that lane. 797 */ 798ASTCENC_SIMD_INLINE vfloat4 min(vfloat4 a, vfloat4 b) 799{ 800 // Do not reorder - second operand will return if either is NaN 801 return vfloat4(_mm_min_ps(a.m, b.m)); 802} 803 804/** 805 * @brief Return the max vector of two vectors. 806 * 807 * If either lane value is NaN, @c b will be returned for that lane. 808 */ 809ASTCENC_SIMD_INLINE vfloat4 max(vfloat4 a, vfloat4 b) 810{ 811 // Do not reorder - second operand will return if either is NaN 812 return vfloat4(_mm_max_ps(a.m, b.m)); 813} 814 815/** 816 * @brief Return the absolute value of the float vector. 817 */ 818ASTCENC_SIMD_INLINE vfloat4 abs(vfloat4 a) 819{ 820 return vfloat4(_mm_max_ps(_mm_sub_ps(_mm_setzero_ps(), a.m), a.m)); 821} 822 823/** 824 * @brief Return a float rounded to the nearest integer value. 825 */ 826ASTCENC_SIMD_INLINE vfloat4 round(vfloat4 a) 827{ 828#if ASTCENC_SSE >= 41 829 constexpr int flags = _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC; 830 return vfloat4(_mm_round_ps(a.m, flags)); 831#else 832 __m128 v = a.m; 833 __m128 neg_zero = _mm_castsi128_ps(_mm_set1_epi32(static_cast<int>(0x80000000))); 834 __m128 no_fraction = _mm_set1_ps(8388608.0f); 835 __m128 abs_mask = _mm_castsi128_ps(_mm_set1_epi32(0x7FFFFFFF)); 836 __m128 sign = _mm_and_ps(v, neg_zero); 837 __m128 s_magic = _mm_or_ps(no_fraction, sign); 838 __m128 r1 = _mm_add_ps(v, s_magic); 839 r1 = _mm_sub_ps(r1, s_magic); 840 __m128 r2 = _mm_and_ps(v, abs_mask); 841 __m128 mask = _mm_cmple_ps(r2, no_fraction); 842 r2 = _mm_andnot_ps(mask, v); 843 r1 = _mm_and_ps(r1, mask); 844 return vfloat4(_mm_xor_ps(r1, r2)); 845#endif 846} 847 848/** 849 * @brief Return the horizontal minimum of a vector. 850 */ 851ASTCENC_SIMD_INLINE vfloat4 hmin(vfloat4 a) 852{ 853 a = min(a, vfloat4(_mm_shuffle_ps(a.m, a.m, _MM_SHUFFLE(0, 0, 3, 2)))); 854 a = min(a, vfloat4(_mm_shuffle_ps(a.m, a.m, _MM_SHUFFLE(0, 0, 0, 1)))); 855 return vfloat4(_mm_shuffle_ps(a.m, a.m, _MM_SHUFFLE(0, 0, 0, 0))); 856} 857 858/** 859 * @brief Return the horizontal maximum of a vector. 860 */ 861ASTCENC_SIMD_INLINE vfloat4 hmax(vfloat4 a) 862{ 863 a = max(a, vfloat4(_mm_shuffle_ps(a.m, a.m, _MM_SHUFFLE(0, 0, 3, 2)))); 864 a = max(a, vfloat4(_mm_shuffle_ps(a.m, a.m, _MM_SHUFFLE(0, 0, 0, 1)))); 865 return vfloat4(_mm_shuffle_ps(a.m, a.m, _MM_SHUFFLE(0, 0, 0, 0))); 866} 867 868/** 869 * @brief Return the horizontal sum of a vector as a scalar. 870 */ 871ASTCENC_SIMD_INLINE float hadd_s(vfloat4 a) 872{ 873 // Add top and bottom halves, lane 1/0 874 __m128 t = _mm_add_ps(a.m, _mm_movehl_ps(a.m, a.m)); 875 876 // Add top and bottom halves, lane 0 (_mm_hadd_ps exists but slow) 877 t = _mm_add_ss(t, _mm_shuffle_ps(t, t, 0x55)); 878 879 return _mm_cvtss_f32(t); 880} 881 882/** 883 * @brief Return the sqrt of the lanes in the vector. 884 */ 885ASTCENC_SIMD_INLINE vfloat4 sqrt(vfloat4 a) 886{ 887 return vfloat4(_mm_sqrt_ps(a.m)); 888} 889 890/** 891 * @brief Return lanes from @c b if @c cond is set, else @c a. 892 */ 893ASTCENC_SIMD_INLINE vfloat4 select(vfloat4 a, vfloat4 b, vmask4 cond) 894{ 895#if ASTCENC_SSE >= 41 896 return vfloat4(_mm_blendv_ps(a.m, b.m, cond.m)); 897#else 898 return vfloat4(_mm_or_ps(_mm_and_ps(cond.m, b.m), _mm_andnot_ps(cond.m, a.m))); 899#endif 900} 901 902/** 903 * @brief Return lanes from @c b if MSB of @c cond is set, else @c a. 904 */ 905ASTCENC_SIMD_INLINE vfloat4 select_msb(vfloat4 a, vfloat4 b, vmask4 cond) 906{ 907#if ASTCENC_SSE >= 41 908 return vfloat4(_mm_blendv_ps(a.m, b.m, cond.m)); 909#else 910 __m128 d = _mm_castsi128_ps(_mm_srai_epi32(_mm_castps_si128(cond.m), 31)); 911 return vfloat4(_mm_or_ps(_mm_and_ps(d, b.m), _mm_andnot_ps(d, a.m))); 912#endif 913} 914 915/** 916 * @brief Load a vector of gathered results from an array; 917 */ 918ASTCENC_SIMD_INLINE vfloat4 gatherf(const float* base, vint4 indices) 919{ 920#if ASTCENC_AVX >= 2 921 return vfloat4(_mm_i32gather_ps(base, indices.m, 4)); 922#else 923 alignas(16) int idx[4]; 924 storea(indices, idx); 925 return vfloat4(base[idx[0]], base[idx[1]], base[idx[2]], base[idx[3]]); 926#endif 927} 928 929/** 930 * @brief Store a vector to an unaligned memory address. 931 */ 932ASTCENC_SIMD_INLINE void store(vfloat4 a, float* p) 933{ 934 _mm_storeu_ps(p, a.m); 935} 936 937/** 938 * @brief Store a vector to a 16B aligned memory address. 939 */ 940ASTCENC_SIMD_INLINE void storea(vfloat4 a, float* p) 941{ 942 _mm_store_ps(p, a.m); 943} 944 945/** 946 * @brief Return a integer value for a float vector, using truncation. 947 */ 948ASTCENC_SIMD_INLINE vint4 float_to_int(vfloat4 a) 949{ 950 return vint4(_mm_cvttps_epi32(a.m)); 951} 952 953/** 954 * @brief Return a integer value for a float vector, using round-to-nearest. 955 */ 956ASTCENC_SIMD_INLINE vint4 float_to_int_rtn(vfloat4 a) 957{ 958 a = a + vfloat4(0.5f); 959 return vint4(_mm_cvttps_epi32(a.m)); 960} 961 962/** 963 * @brief Return a float value for an integer vector. 964 */ 965ASTCENC_SIMD_INLINE vfloat4 int_to_float(vint4 a) 966{ 967 return vfloat4(_mm_cvtepi32_ps(a.m)); 968} 969 970/** 971 * @brief Return a float16 value for a float vector, using round-to-nearest. 972 */ 973ASTCENC_SIMD_INLINE vint4 float_to_float16(vfloat4 a) 974{ 975#if ASTCENC_F16C >= 1 976 __m128i packedf16 = _mm_cvtps_ph(a.m, 0); 977 __m128i f16 = _mm_cvtepu16_epi32(packedf16); 978 return vint4(f16); 979#else 980 return vint4( 981 float_to_sf16(a.lane<0>()), 982 float_to_sf16(a.lane<1>()), 983 float_to_sf16(a.lane<2>()), 984 float_to_sf16(a.lane<3>())); 985#endif 986} 987 988/** 989 * @brief Return a float16 value for a float scalar, using round-to-nearest. 990 */ 991static inline uint16_t float_to_float16(float a) 992{ 993#if ASTCENC_F16C >= 1 994 __m128i f16 = _mm_cvtps_ph(_mm_set1_ps(a), 0); 995 return static_cast<uint16_t>(_mm_cvtsi128_si32(f16)); 996#else 997 return float_to_sf16(a); 998#endif 999} 1000 1001/** 1002 * @brief Return a float value for a float16 vector. 1003 */ 1004ASTCENC_SIMD_INLINE vfloat4 float16_to_float(vint4 a) 1005{ 1006#if ASTCENC_F16C >= 1 1007 __m128i packed = _mm_packs_epi32(a.m, a.m); 1008 __m128 f32 = _mm_cvtph_ps(packed); 1009 return vfloat4(f32); 1010#else 1011 return vfloat4( 1012 sf16_to_float(static_cast<uint16_t>(a.lane<0>())), 1013 sf16_to_float(static_cast<uint16_t>(a.lane<1>())), 1014 sf16_to_float(static_cast<uint16_t>(a.lane<2>())), 1015 sf16_to_float(static_cast<uint16_t>(a.lane<3>()))); 1016#endif 1017} 1018 1019/** 1020 * @brief Return a float value for a float16 scalar. 1021 */ 1022ASTCENC_SIMD_INLINE float float16_to_float(uint16_t a) 1023{ 1024#if ASTCENC_F16C >= 1 1025 __m128i packed = _mm_set1_epi16(static_cast<short>(a)); 1026 __m128 f32 = _mm_cvtph_ps(packed); 1027 return _mm_cvtss_f32(f32); 1028#else 1029 return sf16_to_float(a); 1030#endif 1031} 1032 1033/** 1034 * @brief Return a float value as an integer bit pattern (i.e. no conversion). 1035 * 1036 * It is a common trick to convert floats into integer bit patterns, perform 1037 * some bit hackery based on knowledge they are IEEE 754 layout, and then 1038 * convert them back again. This is the first half of that flip. 1039 */ 1040ASTCENC_SIMD_INLINE vint4 float_as_int(vfloat4 a) 1041{ 1042 return vint4(_mm_castps_si128(a.m)); 1043} 1044 1045/** 1046 * @brief Return a integer value as a float bit pattern (i.e. no conversion). 1047 * 1048 * It is a common trick to convert floats into integer bit patterns, perform 1049 * some bit hackery based on knowledge they are IEEE 754 layout, and then 1050 * convert them back again. This is the second half of that flip. 1051 */ 1052ASTCENC_SIMD_INLINE vfloat4 int_as_float(vint4 v) 1053{ 1054 return vfloat4(_mm_castsi128_ps(v.m)); 1055} 1056 1057/** 1058 * @brief Prepare a vtable lookup table for use with the native SIMD size. 1059 */ 1060ASTCENC_SIMD_INLINE void vtable_prepare(vint4 t0, vint4& t0p) 1061{ 1062 t0p = t0; 1063} 1064 1065/** 1066 * @brief Prepare a vtable lookup table for use with the native SIMD size. 1067 */ 1068ASTCENC_SIMD_INLINE void vtable_prepare(vint4 t0, vint4 t1, vint4& t0p, vint4& t1p) 1069{ 1070#if ASTCENC_SSE >= 41 1071 t0p = t0; 1072 t1p = t0 ^ t1; 1073#else 1074 t0p = t0; 1075 t1p = t1; 1076#endif 1077} 1078 1079/** 1080 * @brief Prepare a vtable lookup table for use with the native SIMD size. 1081 */ 1082ASTCENC_SIMD_INLINE void vtable_prepare( 1083 vint4 t0, vint4 t1, vint4 t2, vint4 t3, 1084 vint4& t0p, vint4& t1p, vint4& t2p, vint4& t3p) 1085{ 1086#if ASTCENC_SSE >= 41 1087 t0p = t0; 1088 t1p = t0 ^ t1; 1089 t2p = t1 ^ t2; 1090 t3p = t2 ^ t3; 1091#else 1092 t0p = t0; 1093 t1p = t1; 1094 t2p = t2; 1095 t3p = t3; 1096#endif 1097} 1098 1099/** 1100 * @brief Perform an 8-bit 16-entry table lookup, with 32-bit indexes. 1101 */ 1102ASTCENC_SIMD_INLINE vint4 vtable_8bt_32bi(vint4 t0, vint4 idx) 1103{ 1104#if ASTCENC_SSE >= 41 1105 // Set index byte MSB to 1 for unused bytes so shuffle returns zero 1106 __m128i idxx = _mm_or_si128(idx.m, _mm_set1_epi32(static_cast<int>(0xFFFFFF00))); 1107 1108 __m128i result = _mm_shuffle_epi8(t0.m, idxx); 1109 return vint4(result); 1110#else 1111 uint8_t table[16]; 1112 1113 std::memcpy(table + 0, &t0.m, 4 * sizeof(int)); 1114 1115 return vint4(table[idx.lane<0>()], 1116 table[idx.lane<1>()], 1117 table[idx.lane<2>()], 1118 table[idx.lane<3>()]); 1119#endif 1120} 1121 1122/** 1123 * @brief Perform an 8-bit 32-entry table lookup, with 32-bit indexes. 1124 */ 1125ASTCENC_SIMD_INLINE vint4 vtable_8bt_32bi(vint4 t0, vint4 t1, vint4 idx) 1126{ 1127#if ASTCENC_SSE >= 41 1128 // Set index byte MSB to 1 for unused bytes so shuffle returns zero 1129 __m128i idxx = _mm_or_si128(idx.m, _mm_set1_epi32(static_cast<int>(0xFFFFFF00))); 1130 1131 __m128i result = _mm_shuffle_epi8(t0.m, idxx); 1132 idxx = _mm_sub_epi8(idxx, _mm_set1_epi8(16)); 1133 1134 __m128i result2 = _mm_shuffle_epi8(t1.m, idxx); 1135 result = _mm_xor_si128(result, result2); 1136 1137 return vint4(result); 1138#else 1139 uint8_t table[32]; 1140 1141 std::memcpy(table + 0, &t0.m, 4 * sizeof(int)); 1142 std::memcpy(table + 16, &t1.m, 4 * sizeof(int)); 1143 1144 return vint4(table[idx.lane<0>()], 1145 table[idx.lane<1>()], 1146 table[idx.lane<2>()], 1147 table[idx.lane<3>()]); 1148#endif 1149} 1150 1151/** 1152 * @brief Perform an 8-bit 64-entry table lookup, with 32-bit indexes. 1153 */ 1154ASTCENC_SIMD_INLINE vint4 vtable_8bt_32bi(vint4 t0, vint4 t1, vint4 t2, vint4 t3, vint4 idx) 1155{ 1156#if ASTCENC_SSE >= 41 1157 // Set index byte MSB to 1 for unused bytes so shuffle returns zero 1158 __m128i idxx = _mm_or_si128(idx.m, _mm_set1_epi32(static_cast<int>(0xFFFFFF00))); 1159 1160 __m128i result = _mm_shuffle_epi8(t0.m, idxx); 1161 idxx = _mm_sub_epi8(idxx, _mm_set1_epi8(16)); 1162 1163 __m128i result2 = _mm_shuffle_epi8(t1.m, idxx); 1164 result = _mm_xor_si128(result, result2); 1165 idxx = _mm_sub_epi8(idxx, _mm_set1_epi8(16)); 1166 1167 result2 = _mm_shuffle_epi8(t2.m, idxx); 1168 result = _mm_xor_si128(result, result2); 1169 idxx = _mm_sub_epi8(idxx, _mm_set1_epi8(16)); 1170 1171 result2 = _mm_shuffle_epi8(t3.m, idxx); 1172 result = _mm_xor_si128(result, result2); 1173 1174 return vint4(result); 1175#else 1176 uint8_t table[64]; 1177 1178 std::memcpy(table + 0, &t0.m, 4 * sizeof(int)); 1179 std::memcpy(table + 16, &t1.m, 4 * sizeof(int)); 1180 std::memcpy(table + 32, &t2.m, 4 * sizeof(int)); 1181 std::memcpy(table + 48, &t3.m, 4 * sizeof(int)); 1182 1183 return vint4(table[idx.lane<0>()], 1184 table[idx.lane<1>()], 1185 table[idx.lane<2>()], 1186 table[idx.lane<3>()]); 1187#endif 1188} 1189 1190/** 1191 * @brief Return a vector of interleaved RGBA data. 1192 * 1193 * Input vectors have the value stored in the bottom 8 bits of each lane, 1194 * with high bits set to zero. 1195 * 1196 * Output vector stores a single RGBA texel packed in each lane. 1197 */ 1198ASTCENC_SIMD_INLINE vint4 interleave_rgba8(vint4 r, vint4 g, vint4 b, vint4 a) 1199{ 1200// Workaround an XCode compiler internal fault; note is slower than slli_epi32 1201// so we should revert this when we get the opportunity 1202#if defined(__APPLE__) 1203 __m128i value = r.m; 1204 value = _mm_add_epi32(value, _mm_bslli_si128(g.m, 1)); 1205 value = _mm_add_epi32(value, _mm_bslli_si128(b.m, 2)); 1206 value = _mm_add_epi32(value, _mm_bslli_si128(a.m, 3)); 1207 return vint4(value); 1208#else 1209 __m128i value = r.m; 1210 value = _mm_add_epi32(value, _mm_slli_epi32(g.m, 8)); 1211 value = _mm_add_epi32(value, _mm_slli_epi32(b.m, 16)); 1212 value = _mm_add_epi32(value, _mm_slli_epi32(a.m, 24)); 1213 return vint4(value); 1214#endif 1215} 1216 1217/** 1218 * @brief Store a single vector lane to an unaligned address. 1219 */ 1220ASTCENC_SIMD_INLINE void store_lane(uint8_t* base, int data) 1221{ 1222 std::memcpy(base, &data, sizeof(int)); 1223} 1224 1225/** 1226 * @brief Store a vector, skipping masked lanes. 1227 * 1228 * All masked lanes must be at the end of vector, after all non-masked lanes. 1229 */ 1230ASTCENC_SIMD_INLINE void store_lanes_masked(uint8_t* base, vint4 data, vmask4 mask) 1231{ 1232#if ASTCENC_AVX >= 2 1233 _mm_maskstore_epi32(reinterpret_cast<int*>(base), _mm_castps_si128(mask.m), data.m); 1234#else 1235 // Note - we cannot use _mm_maskmoveu_si128 as the underlying hardware doesn't guarantee 1236 // fault suppression on masked lanes so we can get page faults at the end of an image. 1237 if (mask.lane<3>() != 0.0f) 1238 { 1239 store(data, base); 1240 } 1241 else if (mask.lane<2>() != 0.0f) 1242 { 1243 store_lane(base + 0, data.lane<0>()); 1244 store_lane(base + 4, data.lane<1>()); 1245 store_lane(base + 8, data.lane<2>()); 1246 } 1247 else if (mask.lane<1>() != 0.0f) 1248 { 1249 store_lane(base + 0, data.lane<0>()); 1250 store_lane(base + 4, data.lane<1>()); 1251 } 1252 else if (mask.lane<0>() != 0.0f) 1253 { 1254 store_lane(base + 0, data.lane<0>()); 1255 } 1256#endif 1257} 1258 1259#if defined(ASTCENC_NO_INVARIANCE) && (ASTCENC_SSE >= 41) 1260 1261#define ASTCENC_USE_NATIVE_DOT_PRODUCT 1 1262 1263/** 1264 * @brief Return the dot product for the full 4 lanes, returning scalar. 1265 */ 1266ASTCENC_SIMD_INLINE float dot_s(vfloat4 a, vfloat4 b) 1267{ 1268 return _mm_cvtss_f32(_mm_dp_ps(a.m, b.m, 0xFF)); 1269} 1270 1271/** 1272 * @brief Return the dot product for the full 4 lanes, returning vector. 1273 */ 1274ASTCENC_SIMD_INLINE vfloat4 dot(vfloat4 a, vfloat4 b) 1275{ 1276 return vfloat4(_mm_dp_ps(a.m, b.m, 0xFF)); 1277} 1278 1279/** 1280 * @brief Return the dot product for the bottom 3 lanes, returning scalar. 1281 */ 1282ASTCENC_SIMD_INLINE float dot3_s(vfloat4 a, vfloat4 b) 1283{ 1284 return _mm_cvtss_f32(_mm_dp_ps(a.m, b.m, 0x77)); 1285} 1286 1287/** 1288 * @brief Return the dot product for the bottom 3 lanes, returning vector. 1289 */ 1290ASTCENC_SIMD_INLINE vfloat4 dot3(vfloat4 a, vfloat4 b) 1291{ 1292 return vfloat4(_mm_dp_ps(a.m, b.m, 0x77)); 1293} 1294 1295#endif // #if defined(ASTCENC_NO_INVARIANCE) && (ASTCENC_SSE >= 41) 1296 1297#if ASTCENC_POPCNT >= 1 1298 1299#define ASTCENC_USE_NATIVE_POPCOUNT 1 1300 1301/** 1302 * @brief Population bit count. 1303 * 1304 * @param v The value to population count. 1305 * 1306 * @return The number of 1 bits. 1307 */ 1308ASTCENC_SIMD_INLINE int popcount(uint64_t v) 1309{ 1310 return static_cast<int>(_mm_popcnt_u64(v)); 1311} 1312 1313#endif // ASTCENC_POPCNT >= 1 1314 1315#endif // #ifndef ASTC_VECMATHLIB_SSE_4_H_INCLUDED 1316