xref: /third_party/node/deps/simdutf/simdutf.cpp

67   simdutf_really_inline implementation() : simdutf::implementation("arm64", "ARM NEON", internal::instruction_set::NEON) {}
178 simdutf_really_inline int count_ones(uint64_t input_num) {
270     simdutf_really_inline base_u8(const uint8x16_t _value) : value(_value) {}
271     simdutf_really_inline operator const uint8x16_t&() const { return this->value; }
272     simdutf_really_inline operator uint8x16_t&() { return this->value; }
273     simdutf_really_inline T first() const { return vgetq_lane_u8(*this,0); }
274     simdutf_really_inline T last() const { return vgetq_lane_u8(*this,15); }
277     simdutf_really_inline simd8<T> operator|(const simd8<T> other) const { return vorrq_u8(*this, other); }
278     simdutf_really_inline simd8<T> operator&(const simd8<T> other) const { return vandq_u8(*this, other); }
279     simdutf_really_inline simd8<T> operator^(const simd8<T> other) const { return veorq_u8(*this, other); }
280     simdutf_really_inline simd8<T> bit_andnot(const simd8<T> other) const { return vbicq_u8(*this, other); }
281     simdutf_really_inline simd8<T> operator~() const { return *this ^ 0xFFu; }
282     simdutf_really_inline simd8<T>& operator|=(const simd8<T> other) { auto this_cast = static_cast<simd8<T>*>(this); *this_cast = *this_cast | other; return *this_cast; }
283     simdutf_really_inline simd8<T>& operator&=(const simd8<T> other) { auto this_cast = static_cast<simd8<T>*>(this); *this_cast = *this_cast & other; return *this_cast; }
284     simdutf_really_inline simd8<T>& operator^=(const simd8<T> other) { auto this_cast = static_cast<simd8<T>*>(this); *this_cast = *this_cast ^ other; return *this_cast; }
286     friend simdutf_really_inline Mask operator==(const simd8<T> lhs, const simd8<T> rhs) { return vceqq_u8(lhs, rhs); }
289     simdutf_really_inline simd8<T> prev(const simd8<T> prev_chunk) const {
300     static simdutf_really_inline simd8<bool> splat(bool _value) { return vmovq_n_u8(uint8_t(-(!!_value))); }
302     simdutf_really_inline simd8(const uint8x16_t _value) : base_u8<bool>(_value) {}
304     simdutf_really_inline simd8() : simd8(vdupq_n_u8(0)) {}
306     simdutf_really_inline simd8(bool _value) : simd8(splat(_value)) {}
307     simdutf_really_inline void store(uint8_t dst[16]) const { return vst1q_u8(dst, *this); }
311     simdutf_really_inline uint32_t to_bitmask() const {
331     simdutf_really_inline uint64_t to_bitmask64() const {
335     simdutf_really_inline bool any() const { return vmaxvq_u8(*this) != 0; }
336     simdutf_really_inline bool none() const { return vmaxvq_u8(*this) == 0; }
337     simdutf_really_inline bool all() const { return vminvq_u8(*this) == 0xFF; }
345     static simdutf_really_inline simd8<uint8_t> splat(uint8_t _value) { return vmovq_n_u8(_value); }
346     static simdutf_really_inline simd8<uint8_t> zero() { return vdupq_n_u8(0); }
347     static simdutf_really_inline simd8<uint8_t> load(const uint8_t* values) { return vld1q_u8(values); }
348     simdutf_really_inline simd8(const uint8x16_t _value) : base_u8<uint8_t>(_value) {}
350     simdutf_really_inline simd8() : simd8(zero()) {}
352     simdutf_really_inline simd8(const uint8_t values[16]) : simd8(load(values)) {}
354     simdutf_really_inline simd8(uint8_t _value) : simd8(splat(_value)) {}
357     simdutf_really_inline simd8(
365     simdutf_really_inline simd8(
375     simdutf_really_inline static simd8<uint8_t> repeat_16(
386     simdutf_really_inline void store(uint8_t dst[16]) const { return vst1q_u8(dst, *this); }
389     simdutf_really_inline simd8<uint8_t> saturating_add(const simd8<uint8_t> other) const { return vqaddq_u8(*this, other); }
390     simdutf_really_inline simd8<uint8_t> saturating_sub(const simd8<uint8_t> other) const { return vqsubq_u8(*this, other); }
393     simdutf_really_inline simd8<uint8_t> operator+(const simd8<uint8_t> other) const { return vaddq_u8(*this, other); }
394     simdutf_really_inline simd8<uint8_t> operator-(const simd8<uint8_t> other) const { return vsubq_u8(*this, other); }
395     simdutf_really_inline simd8<uint8_t>& operator+=(const simd8<uint8_t> other) { *this = *this + other; return *this; }
396     simdutf_really_inline simd8<uint8_t>& operator-=(const simd8<uint8_t> other) { *this = *this - other; return *this; }
399     simdutf_really_inline uint8_t max_val() const { return vmaxvq_u8(*this); }
400     simdutf_really_inline uint8_t min_val() const { return vminvq_u8(*this); }
401     simdutf_really_inline simd8<uint8_t> max_val(const simd8<uint8_t> other) const { return vmaxq_u8(*this, other); }
402     simdutf_really_inline simd8<uint8_t> min_val(const simd8<uint8_t> other) const { return vminq_u8(*this, other); }
403     simdutf_really_inline simd8<bool> operator<=(const simd8<uint8_t> other) const { return vcleq_u8(*this, other); }
404     simdutf_really_inline simd8<bool> operator>=(const simd8<uint8_t> other) const { return vcgeq_u8(*this, other); }
405     simdutf_really_inline simd8<bool> operator<(const simd8<uint8_t> other) const { return vcltq_u8(*this, other); }
406     simdutf_really_inline simd8<bool> operator>(const simd8<uint8_t> other) const { return vcgtq_u8(*this, other); }
408     simdutf_really_inline simd8<uint8_t> gt_bits(const simd8<uint8_t> other) const { return simd8<uint8_t>(*this > other); }
410     simdutf_really_inline simd8<uint8_t> lt_bits(const simd8<uint8_t> other) const { return simd8<uint8_t>(*this < other); }
413     simdutf_really_inline simd8<bool> any_bits_set(simd8<uint8_t> bits) const { return vtstq_u8(*this, bits); }
414     simdutf_really_inline bool is_ascii() const { return this->max_val() < 0b10000000u; }
416     simdutf_really_inline bool any_bits_set_anywhere() const { return this->max_val() != 0; }
417     simdutf_really_inline bool any_bits_set_anywhere(simd8<uint8_t> bits) const { return (*this & bits).any_bits_set_anywhere(); }
419     simdutf_really_inline simd8<uint8_t> shr() const { return vshrq_n_u8(*this, N); }
421     simdutf_really_inline simd8<uint8_t> shl() const { return vshlq_n_u8(*this, N); }
425     simdutf_really_inline simd8<L> lookup_16(simd8<L> lookup_table) const {
431     simdutf_really_inline simd8<L> lookup_16(
445     simdutf_really_inline simd8<uint8_t> apply_lookup_16_to(const simd8<T> original) const {
455     static simdutf_really_inline simd8<int8_t> splat(int8_t _value) { return vmovq_n_s8(_value); }
456     static simdutf_really_inline simd8<int8_t> zero() { return vdupq_n_s8(0); }
457     static simdutf_really_inline simd8<int8_t> load(const int8_t values[16]) { return vld1q_s8(values); }
471     simdutf_really_inline void store_ascii_as_utf16(char16_t * p) const {
486     simdutf_really_inline void store_ascii_as_utf32(char32_t * p) const {
497     simdutf_really_inline void store_ascii_as_utf32_tbl(char32_t * p) const {
515     simdutf_really_inline simd8(const int8x16_t _value) : value{_value} {}
516     simdutf_really_inline operator const int8x16_t&() const { return this->value; }
518     simdutf_really_inline operator const uint8x16_t() const { return vreinterpretq_u8_s8(this->value); }
520     simdutf_really_inline operator int8x16_t&() { return this->value; }
523     simdutf_really_inline simd8() : simd8(zero()) {}
525     simdutf_really_inline simd8(int8_t _value) : simd8(splat(_value)) {}
527     simdutf_really_inline simd8(const int8_t* values) : simd8(load(values)) {}
530     simdutf_really_inline simd8(
538     simdutf_really_inline simd8(
547     simdutf_really_inline static simd8<int8_t> repeat_16(
558     simdutf_really_inline void store(int8_t dst[16]) const { return vst1q_s8(dst, value); }
565     simdutf_really_inline explicit simd8(const uint8x16_t other): simd8(vreinterpretq_s8_u8(other)) {}
567     simdutf_really_inline operator simd8<uint8_t>() const { return vreinterpretq_u8_s8(this->value); }
569     simdutf_really_inline simd8<int8_t> operator|(const simd8<int8_t> other) const { return vorrq_s8(value, other.value); }
570     simdutf_really_inline simd8<int8_t> operator&(const simd8<int8_t> other) const { return vandq_s8(value, other.value); }
571     simdutf_really_inline simd8<int8_t> operator^(const simd8<int8_t> other) const { return veorq_s8(value, other.value); }
572     simdutf_really_inline simd8<int8_t> bit_andnot(const simd8<int8_t> other) const { return vbicq_s8(value, other.value); }
575     simdutf_really_inline simd8<int8_t> operator+(const simd8<int8_t> other) const { return vaddq_s8(value, other.value); }
576     simdutf_really_inline simd8<int8_t> operator-(const simd8<int8_t> other) const { return vsubq_s8(value, other.value); }
577     simdutf_really_inline simd8<int8_t>& operator+=(const simd8<int8_t> other) { *this = *this + other; return *this; }
578     simdutf_really_inline simd8<int8_t>& operator-=(const simd8<int8_t> other) { *this = *this - other; return *this; }
580     simdutf_really_inline int8_t max_val() const { return vmaxvq_s8(value); }
581     simdutf_really_inline int8_t min_val() const { return vminvq_s8(value); }
582     simdutf_really_inline bool is_ascii() const { return this->min_val() >= 0; }
585     simdutf_really_inline simd8<int8_t> max_val(const simd8<int8_t> other) const { return vmaxq_s8(value, other.value); }
586     simdutf_really_inline simd8<int8_t> min_val(const simd8<int8_t> other) const { return vminq_s8(value, other.value); }
587     simdutf_really_inline simd8<bool> operator>(const simd8<int8_t> other) const { return vcgtq_s8(value, other.value); }
588     simdutf_really_inline simd8<bool> operator<(const simd8<int8_t> other) const { return vcltq_s8(value, other.value); }
589     simdutf_really_inline simd8<bool> operator==(const simd8<int8_t> other) const { return vceqq_s8(value, other.value); }
592     simdutf_really_inline simd8<int8_t> prev(const simd8<int8_t> prev_chunk) const {
598     simdutf_really_inline simd8<L> lookup_16(simd8<L> lookup_table) const {
602     simdutf_really_inline simd8<L> lookup_16(
616     simdutf_really_inline simd8<int8_t> apply_lookup_16_to(const simd8<T> original) const {
631     simdutf_really_inline simd8x64(const simd8<T> chunk0, const simd8<T> chunk1, const simd8<T> chunk2, const simd8<T> chunk3) : chunks{chunk0, chunk1, chunk2, chunk3} {}
632     simdutf_really_inline simd8x64(const T* ptr) : chunks{simd8<T>::load(ptr), simd8<T>::load(ptr+sizeof(simd8<T>)/sizeof(T)), simd8<T>::load(ptr+2*sizeof(simd8<T>)/sizeof(T)), simd8<T>::load(ptr+3*sizeof(simd8<T>)/sizeof(T))} {}
634     simdutf_really_inline void store(T* ptr) const {
642     simdutf_really_inline simd8x64<T>& operator |=(const simd8x64<T> &other) {
650     simdutf_really_inline simd8<T> reduce_or() const {
654     simdutf_really_inline bool is_ascii() const {
659     simdutf_really_inline void store_ascii_as_utf16(char16_t * ptr) const {
666     simdutf_really_inline void store_ascii_as_utf32(char32_t * ptr) const {
673     simdutf_really_inline uint64_t to_bitmask() const {
693     simdutf_really_inline uint64_t eq(const T m) const {
703   simdutf_really_inline uint64_t lteq(const T m) const {
713     simdutf_really_inline uint64_t in_range(const T low, const T high) const {
724     simdutf_really_inline uint64_t not_in_range(const T low, const T high) const {
734     simdutf_really_inline uint64_t lt(const T m) const {
743     simdutf_really_inline uint64_t gt(const T m) const {
752     simdutf_really_inline uint64_t gteq(const T m) const {
761     simdutf_really_inline uint64_t gteq_unsigned(const uint8_t m) const {
781     simdutf_really_inline base_u16() = default;
782     simdutf_really_inline base_u16(const uint16x8_t _value) : value(_value) {}
783     simdutf_really_inline operator const uint16x8_t&() const { return this->value; }
784     simdutf_really_inline operator uint16x8_t&() { return this->value; }
786     simdutf_really_inline simd16<T> operator|(const simd16<T> other) const { return vorrq_u16(*this, other); }
787     simdutf_really_inline simd16<T> operator&(const simd16<T> other) const { return vandq_u16(*this, other); }
788     simdutf_really_inline simd16<T> operator^(const simd16<T> other) const { return veorq_u16(*this, other); }
789     simdutf_really_inline simd16<T> bit_andnot(const simd16<T> other) const { return vbicq_u16(*this, other); }
790     simdutf_really_inline simd16<T> operator~() const { return *this ^ 0xFFu; }
791     simdutf_really_inline simd16<T>& operator|=(const simd16<T> other) { auto this_cast = static_cast<simd16<T>*>(this); *this_cast = *this_cast | other; return *this_cast; }
792     simdutf_really_inline simd16<T>& operator&=(const simd16<T> other) { auto this_cast = static_cast<simd16<T>*>(this); *this_cast = *this_cast & other; return *this_cast; }
793     simdutf_really_inline simd16<T>& operator^=(const simd16<T> other) { auto this_cast = static_cast<simd16<T>*>(this); *this_cast = *this_cast ^ other; return *this_cast; }
795     friend simdutf_really_inline Mask operator==(const simd16<T> lhs, const simd16<T> rhs) { return vceqq_u16(lhs, rhs); }
798     simdutf_really_inline simd16<T> prev(const simd16<T> prev_chunk) const {
808   simdutf_really_inline base16() : base_u16<T>() {}
809   simdutf_really_inline base16(const uint16x8_t _value) : base_u16<T>(_value) {}
811   simdutf_really_inline base16(const Pointer* ptr) : base16(vld1q_u16(ptr)) {}
816   simdutf_really_inline simd16<T> prev(const simd16<T> prev_chunk) const {
824   static simdutf_really_inline simd16<bool> splat(bool _value) { return vmovq_n_u16(uint16_t(-(!!_value))); }
826   simdutf_really_inline simd16<bool>() : base16() {}
827   simdutf_really_inline simd16<bool>(const uint16x8_t _value) : base16<bool>(_value) {}
829   simdutf_really_inline simd16<bool>(bool _value) : base16<bool>(splat(_value)) {}
835   static simdutf_really_inline simd16<T> splat(T _value) { return vmovq_n_u16(_value); }
836   static simdutf_really_inline simd16<T> zero() { return vdupq_n_u16(0); }
837   static simdutf_really_inline simd16<T> load(const T values[8]) {
841   simdutf_really_inline base16_numeric() : base16<T>() {}
842   simdutf_really_inline base16_numeric(const uint16x8_t _value) : base16<T>(_value) {}
845   simdutf_really_inline void store(T dst[8]) const { return vst1q_u16(dst, *this); }
848   simdutf_really_inline simd16<T> operator~() const { return *this ^ 0xFFu; }
851   simdutf_really_inline simd16<T> operator+(const simd16<T> other) const { return vaddq_u8(*this, other); }
852   simdutf_really_inline simd16<T> operator-(const simd16<T> other) const { return vsubq_u8(*this, other); }
853   simdutf_really_inline simd16<T>& operator+=(const simd16<T> other) { *this = *this + other; return *static_cast<simd16<T>*>(this); }
854   simdutf_really_inline simd16<T>& operator-=(const simd16<T> other) { *this = *this - other; return *static_cast<simd16<T>*>(this); }
860   simdutf_really_inline simd16() : base16_numeric<int16_t>() {}
862   simdutf_really_inline simd16(const uint16x8_t _value) : base16_numeric<int16_t>(_value) {}
864   simdutf_really_inline simd16(const int16x8_t _value) : base16_numeric<int16_t>(vreinterpretq_u16_s16(_value)) {}
867   simdutf_really_inline simd16(int16_t _value) : simd16(splat(_value)) {}
869   simdutf_really_inline simd16(const int16_t* values) : simd16(load(values)) {}
870   simdutf_really_inline simd16(const char16_t* values) : simd16(load(reinterpret_cast<const int16_t*>(values))) {}
871   simdutf_really_inline operator simd16<uint16_t>() const;
872   simdutf_really_inline operator const uint16x8_t&() const { return this->value; }
873   simdutf_really_inline operator const int16x8_t() const { return vreinterpretq_s16_u16(this->value); }
875   simdutf_really_inline int16_t max_val() const { return vmaxvq_s16(vreinterpretq_s16_u16(this->value)); }
876   simdutf_really_inline int16_t min_val() const { return vminvq_s16(vreinterpretq_s16_u16(this->value)); }
878   simdutf_really_inline simd16<int16_t> max_val(const simd16<int16_t> other) const { return vmaxq_s16(vreinterpretq_s16_u16(this->value), vreinterpretq_s16_u16(other.value)); }
879   simdutf_really_inline simd16<int16_t> min_val(const simd16<int16_t> other) const { return vmaxq_s16(vreinterpretq_s16_u16(this->value), vreinterpretq_s16_u16(other.value)); }
880   simdutf_really_inline simd16<bool> operator>(const simd16<int16_t> other) const { return vcgtq_s16(vreinterpretq_s16_u16(this->value), vreinterpretq_s16_u16(other.value)); }
881   simdutf_really_inline simd16<bool> operator<(const simd16<int16_t> other) const { return vcltq_s16(vreinterpretq_s16_u16(this->value), vreinterpretq_s16_u16(other.value)); }
890   simdutf_really_inline simd16() : base16_numeric<uint16_t>() {}
891   simdutf_really_inline simd16(const uint16x8_t _value) : base16_numeric<uint16_t>(_value) {}
894   simdutf_really_inline simd16(uint16_t _value) : simd16(splat(_value)) {}
896   simdutf_really_inline simd16(const uint16_t* values) : simd16(load(values)) {}
897   simdutf_really_inline simd16(const char16_t* values) : simd16(load(reinterpret_cast<const uint16_t*>(values))) {}
900   simdutf_really_inline int16_t max_val() const { return vmaxvq_u16(*this); }
901   simdutf_really_inline int16_t min_val() const { return vminvq_u16(*this); }
903   simdutf_really_inline simd16<uint16_t> saturating_add(const simd16<uint16_t> other) const { return vqaddq_u16(*this, other); }
904   simdutf_really_inline simd16<uint16_t> saturating_sub(const simd16<uint16_t> other) const { return vqsubq_u16(*this, other); }
907   simdutf_really_inline simd16<uint16_t> max_val(const simd16<uint16_t> other) const { return vmaxq_u16(*this, other); }
908   simdutf_really_inline simd16<uint16_t> min_val(const simd16<uint16_t> other) const { return vminq_u16(*this, other); }
910   simdutf_really_inline simd16<uint16_t> gt_bits(const simd16<uint16_t> other) const { return this->saturating_sub(other); }
912   simdutf_really_inline simd16<uint16_t> lt_bits(const simd16<uint16_t> other) const { return other.saturating_sub(*this); }
913   simdutf_really_inline simd16<bool> operator<=(const simd16<uint16_t> other) const { return vcleq_u16(*this, other); }
914   simdutf_really_inline simd16<bool> operator>=(const simd16<uint16_t> other) const { return vcgeq_u16(*this, other); }
915   simdutf_really_inline simd16<bool> operator>(const simd16<uint16_t> other) const { return  vcgtq_u16(*this, other); }
916   simdutf_really_inline simd16<bool> operator<(const simd16<uint16_t> other) const { return vcltq_u16(*this, other); }
919   simdutf_really_inline simd16<bool> bits_not_set() const { return *this == uint16_t(0); }
921   simdutf_really_inline simd16<uint16_t> shr() const { return simd16<uint16_t>(vshrq_n_u16(*this, N)); }
923   simdutf_really_inline simd16<uint16_t> shl() const { return simd16<uint16_t>(vshlq_n_u16(*this, N)); }
926   simdutf_really_inline simd16<uint16_t> operator|(const simd16<uint16_t> other) const { return vorrq_u16(*this, other); }
927   simdutf_really_inline simd16<uint16_t> operator&(const simd16<uint16_t> other) const { return vandq_u16(*this, other); }
928   simdutf_really_inline simd16<uint16_t> operator^(const simd16<uint16_t> other) const { return veorq_u16(*this, other); }
931   static simdutf_really_inline simd8<uint8_t> pack(const simd16<uint16_t>& v0, const simd16<uint16_t>& v1) {
936   simdutf_really_inline simd16<uint16_t> swap_bytes() const {
940 simdutf_really_inline simd16<int16_t>::operator simd16<uint16_t>() const { return this->value; }
953     simdutf_really_inline simd16x32(const simd16<T> chunk0, const simd16<T> chunk1, const simd16<T> chunk2, const simd16<T> chunk3) : chunks{chunk0, chunk1, chunk2, chunk3} {}
954     simdutf_really_inline simd16x32(const T* ptr) : chunks{simd16<T>::load(ptr), simd16<T>::load(ptr+sizeof(simd16<T>)/sizeof(T)), simd16<T>::load(ptr+2*sizeof(simd16<T>)/sizeof(T)), simd16<T>::load(ptr+3*sizeof(simd16<T>)/sizeof(T))} {}
956     simdutf_really_inline void store(T* ptr) const {
963     simdutf_really_inline simd16<T> reduce_or() const {
967     simdutf_really_inline bool is_ascii() const {
971     simdutf_really_inline void store_ascii_as_utf16(char16_t * ptr) const {
978     simdutf_really_inline uint64_t to_bitmask() const {
998     simdutf_really_inline void swap_bytes() {
1005     simdutf_really_inline uint64_t eq(const T m) const {
1015   simdutf_really_inline uint64_t lteq(const T m) const {
1025     simdutf_really_inline uint64_t in_range(const T low, const T high) const {
1036     simdutf_really_inline uint64_t not_in_range(const T low, const T high) const {
1046     simdutf_really_inline uint64_t lt(const T m) const {
1058   simdutf_really_inline uint64_t simd16x32<uint16_t>::not_in_range(const uint16_t low, const uint16_t high) const {
1264   simdutf_really_inline implementation() : simdutf::implementation(
1378 simdutf_really_inline unsigned __int64 count_ones(uint64_t input_num) {
1383 simdutf_really_inline long long int count_ones(uint64_t input_num) {
1470   simdutf_really_inline implementation() : simdutf::implementation(
1649 simdutf_really_inline unsigned __int64 count_ones(uint64_t input_num) {
1654 simdutf_really_inline long long int count_ones(uint64_t input_num) {
1681     simdutf_really_inline base() : value{__m256i()} {}
1684     simdutf_really_inline base(const __m256i _value) : value(_value) {}
1686     simdutf_really_inline operator const __m256i&() const { return this->value; }
1687     simdutf_really_inline operator __m256i&() { return this->value; }
1689     simdutf_really_inline void store_ascii_as_utf16(char16_t * ptr) const {
1701     simdutf_really_inline void store_ascii_as_utf32(char32_t * ptr) const {
1708     simdutf_really_inline Child operator|(const Child other) const { return _mm256_or_si256(*this, other); }
1709     simdutf_really_inline Child operator&(const Child other) const { return _mm256_and_si256(*this, other); }
1710     simdutf_really_inline Child operator^(const Child other) const { return _mm256_xor_si256(*this, other); }
1711     simdutf_really_inline Child bit_andnot(const Child other) const { return _mm256_andnot_si256(other, *this); }
1712     simdutf_really_inline Child& operator|=(const Child other) { auto this_cast = static_cast<Child*>(this); *this_cast = *this_cast | other; return *this_cast; }
1713     simdutf_really_inline Child& operator&=(const Child other) { auto this_cast = static_cast<Child*>(this); *this_cast = *this_cast & other; return *this_cast; }
1714     simdutf_really_inline Child& operator^=(const Child other) { auto this_cast = static_cast<Child*>(this); *this_cast = *this_cast ^ other; return *this_cast; }
1726     simdutf_really_inline base8() : base<simd8<T>>() {}
1727     simdutf_really_inline base8(const __m256i _value) : base<simd8<T>>(_value) {}
1728     simdutf_really_inline T first() const { return _mm256_extract_epi8(*this,0); }
1729     simdutf_really_inline T last() const { return _mm256_extract_epi8(*this,31); }
1730     friend simdutf_really_inline Mask operator==(const simd8<T> lhs, const simd8<T> rhs) { return _mm256_cmpeq_epi8(lhs, rhs); }
1735     simdutf_really_inline simd8<T> prev(const simd8<T> prev_chunk) const {
1743     static simdutf_really_inline simd8<bool> splat(bool _value) { return _mm256_set1_epi8(uint8_t(-(!!_value))); }
1745     simdutf_really_inline simd8<bool>() : base8() {}
1746     simdutf_really_inline simd8<bool>(const __m256i _value) : base8<bool>(_value) {}
1748     simdutf_really_inline simd8<bool>(bool _value) : base8<bool>(splat(_value)) {}
1750     simdutf_really_inline uint32_t to_bitmask() const { return uint32_t(_mm256_movemask_epi8(*this)); }
1751     simdutf_really_inline bool any() const { return !_mm256_testz_si256(*this, *this); }
1752     simdutf_really_inline bool none() const { return _mm256_testz_si256(*this, *this); }
1753     simdutf_really_inline bool all() const { return static_cast<uint32_t>(_mm256_movemask_epi8(*this)) == 0xFFFFFFFF; }
1754     simdutf_really_inline simd8<bool> operator~() const { return *this ^ true; }
1759     static simdutf_really_inline simd8<T> splat(T _value) { return _mm256_set1_epi8(_value); }
1760     static simdutf_really_inline simd8<T> zero() { return _mm256_setzero_si256(); }
1761     static simdutf_really_inline simd8<T> load(const T values[32]) {
1765     static simdutf_really_inline simd8<T> repeat_16(
1777     simdutf_really_inline base8_numeric() : base8<T>() {}
1778     simdutf_really_inline base8_numeric(const __m256i _value) : base8<T>(_value) {}
1781     simdutf_really_inline void store(T dst[32]) const { return _mm256_storeu_si256(reinterpret_cast<__m256i *>(dst), *this); }
1784     simdutf_really_inline simd8<T> operator+(const simd8<T> other) const { return _mm256_add_epi8(*this, other); }
1785     simdutf_really_inline simd8<T> operator-(const simd8<T> other) const { return _mm256_sub_epi8(*this, other); }
1786     simdutf_really_inline simd8<T>& operator+=(const simd8<T> other) { *this = *this + other; return *static_cast<simd8<T>*>(this); }
1787     simdutf_really_inline simd8<T>& operator-=(const simd8<T> other) { *this = *this - other; return *static_cast<simd8<T>*>(this); }
1790     simdutf_really_inline simd8<T> operator~() const { return *this ^ 0xFFu; }
1794     simdutf_really_inline simd8<L> lookup_16(simd8<L> lookup_table) const {
1799     simdutf_really_inline simd8<L> lookup_16(
1817     simdutf_really_inline simd8() : base8_numeric<int8_t>() {}
1818     simdutf_really_inline simd8(const __m256i _value) : base8_numeric<int8_t>(_value) {}
1821     simdutf_really_inline simd8(int8_t _value) : simd8(splat(_value)) {}
1823     simdutf_really_inline simd8(const int8_t values[32]) : simd8(load(values)) {}
1824     simdutf_really_inline operator simd8<uint8_t>() const;
1826     simdutf_really_inline simd8(
1838     simdutf_really_inline static simd8<int8_t> repeat_16(
1849     simdutf_really_inline bool is_ascii() const { return _mm256_movemask_epi8(*this) == 0; }
1851     simdutf_really_inline simd8<int8_t> max_val(const simd8<int8_t> other) const { return _mm256_max_epi8(*this, other); }
1852     simdutf_really_inline simd8<int8_t> min_val(const simd8<int8_t> other) const { return _mm256_min_epi8(*this, other); }
1853     simdutf_really_inline simd8<bool> operator>(const simd8<int8_t> other) const { return _mm256_cmpgt_epi8(*this, other); }
1854     simdutf_really_inline simd8<bool> operator<(const simd8<int8_t> other) const { return _mm256_cmpgt_epi8(other, *this); }
1860     simdutf_really_inline simd8() : base8_numeric<uint8_t>() {}
1861     simdutf_really_inline simd8(const __m256i _value) : base8_numeric<uint8_t>(_value) {}
1863     simdutf_really_inline simd8(uint8_t _value) : simd8(splat(_value)) {}
1865     simdutf_really_inline simd8(const uint8_t values[32]) : simd8(load(values)) {}
1867     simdutf_really_inline simd8(
1879     simdutf_really_inline static simd8<uint8_t> repeat_16(
1893     simdutf_really_inline simd8<uint8_t> saturating_add(const simd8<uint8_t> other) const { return _mm256_adds_epu8(*this, other); }
1894     simdutf_really_inline simd8<uint8_t> saturating_sub(const simd8<uint8_t> other) const { return _mm256_subs_epu8(*this, other); }
1897     simdutf_really_inline simd8<uint8_t> max_val(const simd8<uint8_t> other) const { return _mm256_max_epu8(*this, other); }
1898     simdutf_really_inline simd8<uint8_t> min_val(const simd8<uint8_t> other) const { return _mm256_min_epu8(other, *this); }
1900     simdutf_really_inline simd8<uint8_t> gt_bits(const simd8<uint8_t> other) const { return this->saturating_sub(other); }
1902     simdutf_really_inline simd8<uint8_t> lt_bits(const simd8<uint8_t> other) const { return other.saturating_sub(*this); }
1903     simdutf_really_inline simd8<bool> operator<=(const simd8<uint8_t> other) const { return other.max_val(*this) == other; }
1904     simdutf_really_inline simd8<bool> operator>=(const simd8<uint8_t> other) const { return other.min_val(*this) == other; }
1905     simdutf_really_inline simd8<bool> operator>(const simd8<uint8_t> other) const { return this->gt_bits(other).any_bits_set(); }
1906     simdutf_really_inline simd8<bool> operator<(const simd8<uint8_t> other) const { return this->lt_bits(other).any_bits_set(); }
1909     simdutf_really_inline simd8<bool> bits_not_set() const { return *this == uint8_t(0); }
1910     simdutf_really_inline simd8<bool> bits_not_set(simd8<uint8_t> bits) const { return (*this & bits).bits_not_set(); }
1911     simdutf_really_inline simd8<bool> any_bits_set() const { return ~this->bits_not_set(); }
1912     simdutf_really_inline simd8<bool> any_bits_set(simd8<uint8_t> bits) const { return ~this->bits_not_set(bits); }
1913     simdutf_really_inline bool is_ascii() const { return _mm256_movemask_epi8(*this) == 0; }
1914     simdutf_really_inline bool bits_not_set_anywhere() const { return _mm256_testz_si256(*this, *this); }
1915     simdutf_really_inline bool any_bits_set_anywhere() const { return !bits_not_set_anywhere(); }
1916     simdutf_really_inline bool bits_not_set_anywhere(simd8<uint8_t> bits) const { return _mm256_testz_si256(*this, bits); }
1917     simdutf_really_inline bool any_bits_set_anywhere(simd8<uint8_t> bits) const { return !bits_not_set_anywhere(bits); }
1919     simdutf_really_inline simd8<uint8_t> shr() const { return simd8<uint8_t>(_mm256_srli_epi16(*this, N)) & uint8_t(0xFFu >> N); }
1921     simdutf_really_inline simd8<uint8_t> shl() const { return simd8<uint8_t>(_mm256_slli_epi16(*this, N)) & uint8_t(0xFFu << N); }
1925     simdutf_really_inline int get_bit() const { return _mm256_movemask_epi8(_mm256_slli_epi16(*this, 7-N)); }
1927   simdutf_really_inline simd8<int8_t>::operator simd8<uint8_t>() const { return this->value; }
1940     simdutf_really_inline simd8x64(const simd8<T> chunk0, const simd8<T> chunk1) : chunks{chunk0, chunk1} {}
1941     simdutf_really_inline simd8x64(const T* ptr) : chunks{simd8<T>::load(ptr), simd8<T>::load(ptr+sizeof(simd8<T>)/sizeof(T))} {}
1943     simdutf_really_inline void store(T* ptr) const {
1948     simdutf_really_inline uint64_t to_bitmask() const {
1954     simdutf_really_inline simd8x64<T>& operator|=(const simd8x64<T> &other) {
1960     simdutf_really_inline simd8<T> reduce_or() const {
1964     simdutf_really_inline bool is_ascii() const {
1969     simdutf_really_inline void store_ascii_as_utf16(char16_t * ptr) const {
1974     simdutf_really_inline void store_ascii_as_utf32(char32_t * ptr) const {
1979     simdutf_really_inline simd8x64<T> bit_or(const T m) const {
1987     simdutf_really_inline uint64_t eq(const T m) const {
1995     simdutf_really_inline uint64_t eq(const simd8x64<uint8_t> &other) const {
2002     simdutf_really_inline uint64_t lteq(const T m) const {
2010     simdutf_really_inline uint64_t in_range(const T low, const T high) const {
2019     simdutf_really_inline uint64_t not_in_range(const T low, const T high) const {
2027     simdutf_really_inline uint64_t lt(const T m) const {
2035     simdutf_really_inline uint64_t gt(const T m) const {
2042     simdutf_really_inline uint64_t gteq(const T m) const {
2049     simdutf_really_inline uint64_t gteq_unsigned(const uint8_t m) const {
2073   simdutf_really_inline base16() : base<simd16<T>>() {}
2074   simdutf_really_inline base16(const __m256i _value) : base<simd16<T>>(_value) {}
2076   simdutf_really_inline base16(const Pointer* ptr) : base16(_mm256_loadu_si256(reinterpret_cast<const __m256i*>(ptr))) {}
2077   friend simdutf_really_inline Mask operator==(const simd16<T> lhs, const simd16<T> rhs) { return _mm256_cmpeq_epi16(lhs, rhs); }
2086   simdutf_really_inline simd16<T> prev(const simd16<T> prev_chunk) const {
2094   static simdutf_really_inline simd16<bool> splat(bool _value) { return _mm256_set1_epi16(uint16_t(-(!!_value))); }
2096   simdutf_really_inline simd16<bool>() : base16() {}
2097   simdutf_really_inline simd16<bool>(const __m256i _value) : base16<bool>(_value) {}
2099   simdutf_really_inline simd16<bool>(bool _value) : base16<bool>(splat(_value)) {}
2101   simdutf_really_inline bitmask_type to_bitmask() const { return _mm256_movemask_epi8(*this); }
2102   simdutf_really_inline bool any() const { return !_mm256_testz_si256(*this, *this); }
2103   simdutf_really_inline simd16<bool> operator~() const { return *this ^ true; }
2108   static simdutf_really_inline simd16<T> splat(T _value) { return _mm256_set1_epi16(_value); }
2109   static simdutf_really_inline simd16<T> zero() { return _mm256_setzero_si256(); }
2110   static simdutf_really_inline simd16<T> load(const T values[8]) {
2114   simdutf_really_inline base16_numeric() : base16<T>() {}
2115   simdutf_really_inline base16_numeric(const __m256i _value) : base16<T>(_value) {}
2118   simdutf_really_inline void store(T dst[8]) const { return _mm256_storeu_si256(reinterpret_cast<__m256i *>(dst), *this); }
2121   simdutf_really_inline simd16<T> operator~() const { return *this ^ 0xFFFFu; }
2124   simdutf_really_inline simd16<T> operator+(const simd16<T> other) const { return _mm256_add_epi16(*this, other); }
2125   simdutf_really_inline simd16<T> operator-(const simd16<T> other) const { return _mm256_sub_epi16(*this, other); }
2126   simdutf_really_inline simd16<T>& operator+=(const simd16<T> other) { *this = *this + other; return *static_cast<simd16<T>*>(this); }
2127   simdutf_really_inline simd16<T>& operator-=(const simd16<T> other) { *this = *this - other; return *static_cast<simd16<T>*>(this); }
2133   simdutf_really_inline simd16() : base16_numeric<int16_t>() {}
2134   simdutf_really_inline simd16(const __m256i _value) : base16_numeric<int16_t>(_value) {}
2136   simdutf_really_inline simd16(int16_t _value) : simd16(splat(_value)) {}
2138   simdutf_really_inline simd16(const int16_t* values) : simd16(load(values)) {}
2139   simdutf_really_inline simd16(const char16_t* values) : simd16(load(reinterpret_cast<const int16_t*>(values))) {}
2141   simdutf_really_inline simd16<int16_t> max_val(const simd16<int16_t> other) const { return _mm256_max_epi16(*this, other); }
2142   simdutf_really_inline simd16<int16_t> min_val(const simd16<int16_t> other) const { return _mm256_min_epi16(*this, other); }
2143   simdutf_really_inline simd16<bool> operator>(const simd16<int16_t> other) const { return _mm256_cmpgt_epi16(*this, other); }
2144   simdutf_really_inline simd16<bool> operator<(const simd16<int16_t> other) const { return _mm256_cmpgt_epi16(other, *this); }
2150   simdutf_really_inline simd16() : base16_numeric<uint16_t>() {}
2151   simdutf_really_inline simd16(const __m256i _value) : base16_numeric<uint16_t>(_value) {}
2154   simdutf_really_inline simd16(uint16_t _value) : simd16(splat(_value)) {}
2156   simdutf_really_inline simd16(const uint16_t* values) : simd16(load(values)) {}
2157   simdutf_really_inline simd16(const char16_t* values) : simd16(load(reinterpret_cast<const uint16_t*>(values))) {}
2160   simdutf_really_inline simd16<uint16_t> saturating_add(const simd16<uint16_t> other) const { return _mm256_adds_epu16(*this, other); }
2161   simdutf_really_inline simd16<uint16_t> saturating_sub(const simd16<uint16_t> other) const { return _mm256_subs_epu16(*this, other); }
2164   simdutf_really_inline simd16<uint16_t> max_val(const simd16<uint16_t> other) const { return _mm256_max_epu16(*this, other); }
2165   simdutf_really_inline simd16<uint16_t> min_val(const simd16<uint16_t> other) const { return _mm256_min_epu16(*this, other); }
2167   simdutf_really_inline simd16<uint16_t> gt_bits(const simd16<uint16_t> other) const { return this->saturating_sub(other); }
2169   simdutf_really_inline simd16<uint16_t> lt_bits(const simd16<uint16_t> other) const { return other.saturating_sub(*this); }
2170   simdutf_really_inline simd16<bool> operator<=(const simd16<uint16_t> other) const { return other.max_val(*this) == other; }
2171   simdutf_really_inline simd16<bool> operator>=(const simd16<uint16_t> other) const { return other.min_val(*this) == other; }
2172   simdutf_really_inline simd16<bool> operator>(const simd16<uint16_t> other) const { return this->gt_bits(other).any_bits_set(); }
2173   simdutf_really_inline simd16<bool> operator<(const simd16<uint16_t> other) const { return this->gt_bits(other).any_bits_set(); }
2176   simdutf_really_inline simd16<bool> bits_not_set() const { return *this == uint16_t(0); }
2177   simdutf_really_inline simd16<bool> bits_not_set(simd16<uint16_t> bits) const { return (*this & bits).bits_not_set(); }
2178   simdutf_really_inline simd16<bool> any_bits_set() const { return ~this->bits_not_set(); }
2179   simdutf_really_inline simd16<bool> any_bits_set(simd16<uint16_t> bits) const { return ~this->bits_not_set(bits); }
2181   simdutf_really_inline bool bits_not_set_anywhere() const { return _mm256_testz_si256(*this, *this); }
2182   simdutf_really_inline bool any_bits_set_anywhere() const { return !bits_not_set_anywhere(); }
2183   simdutf_really_inline bool bits_not_set_anywhere(simd16<uint16_t> bits) const { return _mm256_testz_si256(*this, bits); }
2184   simdutf_really_inline bool any_bits_set_anywhere(simd16<uint16_t> bits) const { return !bits_not_set_anywhere(bits); }
2186   simdutf_really_inline simd16<uint16_t> shr() const { return simd16<uint16_t>(_mm256_srli_epi16(*this, N)); }
2188   simdutf_really_inline simd16<uint16_t> shl() const { return simd16<uint16_t>(_mm256_slli_epi16(*this, N)); }
2192   simdutf_really_inline int get_bit() const { return _mm256_movemask_epi8(_mm256_slli_epi16(*this, 15-N)); }
2195   simdutf_really_inline simd16<uint16_t> swap_bytes() const {
2202   static simdutf_really_inline simd8<uint8_t> pack(const simd16<uint16_t>& v0, const simd16<uint16_t>& v1) {
2235     simdutf_really_inline simd16x32(const simd16<T> chunk0, const simd16<T> chunk1) : chunks{chunk0, chunk1} {}
2236     simdutf_really_inline simd16x32(const T* ptr) : chunks{simd16<T>::load(ptr), simd16<T>::load(ptr+sizeof(simd16<T>)/sizeof(T))} {}
2238     simdutf_really_inline void store(T* ptr) const {
2243     simdutf_really_inline uint64_t to_bitmask() const {
2249     simdutf_really_inline simd16<T> reduce_or() const {
2253     simdutf_really_inline bool is_ascii() const {
2257     simdutf_really_inline void store_ascii_as_utf16(char16_t * ptr) const {
2262     simdutf_really_inline simd16x32<T> bit_or(const T m) const {
2270     simdutf_really_inline void swap_bytes() {
2275     simdutf_really_inline uint64_t eq(const T m) const {
2283     simdutf_really_inline uint64_t eq(const simd16x32<uint16_t> &other) const {
2290     simdutf_really_inline uint64_t lteq(const T m) const {
2298     simdutf_really_inline uint64_t in_range(const T low, const T high) const {
2307     simdutf_really_inline uint64_t not_in_range(const T low, const T high) const {
2315     simdutf_really_inline uint64_t lt(const T m) const {
2405   simdutf_really_inline implementation() : simdutf::implementation("westmere", "Intel/AMD SSE4.2", internal::instruction_set::SSE42) {}
2557 simdutf_really_inline unsigned __int64 count_ones(uint64_t input_num) {
2562 simdutf_really_inline long long int count_ones(uint64_t input_num) {
2587     simdutf_really_inline base() : value{__m128i()} {}
2590     simdutf_really_inline base(const __m128i _value) : value(_value) {}
2592     simdutf_really_inline operator const __m128i&() const { return this->value; }
2593     simdutf_really_inline operator __m128i&() { return this->value; }
2595     simdutf_really_inline void store_ascii_as_utf16(char16_t * p) const {
2606     simdutf_really_inline void store_ascii_as_utf32(char32_t * p) const {
2613     simdutf_really_inline Child operator|(const Child other) const { return _mm_or_si128(*this, other); }
2614     simdutf_really_inline Child operator&(const Child other) const { return _mm_and_si128(*this, other); }
2615     simdutf_really_inline Child operator^(const Child other) const { return _mm_xor_si128(*this, other); }
2616     simdutf_really_inline Child bit_andnot(const Child other) const { return _mm_andnot_si128(other, *this); }
2617     simdutf_really_inline Child& operator|=(const Child other) { auto this_cast = static_cast<Child*>(this); *this_cast = *this_cast | other; return *this_cast; }
2618     simdutf_really_inline Child& operator&=(const Child other) { auto this_cast = static_cast<Child*>(this); *this_cast = *this_cast & other; return *this_cast; }
2619     simdutf_really_inline Child& operator^=(const Child other) { auto this_cast = static_cast<Child*>(this); *this_cast = *this_cast ^ other; return *this_cast; }
2631     simdutf_really_inline T first() const { return _mm_extract_epi8(*this,0); }
2632     simdutf_really_inline T last() const { return _mm_extract_epi8(*this,15); }
2633     simdutf_really_inline base8() : base<simd8<T>>() {}
2634     simdutf_really_inline base8(const __m128i _value) : base<simd8<T>>(_value) {}
2636     friend simdutf_really_inline Mask operator==(const simd8<T> lhs, const simd8<T> rhs) { return _mm_cmpeq_epi8(lhs, rhs); }
2641     simdutf_really_inline simd8<T> prev(const simd8<T> prev_chunk) const {
2649     static simdutf_really_inline simd8<bool> splat(bool _value) { return _mm_set1_epi8(uint8_t(-(!!_value))); }
2651     simdutf_really_inline simd8<bool>() : base8() {}
2652     simdutf_really_inline simd8<bool>(const __m128i _value) : base8<bool>(_value) {}
2654     simdutf_really_inline simd8<bool>(bool _value) : base8<bool>(splat(_value)) {}
2656     simdutf_really_inline int to_bitmask() const { return _mm_movemask_epi8(*this); }
2657     simdutf_really_inline bool any() const { return !_mm_testz_si128(*this, *this); }
2658     simdutf_really_inline bool none() const { return _mm_testz_si128(*this, *this); }
2659     simdutf_really_inline bool all() const { return _mm_movemask_epi8(*this) == 0xFFFF; }
2660     simdutf_really_inline simd8<bool> operator~() const { return *this ^ true; }
2665     static simdutf_really_inline simd8<T> splat(T _value) { return _mm_set1_epi8(_value); }
2666     static simdutf_really_inline simd8<T> zero() { return _mm_setzero_si128(); }
2667     static simdutf_really_inline simd8<T> load(const T values[16]) {
2671     static simdutf_really_inline simd8<T> repeat_16(
2681     simdutf_really_inline base8_numeric() : base8<T>() {}
2682     simdutf_really_inline base8_numeric(const __m128i _value) : base8<T>(_value) {}
2685     simdutf_really_inline void store(T dst[16]) const { return _mm_storeu_si128(reinterpret_cast<__m128i *>(dst), *this); }
2688     simdutf_really_inline simd8<T> operator~() const { return *this ^ 0xFFu; }
2691     simdutf_really_inline simd8<T> operator+(const simd8<T> other) const { return _mm_add_epi8(*this, other); }
2692     simdutf_really_inline simd8<T> operator-(const simd8<T> other) const { return _mm_sub_epi8(*this, other); }
2693     simdutf_really_inline simd8<T>& operator+=(const simd8<T> other) { *this = *this + other; return *static_cast<simd8<T>*>(this); }
2694     simdutf_really_inline simd8<T>& operator-=(const simd8<T> other) { *this = *this - other; return *static_cast<simd8<T>*>(this); }
2698     simdutf_really_inline simd8<L> lookup_16(simd8<L> lookup_table) const {
2703     simdutf_really_inline simd8<L> lookup_16(
2720     simdutf_really_inline simd8() : base8_numeric<int8_t>() {}
2721     simdutf_really_inline simd8(const __m128i _value) : base8_numeric<int8_t>(_value) {}
2723     simdutf_really_inline simd8(int8_t _value) : simd8(splat(_value)) {}
2725     simdutf_really_inline simd8(const int8_t* values) : simd8(load(values)) {}
2727     simdutf_really_inline simd8(
2735     simdutf_really_inline static simd8<int8_t> repeat_16(
2744     simdutf_really_inline operator simd8<uint8_t>() const;
2745     simdutf_really_inline bool is_ascii() const { return _mm_movemask_epi8(*this) == 0; }
2748     simdutf_really_inline simd8<int8_t> max_val(const simd8<int8_t> other) const { return _mm_max_epi8(*this, other); }
2749     simdutf_really_inline simd8<int8_t> min_val(const simd8<int8_t> other) const { return _mm_min_epi8(*this, other); }
2750     simdutf_really_inline simd8<bool> operator>(const simd8<int8_t> other) const { return _mm_cmpgt_epi8(*this, other); }
2751     simdutf_really_inline simd8<bool> operator<(const simd8<int8_t> other) const { return _mm_cmpgt_epi8(other, *this); }
2757     simdutf_really_inline simd8() : base8_numeric<uint8_t>() {}
2758     simdutf_really_inline simd8(const __m128i _value) : base8_numeric<uint8_t>(_value) {}
2761     simdutf_really_inline simd8(uint8_t _value) : simd8(splat(_value)) {}
2763     simdutf_really_inline simd8(const uint8_t* values) : simd8(load(values)) {}
2765     simdutf_really_inline simd8(
2773     simdutf_really_inline static simd8<uint8_t> repeat_16(
2784     simdutf_really_inline simd8<uint8_t> saturating_add(const simd8<uint8_t> other) const { return _mm_adds_epu8(*this, other); }
2785     simdutf_really_inline simd8<uint8_t> saturating_sub(const simd8<uint8_t> other) const { return _mm_subs_epu8(*this, other); }
2788     simdutf_really_inline simd8<uint8_t> max_val(const simd8<uint8_t> other) const { return _mm_max_epu8(*this, other); }
2789     simdutf_really_inline simd8<uint8_t> min_val(const simd8<uint8_t> other) const { return _mm_min_epu8(*this, other); }
2791     simdutf_really_inline simd8<uint8_t> gt_bits(const simd8<uint8_t> other) const { return this->saturating_sub(other); }
2793     simdutf_really_inline simd8<uint8_t> lt_bits(const simd8<uint8_t> other) const { return other.saturating_sub(*this); }
2794     simdutf_really_inline simd8<bool> operator<=(const simd8<uint8_t> other) const { return other.max_val(*this) == other; }
2795     simdutf_really_inline simd8<bool> operator>=(const simd8<uint8_t> other) const { return other.min_val(*this) == other; }
2796     simdutf_really_inline simd8<bool> operator>(const simd8<uint8_t> other) const { return this->gt_bits(other).any_bits_set(); }
2797     simdutf_really_inline simd8<bool> operator<(const simd8<uint8_t> other) const { return this->gt_bits(other).any_bits_set(); }
2800     simdutf_really_inline simd8<bool> bits_not_set() const { return *this == uint8_t(0); }
2801     simdutf_really_inline simd8<bool> bits_not_set(simd8<uint8_t> bits) const { return (*this & bits).bits_not_set(); }
2802     simdutf_really_inline simd8<bool> any_bits_set() const { return ~this->bits_not_set(); }
2803     simdutf_really_inline simd8<bool> any_bits_set(simd8<uint8_t> bits) const { return ~this->bits_not_set(bits); }
2804     simdutf_really_inline bool is_ascii() const { return _mm_movemask_epi8(*this) == 0; }
2806     simdutf_really_inline bool bits_not_set_anywhere() const { return _mm_testz_si128(*this, *this); }
2807     simdutf_really_inline bool any_bits_set_anywhere() const { return !bits_not_set_anywhere(); }
2808     simdutf_really_inline bool bits_not_set_anywhere(simd8<uint8_t> bits) const { return _mm_testz_si128(*this, bits); }
2809     simdutf_really_inline bool any_bits_set_anywhere(simd8<uint8_t> bits) const { return !bits_not_set_anywhere(bits); }
2811     simdutf_really_inline simd8<uint8_t> shr() const { return simd8<uint8_t>(_mm_srli_epi16(*this, N)) & uint8_t(0xFFu >> N); }
2813     simdutf_really_inline simd8<uint8_t> shl() const { return simd8<uint8_t>(_mm_slli_epi16(*this, N)) & uint8_t(0xFFu << N); }
2817     simdutf_really_inline int get_bit() const { return _mm_movemask_epi8(_mm_slli_epi16(*this, 7-N)); }
2819   simdutf_really_inline simd8<int8_t>::operator simd8<uint8_t>() const { return this->value; }
2824     static simdutf_really_inline simd8<uint16_t> splat(uint16_t _value) { return _mm_set1_epi16(_value); }
2825     static simdutf_really_inline simd8<uint16_t> load(const uint16_t values[8]) {
2829     simdutf_really_inline simd8() : base<uint16_t>() {}
2830     simdutf_really_inline simd8(const __m128i _value) : base<uint16_t>(_value) {}
2832     simdutf_really_inline simd8(uint16_t _value) : simd8(splat(_value)) {}
2834     simdutf_really_inline simd8(const uint16_t* values) : simd8(load(values)) {}
2836     simdutf_really_inline simd8(
2843     simdutf_really_inline simd8<uint16_t> saturating_add(const simd8<uint16_t> other) const { return _mm_adds_epu16(*this, other); }
2844     simdutf_really_inline simd8<uint16_t> saturating_sub(const simd8<uint16_t> other) const { return _mm_subs_epu16(*this, other); }
2847     simdutf_really_inline simd8<uint16_t> max_val(const simd8<uint16_t> other) const { return _mm_max_epu16(*this, other); }
2848     simdutf_really_inline simd8<uint16_t> min_val(const simd8<uint16_t> other) const { return _mm_min_epu16(*this, other); }
2850     simdutf_really_inline simd8<uint16_t> gt_bits(const simd8<uint16_t> other) const { return this->saturating_sub(other); }
2852     simdutf_really_inline simd8<uint16_t> lt_bits(const simd8<uint16_t> other) const { return other.saturating_sub(*this); }
2853     simdutf_really_inline simd8<bool> operator<=(const simd8<uint16_t> other) const { return other.max_val(*this) == other; }
2854     simdutf_really_inline simd8<bool> operator>=(const simd8<uint16_t> other) const { return other.min_val(*this) == other; }
2855     simdutf_really_inline simd8<bool> operator==(const simd8<uint16_t> other) const { return _mm_cmpeq_epi16(*this, other); }
2856     simdutf_really_inline simd8<bool> operator&(const simd8<uint16_t> other) const { return _mm_and_si128(*this, other); }
2857     simdutf_really_inline simd8<bool> operator|(const simd8<uint16_t> other) const { return _mm_or_si128(*this, other); }
2860     simdutf_really_inline simd8<bool> bits_not_set() const { return *this == uint16_t(0); }
2861     simdutf_really_inline simd8<bool> any_bits_set() const { return ~this->bits_not_set(); }
2863     simdutf_really_inline bool bits_not_set_anywhere() const { return _mm_testz_si128(*this, *this); }
2864     simdutf_really_inline bool any_bits_set_anywhere() const { return !bits_not_set_anywhere(); }
2865     simdutf_really_inline bool bits_not_set_anywhere(simd8<uint16_t> bits) const { return _mm_testz_si128(*this, bits); }
2866     simdutf_really_inline bool any_bits_set_anywhere(simd8<uint16_t> bits) const { return !bits_not_set_anywhere(bits); }
2878     simdutf_really_inline simd8x64(const simd8<T> chunk0, const simd8<T> chunk1, const simd8<T> chunk2, const simd8<T> chunk3) : chunks{chunk0, chunk1, chunk2, chunk3} {}
2879     simdutf_really_inline simd8x64(const T* ptr) : chunks{simd8<T>::load(ptr), simd8<T>::load(ptr+sizeof(simd8<T>)/sizeof(T)), simd8<T>::load(ptr+2*sizeof(simd8<T>)/sizeof(T)), simd8<T>::load(ptr+3*sizeof(simd8<T>)/sizeof(T))} {}
2881     simdutf_really_inline void store(T* ptr) const {
2888     simdutf_really_inline simd8x64<T>& operator |=(const simd8x64<T> &other) {
2896     simdutf_really_inline simd8<T> reduce_or() const {
2900     simdutf_really_inline bool is_ascii() const {
2905     simdutf_really_inline void store_ascii_as_utf16(char16_t * ptr) const {
2912     simdutf_really_inline void store_ascii_as_utf32(char32_t * ptr) const {
2919     simdutf_really_inline uint64_t to_bitmask() const {
2927     simdutf_really_inline uint64_t eq(const T m) const {
2937     simdutf_really_inline uint64_t eq(const simd8x64<uint8_t> &other) const {
2946     simdutf_really_inline uint64_t lteq(const T m) const {
2956     simdutf_really_inline uint64_t in_range(const T low, const T high) const {
2967     simdutf_really_inline uint64_t not_in_range(const T low, const T high) const {
2977     simdutf_really_inline uint64_t lt(const T m) const {
2987     simdutf_really_inline uint64_t gt(const T m) const {
2996     simdutf_really_inline uint64_t gteq(const T m) const {
3005     simdutf_really_inline uint64_t gteq_unsigned(const uint8_t m) const {
3025   simdutf_really_inline base16() : base<simd16<T>>() {}
3026   simdutf_really_inline base16(const __m128i _value) : base<simd16<T>>(_value) {}
3028   simdutf_really_inline base16(const Pointer* ptr) : base16(_mm_loadu_si128(reinterpret_cast<const __m128i*>(ptr))) {}
3030   friend simdutf_really_inline Mask operator==(const simd16<T> lhs, const simd16<T> rhs) { return _mm_cmpeq_epi16(lhs, rhs); }
3035   simdutf_really_inline simd16<T> prev(const simd16<T> prev_chunk) const {
3043   static simdutf_really_inline simd16<bool> splat(bool _value) { return _mm_set1_epi16(uint16_t(-(!!_value))); }
3045   simdutf_really_inline simd16<bool>() : base16() {}
3046   simdutf_really_inline simd16<bool>(const __m128i _value) : base16<bool>(_value) {}
3048   simdutf_really_inline simd16<bool>(bool _value) : base16<bool>(splat(_value)) {}
3050   simdutf_really_inline int to_bitmask() const { return _mm_movemask_epi8(*this); }
3051   simdutf_really_inline bool any() const { return !_mm_testz_si128(*this, *this); }
3052   simdutf_really_inline simd16<bool> operator~() const { return *this ^ true; }
3057   static simdutf_really_inline simd16<T> splat(T _value) { return _mm_set1_epi16(_value); }
3058   static simdutf_really_inline simd16<T> zero() { return _mm_setzero_si128(); }
3059   static simdutf_really_inline simd16<T> load(const T values[8]) {
3063   simdutf_really_inline base16_numeric() : base16<T>() {}
3064   simdutf_really_inline base16_numeric(const __m128i _value) : base16<T>(_value) {}
3067   simdutf_really_inline void store(T dst[8]) const { return _mm_storeu_si128(reinterpret_cast<__m128i *>(dst), *this); }
3070   simdutf_really_inline simd16<T> operator~() const { return *this ^ 0xFFu; }
3073   simdutf_really_inline simd16<T> operator+(const simd16<T> other) const { return _mm_add_epi16(*this, other); }
3074   simdutf_really_inline simd16<T> operator-(const simd16<T> other) const { return _mm_sub_epi16(*this, other); }
3075   simdutf_really_inline simd16<T>& operator+=(const simd16<T> other) { *this = *this + other; return *static_cast<simd16<T>*>(this); }
3076   simdutf_really_inline simd16<T>& operator-=(const simd16<T> other) { *this = *this - other; return *static_cast<simd16<T>*>(this); }
3082   simdutf_really_inline simd16() : base16_numeric<int16_t>() {}
3083   simdutf_really_inline simd16(const __m128i _value) : base16_numeric<int16_t>(_value) {}
3085   simdutf_really_inline simd16(int16_t _value) : simd16(splat(_value)) {}
3087   simdutf_really_inline simd16(const int16_t* values) : simd16(load(values)) {}
3088   simdutf_really_inline simd16(const char16_t* values) : simd16(load(reinterpret_cast<const int16_t*>(values))) {}
3090   simdutf_really_inline simd16(
3093   simdutf_really_inline operator simd16<uint16_t>() const;
3096   simdutf_really_inline simd16<int16_t> max_val(const simd16<int16_t> other) const { return _mm_max_epi16(*this, other); }
3097   simdutf_really_inline simd16<int16_t> min_val(const simd16<int16_t> other) const { return _mm_min_epi16(*this, other); }
3098   simdutf_really_inline simd16<bool> operator>(const simd16<int16_t> other) const { return _mm_cmpgt_epi16(*this, other); }
3099   simdutf_really_inline simd16<bool> operator<(const simd16<int16_t> other) const { return _mm_cmpgt_epi16(other, *this); }
3105   simdutf_really_inline simd16() : base16_numeric<uint16_t>() {}
3106   simdutf_really_inline simd16(const __m128i _value) : base16_numeric<uint16_t>(_value) {}
3109   simdutf_really_inline simd16(uint16_t _value) : simd16(splat(_value)) {}
3111   simdutf_really_inline simd16(const uint16_t* values) : simd16(load(values)) {}
3112   simdutf_really_inline simd16(const char16_t* values) : simd16(load(reinterpret_cast<const uint16_t*>(values))) {}
3114   simdutf_really_inline simd16(
3118   simdutf_really_inline static simd16<uint16_t> repeat_16(
3125   simdutf_really_inline simd16<uint16_t> saturating_add(const simd16<uint16_t> other) const { return _mm_adds_epu16(*this, other); }
3126   simdutf_really_inline simd16<uint16_t> saturating_sub(const simd16<uint16_t> other) const { return _mm_subs_epu16(*this, other); }
3129   simdutf_really_inline simd16<uint16_t> max_val(const simd16<uint16_t> other) const { return _mm_max_epu16(*this, other); }
3130   simdutf_really_inline simd16<uint16_t> min_val(const simd16<uint16_t> other) const { return _mm_min_epu16(*this, other); }
3132   simdutf_really_inline simd16<uint16_t> gt_bits(const simd16<uint16_t> other) const { return this->saturating_sub(other); }
3134   simdutf_really_inline simd16<uint16_t> lt_bits(const simd16<uint16_t> other) const { return other.saturating_sub(*this); }
3135   simdutf_really_inline simd16<bool> operator<=(const simd16<uint16_t> other) const { return other.max_val(*this) == other; }
3136   simdutf_really_inline simd16<bool> operator>=(const simd16<uint16_t> other) const { return other.min_val(*this) == other; }
3137   simdutf_really_inline simd16<bool> operator>(const simd16<uint16_t> other) const { return this->gt_bits(other).any_bits_set(); }
3138   simdutf_really_inline simd16<bool> operator<(const simd16<uint16_t> other) const { return this->gt_bits(other).any_bits_set(); }
3141   simdutf_really_inline simd16<bool> bits_not_set() const { return *this == uint16_t(0); }
3142   simdutf_really_inline simd16<bool> bits_not_set(simd16<uint16_t> bits) const { return (*this & bits).bits_not_set(); }
3143   simdutf_really_inline simd16<bool> any_bits_set() const { return ~this->bits_not_set(); }
3144   simdutf_really_inline simd16<bool> any_bits_set(simd16<uint16_t> bits) const { return ~this->bits_not_set(bits); }
3146   simdutf_really_inline bool bits_not_set_anywhere() const { return _mm_testz_si128(*this, *this); }
3147   simdutf_really_inline bool any_bits_set_anywhere() const { return !bits_not_set_anywhere(); }
3148   simdutf_really_inline bool bits_not_set_anywhere(simd16<uint16_t> bits) const { return _mm_testz_si128(*this, bits); }
3149   simdutf_really_inline bool any_bits_set_anywhere(simd16<uint16_t> bits) const { return !bits_not_set_anywhere(bits); }
3151   simdutf_really_inline simd16<uint16_t> shr() const { return simd16<uint16_t>(_mm_srli_epi16(*this, N)); }
3153   simdutf_really_inline simd16<uint16_t> shl() const { return simd16<uint16_t>(_mm_slli_epi16(*this, N)); }
3157   simdutf_really_inline int get_bit() const { return _mm_movemask_epi8(_mm_slli_epi16(*this, 7-N)); }
3160   simdutf_really_inline simd16<uint16_t> swap_bytes() const {
3166   static simdutf_really_inline simd8<uint8_t> pack(const simd16<uint16_t>& v0, const simd16<uint16_t>& v1) {
3170 simdutf_really_inline simd16<int16_t>::operator simd16<uint16_t>() const { return this->value; }
3182     simdutf_really_inline simd16x32(const simd16<T> chunk0, const simd16<T> chunk1, const simd16<T> chunk2, const simd16<T> chunk3) : chunks{chunk0, chunk1, chunk2, chunk3} {}
3183     simdutf_really_inline simd16x32(const T* ptr) : chunks{simd16<T>::load(ptr), simd16<T>::load(ptr+sizeof(simd16<T>)/sizeof(T)), simd16<T>::load(ptr+2*sizeof(simd16<T>)/sizeof(T)), simd16<T>::load(ptr+3*sizeof(simd16<T>)/sizeof(T))} {}
3185     simdutf_really_inline void store(T* ptr) const {
3192     simdutf_really_inline simd16<T> reduce_or() const {
3196     simdutf_really_inline bool is_ascii() const {
3200     simdutf_really_inline void store_ascii_as_utf16(char16_t * ptr) const {
3207     simdutf_really_inline uint64_t to_bitmask() const {
3215     simdutf_really_inline void swap_bytes() {
3222     simdutf_really_inline uint64_t eq(const T m) const {
3232     simdutf_really_inline uint64_t eq(const simd16x32<uint16_t> &other) const {
3241     simdutf_really_inline uint64_t lteq(const T m) const {
3251     simdutf_really_inline uint64_t in_range(const T low, const T high) const {
3262     simdutf_really_inline uint64_t not_in_range(const T low, const T high) const {
3272     simdutf_really_inline uint64_t lt(const T m) const {
3344   simdutf_really_inline implementation()
3443 simdutf_really_inline int count_ones(uint64_t input_num) {
3448 simdutf_really_inline int count_ones(uint64_t input_num) {
3476   simdutf_really_inline base() : value{__m128i()} {}
3479   simdutf_really_inline base(const __m128i _value) : value(_value) {}
3482   simdutf_really_inline operator const __m128i &() const {
3485   simdutf_really_inline operator __m128i &() { return this->value; }
3488   simdutf_really_inline Child operator|(const Child other) const {
3491   simdutf_really_inline Child operator&(const Child other) const {
3494   simdutf_really_inline Child operator^(const Child other) const {
3497   simdutf_really_inline Child bit_andnot(const Child other) const {
3500   simdutf_really_inline Child &operator|=(const Child other) {
3505   simdutf_really_inline Child &operator&=(const Child other) {
3510   simdutf_really_inline Child &operator^=(const Child other) {
3525   simdutf_really_inline base8() : base<simd8<T>>() {}
3526   simdutf_really_inline base8(const __m128i _value) : base<simd8<T>>(_value) {}
3528   friend simdutf_really_inline Mask operator==(const simd8<T> lhs, const simd8<T> rhs) {
3535   simdutf_really_inline simd8<T> prev(simd8<T> prev_chunk) const {
3551   static simdutf_really_inline simd8<bool> splat(bool _value) {
3555   simdutf_really_inline simd8<bool>() : base8() {}
3556   simdutf_really_inline simd8<bool>(const __m128i _value)
3559   simdutf_really_inline simd8<bool>(bool _value)
3562   simdutf_really_inline int to_bitmask() const {
3575   simdutf_really_inline bool any() const {
3578   simdutf_really_inline simd8<bool> operator~() const {
3584   static simdutf_really_inline simd8<T> splat(T value) {
3588   static simdutf_really_inline simd8<T> zero() { return splat(0); }
3589   static simdutf_really_inline simd8<T> load(const T values[16]) {
3593   static simdutf_really_inline simd8<T> repeat_16(T v0, T v1, T v2, T v3, T v4,
3601   simdutf_really_inline base8_numeric() : base8<T>() {}
3602   simdutf_really_inline base8_numeric(const __m128i _value)
3606   simdutf_really_inline void store(T dst[16]) const {
3611   simdutf_really_inline simd8<T> operator~() const { return *this ^ 0xFFu; }
3614   simdutf_really_inline simd8<T> operator+(const simd8<T> other) const {
3617   simdutf_really_inline simd8<T> operator-(const simd8<T> other) const {
3620   simdutf_really_inline simd8<T> &operator+=(const simd8<T> other) {
3624   simdutf_really_inline simd8<T> &operator-=(const simd8<T> other) {
3632   simdutf_really_inline simd8<L> lookup_16(simd8<L> lookup_table) const {
3637   simdutf_really_inline simd8<L>
3651   simdutf_really_inline simd8() : base8_numeric<int8_t>() {}
3652   simdutf_really_inline simd8(const __m128i _value)
3656   simdutf_really_inline simd8(int8_t _value) : simd8(splat(_value)) {}
3658   simdutf_really_inline simd8(const int8_t *values) : simd8(load(values)) {}
3660   simdutf_really_inline simd8(int8_t v0, int8_t v1, int8_t v2, int8_t v3,
3668   simdutf_really_inline static simd8<int8_t>
3677   simdutf_really_inline simd8<int8_t>
3682   simdutf_really_inline simd8<int8_t>
3687   simdutf_really_inline simd8<bool>
3692   simdutf_really_inline simd8<bool>
3701   simdutf_really_inline simd8() : base8_numeric<uint8_t>() {}
3702   simdutf_really_inline simd8(const __m128i _value)
3705   simdutf_really_inline simd8(uint8_t _value) : simd8(splat(_value)) {}
3707   simdutf_really_inline simd8(const uint8_t *values) : simd8(load(values)) {}
3709   simdutf_really_inline
3716   simdutf_really_inline static simd8<uint8_t>
3726   simdutf_really_inline simd8<uint8_t>
3730   simdutf_really_inline simd8<uint8_t>
3736   simdutf_really_inline simd8<uint8_t>
3740   simdutf_really_inline simd8<uint8_t>
3745   simdutf_really_inline simd8<uint8_t>
3750   simdutf_really_inline simd8<uint8_t>
3754   simdutf_really_inline simd8<bool>
3758   simdutf_really_inline simd8<bool>
3762   simdutf_really_inline simd8<bool>
3766   simdutf_really_inline simd8<bool>
3772   simdutf_really_inline simd8<bool> bits_not_set() const {
3775   simdutf_really_inline simd8<bool> bits_not_set(simd8<uint8_t> bits) const {
3778   simdutf_really_inline simd8<bool> any_bits_set() const {
3781   simdutf_really_inline simd8<bool> any_bits_set(simd8<uint8_t> bits) const {
3785   simdutf_really_inline bool is_ascii() const {
3789   simdutf_really_inline bool bits_not_set_anywhere() const {
3792   simdutf_really_inline bool any_bits_set_anywhere() const {
3795   simdutf_really_inline bool bits_not_set_anywhere(simd8<uint8_t> bits) const {
3799   simdutf_really_inline bool any_bits_set_anywhere(simd8<uint8_t> bits) const {
3802   template <int N> simdutf_really_inline simd8<uint8_t> shr() const {
3806   template <int N> simdutf_really_inline simd8<uint8_t> shl() const {
3823   simdutf_really_inline simd8x64(const simd8<T> chunk0, const simd8<T> chunk1,
3827   simdutf_really_inline simd8x64(const T* ptr) : chunks{simd8<T>::load(ptr), simd8<T>::load(ptr+sizeof(simd8<T>)/sizeof(T)), simd8<T>::load(ptr+2*sizeof(simd8<T>)/sizeof(T)), simd8<T>::load(ptr+3*sizeof(simd8<T>)/sizeof(T))} {}
3829   simdutf_really_inline void store(T* ptr) const {
3837   simdutf_really_inline simd8x64<T>& operator |=(const simd8x64<T> &other) {
3845   simdutf_really_inline simd8<T> reduce_or() const {
3851   simdutf_really_inline bool is_ascii() const {
3855   simdutf_really_inline uint64_t to_bitmask() const {
3863   simdutf_really_inline uint64_t eq(const T m) const {
3870   simdutf_really_inline uint64_t eq(const simd8x64<uint8_t> &other) const {
3878   simdutf_really_inline uint64_t lteq(const T m) const {
3885   simdutf_really_inline uint64_t in_range(const T low, const T high) const {
3896   simdutf_really_inline uint64_t not_in_range(const T low, const T high) const {
3906   simdutf_really_inline uint64_t lt(const T m) const {
3913   simdutf_really_inline uint64_t gt(const T m) const {
3922   simdutf_really_inline uint64_t gteq(const T m) const {
3931   simdutf_really_inline uint64_t gteq_unsigned(const uint8_t m) const {
3999   simdutf_really_inline implementation() : simdutf::implementation(
4429 simdutf_really_inline void change_endianness_utf16(const char16_t* in, size_t size, char16_t* out) {
4840   simdutf_really_inline detect_best_supported_implementation_on_first_use() noexcept : implementation("best_supported_detector", "Detects the best supported implementation and sets it", 0) {}
5698   simdutf_really_inline result::result() : error{error_code::SUCCESS}, count{0} {}
5700   simdutf_really_inline result::result(error_code _err, size_t _pos) : error{_err}, count{_pos} {}
12576 simdutf_really_inline bool is_ascii(const simd8x64<uint8_t>& input) {
12581 simdutf_unused simdutf_really_inline simd8<bool> must_be_continuation(const simd8<uint8_t> prev1, const simd8<uint8_t> prev2, const simd8<uint8_t> prev3) {
12593 simdutf_really_inline simd8<bool> must_be_2_3_continuation(const simd8<uint8_t> prev2, const simd8<uint8_t> prev3) {
12600 simdutf_really_inline uint16x4_t convert_utf8_3_byte_to_utf16(uint8x16_t in) {
12630 simdutf_really_inline uint16x8_t convert_utf8_2_byte_to_utf16(uint8x16_t in) {
12649 simdutf_really_inline uint16x8_t convert_utf8_1_to_2_byte_to_utf16(uint8x16_t in, size_t shufutf8_idx) {
15126   simdutf_really_inline buf_block_reader(const uint8_t *_buf, size_t _len);
15127   simdutf_really_inline size_t block_index();
15128   simdutf_really_inline bool has_full_block() const;
15129   simdutf_really_inline const uint8_t *full_block() const;
15139   simdutf_really_inline size_t get_remainder(uint8_t *dst) const;
15140   simdutf_really_inline void advance();
15179 simdutf_really_inline buf_block_reader<STEP_SIZE>::buf_block_reader(const uint8_t *_buf, size_t _len) : buf{_buf}, len{_len}, lenminusstep{len < STEP_SIZE ? 0 : len - STEP_SIZE}, idx{0} {}
15182 simdutf_really_inline size_t buf_block_reader<STEP_SIZE>::block_index() { return idx; }
15185 simdutf_really_inline bool buf_block_reader<STEP_SIZE>::has_full_block() const {
15190 simdutf_really_inline const uint8_t *buf_block_reader<STEP_SIZE>::full_block() const {
15195 simdutf_really_inline size_t buf_block_reader<STEP_SIZE>::get_remainder(uint8_t *dst) const {
15203 simdutf_really_inline void buf_block_reader<STEP_SIZE>::advance() {
15219   simdutf_really_inline simd8<uint8_t> check_special_cases(const simd8<uint8_t> input, const simd8<uint8_t> prev1) {
15309   simdutf_really_inline simd8<uint8_t> check_multibyte_lengths(const simd8<uint8_t> input,
15322   simdutf_really_inline simd8<uint8_t> is_incomplete(const simd8<uint8_t> input) {
15346     simdutf_really_inline void check_utf8_bytes(const simd8<uint8_t> input, const simd8<uint8_t> prev_input) {
15357     simdutf_really_inline void check_eof() {
15363     simdutf_really_inline void check_next_input(const simd8x64<uint8_t>& input) {
15386     simdutf_really_inline bool errors() const {
15610   simdutf_really_inline simd8<uint8_t> check_special_cases(const simd8<uint8_t> input, const simd8<uint8_t> prev1) {
15700   simdutf_really_inline simd8<uint8_t> check_multibyte_lengths(const simd8<uint8_t> input,
15718     simdutf_really_inline void check_utf8_bytes(const simd8<uint8_t> input, const simd8<uint8_t> prev_input) {
15728     simdutf_really_inline size_t convert(const char* in, size_t size, char16_t* utf16_output) {
15803     simdutf_really_inline result convert_with_errors(const char* in, size_t size, char16_t* utf16_output) {
15896     simdutf_really_inline bool errors() const {
15962   simdutf_really_inline simd8<uint8_t> check_special_cases(const simd8<uint8_t> input, const simd8<uint8_t> prev1) {
16052   simdutf_really_inline simd8<uint8_t> check_multibyte_lengths(const simd8<uint8_t> input,
16070     simdutf_really_inline void check_utf8_bytes(const simd8<uint8_t> input, const simd8<uint8_t> prev_input) {
16080     simdutf_really_inline size_t convert(const char* in, size_t size, char32_t* utf32_output) {
16154     simdutf_really_inline result convert_with_errors(const char* in, size_t size, char32_t* utf32_output) {
16241     simdutf_really_inline bool errors() const {
16261 simdutf_really_inline size_t count_code_points(const char* in, size_t size) {
16272 simdutf_really_inline size_t utf16_length_from_utf8(const char* in, size_t size) {
16299 simdutf_really_inline size_t count_code_points(const char16_t* in, size_t size) {
16312 simdutf_really_inline size_t utf8_length_from_utf16(const char16_t* in, size_t size) {
16333 simdutf_really_inline size_t utf32_length_from_utf16(const char16_t* in, size_t size) {
16337 simdutf_really_inline void change_endianness_utf16(const char16_t* in, size_t size, char16_t* output) {
16367   simdutf_really_inline simd8<uint8_t> check_special_cases(const simd8<uint8_t> input, const simd8<uint8_t> prev1) {
16470     simdutf_really_inline void check_utf8_bytes(const simd8<uint8_t> input, const simd8<uint8_t> prev_input) {
16478     simdutf_really_inline size_t convert(const char* in, size_t size, char* latin1_output) {
16552     simdutf_really_inline result convert_with_errors(const char* in, size_t size, char* latin1_output) {
16645     simdutf_really_inline bool errors() const {
16665     simdutf_really_inline size_t convert_valid(const char* in, size_t size, char* latin1_output) {
17809 simdutf_really_inline bool process_block_utf8_to_utf16(const char *&in, char16_t *&out, size_t gap) {
18127 simdutf_really_inline size_t utf32_to_utf16_masked(const __m512i byteflip, __m512i utf32, unsigned int count, char16_t* output) {
18202 simdutf_really_inline size_t utf32_to_utf16(const __m512i byteflip, __m512i utf32, unsigned int count, char16_t* output) {
18302 simdutf_really_inline __m512i expanded_utf8_to_utf32(__m512i char_class, __m512i utf8) {
18424 simdutf_really_inline __m512i expand_and_identify(__m512i lane0, __m512i lane1, int &count) {
18445 simdutf_really_inline __m512i expand_utf8_to_utf32(__m512i input) {
18731 simdutf_really_inline __m512i check_special_cases(__m512i input, const __m512i prev1) {
18772   simdutf_really_inline __m512i check_multibyte_lengths(const __m512i input,
18791   simdutf_really_inline __m512i is_incomplete(const __m512i input) {
18818     simdutf_really_inline void check_utf8_bytes(const __m512i input, const __m512i prev_input) {
18829     simdutf_really_inline void check_eof() {
18836     simdutf_really_inline bool check_next_input(const __m512i input) {
18850     simdutf_really_inline bool errors() const {
19165 simdutf_really_inline size_t process_block_from_utf8_to_latin1(const char *buf, size_t len,
19262 simdutf_really_inline size_t process_valid_block_from_utf8_to_latin1(const char *buf, size_t len,
22018 simdutf_really_inline bool is_ascii(const simd8x64<uint8_t>& input) {
22022 simdutf_unused simdutf_really_inline simd8<bool> must_be_continuation(const simd8<uint8_t> prev1, const simd8<uint8_t> prev2, const simd8<uint8_t> prev3) {
22030 simdutf_really_inline simd8<bool> must_be_2_3_continuation(const simd8<uint8_t> prev2, const simd8<uint8_t> prev3) {
24537   simdutf_really_inline buf_block_reader(const uint8_t *_buf, size_t _len);
24538   simdutf_really_inline size_t block_index();
24539   simdutf_really_inline bool has_full_block() const;
24540   simdutf_really_inline const uint8_t *full_block() const;
24550   simdutf_really_inline size_t get_remainder(uint8_t *dst) const;
24551   simdutf_really_inline void advance();
24590 simdutf_really_inline buf_block_reader<STEP_SIZE>::buf_block_reader(const uint8_t *_buf, size_t _len) : buf{_buf}, len{_len}, lenminusstep{len < STEP_SIZE ? 0 : len - STEP_SIZE}, idx{0} {}
24593 simdutf_really_inline size_t buf_block_reader<STEP_SIZE>::block_index() { return idx; }
24596 simdutf_really_inline bool buf_block_reader<STEP_SIZE>::has_full_block() const {
24601 simdutf_really_inline const uint8_t *buf_block_reader<STEP_SIZE>::full_block() const {
24606 simdutf_really_inline size_t buf_block_reader<STEP_SIZE>::get_remainder(uint8_t *dst) const {
24614 simdutf_really_inline void buf_block_reader<STEP_SIZE>::advance() {
24630   simdutf_really_inline simd8<uint8_t> check_special_cases(const simd8<uint8_t> input, const simd8<uint8_t> prev1) {
24720   simdutf_really_inline simd8<uint8_t> check_multibyte_lengths(const simd8<uint8_t> input,
24733   simdutf_really_inline simd8<uint8_t> is_incomplete(const simd8<uint8_t> input) {
24757     simdutf_really_inline void check_utf8_bytes(const simd8<uint8_t> input, const simd8<uint8_t> prev_input) {
24768     simdutf_really_inline void check_eof() {
24774     simdutf_really_inline void check_next_input(const simd8x64<uint8_t>& input) {
24797     simdutf_really_inline bool errors() const {
25021   simdutf_really_inline simd8<uint8_t> check_special_cases(const simd8<uint8_t> input, const simd8<uint8_t> prev1) {
25111   simdutf_really_inline simd8<uint8_t> check_multibyte_lengths(const simd8<uint8_t> input,
25129     simdutf_really_inline void check_utf8_bytes(const simd8<uint8_t> input, const simd8<uint8_t> prev_input) {
25139     simdutf_really_inline size_t convert(const char* in, size_t size, char16_t* utf16_output) {
25214     simdutf_really_inline result convert_with_errors(const char* in, size_t size, char16_t* utf16_output) {
25307     simdutf_really_inline bool errors() const {
25373   simdutf_really_inline simd8<uint8_t> check_special_cases(const simd8<uint8_t> input, const simd8<uint8_t> prev1) {
25463   simdutf_really_inline simd8<uint8_t> check_multibyte_lengths(const simd8<uint8_t> input,
25481     simdutf_really_inline void check_utf8_bytes(const simd8<uint8_t> input, const simd8<uint8_t> prev_input) {
25491     simdutf_really_inline size_t convert(const char* in, size_t size, char32_t* utf32_output) {
25565     simdutf_really_inline result convert_with_errors(const char* in, size_t size, char32_t* utf32_output) {
25652     simdutf_really_inline bool errors() const {
25672 simdutf_really_inline size_t count_code_points(const char* in, size_t size) {
25683 simdutf_really_inline size_t utf16_length_from_utf8(const char* in, size_t size) {
25710 simdutf_really_inline size_t count_code_points(const char16_t* in, size_t size) {
25723 simdutf_really_inline size_t utf8_length_from_utf16(const char16_t* in, size_t size) {
25744 simdutf_really_inline size_t utf32_length_from_utf16(const char16_t* in, size_t size) {
25748 simdutf_really_inline void change_endianness_utf16(const char16_t* in, size_t size, char16_t* output) {
25780   simdutf_really_inline simd8<uint8_t> check_special_cases(const simd8<uint8_t> input, const simd8<uint8_t> prev1) {
25883     simdutf_really_inline void check_utf8_bytes(const simd8<uint8_t> input, const simd8<uint8_t> prev_input) {
25891     simdutf_really_inline size_t convert(const char* in, size_t size, char* latin1_output) {
25965     simdutf_really_inline result convert_with_errors(const char* in, size_t size, char* latin1_output) {
26058     simdutf_really_inline bool errors() const {
26078     simdutf_really_inline size_t convert_valid(const char* in, size_t size, char* latin1_output) {
26876 simdutf_really_inline bool is_ascii(const simd8x64<uint8_t>& input) {
26881 simdutf_unused simdutf_really_inline simd8<bool> must_be_continuation(const simd8<uint8_t> prev1, const simd8<uint8_t> prev2, const simd8<uint8_t> prev3) {
26889 simdutf_really_inline simd8<bool> must_be_2_3_continuation(const simd8<uint8_t> prev2, const simd8<uint8_t> prev3) {
26909   simdutf_really_inline buf_block_reader(const uint8_t *_buf, size_t _len);
26910   simdutf_really_inline size_t block_index();
26911   simdutf_really_inline bool has_full_block() const;
26912   simdutf_really_inline const uint8_t *full_block() const;
26922   simdutf_really_inline size_t get_remainder(uint8_t *dst) const;
26923   simdutf_really_inline void advance();
26962 simdutf_really_inline buf_block_reader<STEP_SIZE>::buf_block_reader(const uint8_t *_buf, size_t _len) : buf{_buf}, len{_len}, lenminusstep{len < STEP_SIZE ? 0 : len - STEP_SIZE}, idx{0} {}
26965 simdutf_really_inline size_t buf_block_reader<STEP_SIZE>::block_index() { return idx; }
26968 simdutf_really_inline bool buf_block_reader<STEP_SIZE>::has_full_block() const {
26973 simdutf_really_inline const uint8_t *buf_block_reader<STEP_SIZE>::full_block() const {
26978 simdutf_really_inline size_t buf_block_reader<STEP_SIZE>::get_remainder(uint8_t *dst) const {
26986 simdutf_really_inline void buf_block_reader<STEP_SIZE>::advance() {
27002   simdutf_really_inline simd8<uint8_t> check_special_cases(const simd8<uint8_t> input, const simd8<uint8_t> prev1) {
27092   simdutf_really_inline simd8<uint8_t> check_multibyte_lengths(const simd8<uint8_t> input,
27105   simdutf_really_inline simd8<uint8_t> is_incomplete(const simd8<uint8_t> input) {
27129     simdutf_really_inline void check_utf8_bytes(const simd8<uint8_t> input, const simd8<uint8_t> prev_input) {
27140     simdutf_really_inline void check_eof() {
27146     simdutf_really_inline void check_next_input(const simd8x64<uint8_t>& input) {
27169     simdutf_really_inline bool errors() const {
27393   simdutf_really_inline simd8<uint8_t> check_special_cases(const simd8<uint8_t> input, const simd8<uint8_t> prev1) {
27483   simdutf_really_inline simd8<uint8_t> check_multibyte_lengths(const simd8<uint8_t> input,
27501     simdutf_really_inline void check_utf8_bytes(const simd8<uint8_t> input, const simd8<uint8_t> prev_input) {
27511     simdutf_really_inline size_t convert(const char* in, size_t size, char16_t* utf16_output) {
27586     simdutf_really_inline result convert_with_errors(const char* in, size_t size, char16_t* utf16_output) {
27679     simdutf_really_inline bool errors() const {
27745   simdutf_really_inline simd8<uint8_t> check_special_cases(const simd8<uint8_t> input, const simd8<uint8_t> prev1) {
27835   simdutf_really_inline simd8<uint8_t> check_multibyte_lengths(const simd8<uint8_t> input,
27853     simdutf_really_inline void check_utf8_bytes(const simd8<uint8_t> input, const simd8<uint8_t> prev_input) {
27863     simdutf_really_inline size_t convert(const char* in, size_t size, char32_t* utf32_output) {
27937     simdutf_really_inline result convert_with_errors(const char* in, size_t size, char32_t* utf32_output) {
28024     simdutf_really_inline bool errors() const {
28044 simdutf_really_inline size_t count_code_points(const char* in, size_t size) {
28055 simdutf_really_inline size_t utf16_length_from_utf8(const char* in, size_t size) {
28082 simdutf_really_inline size_t count_code_points(const char16_t* in, size_t size) {
28095 simdutf_really_inline size_t utf8_length_from_utf16(const char16_t* in, size_t size) {
28116 simdutf_really_inline size_t utf32_length_from_utf16(const char16_t* in, size_t size) {
28120 simdutf_really_inline void change_endianness_utf16(const char16_t* in, size_t size, char16_t* output) {
28397 simdutf_really_inline bool is_ascii(const simd8x64<uint8_t>& input) {
28401 simdutf_unused simdutf_really_inline simd8<bool> must_be_continuation(const simd8<uint8_t> prev1, const simd8<uint8_t> prev2, const simd8<uint8_t> prev3) {
28409 simdutf_really_inline simd8<bool> must_be_2_3_continuation(const simd8<uint8_t> prev2, const simd8<uint8_t> prev3) {
30916   simdutf_really_inline buf_block_reader(const uint8_t *_buf, size_t _len);
30917   simdutf_really_inline size_t block_index();
30918   simdutf_really_inline bool has_full_block() const;
30919   simdutf_really_inline const uint8_t *full_block() const;
30929   simdutf_really_inline size_t get_remainder(uint8_t *dst) const;
30930   simdutf_really_inline void advance();
30969 simdutf_really_inline buf_block_reader<STEP_SIZE>::buf_block_reader(const uint8_t *_buf, size_t _len) : buf{_buf}, len{_len}, lenminusstep{len < STEP_SIZE ? 0 : len - STEP_SIZE}, idx{0} {}
30972 simdutf_really_inline size_t buf_block_reader<STEP_SIZE>::block_index() { return idx; }
30975 simdutf_really_inline bool buf_block_reader<STEP_SIZE>::has_full_block() const {
30980 simdutf_really_inline const uint8_t *buf_block_reader<STEP_SIZE>::full_block() const {
30985 simdutf_really_inline size_t buf_block_reader<STEP_SIZE>::get_remainder(uint8_t *dst) const {
30993 simdutf_really_inline void buf_block_reader<STEP_SIZE>::advance() {
31009   simdutf_really_inline simd8<uint8_t> check_special_cases(const simd8<uint8_t> input, const simd8<uint8_t> prev1) {
31099   simdutf_really_inline simd8<uint8_t> check_multibyte_lengths(const simd8<uint8_t> input,
31112   simdutf_really_inline simd8<uint8_t> is_incomplete(const simd8<uint8_t> input) {
31136     simdutf_really_inline void check_utf8_bytes(const simd8<uint8_t> input, const simd8<uint8_t> prev_input) {
31147     simdutf_really_inline void check_eof() {
31153     simdutf_really_inline void check_next_input(const simd8x64<uint8_t>& input) {
31176     simdutf_really_inline bool errors() const {
31400   simdutf_really_inline simd8<uint8_t> check_special_cases(const simd8<uint8_t> input, const simd8<uint8_t> prev1) {
31490   simdutf_really_inline simd8<uint8_t> check_multibyte_lengths(const simd8<uint8_t> input,
31508     simdutf_really_inline void check_utf8_bytes(const simd8<uint8_t> input, const simd8<uint8_t> prev_input) {
31518     simdutf_really_inline size_t convert(const char* in, size_t size, char16_t* utf16_output) {
31593     simdutf_really_inline result convert_with_errors(const char* in, size_t size, char16_t* utf16_output) {
31686     simdutf_really_inline bool errors() const {
31752   simdutf_really_inline simd8<uint8_t> check_special_cases(const simd8<uint8_t> input, const simd8<uint8_t> prev1) {
31842   simdutf_really_inline simd8<uint8_t> check_multibyte_lengths(const simd8<uint8_t> input,
31860     simdutf_really_inline void check_utf8_bytes(const simd8<uint8_t> input, const simd8<uint8_t> prev_input) {
31870     simdutf_really_inline size_t convert(const char* in, size_t size, char32_t* utf32_output) {
31944     simdutf_really_inline result convert_with_errors(const char* in, size_t size, char32_t* utf32_output) {
32031     simdutf_really_inline bool errors() const {
32051 simdutf_really_inline size_t count_code_points(const char* in, size_t size) {
32062 simdutf_really_inline size_t utf16_length_from_utf8(const char* in, size_t size) {
32089 simdutf_really_inline size_t count_code_points(const char16_t* in, size_t size) {
32102 simdutf_really_inline size_t utf8_length_from_utf16(const char16_t* in, size_t size) {
32123 simdutf_really_inline size_t utf32_length_from_utf16(const char16_t* in, size_t size) {
32127 simdutf_really_inline void change_endianness_utf16(const char16_t* in, size_t size, char16_t* output) {
32157   simdutf_really_inline simd8<uint8_t> check_special_cases(const simd8<uint8_t> input, const simd8<uint8_t> prev1) {
32260     simdutf_really_inline void check_utf8_bytes(const simd8<uint8_t> input, const simd8<uint8_t> prev_input) {
32268     simdutf_really_inline size_t convert(const char* in, size_t size, char* latin1_output) {
32342     simdutf_really_inline result convert_with_errors(const char* in, size_t size, char* latin1_output) {
32435     simdutf_really_inline bool errors() const {
32455     simdutf_really_inline size_t convert_valid(const char* in, size_t size, char* latin1_output) {

Indexes created Thu Nov 07 10:32:03 CST 2024