1// Copyright 2017 the V8 project authors. All rights reserved. 2// Use of this source code is governed by a BSD-style license that can be 3// found in the LICENSE file. 4 5#ifndef V8_WASM_BASELINE_LIFTOFF_ASSEMBLER_H_ 6#define V8_WASM_BASELINE_LIFTOFF_ASSEMBLER_H_ 7 8#include <iosfwd> 9#include <memory> 10 11#include "src/base/bits.h" 12#include "src/base/small-vector.h" 13#include "src/codegen/macro-assembler.h" 14#include "src/wasm/baseline/liftoff-assembler-defs.h" 15#include "src/wasm/baseline/liftoff-compiler.h" 16#include "src/wasm/baseline/liftoff-register.h" 17#include "src/wasm/function-body-decoder.h" 18#include "src/wasm/wasm-code-manager.h" 19#include "src/wasm/wasm-module.h" 20#include "src/wasm/wasm-opcodes.h" 21#include "src/wasm/wasm-value.h" 22 23namespace v8 { 24namespace internal { 25 26// Forward declarations. 27namespace compiler { 28class CallDescriptor; 29} // namespace compiler 30 31namespace wasm { 32 33enum LiftoffCondition { 34 kEqual, 35 kEqualZero = kEqual, // When used in a unary operation. 36 kUnequal, 37 kNotEqualZero = kUnequal, // When used in a unary operation. 38 kSignedLessThan, 39 kSignedLessEqual, 40 kSignedGreaterThan, 41 kSignedGreaterEqual, 42 kUnsignedLessThan, 43 kUnsignedLessEqual, 44 kUnsignedGreaterThan, 45 kUnsignedGreaterEqual 46}; 47 48inline constexpr LiftoffCondition Negate(LiftoffCondition cond) { 49 switch (cond) { 50 case kEqual: 51 return kUnequal; 52 case kUnequal: 53 return kEqual; 54 case kSignedLessThan: 55 return kSignedGreaterEqual; 56 case kSignedLessEqual: 57 return kSignedGreaterThan; 58 case kSignedGreaterEqual: 59 return kSignedLessThan; 60 case kSignedGreaterThan: 61 return kSignedLessEqual; 62 case kUnsignedLessThan: 63 return kUnsignedGreaterEqual; 64 case kUnsignedLessEqual: 65 return kUnsignedGreaterThan; 66 case kUnsignedGreaterEqual: 67 return kUnsignedLessThan; 68 case kUnsignedGreaterThan: 69 return kUnsignedLessEqual; 70 } 71} 72 73inline constexpr LiftoffCondition Flip(LiftoffCondition cond) { 74 switch (cond) { 75 case kEqual: 76 return kEqual; 77 case kUnequal: 78 return kUnequal; 79 case kSignedLessThan: 80 return kSignedGreaterThan; 81 case kSignedLessEqual: 82 return kSignedGreaterEqual; 83 case kSignedGreaterEqual: 84 return kSignedLessEqual; 85 case kSignedGreaterThan: 86 return kSignedLessThan; 87 case kUnsignedLessThan: 88 return kUnsignedGreaterThan; 89 case kUnsignedLessEqual: 90 return kUnsignedGreaterEqual; 91 case kUnsignedGreaterEqual: 92 return kUnsignedLessEqual; 93 case kUnsignedGreaterThan: 94 return kUnsignedLessThan; 95 } 96} 97 98class LiftoffAssembler : public TurboAssembler { 99 public: 100 // Each slot in our stack frame currently has exactly 8 bytes. 101 static constexpr int kStackSlotSize = 8; 102 103 static constexpr ValueKind kPointerKind = 104 kSystemPointerSize == kInt32Size ? kI32 : kI64; 105 static constexpr ValueKind kTaggedKind = 106 kTaggedSize == kInt32Size ? kI32 : kI64; 107 static constexpr ValueKind kSmiKind = kTaggedKind; 108 109 using ValueKindSig = Signature<ValueKind>; 110 111 class VarState { 112 public: 113 enum Location : uint8_t { kStack, kRegister, kIntConst }; 114 115 explicit VarState(ValueKind kind, int offset) 116 : loc_(kStack), kind_(kind), spill_offset_(offset) {} 117 explicit VarState(ValueKind kind, LiftoffRegister r, int offset) 118 : loc_(kRegister), kind_(kind), reg_(r), spill_offset_(offset) { 119 DCHECK_EQ(r.reg_class(), reg_class_for(kind)); 120 } 121 explicit VarState(ValueKind kind, int32_t i32_const, int offset) 122 : loc_(kIntConst), 123 kind_(kind), 124 i32_const_(i32_const), 125 spill_offset_(offset) { 126 DCHECK(kind_ == kI32 || kind_ == kI64); 127 } 128 129 bool is_stack() const { return loc_ == kStack; } 130 bool is_gp_reg() const { return loc_ == kRegister && reg_.is_gp(); } 131 bool is_fp_reg() const { return loc_ == kRegister && reg_.is_fp(); } 132 bool is_reg() const { return loc_ == kRegister; } 133 bool is_const() const { return loc_ == kIntConst; } 134 135 ValueKind kind() const { return kind_; } 136 137 Location loc() const { return loc_; } 138 139 int32_t i32_const() const { 140 DCHECK_EQ(loc_, kIntConst); 141 return i32_const_; 142 } 143 WasmValue constant() const { 144 DCHECK(kind_ == kI32 || kind_ == kI64); 145 DCHECK_EQ(loc_, kIntConst); 146 return kind_ == kI32 ? WasmValue(i32_const_) 147 : WasmValue(int64_t{i32_const_}); 148 } 149 150 int offset() const { return spill_offset_; } 151 void set_offset(int offset) { spill_offset_ = offset; } 152 153 Register gp_reg() const { return reg().gp(); } 154 DoubleRegister fp_reg() const { return reg().fp(); } 155 LiftoffRegister reg() const { 156 DCHECK_EQ(loc_, kRegister); 157 return reg_; 158 } 159 RegClass reg_class() const { return reg().reg_class(); } 160 161 void MakeStack() { loc_ = kStack; } 162 163 void MakeRegister(LiftoffRegister r) { 164 loc_ = kRegister; 165 reg_ = r; 166 } 167 168 void MakeConstant(int32_t i32_const) { 169 DCHECK(kind_ == kI32 || kind_ == kI64); 170 loc_ = kIntConst; 171 i32_const_ = i32_const; 172 } 173 174 // Copy src to this, except for offset, since src and this could have been 175 // from different stack states. 176 void Copy(VarState src) { 177 loc_ = src.loc(); 178 kind_ = src.kind(); 179 if (loc_ == kRegister) { 180 reg_ = src.reg(); 181 } else if (loc_ == kIntConst) { 182 i32_const_ = src.i32_const(); 183 } 184 } 185 186 private: 187 Location loc_; 188 // TODO(wasm): This is redundant, the decoder already knows the type of each 189 // stack value. Try to collapse. 190 ValueKind kind_; 191 192 union { 193 LiftoffRegister reg_; // used if loc_ == kRegister 194 int32_t i32_const_; // used if loc_ == kIntConst 195 }; 196 int spill_offset_; 197 }; 198 199 ASSERT_TRIVIALLY_COPYABLE(VarState); 200 201 struct CacheState { 202 // Allow default construction, move construction, and move assignment. 203 CacheState() = default; 204 CacheState(CacheState&&) V8_NOEXCEPT = default; 205 CacheState& operator=(CacheState&&) V8_NOEXCEPT = default; 206 // Disallow copy construction. 207 CacheState(const CacheState&) = delete; 208 209 enum class SpillLocation { kTopOfStack, kStackSlots }; 210 // Generates two lists of locations that contain references. {slots} 211 // contains the indices of slots on the value stack that contain references. 212 // {spills} contains all registers that contain references. The 213 // {spill_location} defines where register values will be spilled for a 214 // function call within the out-of-line code. {kStackSlots} means that the 215 // values in the registers will be written back to their stack slots. 216 // {kTopOfStack} means that the registers will be spilled on the stack with 217 // a {push} instruction. 218 void GetTaggedSlotsForOOLCode(/*out*/ ZoneVector<int>* slots, 219 /*out*/ LiftoffRegList* spills, 220 SpillLocation spill_location); 221 222 void DefineSafepoint(SafepointTableBuilder::Safepoint& safepoint); 223 224 void DefineSafepointWithCalleeSavedRegisters( 225 SafepointTableBuilder::Safepoint& safepoint); 226 227 base::SmallVector<VarState, 8> stack_state; 228 LiftoffRegList used_registers; 229 uint32_t register_use_count[kAfterMaxLiftoffRegCode] = {0}; 230 LiftoffRegList last_spilled_regs; 231 Register cached_instance = no_reg; 232 Register cached_mem_start = no_reg; 233 234 bool has_unused_register(RegClass rc, LiftoffRegList pinned = {}) const { 235 if (kNeedI64RegPair && rc == kGpRegPair) { 236 LiftoffRegList available_regs = 237 kGpCacheRegList.MaskOut(used_registers).MaskOut(pinned); 238 return available_regs.GetNumRegsSet() >= 2; 239 } else if (kNeedS128RegPair && rc == kFpRegPair) { 240 LiftoffRegList available_regs = 241 kFpCacheRegList.MaskOut(used_registers).MaskOut(pinned); 242 return available_regs.HasAdjacentFpRegsSet(); 243 } 244 DCHECK(rc == kGpReg || rc == kFpReg); 245 LiftoffRegList candidates = GetCacheRegList(rc); 246 return has_unused_register(candidates.MaskOut(pinned)); 247 } 248 249 bool has_unused_register(LiftoffRegList candidates) const { 250 LiftoffRegList available_regs = candidates.MaskOut(used_registers); 251 return !available_regs.is_empty(); 252 } 253 254 LiftoffRegister unused_register(RegClass rc, 255 LiftoffRegList pinned = {}) const { 256 if (kNeedI64RegPair && rc == kGpRegPair) { 257 Register low = pinned.set(unused_register(kGpReg, pinned)).gp(); 258 Register high = unused_register(kGpReg, pinned).gp(); 259 return LiftoffRegister::ForPair(low, high); 260 } else if (kNeedS128RegPair && rc == kFpRegPair) { 261 LiftoffRegList available_regs = 262 kFpCacheRegList.MaskOut(used_registers).MaskOut(pinned); 263 DoubleRegister low = 264 available_regs.GetAdjacentFpRegsSet().GetFirstRegSet().fp(); 265 DCHECK(is_free(LiftoffRegister::ForFpPair(low))); 266 return LiftoffRegister::ForFpPair(low); 267 } 268 DCHECK(rc == kGpReg || rc == kFpReg); 269 LiftoffRegList candidates = GetCacheRegList(rc); 270 return unused_register(candidates, pinned); 271 } 272 273 LiftoffRegister unused_register(LiftoffRegList candidates, 274 LiftoffRegList pinned = {}) const { 275 LiftoffRegList available_regs = 276 candidates.MaskOut(used_registers).MaskOut(pinned); 277 return available_regs.GetFirstRegSet(); 278 } 279 280 // Volatile registers are registers which are used for caching values that 281 // can easily be reloaded. Those are returned first if we run out of free 282 // registers. 283 bool has_volatile_register(LiftoffRegList candidates) { 284 return (cached_instance != no_reg && candidates.has(cached_instance)) || 285 (cached_mem_start != no_reg && candidates.has(cached_mem_start)); 286 } 287 288 LiftoffRegister take_volatile_register(LiftoffRegList candidates) { 289 DCHECK(has_volatile_register(candidates)); 290 Register reg = no_reg; 291 if (cached_instance != no_reg && candidates.has(cached_instance)) { 292 reg = cached_instance; 293 cached_instance = no_reg; 294 } else { 295 DCHECK(candidates.has(cached_mem_start)); 296 reg = cached_mem_start; 297 cached_mem_start = no_reg; 298 } 299 300 LiftoffRegister ret{reg}; 301 DCHECK_EQ(1, register_use_count[ret.liftoff_code()]); 302 register_use_count[ret.liftoff_code()] = 0; 303 used_registers.clear(ret); 304 return ret; 305 } 306 307 void SetCacheRegister(Register* cache, Register reg) { 308 DCHECK_EQ(no_reg, *cache); 309 *cache = reg; 310 int liftoff_code = LiftoffRegister{reg}.liftoff_code(); 311 DCHECK_EQ(0, register_use_count[liftoff_code]); 312 register_use_count[liftoff_code] = 1; 313 used_registers.set(reg); 314 } 315 316 void SetInstanceCacheRegister(Register reg) { 317 SetCacheRegister(&cached_instance, reg); 318 } 319 320 void SetMemStartCacheRegister(Register reg) { 321 SetCacheRegister(&cached_mem_start, reg); 322 } 323 324 Register TrySetCachedInstanceRegister(LiftoffRegList pinned) { 325 DCHECK_EQ(no_reg, cached_instance); 326 LiftoffRegList available_regs = 327 kGpCacheRegList.MaskOut(pinned).MaskOut(used_registers); 328 if (available_regs.is_empty()) return no_reg; 329 // Prefer the {kWasmInstanceRegister}, because that's where the instance 330 // initially is, and where it needs to be for calls. 331 Register new_cache_reg = available_regs.has(kWasmInstanceRegister) 332 ? kWasmInstanceRegister 333 : available_regs.GetFirstRegSet().gp(); 334 SetInstanceCacheRegister(new_cache_reg); 335 DCHECK_EQ(new_cache_reg, cached_instance); 336 return new_cache_reg; 337 } 338 339 void ClearCacheRegister(Register* cache) { 340 DCHECK(cache == &cached_instance || cache == &cached_mem_start); 341 if (*cache == no_reg) return; 342 int liftoff_code = LiftoffRegister{*cache}.liftoff_code(); 343 DCHECK_EQ(1, register_use_count[liftoff_code]); 344 register_use_count[liftoff_code] = 0; 345 used_registers.clear(*cache); 346 *cache = no_reg; 347 } 348 349 void ClearCachedInstanceRegister() { ClearCacheRegister(&cached_instance); } 350 351 void ClearCachedMemStartRegister() { 352 ClearCacheRegister(&cached_mem_start); 353 } 354 355 void ClearAllCacheRegisters() { 356 ClearCacheRegister(&cached_instance); 357 ClearCacheRegister(&cached_mem_start); 358 } 359 360 void inc_used(LiftoffRegister reg) { 361 if (reg.is_pair()) { 362 inc_used(reg.low()); 363 inc_used(reg.high()); 364 return; 365 } 366 used_registers.set(reg); 367 DCHECK_GT(kMaxInt, register_use_count[reg.liftoff_code()]); 368 ++register_use_count[reg.liftoff_code()]; 369 } 370 371 // Returns whether this was the last use. 372 void dec_used(LiftoffRegister reg) { 373 DCHECK(is_used(reg)); 374 if (reg.is_pair()) { 375 dec_used(reg.low()); 376 dec_used(reg.high()); 377 return; 378 } 379 int code = reg.liftoff_code(); 380 DCHECK_LT(0, register_use_count[code]); 381 if (--register_use_count[code] == 0) used_registers.clear(reg); 382 } 383 384 bool is_used(LiftoffRegister reg) const { 385 if (reg.is_pair()) return is_used(reg.low()) || is_used(reg.high()); 386 bool used = used_registers.has(reg); 387 DCHECK_EQ(used, register_use_count[reg.liftoff_code()] != 0); 388 return used; 389 } 390 391 uint32_t get_use_count(LiftoffRegister reg) const { 392 if (reg.is_pair()) { 393 DCHECK_EQ(register_use_count[reg.low().liftoff_code()], 394 register_use_count[reg.high().liftoff_code()]); 395 reg = reg.low(); 396 } 397 DCHECK_GT(arraysize(register_use_count), reg.liftoff_code()); 398 return register_use_count[reg.liftoff_code()]; 399 } 400 401 void clear_used(LiftoffRegister reg) { 402 if (reg.is_pair()) { 403 clear_used(reg.low()); 404 clear_used(reg.high()); 405 return; 406 } 407 register_use_count[reg.liftoff_code()] = 0; 408 used_registers.clear(reg); 409 } 410 411 bool is_free(LiftoffRegister reg) const { return !is_used(reg); } 412 413 void reset_used_registers() { 414 used_registers = {}; 415 memset(register_use_count, 0, sizeof(register_use_count)); 416 } 417 418 LiftoffRegister GetNextSpillReg(LiftoffRegList candidates) { 419 DCHECK(!candidates.is_empty()); 420 // This method should only be called if none of the candidates is free. 421 DCHECK(candidates.MaskOut(used_registers).is_empty()); 422 LiftoffRegList unspilled = candidates.MaskOut(last_spilled_regs); 423 if (unspilled.is_empty()) { 424 unspilled = candidates; 425 last_spilled_regs = {}; 426 } 427 LiftoffRegister reg = unspilled.GetFirstRegSet(); 428 return reg; 429 } 430 431 // TODO(clemensb): Don't copy the full parent state (this makes us N^2). 432 void InitMerge(const CacheState& source, uint32_t num_locals, 433 uint32_t arity, uint32_t stack_depth); 434 435 void Steal(const CacheState& source); 436 437 void Split(const CacheState& source); 438 439 uint32_t stack_height() const { 440 return static_cast<uint32_t>(stack_state.size()); 441 } 442 443 private: 444 // Make the copy assignment operator private (to be used from {Split()}). 445 CacheState& operator=(const CacheState&) V8_NOEXCEPT = default; 446 }; 447 448 explicit LiftoffAssembler(std::unique_ptr<AssemblerBuffer>); 449 ~LiftoffAssembler() override; 450 451 LiftoffRegister LoadToRegister(VarState slot, LiftoffRegList pinned); 452 453 LiftoffRegister PopToRegister(LiftoffRegList pinned = {}) { 454 DCHECK(!cache_state_.stack_state.empty()); 455 VarState slot = cache_state_.stack_state.back(); 456 cache_state_.stack_state.pop_back(); 457 if (slot.is_reg()) { 458 cache_state_.dec_used(slot.reg()); 459 return slot.reg(); 460 } 461 return LoadToRegister(slot, pinned); 462 } 463 464 // Use this to pop a value into a register that has no other uses, so it 465 // can be modified. 466 LiftoffRegister PopToModifiableRegister(LiftoffRegList pinned = {}) { 467 ValueKind kind = cache_state_.stack_state.back().kind(); 468 LiftoffRegister reg = PopToRegister(pinned); 469 if (cache_state()->is_free(reg)) return reg; 470 471 pinned.set(reg); 472 LiftoffRegister new_reg = GetUnusedRegister(reg.reg_class(), pinned); 473 Move(new_reg, reg, kind); 474 return new_reg; 475 } 476 477 // Returns the register which holds the value of stack slot {index}. If the 478 // value is not stored in a register yet, a register is allocated for it. The 479 // register is then assigned to the stack slot. The value stack height is not 480 // modified. The top of the stack is index 0, i.e. {PopToRegister()} and 481 // {PeekToRegister(0)} should result in the same register. 482 // When the value is finally popped, the use counter of its register has to be 483 // decremented. This can be done by popping the value with {DropValues}. 484 LiftoffRegister PeekToRegister(int index, LiftoffRegList pinned); 485 486 void DropValues(int count); 487 488 // Careful: this indexes "from the other end", i.e. depth=0 is the value 489 // at the bottom of the stack! 490 void DropValue(int depth); 491 492 // Ensure that the loop inputs are either in a register or spilled to the 493 // stack, so that we can merge different values on the back-edge. 494 void PrepareLoopArgs(int num); 495 496 V8_INLINE static int NextSpillOffset(ValueKind kind, int top_spill_offset) { 497 int offset = top_spill_offset + SlotSizeForType(kind); 498 if (NeedsAlignment(kind)) { 499 offset = RoundUp(offset, SlotSizeForType(kind)); 500 } 501 return offset; 502 } 503 504 int NextSpillOffset(ValueKind kind) { 505 return NextSpillOffset(kind, TopSpillOffset()); 506 } 507 508 int TopSpillOffset() const { 509 return cache_state_.stack_state.empty() 510 ? StaticStackFrameSize() 511 : cache_state_.stack_state.back().offset(); 512 } 513 514 void PushRegister(ValueKind kind, LiftoffRegister reg) { 515 DCHECK_EQ(reg_class_for(kind), reg.reg_class()); 516 cache_state_.inc_used(reg); 517 cache_state_.stack_state.emplace_back(kind, reg, NextSpillOffset(kind)); 518 } 519 520 // Assumes that the exception is in {kReturnRegister0}. This is where the 521 // exception is stored by the unwinder after a throwing call. 522 void PushException() { 523 LiftoffRegister reg{kReturnRegister0}; 524 // This is used after a call, so {kReturnRegister0} is not used yet. 525 DCHECK(cache_state_.is_free(reg)); 526 cache_state_.inc_used(reg); 527 cache_state_.stack_state.emplace_back(kRef, reg, NextSpillOffset(kRef)); 528 } 529 530 void PushConstant(ValueKind kind, int32_t i32_const) { 531 DCHECK(kind == kI32 || kind == kI64); 532 cache_state_.stack_state.emplace_back(kind, i32_const, 533 NextSpillOffset(kind)); 534 } 535 536 void PushStack(ValueKind kind) { 537 cache_state_.stack_state.emplace_back(kind, NextSpillOffset(kind)); 538 } 539 540 void SpillRegister(LiftoffRegister); 541 542 uint32_t GetNumUses(LiftoffRegister reg) const { 543 return cache_state_.get_use_count(reg); 544 } 545 546 // Get an unused register for class {rc}, reusing one of {try_first} if 547 // possible. 548 LiftoffRegister GetUnusedRegister( 549 RegClass rc, std::initializer_list<LiftoffRegister> try_first, 550 LiftoffRegList pinned) { 551 for (LiftoffRegister reg : try_first) { 552 DCHECK_EQ(reg.reg_class(), rc); 553 if (cache_state_.is_free(reg)) return reg; 554 } 555 return GetUnusedRegister(rc, pinned); 556 } 557 558 // Get an unused register for class {rc}, potentially spilling to free one. 559 LiftoffRegister GetUnusedRegister(RegClass rc, LiftoffRegList pinned) { 560 if (kNeedI64RegPair && rc == kGpRegPair) { 561 LiftoffRegList candidates = kGpCacheRegList.MaskOut(pinned); 562 Register low = candidates.clear(GetUnusedRegister(candidates)).gp(); 563 Register high = GetUnusedRegister(candidates).gp(); 564 return LiftoffRegister::ForPair(low, high); 565 } else if (kNeedS128RegPair && rc == kFpRegPair) { 566 // kFpRegPair specific logic here because we need adjacent registers, not 567 // just any two registers (like kGpRegPair). 568 if (cache_state_.has_unused_register(rc, pinned)) { 569 return cache_state_.unused_register(rc, pinned); 570 } 571 DoubleRegister low_fp = SpillAdjacentFpRegisters(pinned).fp(); 572 return LiftoffRegister::ForFpPair(low_fp); 573 } 574 DCHECK(rc == kGpReg || rc == kFpReg); 575 LiftoffRegList candidates = GetCacheRegList(rc).MaskOut(pinned); 576 return GetUnusedRegister(candidates); 577 } 578 579 // Get an unused register of {candidates}, potentially spilling to free one. 580 LiftoffRegister GetUnusedRegister(LiftoffRegList candidates) { 581 DCHECK(!candidates.is_empty()); 582 if (cache_state_.has_unused_register(candidates)) { 583 return cache_state_.unused_register(candidates); 584 } 585 if (cache_state_.has_volatile_register(candidates)) { 586 return cache_state_.take_volatile_register(candidates); 587 } 588 return SpillOneRegister(candidates); 589 } 590 591 void MaterializeMergedConstants(uint32_t arity); 592 593 enum JumpDirection { kForwardJump, kBackwardJump }; 594 void MergeFullStackWith(CacheState& target, const CacheState& source); 595 void MergeStackWith(CacheState& target, uint32_t arity, JumpDirection); 596 597 void Spill(VarState* slot); 598 void SpillLocals(); 599 void SpillAllRegisters(); 600 601 // Clear any uses of {reg} in both the cache and in {possible_uses}. 602 // Any use in the stack is spilled. If any register in {possible_uses} matches 603 // {reg}, then the content of {reg} is moved to a new temporary register, and 604 // all matches in {possible_uses} are rewritten to that temporary register. 605 void ClearRegister(Register reg, 606 std::initializer_list<Register*> possible_uses, 607 LiftoffRegList pinned); 608 609 // Spills all passed registers. 610 template <typename... Regs> 611 void SpillRegisters(Regs... regs) { 612 for (LiftoffRegister r : {LiftoffRegister(regs)...}) { 613 if (cache_state_.is_free(r)) continue; 614 if (r.is_gp() && cache_state_.cached_instance == r.gp()) { 615 cache_state_.ClearCachedInstanceRegister(); 616 } else if (r.is_gp() && cache_state_.cached_mem_start == r.gp()) { 617 cache_state_.ClearCachedMemStartRegister(); 618 } else { 619 SpillRegister(r); 620 } 621 } 622 } 623 624 // Call this method whenever spilling something, such that the number of used 625 // spill slot can be tracked and the stack frame will be allocated big enough. 626 void RecordUsedSpillOffset(int offset) { 627 if (offset >= max_used_spill_offset_) max_used_spill_offset_ = offset; 628 } 629 630 void RecordOolSpillSpaceSize(int size) { 631 if (size > ool_spill_space_size_) ool_spill_space_size_ = size; 632 } 633 634 // Load parameters into the right registers / stack slots for the call. 635 void PrepareBuiltinCall(const ValueKindSig* sig, 636 compiler::CallDescriptor* call_descriptor, 637 std::initializer_list<VarState> params); 638 639 // Load parameters into the right registers / stack slots for the call. 640 // Move {*target} into another register if needed and update {*target} to that 641 // register, or {no_reg} if target was spilled to the stack. 642 void PrepareCall(const ValueKindSig*, compiler::CallDescriptor*, 643 Register* target = nullptr, 644 Register* target_instance = nullptr); 645 // Process return values of the call. 646 void FinishCall(const ValueKindSig*, compiler::CallDescriptor*); 647 648 // Move {src} into {dst}. {src} and {dst} must be different. 649 void Move(LiftoffRegister dst, LiftoffRegister src, ValueKind); 650 651 // Parallel register move: For a list of tuples <dst, src, kind>, move the 652 // {src} register of kind {kind} into {dst}. If {src} equals {dst}, ignore 653 // that tuple. 654 struct ParallelRegisterMoveTuple { 655 LiftoffRegister dst; 656 LiftoffRegister src; 657 ValueKind kind; 658 template <typename Dst, typename Src> 659 ParallelRegisterMoveTuple(Dst dst, Src src, ValueKind kind) 660 : dst(dst), src(src), kind(kind) {} 661 }; 662 663 void ParallelRegisterMove(base::Vector<const ParallelRegisterMoveTuple>); 664 665 void ParallelRegisterMove( 666 std::initializer_list<ParallelRegisterMoveTuple> moves) { 667 ParallelRegisterMove(base::VectorOf(moves)); 668 } 669 670 void MoveToReturnLocations(const FunctionSig*, 671 compiler::CallDescriptor* descriptor); 672 673#ifdef ENABLE_SLOW_DCHECKS 674 // Validate that the register use counts reflect the state of the cache. 675 bool ValidateCacheState() const; 676#endif 677 678 //////////////////////////////////// 679 // Platform-specific part. // 680 //////////////////////////////////// 681 682 // This function emits machine code to prepare the stack frame, before the 683 // size of the stack frame is known. It returns an offset in the machine code 684 // which can later be patched (via {PatchPrepareStackFrame)} when the size of 685 // the frame is known. 686 inline int PrepareStackFrame(); 687 inline void PrepareTailCall(int num_callee_stack_params, 688 int stack_param_delta); 689 inline void AlignFrameSize(); 690 inline void PatchPrepareStackFrame(int offset, SafepointTableBuilder*); 691 inline void FinishCode(); 692 inline void AbortCompilation(); 693 inline static constexpr int StaticStackFrameSize(); 694 inline static int SlotSizeForType(ValueKind kind); 695 inline static bool NeedsAlignment(ValueKind kind); 696 697 inline void LoadConstant(LiftoffRegister, WasmValue, 698 RelocInfo::Mode rmode = RelocInfo::NO_INFO); 699 inline void LoadInstanceFromFrame(Register dst); 700 inline void LoadFromInstance(Register dst, Register instance, int offset, 701 int size); 702 inline void LoadTaggedPointerFromInstance(Register dst, Register instance, 703 int offset); 704 inline void LoadExternalPointer(Register dst, Register instance, int offset, 705 ExternalPointerTag tag, 706 Register isolate_root); 707 inline void SpillInstance(Register instance); 708 inline void ResetOSRTarget(); 709 inline void LoadTaggedPointer(Register dst, Register src_addr, 710 Register offset_reg, int32_t offset_imm, 711 LiftoffRegList pinned); 712 inline void LoadFullPointer(Register dst, Register src_addr, 713 int32_t offset_imm); 714 enum SkipWriteBarrier : bool { 715 kSkipWriteBarrier = true, 716 kNoSkipWriteBarrier = false 717 }; 718 inline void StoreTaggedPointer(Register dst_addr, Register offset_reg, 719 int32_t offset_imm, LiftoffRegister src, 720 LiftoffRegList pinned, 721 SkipWriteBarrier = kNoSkipWriteBarrier); 722 void LoadFixedArrayLengthAsInt32(LiftoffRegister dst, Register array, 723 LiftoffRegList pinned) { 724 int offset = FixedArray::kLengthOffset - kHeapObjectTag; 725 LoadSmiAsInt32(dst, array, offset, pinned); 726 } 727 void LoadSmiAsInt32(LiftoffRegister dst, Register src_addr, int32_t offset, 728 LiftoffRegList pinned) { 729 if (SmiValuesAre32Bits()) { 730#if V8_TARGET_LITTLE_ENDIAN 731 DCHECK_EQ(kSmiShiftSize + kSmiTagSize, 4 * kBitsPerByte); 732 offset += 4; 733#endif 734 Load(dst, src_addr, no_reg, offset, LoadType::kI32Load, pinned); 735 } else { 736 DCHECK(SmiValuesAre31Bits()); 737 Load(dst, src_addr, no_reg, offset, LoadType::kI32Load, pinned); 738 emit_i32_sari(dst.gp(), dst.gp(), kSmiTagSize); 739 } 740 } 741 inline void IncrementSmi(LiftoffRegister dst, int offset); 742 inline void Load(LiftoffRegister dst, Register src_addr, Register offset_reg, 743 uintptr_t offset_imm, LoadType type, LiftoffRegList pinned, 744 uint32_t* protected_load_pc = nullptr, 745 bool is_load_mem = false, bool i64_offset = false); 746 inline void Store(Register dst_addr, Register offset_reg, 747 uintptr_t offset_imm, LiftoffRegister src, StoreType type, 748 LiftoffRegList pinned, 749 uint32_t* protected_store_pc = nullptr, 750 bool is_store_mem = false); 751 inline void AtomicLoad(LiftoffRegister dst, Register src_addr, 752 Register offset_reg, uintptr_t offset_imm, 753 LoadType type, LiftoffRegList pinned); 754 inline void AtomicStore(Register dst_addr, Register offset_reg, 755 uintptr_t offset_imm, LiftoffRegister src, 756 StoreType type, LiftoffRegList pinned); 757 758 inline void AtomicAdd(Register dst_addr, Register offset_reg, 759 uintptr_t offset_imm, LiftoffRegister value, 760 LiftoffRegister result, StoreType type); 761 762 inline void AtomicSub(Register dst_addr, Register offset_reg, 763 uintptr_t offset_imm, LiftoffRegister value, 764 LiftoffRegister result, StoreType type); 765 766 inline void AtomicAnd(Register dst_addr, Register offset_reg, 767 uintptr_t offset_imm, LiftoffRegister value, 768 LiftoffRegister result, StoreType type); 769 770 inline void AtomicOr(Register dst_addr, Register offset_reg, 771 uintptr_t offset_imm, LiftoffRegister value, 772 LiftoffRegister result, StoreType type); 773 774 inline void AtomicXor(Register dst_addr, Register offset_reg, 775 uintptr_t offset_imm, LiftoffRegister value, 776 LiftoffRegister result, StoreType type); 777 778 inline void AtomicExchange(Register dst_addr, Register offset_reg, 779 uintptr_t offset_imm, LiftoffRegister value, 780 LiftoffRegister result, StoreType type); 781 782 inline void AtomicCompareExchange(Register dst_addr, Register offset_reg, 783 uintptr_t offset_imm, 784 LiftoffRegister expected, 785 LiftoffRegister new_value, 786 LiftoffRegister value, StoreType type); 787 788 inline void AtomicFence(); 789 790 inline void LoadCallerFrameSlot(LiftoffRegister, uint32_t caller_slot_idx, 791 ValueKind); 792 inline void StoreCallerFrameSlot(LiftoffRegister, uint32_t caller_slot_idx, 793 ValueKind); 794 inline void LoadReturnStackSlot(LiftoffRegister, int offset, ValueKind); 795 inline void MoveStackValue(uint32_t dst_offset, uint32_t src_offset, 796 ValueKind); 797 798 inline void Move(Register dst, Register src, ValueKind); 799 inline void Move(DoubleRegister dst, DoubleRegister src, ValueKind); 800 801 inline void Spill(int offset, LiftoffRegister, ValueKind); 802 inline void Spill(int offset, WasmValue); 803 inline void Fill(LiftoffRegister, int offset, ValueKind); 804 // Only used on 32-bit systems: Fill a register from a "half stack slot", i.e. 805 // 4 bytes on the stack holding half of a 64-bit value. 806 inline void FillI64Half(Register, int offset, RegPairHalf); 807 inline void FillStackSlotsWithZero(int start, int size); 808 809 // i32 binops. 810 inline void emit_i32_add(Register dst, Register lhs, Register rhs); 811 inline void emit_i32_addi(Register dst, Register lhs, int32_t imm); 812 inline void emit_i32_sub(Register dst, Register lhs, Register rhs); 813 inline void emit_i32_subi(Register dst, Register lhs, int32_t imm); 814 inline void emit_i32_mul(Register dst, Register lhs, Register rhs); 815 inline void emit_i32_divs(Register dst, Register lhs, Register rhs, 816 Label* trap_div_by_zero, 817 Label* trap_div_unrepresentable); 818 inline void emit_i32_divu(Register dst, Register lhs, Register rhs, 819 Label* trap_div_by_zero); 820 inline void emit_i32_rems(Register dst, Register lhs, Register rhs, 821 Label* trap_rem_by_zero); 822 inline void emit_i32_remu(Register dst, Register lhs, Register rhs, 823 Label* trap_rem_by_zero); 824 inline void emit_i32_and(Register dst, Register lhs, Register rhs); 825 inline void emit_i32_andi(Register dst, Register lhs, int32_t imm); 826 inline void emit_i32_or(Register dst, Register lhs, Register rhs); 827 inline void emit_i32_ori(Register dst, Register lhs, int32_t imm); 828 inline void emit_i32_xor(Register dst, Register lhs, Register rhs); 829 inline void emit_i32_xori(Register dst, Register lhs, int32_t imm); 830 inline void emit_i32_shl(Register dst, Register src, Register amount); 831 inline void emit_i32_shli(Register dst, Register src, int32_t amount); 832 inline void emit_i32_sar(Register dst, Register src, Register amount); 833 inline void emit_i32_sari(Register dst, Register src, int32_t amount); 834 inline void emit_i32_shr(Register dst, Register src, Register amount); 835 inline void emit_i32_shri(Register dst, Register src, int32_t amount); 836 837 // i32 unops. 838 inline void emit_i32_clz(Register dst, Register src); 839 inline void emit_i32_ctz(Register dst, Register src); 840 inline bool emit_i32_popcnt(Register dst, Register src); 841 842 // i64 binops. 843 inline void emit_i64_add(LiftoffRegister dst, LiftoffRegister lhs, 844 LiftoffRegister rhs); 845 inline void emit_i64_addi(LiftoffRegister dst, LiftoffRegister lhs, 846 int64_t imm); 847 inline void emit_i64_sub(LiftoffRegister dst, LiftoffRegister lhs, 848 LiftoffRegister rhs); 849 inline void emit_i64_mul(LiftoffRegister dst, LiftoffRegister lhs, 850 LiftoffRegister rhs); 851 inline bool emit_i64_divs(LiftoffRegister dst, LiftoffRegister lhs, 852 LiftoffRegister rhs, Label* trap_div_by_zero, 853 Label* trap_div_unrepresentable); 854 inline bool emit_i64_divu(LiftoffRegister dst, LiftoffRegister lhs, 855 LiftoffRegister rhs, Label* trap_div_by_zero); 856 inline bool emit_i64_rems(LiftoffRegister dst, LiftoffRegister lhs, 857 LiftoffRegister rhs, Label* trap_rem_by_zero); 858 inline bool emit_i64_remu(LiftoffRegister dst, LiftoffRegister lhs, 859 LiftoffRegister rhs, Label* trap_rem_by_zero); 860 inline void emit_i64_and(LiftoffRegister dst, LiftoffRegister lhs, 861 LiftoffRegister rhs); 862 inline void emit_i64_andi(LiftoffRegister dst, LiftoffRegister lhs, 863 int32_t imm); 864 inline void emit_i64_or(LiftoffRegister dst, LiftoffRegister lhs, 865 LiftoffRegister rhs); 866 inline void emit_i64_ori(LiftoffRegister dst, LiftoffRegister lhs, 867 int32_t imm); 868 inline void emit_i64_xor(LiftoffRegister dst, LiftoffRegister lhs, 869 LiftoffRegister rhs); 870 inline void emit_i64_xori(LiftoffRegister dst, LiftoffRegister lhs, 871 int32_t imm); 872 inline void emit_i64_shl(LiftoffRegister dst, LiftoffRegister src, 873 Register amount); 874 inline void emit_i64_shli(LiftoffRegister dst, LiftoffRegister src, 875 int32_t amount); 876 inline void emit_i64_sar(LiftoffRegister dst, LiftoffRegister src, 877 Register amount); 878 inline void emit_i64_sari(LiftoffRegister dst, LiftoffRegister src, 879 int32_t amount); 880 inline void emit_i64_shr(LiftoffRegister dst, LiftoffRegister src, 881 Register amount); 882 inline void emit_i64_shri(LiftoffRegister dst, LiftoffRegister src, 883 int32_t amount); 884 885 // i64 unops. 886 inline void emit_i64_clz(LiftoffRegister dst, LiftoffRegister src); 887 inline void emit_i64_ctz(LiftoffRegister dst, LiftoffRegister src); 888 inline bool emit_i64_popcnt(LiftoffRegister dst, LiftoffRegister src); 889 890 inline void emit_u32_to_uintptr(Register dst, Register src); 891 892 void emit_ptrsize_add(Register dst, Register lhs, Register rhs) { 893 if (kSystemPointerSize == 8) { 894 emit_i64_add(LiftoffRegister(dst), LiftoffRegister(lhs), 895 LiftoffRegister(rhs)); 896 } else { 897 emit_i32_add(dst, lhs, rhs); 898 } 899 } 900 void emit_ptrsize_sub(Register dst, Register lhs, Register rhs) { 901 if (kSystemPointerSize == 8) { 902 emit_i64_sub(LiftoffRegister(dst), LiftoffRegister(lhs), 903 LiftoffRegister(rhs)); 904 } else { 905 emit_i32_sub(dst, lhs, rhs); 906 } 907 } 908 void emit_ptrsize_and(Register dst, Register lhs, Register rhs) { 909 if (kSystemPointerSize == 8) { 910 emit_i64_and(LiftoffRegister(dst), LiftoffRegister(lhs), 911 LiftoffRegister(rhs)); 912 } else { 913 emit_i32_and(dst, lhs, rhs); 914 } 915 } 916 void emit_ptrsize_shri(Register dst, Register src, int amount) { 917 if (kSystemPointerSize == 8) { 918 emit_i64_shri(LiftoffRegister(dst), LiftoffRegister(src), amount); 919 } else { 920 emit_i32_shri(dst, src, amount); 921 } 922 } 923 924 void emit_ptrsize_addi(Register dst, Register lhs, intptr_t imm) { 925 if (kSystemPointerSize == 8) { 926 emit_i64_addi(LiftoffRegister(dst), LiftoffRegister(lhs), imm); 927 } else { 928 emit_i32_addi(dst, lhs, static_cast<int32_t>(imm)); 929 } 930 } 931 932 void emit_ptrsize_set_cond(LiftoffCondition condition, Register dst, 933 LiftoffRegister lhs, LiftoffRegister rhs) { 934 if (kSystemPointerSize == 8) { 935 emit_i64_set_cond(condition, dst, lhs, rhs); 936 } else { 937 emit_i32_set_cond(condition, dst, lhs.gp(), rhs.gp()); 938 } 939 } 940 941 void emit_ptrsize_zeroextend_i32(Register dst, Register src) { 942 if (kSystemPointerSize == 8) { 943 emit_type_conversion(kExprI64UConvertI32, LiftoffRegister(dst), 944 LiftoffRegister(src)); 945 } else if (dst != src) { 946 Move(dst, src, kI32); 947 } 948 } 949 950 // f32 binops. 951 inline void emit_f32_add(DoubleRegister dst, DoubleRegister lhs, 952 DoubleRegister rhs); 953 inline void emit_f32_sub(DoubleRegister dst, DoubleRegister lhs, 954 DoubleRegister rhs); 955 inline void emit_f32_mul(DoubleRegister dst, DoubleRegister lhs, 956 DoubleRegister rhs); 957 inline void emit_f32_div(DoubleRegister dst, DoubleRegister lhs, 958 DoubleRegister rhs); 959 inline void emit_f32_min(DoubleRegister dst, DoubleRegister lhs, 960 DoubleRegister rhs); 961 inline void emit_f32_max(DoubleRegister dst, DoubleRegister lhs, 962 DoubleRegister rhs); 963 inline void emit_f32_copysign(DoubleRegister dst, DoubleRegister lhs, 964 DoubleRegister rhs); 965 966 // f32 unops. 967 inline void emit_f32_abs(DoubleRegister dst, DoubleRegister src); 968 inline void emit_f32_neg(DoubleRegister dst, DoubleRegister src); 969 inline bool emit_f32_ceil(DoubleRegister dst, DoubleRegister src); 970 inline bool emit_f32_floor(DoubleRegister dst, DoubleRegister src); 971 inline bool emit_f32_trunc(DoubleRegister dst, DoubleRegister src); 972 inline bool emit_f32_nearest_int(DoubleRegister dst, DoubleRegister src); 973 inline void emit_f32_sqrt(DoubleRegister dst, DoubleRegister src); 974 975 // f64 binops. 976 inline void emit_f64_add(DoubleRegister dst, DoubleRegister lhs, 977 DoubleRegister rhs); 978 inline void emit_f64_sub(DoubleRegister dst, DoubleRegister lhs, 979 DoubleRegister rhs); 980 inline void emit_f64_mul(DoubleRegister dst, DoubleRegister lhs, 981 DoubleRegister rhs); 982 inline void emit_f64_div(DoubleRegister dst, DoubleRegister lhs, 983 DoubleRegister rhs); 984 inline void emit_f64_min(DoubleRegister dst, DoubleRegister lhs, 985 DoubleRegister rhs); 986 inline void emit_f64_max(DoubleRegister dst, DoubleRegister lhs, 987 DoubleRegister rhs); 988 inline void emit_f64_copysign(DoubleRegister dst, DoubleRegister lhs, 989 DoubleRegister rhs); 990 991 // f64 unops. 992 inline void emit_f64_abs(DoubleRegister dst, DoubleRegister src); 993 inline void emit_f64_neg(DoubleRegister dst, DoubleRegister src); 994 inline bool emit_f64_ceil(DoubleRegister dst, DoubleRegister src); 995 inline bool emit_f64_floor(DoubleRegister dst, DoubleRegister src); 996 inline bool emit_f64_trunc(DoubleRegister dst, DoubleRegister src); 997 inline bool emit_f64_nearest_int(DoubleRegister dst, DoubleRegister src); 998 inline void emit_f64_sqrt(DoubleRegister dst, DoubleRegister src); 999 1000 inline bool emit_type_conversion(WasmOpcode opcode, LiftoffRegister dst, 1001 LiftoffRegister src, Label* trap = nullptr); 1002 1003 inline void emit_i32_signextend_i8(Register dst, Register src); 1004 inline void emit_i32_signextend_i16(Register dst, Register src); 1005 inline void emit_i64_signextend_i8(LiftoffRegister dst, LiftoffRegister src); 1006 inline void emit_i64_signextend_i16(LiftoffRegister dst, LiftoffRegister src); 1007 inline void emit_i64_signextend_i32(LiftoffRegister dst, LiftoffRegister src); 1008 1009 inline void emit_jump(Label*); 1010 inline void emit_jump(Register); 1011 1012 inline void emit_cond_jump(LiftoffCondition, Label*, ValueKind value, 1013 Register lhs, Register rhs = no_reg); 1014 inline void emit_i32_cond_jumpi(LiftoffCondition, Label*, Register lhs, 1015 int imm); 1016 inline void emit_i32_subi_jump_negative(Register value, int subtrahend, 1017 Label* result_negative); 1018 // Set {dst} to 1 if condition holds, 0 otherwise. 1019 inline void emit_i32_eqz(Register dst, Register src); 1020 inline void emit_i32_set_cond(LiftoffCondition, Register dst, Register lhs, 1021 Register rhs); 1022 inline void emit_i64_eqz(Register dst, LiftoffRegister src); 1023 inline void emit_i64_set_cond(LiftoffCondition condition, Register dst, 1024 LiftoffRegister lhs, LiftoffRegister rhs); 1025 inline void emit_f32_set_cond(LiftoffCondition condition, Register dst, 1026 DoubleRegister lhs, DoubleRegister rhs); 1027 inline void emit_f64_set_cond(LiftoffCondition condition, Register dst, 1028 DoubleRegister lhs, DoubleRegister rhs); 1029 1030 // Optional select support: Returns false if generic code (via branches) 1031 // should be emitted instead. 1032 inline bool emit_select(LiftoffRegister dst, Register condition, 1033 LiftoffRegister true_value, 1034 LiftoffRegister false_value, ValueKind kind); 1035 1036 enum SmiCheckMode { kJumpOnSmi, kJumpOnNotSmi }; 1037 inline void emit_smi_check(Register obj, Label* target, SmiCheckMode mode); 1038 1039 inline void LoadTransform(LiftoffRegister dst, Register src_addr, 1040 Register offset_reg, uintptr_t offset_imm, 1041 LoadType type, LoadTransformationKind transform, 1042 uint32_t* protected_load_pc); 1043 inline void LoadLane(LiftoffRegister dst, LiftoffRegister src, Register addr, 1044 Register offset_reg, uintptr_t offset_imm, LoadType type, 1045 uint8_t lane, uint32_t* protected_load_pc); 1046 inline void StoreLane(Register dst, Register offset, uintptr_t offset_imm, 1047 LiftoffRegister src, StoreType type, uint8_t lane, 1048 uint32_t* protected_store_pc); 1049 inline void emit_i8x16_shuffle(LiftoffRegister dst, LiftoffRegister lhs, 1050 LiftoffRegister rhs, const uint8_t shuffle[16], 1051 bool is_swizzle); 1052 inline void emit_i8x16_swizzle(LiftoffRegister dst, LiftoffRegister lhs, 1053 LiftoffRegister rhs); 1054 inline void emit_i8x16_popcnt(LiftoffRegister dst, LiftoffRegister src); 1055 inline void emit_i8x16_splat(LiftoffRegister dst, LiftoffRegister src); 1056 inline void emit_i16x8_splat(LiftoffRegister dst, LiftoffRegister src); 1057 inline void emit_i32x4_splat(LiftoffRegister dst, LiftoffRegister src); 1058 inline void emit_i64x2_splat(LiftoffRegister dst, LiftoffRegister src); 1059 inline void emit_f32x4_splat(LiftoffRegister dst, LiftoffRegister src); 1060 inline void emit_f64x2_splat(LiftoffRegister dst, LiftoffRegister src); 1061 inline void emit_i8x16_eq(LiftoffRegister dst, LiftoffRegister lhs, 1062 LiftoffRegister rhs); 1063 inline void emit_i8x16_ne(LiftoffRegister dst, LiftoffRegister lhs, 1064 LiftoffRegister rhs); 1065 inline void emit_i8x16_gt_s(LiftoffRegister dst, LiftoffRegister lhs, 1066 LiftoffRegister rhs); 1067 inline void emit_i8x16_gt_u(LiftoffRegister dst, LiftoffRegister lhs, 1068 LiftoffRegister rhs); 1069 inline void emit_i8x16_ge_s(LiftoffRegister dst, LiftoffRegister lhs, 1070 LiftoffRegister rhs); 1071 inline void emit_i8x16_ge_u(LiftoffRegister dst, LiftoffRegister lhs, 1072 LiftoffRegister rhs); 1073 inline void emit_i16x8_eq(LiftoffRegister dst, LiftoffRegister lhs, 1074 LiftoffRegister rhs); 1075 inline void emit_i16x8_ne(LiftoffRegister dst, LiftoffRegister lhs, 1076 LiftoffRegister rhs); 1077 inline void emit_i16x8_gt_s(LiftoffRegister dst, LiftoffRegister lhs, 1078 LiftoffRegister rhs); 1079 inline void emit_i16x8_gt_u(LiftoffRegister dst, LiftoffRegister lhs, 1080 LiftoffRegister rhs); 1081 inline void emit_i16x8_ge_s(LiftoffRegister dst, LiftoffRegister lhs, 1082 LiftoffRegister rhs); 1083 inline void emit_i16x8_ge_u(LiftoffRegister dst, LiftoffRegister lhs, 1084 LiftoffRegister rhs); 1085 inline void emit_i32x4_eq(LiftoffRegister dst, LiftoffRegister lhs, 1086 LiftoffRegister rhs); 1087 inline void emit_i32x4_ne(LiftoffRegister dst, LiftoffRegister lhs, 1088 LiftoffRegister rhs); 1089 inline void emit_i32x4_gt_s(LiftoffRegister dst, LiftoffRegister lhs, 1090 LiftoffRegister rhs); 1091 inline void emit_i32x4_gt_u(LiftoffRegister dst, LiftoffRegister lhs, 1092 LiftoffRegister rhs); 1093 inline void emit_i32x4_ge_s(LiftoffRegister dst, LiftoffRegister lhs, 1094 LiftoffRegister rhs); 1095 inline void emit_i32x4_ge_u(LiftoffRegister dst, LiftoffRegister lhs, 1096 LiftoffRegister rhs); 1097 inline void emit_i64x2_eq(LiftoffRegister dst, LiftoffRegister lhs, 1098 LiftoffRegister rhs); 1099 inline void emit_i64x2_ne(LiftoffRegister dst, LiftoffRegister lhs, 1100 LiftoffRegister rhs); 1101 inline void emit_i64x2_gt_s(LiftoffRegister dst, LiftoffRegister lhs, 1102 LiftoffRegister rhs); 1103 inline void emit_i64x2_ge_s(LiftoffRegister dst, LiftoffRegister lhs, 1104 LiftoffRegister rhs); 1105 inline void emit_f32x4_eq(LiftoffRegister dst, LiftoffRegister lhs, 1106 LiftoffRegister rhs); 1107 inline void emit_f32x4_ne(LiftoffRegister dst, LiftoffRegister lhs, 1108 LiftoffRegister rhs); 1109 inline void emit_f32x4_lt(LiftoffRegister dst, LiftoffRegister lhs, 1110 LiftoffRegister rhs); 1111 inline void emit_f32x4_le(LiftoffRegister dst, LiftoffRegister lhs, 1112 LiftoffRegister rhs); 1113 inline void emit_f64x2_eq(LiftoffRegister dst, LiftoffRegister lhs, 1114 LiftoffRegister rhs); 1115 inline void emit_f64x2_ne(LiftoffRegister dst, LiftoffRegister lhs, 1116 LiftoffRegister rhs); 1117 inline void emit_f64x2_lt(LiftoffRegister dst, LiftoffRegister lhs, 1118 LiftoffRegister rhs); 1119 inline void emit_f64x2_le(LiftoffRegister dst, LiftoffRegister lhs, 1120 LiftoffRegister rhs); 1121 inline void emit_s128_const(LiftoffRegister dst, const uint8_t imms[16]); 1122 inline void emit_s128_not(LiftoffRegister dst, LiftoffRegister src); 1123 inline void emit_s128_and(LiftoffRegister dst, LiftoffRegister lhs, 1124 LiftoffRegister rhs); 1125 inline void emit_s128_or(LiftoffRegister dst, LiftoffRegister lhs, 1126 LiftoffRegister rhs); 1127 inline void emit_s128_xor(LiftoffRegister dst, LiftoffRegister lhs, 1128 LiftoffRegister rhs); 1129 inline void emit_s128_select(LiftoffRegister dst, LiftoffRegister src1, 1130 LiftoffRegister src2, LiftoffRegister mask); 1131 inline void emit_i8x16_neg(LiftoffRegister dst, LiftoffRegister src); 1132 inline void emit_v128_anytrue(LiftoffRegister dst, LiftoffRegister src); 1133 inline void emit_i8x16_alltrue(LiftoffRegister dst, LiftoffRegister src); 1134 inline void emit_i8x16_bitmask(LiftoffRegister dst, LiftoffRegister src); 1135 inline void emit_i8x16_shl(LiftoffRegister dst, LiftoffRegister lhs, 1136 LiftoffRegister rhs); 1137 inline void emit_i8x16_shli(LiftoffRegister dst, LiftoffRegister lhs, 1138 int32_t rhs); 1139 inline void emit_i8x16_shr_s(LiftoffRegister dst, LiftoffRegister lhs, 1140 LiftoffRegister rhs); 1141 inline void emit_i8x16_shri_s(LiftoffRegister dst, LiftoffRegister lhs, 1142 int32_t rhs); 1143 inline void emit_i8x16_shr_u(LiftoffRegister dst, LiftoffRegister lhs, 1144 LiftoffRegister rhs); 1145 inline void emit_i8x16_shri_u(LiftoffRegister dst, LiftoffRegister lhs, 1146 int32_t rhs); 1147 inline void emit_i8x16_add(LiftoffRegister dst, LiftoffRegister lhs, 1148 LiftoffRegister rhs); 1149 inline void emit_i8x16_add_sat_s(LiftoffRegister dst, LiftoffRegister lhs, 1150 LiftoffRegister rhs); 1151 inline void emit_i8x16_add_sat_u(LiftoffRegister dst, LiftoffRegister lhs, 1152 LiftoffRegister rhs); 1153 inline void emit_i8x16_sub(LiftoffRegister dst, LiftoffRegister lhs, 1154 LiftoffRegister rhs); 1155 inline void emit_i8x16_sub_sat_s(LiftoffRegister dst, LiftoffRegister lhs, 1156 LiftoffRegister rhs); 1157 inline void emit_i8x16_sub_sat_u(LiftoffRegister dst, LiftoffRegister lhs, 1158 LiftoffRegister rhs); 1159 inline void emit_i8x16_min_s(LiftoffRegister dst, LiftoffRegister lhs, 1160 LiftoffRegister rhs); 1161 inline void emit_i8x16_min_u(LiftoffRegister dst, LiftoffRegister lhs, 1162 LiftoffRegister rhs); 1163 inline void emit_i8x16_max_s(LiftoffRegister dst, LiftoffRegister lhs, 1164 LiftoffRegister rhs); 1165 inline void emit_i8x16_max_u(LiftoffRegister dst, LiftoffRegister lhs, 1166 LiftoffRegister rhs); 1167 inline void emit_i16x8_neg(LiftoffRegister dst, LiftoffRegister src); 1168 inline void emit_i16x8_alltrue(LiftoffRegister dst, LiftoffRegister src); 1169 inline void emit_i16x8_bitmask(LiftoffRegister dst, LiftoffRegister src); 1170 inline void emit_i16x8_shl(LiftoffRegister dst, LiftoffRegister lhs, 1171 LiftoffRegister rhs); 1172 inline void emit_i16x8_shli(LiftoffRegister dst, LiftoffRegister lhs, 1173 int32_t rhs); 1174 inline void emit_i16x8_shr_s(LiftoffRegister dst, LiftoffRegister lhs, 1175 LiftoffRegister rhs); 1176 inline void emit_i16x8_shri_s(LiftoffRegister dst, LiftoffRegister lhs, 1177 int32_t rhs); 1178 inline void emit_i16x8_shr_u(LiftoffRegister dst, LiftoffRegister lhs, 1179 LiftoffRegister rhs); 1180 inline void emit_i16x8_shri_u(LiftoffRegister dst, LiftoffRegister lhs, 1181 int32_t rhs); 1182 inline void emit_i16x8_add(LiftoffRegister dst, LiftoffRegister lhs, 1183 LiftoffRegister rhs); 1184 inline void emit_i16x8_add_sat_s(LiftoffRegister dst, LiftoffRegister lhs, 1185 LiftoffRegister rhs); 1186 inline void emit_i16x8_add_sat_u(LiftoffRegister dst, LiftoffRegister lhs, 1187 LiftoffRegister rhs); 1188 inline void emit_i16x8_sub(LiftoffRegister dst, LiftoffRegister lhs, 1189 LiftoffRegister rhs); 1190 inline void emit_i16x8_sub_sat_s(LiftoffRegister dst, LiftoffRegister lhs, 1191 LiftoffRegister rhs); 1192 inline void emit_i16x8_sub_sat_u(LiftoffRegister dst, LiftoffRegister lhs, 1193 LiftoffRegister rhs); 1194 inline void emit_i16x8_mul(LiftoffRegister dst, LiftoffRegister lhs, 1195 LiftoffRegister rhs); 1196 inline void emit_i16x8_min_s(LiftoffRegister dst, LiftoffRegister lhs, 1197 LiftoffRegister rhs); 1198 inline void emit_i16x8_min_u(LiftoffRegister dst, LiftoffRegister lhs, 1199 LiftoffRegister rhs); 1200 inline void emit_i16x8_max_s(LiftoffRegister dst, LiftoffRegister lhs, 1201 LiftoffRegister rhs); 1202 inline void emit_i16x8_max_u(LiftoffRegister dst, LiftoffRegister lhs, 1203 LiftoffRegister rhs); 1204 inline void emit_i16x8_extadd_pairwise_i8x16_s(LiftoffRegister dst, 1205 LiftoffRegister src); 1206 inline void emit_i16x8_extadd_pairwise_i8x16_u(LiftoffRegister dst, 1207 LiftoffRegister src); 1208 inline void emit_i16x8_extmul_low_i8x16_s(LiftoffRegister dst, 1209 LiftoffRegister src1, 1210 LiftoffRegister src2); 1211 inline void emit_i16x8_extmul_low_i8x16_u(LiftoffRegister dst, 1212 LiftoffRegister src1, 1213 LiftoffRegister src2); 1214 inline void emit_i16x8_extmul_high_i8x16_s(LiftoffRegister dst, 1215 LiftoffRegister src1, 1216 LiftoffRegister src2); 1217 inline void emit_i16x8_extmul_high_i8x16_u(LiftoffRegister dst, 1218 LiftoffRegister src1, 1219 LiftoffRegister src2); 1220 inline void emit_i16x8_q15mulr_sat_s(LiftoffRegister dst, 1221 LiftoffRegister src1, 1222 LiftoffRegister src2); 1223 inline void emit_i32x4_neg(LiftoffRegister dst, LiftoffRegister src); 1224 inline void emit_i32x4_alltrue(LiftoffRegister dst, LiftoffRegister src); 1225 inline void emit_i32x4_bitmask(LiftoffRegister dst, LiftoffRegister src); 1226 inline void emit_i32x4_shl(LiftoffRegister dst, LiftoffRegister lhs, 1227 LiftoffRegister rhs); 1228 inline void emit_i32x4_shli(LiftoffRegister dst, LiftoffRegister lhs, 1229 int32_t rhs); 1230 inline void emit_i32x4_shr_s(LiftoffRegister dst, LiftoffRegister lhs, 1231 LiftoffRegister rhs); 1232 inline void emit_i32x4_shri_s(LiftoffRegister dst, LiftoffRegister lhs, 1233 int32_t rhs); 1234 inline void emit_i32x4_shr_u(LiftoffRegister dst, LiftoffRegister lhs, 1235 LiftoffRegister rhs); 1236 inline void emit_i32x4_shri_u(LiftoffRegister dst, LiftoffRegister lhs, 1237 int32_t rhs); 1238 inline void emit_i32x4_add(LiftoffRegister dst, LiftoffRegister lhs, 1239 LiftoffRegister rhs); 1240 inline void emit_i32x4_sub(LiftoffRegister dst, LiftoffRegister lhs, 1241 LiftoffRegister rhs); 1242 inline void emit_i32x4_mul(LiftoffRegister dst, LiftoffRegister lhs, 1243 LiftoffRegister rhs); 1244 inline void emit_i32x4_min_s(LiftoffRegister dst, LiftoffRegister lhs, 1245 LiftoffRegister rhs); 1246 inline void emit_i32x4_min_u(LiftoffRegister dst, LiftoffRegister lhs, 1247 LiftoffRegister rhs); 1248 inline void emit_i32x4_max_s(LiftoffRegister dst, LiftoffRegister lhs, 1249 LiftoffRegister rhs); 1250 inline void emit_i32x4_max_u(LiftoffRegister dst, LiftoffRegister lhs, 1251 LiftoffRegister rhs); 1252 inline void emit_i32x4_dot_i16x8_s(LiftoffRegister dst, LiftoffRegister lhs, 1253 LiftoffRegister rhs); 1254 inline void emit_i32x4_extadd_pairwise_i16x8_s(LiftoffRegister dst, 1255 LiftoffRegister src); 1256 inline void emit_i32x4_extadd_pairwise_i16x8_u(LiftoffRegister dst, 1257 LiftoffRegister src); 1258 inline void emit_i32x4_extmul_low_i16x8_s(LiftoffRegister dst, 1259 LiftoffRegister src1, 1260 LiftoffRegister src2); 1261 inline void emit_i32x4_extmul_low_i16x8_u(LiftoffRegister dst, 1262 LiftoffRegister src1, 1263 LiftoffRegister src2); 1264 inline void emit_i32x4_extmul_high_i16x8_s(LiftoffRegister dst, 1265 LiftoffRegister src1, 1266 LiftoffRegister src2); 1267 inline void emit_i32x4_extmul_high_i16x8_u(LiftoffRegister dst, 1268 LiftoffRegister src1, 1269 LiftoffRegister src2); 1270 inline void emit_i64x2_neg(LiftoffRegister dst, LiftoffRegister src); 1271 inline void emit_i64x2_alltrue(LiftoffRegister dst, LiftoffRegister src); 1272 inline void emit_i64x2_shl(LiftoffRegister dst, LiftoffRegister lhs, 1273 LiftoffRegister rhs); 1274 inline void emit_i64x2_shli(LiftoffRegister dst, LiftoffRegister lhs, 1275 int32_t rhs); 1276 inline void emit_i64x2_shr_s(LiftoffRegister dst, LiftoffRegister lhs, 1277 LiftoffRegister rhs); 1278 inline void emit_i64x2_shri_s(LiftoffRegister dst, LiftoffRegister lhs, 1279 int32_t rhs); 1280 inline void emit_i64x2_shr_u(LiftoffRegister dst, LiftoffRegister lhs, 1281 LiftoffRegister rhs); 1282 inline void emit_i64x2_shri_u(LiftoffRegister dst, LiftoffRegister lhs, 1283 int32_t rhs); 1284 inline void emit_i64x2_add(LiftoffRegister dst, LiftoffRegister lhs, 1285 LiftoffRegister rhs); 1286 inline void emit_i64x2_sub(LiftoffRegister dst, LiftoffRegister lhs, 1287 LiftoffRegister rhs); 1288 inline void emit_i64x2_mul(LiftoffRegister dst, LiftoffRegister lhs, 1289 LiftoffRegister rhs); 1290 inline void emit_i64x2_extmul_low_i32x4_s(LiftoffRegister dst, 1291 LiftoffRegister src1, 1292 LiftoffRegister src2); 1293 inline void emit_i64x2_extmul_low_i32x4_u(LiftoffRegister dst, 1294 LiftoffRegister src1, 1295 LiftoffRegister src2); 1296 inline void emit_i64x2_extmul_high_i32x4_s(LiftoffRegister dst, 1297 LiftoffRegister src1, 1298 LiftoffRegister src2); 1299 inline void emit_i64x2_extmul_high_i32x4_u(LiftoffRegister dst, 1300 LiftoffRegister src1, 1301 LiftoffRegister src2); 1302 inline void emit_i64x2_bitmask(LiftoffRegister dst, LiftoffRegister src); 1303 inline void emit_i64x2_sconvert_i32x4_low(LiftoffRegister dst, 1304 LiftoffRegister src); 1305 inline void emit_i64x2_sconvert_i32x4_high(LiftoffRegister dst, 1306 LiftoffRegister src); 1307 inline void emit_i64x2_uconvert_i32x4_low(LiftoffRegister dst, 1308 LiftoffRegister src); 1309 inline void emit_i64x2_uconvert_i32x4_high(LiftoffRegister dst, 1310 LiftoffRegister src); 1311 inline void emit_f32x4_abs(LiftoffRegister dst, LiftoffRegister src); 1312 inline void emit_f32x4_neg(LiftoffRegister dst, LiftoffRegister src); 1313 inline void emit_f32x4_sqrt(LiftoffRegister dst, LiftoffRegister src); 1314 inline bool emit_f32x4_ceil(LiftoffRegister dst, LiftoffRegister src); 1315 inline bool emit_f32x4_floor(LiftoffRegister dst, LiftoffRegister src); 1316 inline bool emit_f32x4_trunc(LiftoffRegister dst, LiftoffRegister src); 1317 inline bool emit_f32x4_nearest_int(LiftoffRegister dst, LiftoffRegister src); 1318 inline void emit_f32x4_add(LiftoffRegister dst, LiftoffRegister lhs, 1319 LiftoffRegister rhs); 1320 inline void emit_f32x4_sub(LiftoffRegister dst, LiftoffRegister lhs, 1321 LiftoffRegister rhs); 1322 inline void emit_f32x4_mul(LiftoffRegister dst, LiftoffRegister lhs, 1323 LiftoffRegister rhs); 1324 inline void emit_f32x4_div(LiftoffRegister dst, LiftoffRegister lhs, 1325 LiftoffRegister rhs); 1326 inline void emit_f32x4_min(LiftoffRegister dst, LiftoffRegister lhs, 1327 LiftoffRegister rhs); 1328 inline void emit_f32x4_max(LiftoffRegister dst, LiftoffRegister lhs, 1329 LiftoffRegister rhs); 1330 inline void emit_f32x4_pmin(LiftoffRegister dst, LiftoffRegister lhs, 1331 LiftoffRegister rhs); 1332 inline void emit_f32x4_pmax(LiftoffRegister dst, LiftoffRegister lhs, 1333 LiftoffRegister rhs); 1334 inline void emit_f64x2_abs(LiftoffRegister dst, LiftoffRegister src); 1335 inline void emit_f64x2_neg(LiftoffRegister dst, LiftoffRegister src); 1336 inline void emit_f64x2_sqrt(LiftoffRegister dst, LiftoffRegister src); 1337 inline bool emit_f64x2_ceil(LiftoffRegister dst, LiftoffRegister src); 1338 inline bool emit_f64x2_floor(LiftoffRegister dst, LiftoffRegister src); 1339 inline bool emit_f64x2_trunc(LiftoffRegister dst, LiftoffRegister src); 1340 inline bool emit_f64x2_nearest_int(LiftoffRegister dst, LiftoffRegister src); 1341 inline void emit_f64x2_add(LiftoffRegister dst, LiftoffRegister lhs, 1342 LiftoffRegister rhs); 1343 inline void emit_f64x2_sub(LiftoffRegister dst, LiftoffRegister lhs, 1344 LiftoffRegister rhs); 1345 inline void emit_f64x2_mul(LiftoffRegister dst, LiftoffRegister lhs, 1346 LiftoffRegister rhs); 1347 inline void emit_f64x2_div(LiftoffRegister dst, LiftoffRegister lhs, 1348 LiftoffRegister rhs); 1349 inline void emit_f64x2_min(LiftoffRegister dst, LiftoffRegister lhs, 1350 LiftoffRegister rhs); 1351 inline void emit_f64x2_max(LiftoffRegister dst, LiftoffRegister lhs, 1352 LiftoffRegister rhs); 1353 inline void emit_f64x2_pmin(LiftoffRegister dst, LiftoffRegister lhs, 1354 LiftoffRegister rhs); 1355 inline void emit_f64x2_pmax(LiftoffRegister dst, LiftoffRegister lhs, 1356 LiftoffRegister rhs); 1357 inline void emit_f64x2_convert_low_i32x4_s(LiftoffRegister dst, 1358 LiftoffRegister src); 1359 inline void emit_f64x2_convert_low_i32x4_u(LiftoffRegister dst, 1360 LiftoffRegister src); 1361 inline void emit_i32x4_trunc_sat_f64x2_s_zero(LiftoffRegister dst, 1362 LiftoffRegister src); 1363 inline void emit_i32x4_trunc_sat_f64x2_u_zero(LiftoffRegister dst, 1364 LiftoffRegister src); 1365 inline void emit_f32x4_demote_f64x2_zero(LiftoffRegister dst, 1366 LiftoffRegister src); 1367 inline void emit_f64x2_promote_low_f32x4(LiftoffRegister dst, 1368 LiftoffRegister src); 1369 inline void emit_i32x4_sconvert_f32x4(LiftoffRegister dst, 1370 LiftoffRegister src); 1371 inline void emit_i32x4_uconvert_f32x4(LiftoffRegister dst, 1372 LiftoffRegister src); 1373 inline void emit_f32x4_sconvert_i32x4(LiftoffRegister dst, 1374 LiftoffRegister src); 1375 inline void emit_f32x4_uconvert_i32x4(LiftoffRegister dst, 1376 LiftoffRegister src); 1377 inline void emit_i8x16_sconvert_i16x8(LiftoffRegister dst, 1378 LiftoffRegister lhs, 1379 LiftoffRegister rhs); 1380 inline void emit_i8x16_uconvert_i16x8(LiftoffRegister dst, 1381 LiftoffRegister lhs, 1382 LiftoffRegister rhs); 1383 inline void emit_i16x8_sconvert_i32x4(LiftoffRegister dst, 1384 LiftoffRegister lhs, 1385 LiftoffRegister rhs); 1386 inline void emit_i16x8_uconvert_i32x4(LiftoffRegister dst, 1387 LiftoffRegister lhs, 1388 LiftoffRegister rhs); 1389 inline void emit_i16x8_sconvert_i8x16_low(LiftoffRegister dst, 1390 LiftoffRegister src); 1391 inline void emit_i16x8_sconvert_i8x16_high(LiftoffRegister dst, 1392 LiftoffRegister src); 1393 inline void emit_i16x8_uconvert_i8x16_low(LiftoffRegister dst, 1394 LiftoffRegister src); 1395 inline void emit_i16x8_uconvert_i8x16_high(LiftoffRegister dst, 1396 LiftoffRegister src); 1397 inline void emit_i32x4_sconvert_i16x8_low(LiftoffRegister dst, 1398 LiftoffRegister src); 1399 inline void emit_i32x4_sconvert_i16x8_high(LiftoffRegister dst, 1400 LiftoffRegister src); 1401 inline void emit_i32x4_uconvert_i16x8_low(LiftoffRegister dst, 1402 LiftoffRegister src); 1403 inline void emit_i32x4_uconvert_i16x8_high(LiftoffRegister dst, 1404 LiftoffRegister src); 1405 inline void emit_s128_and_not(LiftoffRegister dst, LiftoffRegister lhs, 1406 LiftoffRegister rhs); 1407 inline void emit_i8x16_rounding_average_u(LiftoffRegister dst, 1408 LiftoffRegister lhs, 1409 LiftoffRegister rhs); 1410 inline void emit_i16x8_rounding_average_u(LiftoffRegister dst, 1411 LiftoffRegister lhs, 1412 LiftoffRegister rhs); 1413 inline void emit_i8x16_abs(LiftoffRegister dst, LiftoffRegister src); 1414 inline void emit_i16x8_abs(LiftoffRegister dst, LiftoffRegister src); 1415 inline void emit_i32x4_abs(LiftoffRegister dst, LiftoffRegister src); 1416 inline void emit_i64x2_abs(LiftoffRegister dst, LiftoffRegister src); 1417 inline void emit_i8x16_extract_lane_s(LiftoffRegister dst, 1418 LiftoffRegister lhs, 1419 uint8_t imm_lane_idx); 1420 inline void emit_i8x16_extract_lane_u(LiftoffRegister dst, 1421 LiftoffRegister lhs, 1422 uint8_t imm_lane_idx); 1423 inline void emit_i16x8_extract_lane_s(LiftoffRegister dst, 1424 LiftoffRegister lhs, 1425 uint8_t imm_lane_idx); 1426 inline void emit_i16x8_extract_lane_u(LiftoffRegister dst, 1427 LiftoffRegister lhs, 1428 uint8_t imm_lane_idx); 1429 inline void emit_i32x4_extract_lane(LiftoffRegister dst, LiftoffRegister lhs, 1430 uint8_t imm_lane_idx); 1431 inline void emit_i64x2_extract_lane(LiftoffRegister dst, LiftoffRegister lhs, 1432 uint8_t imm_lane_idx); 1433 inline void emit_f32x4_extract_lane(LiftoffRegister dst, LiftoffRegister lhs, 1434 uint8_t imm_lane_idx); 1435 inline void emit_f64x2_extract_lane(LiftoffRegister dst, LiftoffRegister lhs, 1436 uint8_t imm_lane_idx); 1437 inline void emit_i8x16_replace_lane(LiftoffRegister dst, LiftoffRegister src1, 1438 LiftoffRegister src2, 1439 uint8_t imm_lane_idx); 1440 inline void emit_i16x8_replace_lane(LiftoffRegister dst, LiftoffRegister src1, 1441 LiftoffRegister src2, 1442 uint8_t imm_lane_idx); 1443 inline void emit_i32x4_replace_lane(LiftoffRegister dst, LiftoffRegister src1, 1444 LiftoffRegister src2, 1445 uint8_t imm_lane_idx); 1446 inline void emit_i64x2_replace_lane(LiftoffRegister dst, LiftoffRegister src1, 1447 LiftoffRegister src2, 1448 uint8_t imm_lane_idx); 1449 inline void emit_f32x4_replace_lane(LiftoffRegister dst, LiftoffRegister src1, 1450 LiftoffRegister src2, 1451 uint8_t imm_lane_idx); 1452 inline void emit_f64x2_replace_lane(LiftoffRegister dst, LiftoffRegister src1, 1453 LiftoffRegister src2, 1454 uint8_t imm_lane_idx); 1455 1456 inline void StackCheck(Label* ool_code, Register limit_address); 1457 1458 inline void CallTrapCallbackForTesting(); 1459 1460 inline void AssertUnreachable(AbortReason reason); 1461 1462 inline void PushRegisters(LiftoffRegList); 1463 inline void PopRegisters(LiftoffRegList); 1464 1465 inline void RecordSpillsInSafepoint( 1466 SafepointTableBuilder::Safepoint& safepoint, LiftoffRegList all_spills, 1467 LiftoffRegList ref_spills, int spill_offset); 1468 1469 inline void DropStackSlotsAndRet(uint32_t num_stack_slots); 1470 1471 // Execute a C call. Arguments are pushed to the stack and a pointer to this 1472 // region is passed to the C function. If {out_argument_kind != kVoid}, 1473 // this is the return value of the C function, stored in {rets[0]}. Further 1474 // outputs (specified in {sig->returns()}) are read from the buffer and stored 1475 // in the remaining {rets} registers. 1476 inline void CallC(const ValueKindSig* sig, const LiftoffRegister* args, 1477 const LiftoffRegister* rets, ValueKind out_argument_kind, 1478 int stack_bytes, ExternalReference ext_ref); 1479 1480 inline void CallNativeWasmCode(Address addr); 1481 inline void TailCallNativeWasmCode(Address addr); 1482 // Indirect call: If {target == no_reg}, then pop the target from the stack. 1483 inline void CallIndirect(const ValueKindSig* sig, 1484 compiler::CallDescriptor* call_descriptor, 1485 Register target); 1486 inline void TailCallIndirect(Register target); 1487 inline void CallRuntimeStub(WasmCode::RuntimeStubId sid); 1488 1489 // Reserve space in the current frame, store address to space in {addr}. 1490 inline void AllocateStackSlot(Register addr, uint32_t size); 1491 inline void DeallocateStackSlot(uint32_t size); 1492 1493 // Instrumentation for shadow-stack-compatible OSR on x64. 1494 inline void MaybeOSR(); 1495 1496 // Set the i32 at address dst to a non-zero value if src is a NaN. 1497 inline void emit_set_if_nan(Register dst, DoubleRegister src, ValueKind kind); 1498 1499 // Set the i32 at address dst to a non-zero value if src contains a NaN. 1500 inline void emit_s128_set_if_nan(Register dst, LiftoffRegister src, 1501 Register tmp_gp, LiftoffRegister tmp_s128, 1502 ValueKind lane_kind); 1503 1504 //////////////////////////////////// 1505 // End of platform-specific part. // 1506 //////////////////////////////////// 1507 1508 uint32_t num_locals() const { return num_locals_; } 1509 void set_num_locals(uint32_t num_locals); 1510 1511 int GetTotalFrameSlotCountForGC() const; 1512 1513 int GetTotalFrameSize() const { return max_used_spill_offset_; } 1514 1515 ValueKind local_kind(uint32_t index) { 1516 DCHECK_GT(num_locals_, index); 1517 ValueKind* locals = 1518 num_locals_ <= kInlineLocalKinds ? local_kinds_ : more_local_kinds_; 1519 return locals[index]; 1520 } 1521 1522 void set_local_kind(uint32_t index, ValueKind kind) { 1523 ValueKind* locals = 1524 num_locals_ <= kInlineLocalKinds ? local_kinds_ : more_local_kinds_; 1525 locals[index] = kind; 1526 } 1527 1528 CacheState* cache_state() { return &cache_state_; } 1529 const CacheState* cache_state() const { return &cache_state_; } 1530 1531 bool did_bailout() { return bailout_reason_ != kSuccess; } 1532 LiftoffBailoutReason bailout_reason() const { return bailout_reason_; } 1533 const char* bailout_detail() const { return bailout_detail_; } 1534 1535 void bailout(LiftoffBailoutReason reason, const char* detail) { 1536 DCHECK_NE(kSuccess, reason); 1537 if (bailout_reason_ != kSuccess) return; 1538 AbortCompilation(); 1539 bailout_reason_ = reason; 1540 bailout_detail_ = detail; 1541 } 1542 1543 private: 1544 LiftoffRegister LoadI64HalfIntoRegister(VarState slot, RegPairHalf half); 1545 1546 V8_NOINLINE LiftoffRegister SpillOneRegister(LiftoffRegList candidates); 1547 // Spill one or two fp registers to get a pair of adjacent fp registers. 1548 LiftoffRegister SpillAdjacentFpRegisters(LiftoffRegList pinned); 1549 1550 uint32_t num_locals_ = 0; 1551 static constexpr uint32_t kInlineLocalKinds = 16; 1552 union { 1553 ValueKind local_kinds_[kInlineLocalKinds]; 1554 ValueKind* more_local_kinds_; 1555 }; 1556 static_assert(sizeof(ValueKind) == 1, 1557 "Reconsider this inlining if ValueKind gets bigger"); 1558 CacheState cache_state_; 1559 // The maximum spill offset for slots in the value stack. 1560 int max_used_spill_offset_ = StaticStackFrameSize(); 1561 // The amount of memory needed for register spills in OOL code. 1562 int ool_spill_space_size_ = 0; 1563 LiftoffBailoutReason bailout_reason_ = kSuccess; 1564 const char* bailout_detail_ = nullptr; 1565}; 1566 1567std::ostream& operator<<(std::ostream& os, LiftoffAssembler::VarState); 1568 1569// ======================================================================= 1570// Partially platform-independent implementations of the platform-dependent 1571// part. 1572 1573#ifdef V8_TARGET_ARCH_32_BIT 1574 1575namespace liftoff { 1576template <void (LiftoffAssembler::*op)(Register, Register, Register)> 1577void EmitI64IndependentHalfOperation(LiftoffAssembler* assm, 1578 LiftoffRegister dst, LiftoffRegister lhs, 1579 LiftoffRegister rhs) { 1580 // If {dst.low_gp()} does not overlap with {lhs.high_gp()} or {rhs.high_gp()}, 1581 // just first compute the lower half, then the upper half. 1582 if (dst.low() != lhs.high() && dst.low() != rhs.high()) { 1583 (assm->*op)(dst.low_gp(), lhs.low_gp(), rhs.low_gp()); 1584 (assm->*op)(dst.high_gp(), lhs.high_gp(), rhs.high_gp()); 1585 return; 1586 } 1587 // If {dst.high_gp()} does not overlap with {lhs.low_gp()} or {rhs.low_gp()}, 1588 // we can compute this the other way around. 1589 if (dst.high() != lhs.low() && dst.high() != rhs.low()) { 1590 (assm->*op)(dst.high_gp(), lhs.high_gp(), rhs.high_gp()); 1591 (assm->*op)(dst.low_gp(), lhs.low_gp(), rhs.low_gp()); 1592 return; 1593 } 1594 // Otherwise, we need a temporary register. 1595 Register tmp = assm->GetUnusedRegister(kGpReg, LiftoffRegList{lhs, rhs}).gp(); 1596 (assm->*op)(tmp, lhs.low_gp(), rhs.low_gp()); 1597 (assm->*op)(dst.high_gp(), lhs.high_gp(), rhs.high_gp()); 1598 assm->Move(dst.low_gp(), tmp, kI32); 1599} 1600 1601template <void (LiftoffAssembler::*op)(Register, Register, int32_t)> 1602void EmitI64IndependentHalfOperationImm(LiftoffAssembler* assm, 1603 LiftoffRegister dst, 1604 LiftoffRegister lhs, int64_t imm) { 1605 int32_t low_word = static_cast<int32_t>(imm); 1606 int32_t high_word = static_cast<int32_t>(imm >> 32); 1607 // If {dst.low_gp()} does not overlap with {lhs.high_gp()}, 1608 // just first compute the lower half, then the upper half. 1609 if (dst.low() != lhs.high()) { 1610 (assm->*op)(dst.low_gp(), lhs.low_gp(), low_word); 1611 (assm->*op)(dst.high_gp(), lhs.high_gp(), high_word); 1612 return; 1613 } 1614 // If {dst.high_gp()} does not overlap with {lhs.low_gp()}, 1615 // we can compute this the other way around. 1616 if (dst.high() != lhs.low()) { 1617 (assm->*op)(dst.high_gp(), lhs.high_gp(), high_word); 1618 (assm->*op)(dst.low_gp(), lhs.low_gp(), low_word); 1619 return; 1620 } 1621 // Otherwise, we need a temporary register. 1622 Register tmp = assm->GetUnusedRegister(kGpReg, LiftoffRegList{lhs}).gp(); 1623 (assm->*op)(tmp, lhs.low_gp(), low_word); 1624 (assm->*op)(dst.high_gp(), lhs.high_gp(), high_word); 1625 assm->Move(dst.low_gp(), tmp, kI32); 1626} 1627} // namespace liftoff 1628 1629void LiftoffAssembler::emit_i64_and(LiftoffRegister dst, LiftoffRegister lhs, 1630 LiftoffRegister rhs) { 1631 liftoff::EmitI64IndependentHalfOperation<&LiftoffAssembler::emit_i32_and>( 1632 this, dst, lhs, rhs); 1633} 1634 1635void LiftoffAssembler::emit_i64_andi(LiftoffRegister dst, LiftoffRegister lhs, 1636 int32_t imm) { 1637 liftoff::EmitI64IndependentHalfOperationImm<&LiftoffAssembler::emit_i32_andi>( 1638 this, dst, lhs, imm); 1639} 1640 1641void LiftoffAssembler::emit_i64_or(LiftoffRegister dst, LiftoffRegister lhs, 1642 LiftoffRegister rhs) { 1643 liftoff::EmitI64IndependentHalfOperation<&LiftoffAssembler::emit_i32_or>( 1644 this, dst, lhs, rhs); 1645} 1646 1647void LiftoffAssembler::emit_i64_ori(LiftoffRegister dst, LiftoffRegister lhs, 1648 int32_t imm) { 1649 liftoff::EmitI64IndependentHalfOperationImm<&LiftoffAssembler::emit_i32_ori>( 1650 this, dst, lhs, imm); 1651} 1652 1653void LiftoffAssembler::emit_i64_xor(LiftoffRegister dst, LiftoffRegister lhs, 1654 LiftoffRegister rhs) { 1655 liftoff::EmitI64IndependentHalfOperation<&LiftoffAssembler::emit_i32_xor>( 1656 this, dst, lhs, rhs); 1657} 1658 1659void LiftoffAssembler::emit_i64_xori(LiftoffRegister dst, LiftoffRegister lhs, 1660 int32_t imm) { 1661 liftoff::EmitI64IndependentHalfOperationImm<&LiftoffAssembler::emit_i32_xori>( 1662 this, dst, lhs, imm); 1663} 1664 1665void LiftoffAssembler::emit_u32_to_uintptr(Register dst, Register src) { 1666 // This is a no-op on 32-bit systems. 1667} 1668 1669#endif // V8_TARGET_ARCH_32_BIT 1670 1671// End of the partially platform-independent implementations of the 1672// platform-dependent part. 1673// ======================================================================= 1674 1675class LiftoffStackSlots { 1676 public: 1677 explicit LiftoffStackSlots(LiftoffAssembler* wasm_asm) : asm_(wasm_asm) {} 1678 LiftoffStackSlots(const LiftoffStackSlots&) = delete; 1679 LiftoffStackSlots& operator=(const LiftoffStackSlots&) = delete; 1680 1681 void Add(const LiftoffAssembler::VarState& src, uint32_t src_offset, 1682 RegPairHalf half, int dst_slot) { 1683 DCHECK_LE(0, dst_slot); 1684 slots_.emplace_back(src, src_offset, half, dst_slot); 1685 } 1686 1687 void Add(const LiftoffAssembler::VarState& src, int dst_slot) { 1688 DCHECK_LE(0, dst_slot); 1689 slots_.emplace_back(src, dst_slot); 1690 } 1691 1692 void SortInPushOrder() { 1693 std::sort(slots_.begin(), slots_.end(), [](const Slot& a, const Slot& b) { 1694 return a.dst_slot_ > b.dst_slot_; 1695 }); 1696 } 1697 1698 inline void Construct(int param_slots); 1699 1700 private: 1701 // A logical slot, which may occupy multiple stack slots. 1702 struct Slot { 1703 Slot(const LiftoffAssembler::VarState& src, uint32_t src_offset, 1704 RegPairHalf half, int dst_slot) 1705 : src_(src), 1706 src_offset_(src_offset), 1707 half_(half), 1708 dst_slot_(dst_slot) {} 1709 Slot(const LiftoffAssembler::VarState& src, int dst_slot) 1710 : src_(src), half_(kLowWord), dst_slot_(dst_slot) {} 1711 1712 LiftoffAssembler::VarState src_; 1713 uint32_t src_offset_ = 0; 1714 RegPairHalf half_; 1715 int dst_slot_ = 0; 1716 }; 1717 1718 // Returns the size in bytes of the given logical slot. 1719 static int SlotSizeInBytes(const Slot& slot) { 1720 const ValueKind kind = slot.src_.kind(); 1721 if (kind == kS128) return kSimd128Size; 1722 if (kind == kF64) return kDoubleSize; 1723 return kSystemPointerSize; 1724 } 1725 1726 base::SmallVector<Slot, 8> slots_; 1727 LiftoffAssembler* const asm_; 1728}; 1729 1730#if DEBUG 1731bool CheckCompatibleStackSlotTypes(ValueKind a, ValueKind b); 1732#endif 1733 1734} // namespace wasm 1735} // namespace internal 1736} // namespace v8 1737 1738// Include platform specific implementation. 1739#if V8_TARGET_ARCH_IA32 1740#include "src/wasm/baseline/ia32/liftoff-assembler-ia32.h" 1741#elif V8_TARGET_ARCH_X64 1742#include "src/wasm/baseline/x64/liftoff-assembler-x64.h" 1743#elif V8_TARGET_ARCH_ARM64 1744#include "src/wasm/baseline/arm64/liftoff-assembler-arm64.h" 1745#elif V8_TARGET_ARCH_ARM 1746#include "src/wasm/baseline/arm/liftoff-assembler-arm.h" 1747#elif V8_TARGET_ARCH_PPC || V8_TARGET_ARCH_PPC64 1748#include "src/wasm/baseline/ppc/liftoff-assembler-ppc.h" 1749#elif V8_TARGET_ARCH_MIPS 1750#include "src/wasm/baseline/mips/liftoff-assembler-mips.h" 1751#elif V8_TARGET_ARCH_MIPS64 1752#include "src/wasm/baseline/mips64/liftoff-assembler-mips64.h" 1753#elif V8_TARGET_ARCH_LOONG64 1754#include "src/wasm/baseline/loong64/liftoff-assembler-loong64.h" 1755#elif V8_TARGET_ARCH_S390 1756#include "src/wasm/baseline/s390/liftoff-assembler-s390.h" 1757#elif V8_TARGET_ARCH_RISCV64 1758#include "src/wasm/baseline/riscv64/liftoff-assembler-riscv64.h" 1759#else 1760#error Unsupported architecture. 1761#endif 1762 1763#endif // V8_WASM_BASELINE_LIFTOFF_ASSEMBLER_H_ 1764