xref: /third_party/node/deps/v8/src/compiler/int64-lowering.cc

// Copyright 2014 the V8 project authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "src/compiler/int64-lowering.h"

#include "src/compiler/common-operator.h"
#include "src/compiler/diamond.h"
#include "src/compiler/graph.h"
#include "src/compiler/linkage.h"
#include "src/compiler/machine-operator.h"
#include "src/compiler/node-matchers.h"
#include "src/compiler/node-properties.h"
#include "src/compiler/node.h"
#include "src/compiler/wasm-compiler.h"
// TODO(wasm): Remove this include.
#include "src/wasm/wasm-linkage.h"
#include "src/zone/zone.h"

namespace v8 {
namespace internal {
namespace compiler {

Int64Lowering::Int64Lowering(
    Graph* graph, MachineOperatorBuilder* machine,
    CommonOperatorBuilder* common, SimplifiedOperatorBuilder* simplified,
    Zone* zone, Signature<MachineRepresentation>* signature,
    std::unique_ptr<Int64LoweringSpecialCase> special_case)
    : zone_(zone),
      graph_(graph),
      machine_(machine),
      common_(common),
      simplified_(simplified),
      state_(graph->NodeCount(), State::kUnvisited),
      stack_(zone),
      replacements_(nullptr),
      signature_(signature),
      placeholder_(graph->NewNode(common->Dead())),
      special_case_(std::move(special_case)) {
  DCHECK_NOT_NULL(graph);
  DCHECK_NOT_NULL(graph->end());
  replacements_ = zone->NewArray<Replacement>(graph->NodeCount());
  memset(replacements_, 0, sizeof(Replacement) * graph->NodeCount());
}

void Int64Lowering::LowerGraph() {
  if (!machine()->Is32()) {
    return;
  }
  stack_.push_back({graph()->end(), 0});
  state_[graph()->end()->id()] = State::kOnStack;

  while (!stack_.empty()) {
    NodeState& top = stack_.back();
    if (top.input_index == top.node->InputCount()) {
      // All inputs of top have already been lowered, now lower top.
      stack_.pop_back();
      state_[top.node->id()] = State::kVisited;
      LowerNode(top.node);
    } else {
      // Push the next input onto the stack.
      Node* input = top.node->InputAt(top.input_index++);
      if (state_[input->id()] == State::kUnvisited) {
        if (input->opcode() == IrOpcode::kPhi) {
          // To break cycles with phi nodes we push phis on a separate stack so
          // that they are processed after all other nodes.
          PreparePhiReplacement(input);
          stack_.push_front({input, 0});
        } else if (input->opcode() == IrOpcode::kEffectPhi ||
                   input->opcode() == IrOpcode::kLoop) {
          stack_.push_front({input, 0});
        } else {
          stack_.push_back({input, 0});
        }
        state_[input->id()] = State::kOnStack;
      }
    }
  }
}

namespace {

int GetReturnIndexAfterLowering(const CallDescriptor* call_descriptor,
                                int old_index) {
  int result = old_index;
  for (int i = 0; i < old_index; i++) {
    if (call_descriptor->GetReturnType(i).representation() ==
        MachineRepresentation::kWord64) {
      result++;
    }
  }
  return result;
}

int GetReturnCountAfterLowering(const CallDescriptor* call_descriptor) {
  return GetReturnIndexAfterLowering(
      call_descriptor, static_cast<int>(call_descriptor->ReturnCount()));
}

int GetParameterIndexAfterLowering(
    Signature<MachineRepresentation>* signature, int old_index) {
  int result = old_index;
  // Be robust towards special indexes (>= param count).
  int max_to_check =
      std::min(old_index, static_cast<int>(signature->parameter_count()));
  for (int i = 0; i < max_to_check; i++) {
    if (signature->GetParam(i) == MachineRepresentation::kWord64) {
      result++;
    }
  }
  return result;
}

int GetReturnCountAfterLowering(Signature<MachineRepresentation>* signature) {
  int result = static_cast<int>(signature->return_count());
  for (int i = 0; i < static_cast<int>(signature->return_count()); i++) {
    if (signature->GetReturn(i) == MachineRepresentation::kWord64) {
      result++;
    }
  }
  return result;
}

}  // namespace

void Int64Lowering::LowerWord64AtomicBinop(Node* node, const Operator* op) {
  DCHECK_EQ(5, node->InputCount());
  LowerMemoryBaseAndIndex(node);
  Node* value = node->InputAt(2);
  node->ReplaceInput(2, GetReplacementLow(value));
  node->InsertInput(zone(), 3, GetReplacementHigh(value));
  NodeProperties::ChangeOp(node, op);
  ReplaceNodeWithProjections(node);
}

void Int64Lowering::LowerWord64AtomicNarrowOp(Node* node, const Operator* op) {
  DefaultLowering(node, true);
  NodeProperties::ChangeOp(node, op);
  ReplaceNode(node, node, graph()->NewNode(common()->Int32Constant(0)));
}

// static
int Int64Lowering::GetParameterCountAfterLowering(
    Signature<MachineRepresentation>* signature) {
  // GetParameterIndexAfterLowering(parameter_count) returns the parameter count
  // after lowering.
  return GetParameterIndexAfterLowering(
      signature, static_cast<int>(signature->parameter_count()));
}

void Int64Lowering::GetIndexNodes(Node* index, Node** index_low,
                                  Node** index_high) {
#if defined(V8_TARGET_LITTLE_ENDIAN)
  *index_low = index;
  *index_high = graph()->NewNode(machine()->Int32Add(), index,
                                 graph()->NewNode(common()->Int32Constant(4)));
#elif defined(V8_TARGET_BIG_ENDIAN)
  *index_low = graph()->NewNode(machine()->Int32Add(), index,
                                graph()->NewNode(common()->Int32Constant(4)));
  *index_high = index;
#endif
}

void Int64Lowering::LowerLoadOperator(Node* node, MachineRepresentation rep,
                                      const Operator* load_op) {
  if (rep == MachineRepresentation::kWord64) {
    LowerMemoryBaseAndIndex(node);
    Node* base = node->InputAt(0);
    Node* index = node->InputAt(1);
    Node* index_low;
    Node* index_high;
    GetIndexNodes(index, &index_low, &index_high);
    Node* high_node;
    if (node->InputCount() > 2) {
      Node* effect_high = node->InputAt(2);
      Node* control_high = node->InputAt(3);
      high_node = graph()->NewNode(load_op, base, index_high, effect_high,
                                   control_high);
      // change the effect change from old_node --> old_effect to
      // old_node --> high_node --> old_effect.
      node->ReplaceInput(2, high_node);
    } else {
      high_node = graph()->NewNode(load_op, base, index_high);
    }
    node->ReplaceInput(1, index_low);
    NodeProperties::ChangeOp(node, load_op);
    ReplaceNode(node, node, high_node);
  } else {
    DefaultLowering(node);
  }
}

void Int64Lowering::LowerStoreOperator(Node* node, MachineRepresentation rep,
                                       const Operator* store_op) {
  if (rep == MachineRepresentation::kWord64) {
    // We change the original store node to store the low word, and create
    // a new store node to store the high word. The effect and control edges
    // are copied from the original store to the new store node, the effect
    // edge of the original store is redirected to the new store.
    LowerMemoryBaseAndIndex(node);
    Node* base = node->InputAt(0);
    Node* index = node->InputAt(1);
    Node* index_low;
    Node* index_high;
    GetIndexNodes(index, &index_low, &index_high);
    Node* value = node->InputAt(2);
    DCHECK(HasReplacementLow(value));
    DCHECK(HasReplacementHigh(value));

    Node* high_node;
    if (node->InputCount() > 3) {
      Node* effect_high = node->InputAt(3);
      Node* control_high = node->InputAt(4);
      high_node = graph()->NewNode(store_op, base, index_high,
                                   GetReplacementHigh(value), effect_high,
                                   control_high);
      node->ReplaceInput(3, high_node);

    } else {
      high_node = graph()->NewNode(store_op, base, index_high,
                                   GetReplacementHigh(value));
    }

    node->ReplaceInput(1, index_low);
    node->ReplaceInput(2, GetReplacementLow(value));
    NodeProperties::ChangeOp(node, store_op);
    ReplaceNode(node, node, high_node);
  } else {
    DefaultLowering(node, true);
  }
}

void Int64Lowering::LowerNode(Node* node) {
  switch (node->opcode()) {
    case IrOpcode::kInt64Constant: {
      int64_t value = OpParameter<int64_t>(node->op());
      Node* low_node = graph()->NewNode(
          common()->Int32Constant(static_cast<int32_t>(value & 0xFFFFFFFF)));
      Node* high_node = graph()->NewNode(
          common()->Int32Constant(static_cast<int32_t>(value >> 32)));
      ReplaceNode(node, low_node, high_node);
      break;
    }
    case IrOpcode::kLoad: {
      MachineRepresentation rep =
          LoadRepresentationOf(node->op()).representation();
      LowerLoadOperator(node, rep, machine()->Load(MachineType::Int32()));
      break;
    }
    case IrOpcode::kUnalignedLoad: {
      MachineRepresentation rep =
          LoadRepresentationOf(node->op()).representation();
      LowerLoadOperator(node, rep,
                        machine()->UnalignedLoad(MachineType::Int32()));
      break;
    }
    case IrOpcode::kLoadImmutable: {
      MachineRepresentation rep =
          LoadRepresentationOf(node->op()).representation();
      LowerLoadOperator(node, rep,
                        machine()->LoadImmutable(MachineType::Int32()));
      break;
    }
    case IrOpcode::kLoadFromObject: {
      ObjectAccess access = ObjectAccessOf(node->op());
      LowerLoadOperator(node, access.machine_type.representation(),
                        simplified()->LoadFromObject(ObjectAccess(
                            MachineType::Int32(), access.write_barrier_kind)));
      break;
    }
    case IrOpcode::kLoadImmutableFromObject: {
      ObjectAccess access = ObjectAccessOf(node->op());
      LowerLoadOperator(node, access.machine_type.representation(),
                        simplified()->LoadImmutableFromObject(ObjectAccess(
                            MachineType::Int32(), access.write_barrier_kind)));
      break;
    }
    case IrOpcode::kStore: {
      StoreRepresentation store_rep = StoreRepresentationOf(node->op());
      LowerStoreOperator(
          node, store_rep.representation(),
          machine()->Store(StoreRepresentation(
              MachineRepresentation::kWord32, store_rep.write_barrier_kind())));
      break;
    }
    case IrOpcode::kUnalignedStore: {
      UnalignedStoreRepresentation store_rep =
          UnalignedStoreRepresentationOf(node->op());
      LowerStoreOperator(
          node, store_rep,
          machine()->UnalignedStore(MachineRepresentation::kWord32));
      break;
    }
    case IrOpcode::kStoreToObject: {
      ObjectAccess access = ObjectAccessOf(node->op());
      LowerStoreOperator(node, access.machine_type.representation(),
                         simplified()->StoreToObject(ObjectAccess(
                             MachineType::Int32(), access.write_barrier_kind)));
      break;
    }
    case IrOpcode::kInitializeImmutableInObject: {
      ObjectAccess access = ObjectAccessOf(node->op());
      LowerStoreOperator(node, access.machine_type.representation(),
                         simplified()->InitializeImmutableInObject(ObjectAccess(
                             MachineType::Int32(), access.write_barrier_kind)));
      break;
    }
    case IrOpcode::kStart: {
      int parameter_count = GetParameterCountAfterLowering(signature());
      // Only exchange the node if the parameter count actually changed.
      if (parameter_count != static_cast<int>(signature()->parameter_count())) {
        int delta =
            parameter_count - static_cast<int>(signature()->parameter_count());
        int new_output_count = node->op()->ValueOutputCount() + delta;
        NodeProperties::ChangeOp(node, common()->Start(new_output_count));
      }
      break;
    }
    case IrOpcode::kParameter: {
      DCHECK_EQ(1, node->InputCount());
      int param_count = static_cast<int>(signature()->parameter_count());
      // Only exchange the node if the parameter count actually changed. We do
      // not even have to do the default lowering because the the start node,
      // the only input of a parameter node, only changes if the parameter count
      // changes.
      if (GetParameterCountAfterLowering(signature()) != param_count) {
        int old_index = ParameterIndexOf(node->op());
        // Adjust old_index to be compliant with the signature.
        --old_index;
        int new_index = GetParameterIndexAfterLowering(signature(), old_index);
        // Adjust new_index to consider the instance parameter.
        ++new_index;
        NodeProperties::ChangeOp(node, common()->Parameter(new_index));

        if (old_index < 0 || old_index >= param_count) {
          // Special parameters (JS closure/context) don't have kWord64
          // representation anyway.
          break;
        }

        if (signature()->GetParam(old_index) ==
            MachineRepresentation::kWord64) {
          Node* high_node = graph()->NewNode(common()->Parameter(new_index + 1),
                                             graph()->start());
          ReplaceNode(node, node, high_node);
        }
      }
      break;
    }
    case IrOpcode::kReturn: {
      int input_count = node->InputCount();
      DefaultLowering(node);
      if (input_count != node->InputCount()) {
        int new_return_count = GetReturnCountAfterLowering(signature());
        if (static_cast<int>(signature()->return_count()) != new_return_count) {
          NodeProperties::ChangeOp(node, common()->Return(new_return_count));
        }
      }
      break;
    }
    case IrOpcode::kTailCall: {
      auto call_descriptor =
          const_cast<CallDescriptor*>(CallDescriptorOf(node->op()));
      bool returns_require_lowering =
          GetReturnCountAfterLowering(call_descriptor) !=
          static_cast<int>(call_descriptor->ReturnCount());
      if (DefaultLowering(node) || returns_require_lowering) {
        // Tail calls do not have return values, so adjusting the call
        // descriptor is enough.
        NodeProperties::ChangeOp(
            node, common()->TailCall(LowerCallDescriptor(call_descriptor)));
      }
      break;
    }
    case IrOpcode::kCall: {
      auto call_descriptor = CallDescriptorOf(node->op());

      bool returns_require_lowering =
          GetReturnCountAfterLowering(call_descriptor) !=
          static_cast<int>(call_descriptor->ReturnCount());
      if (DefaultLowering(node) || returns_require_lowering) {
        // We have to adjust the call descriptor.
        NodeProperties::ChangeOp(
            node, common()->Call(LowerCallDescriptor(call_descriptor)));
      }
      if (returns_require_lowering) {
        size_t return_arity = call_descriptor->ReturnCount();
        if (return_arity == 1) {
          // We access the additional return values through projections.
          ReplaceNodeWithProjections(node);
        } else {
          ZoneVector<Node*> projections(return_arity, zone());
          NodeProperties::CollectValueProjections(node, projections.data(),
                                                  return_arity);
          for (size_t old_index = 0, new_index = 0; old_index < return_arity;
               ++old_index, ++new_index) {
            Node* use_node = projections[old_index];
            DCHECK_EQ(ProjectionIndexOf(use_node->op()), old_index);
            DCHECK_EQ(GetReturnIndexAfterLowering(call_descriptor,
                                                  static_cast<int>(old_index)),
                      static_cast<int>(new_index));
            if (new_index != old_index) {
              NodeProperties::ChangeOp(
                  use_node, common()->Projection(new_index));
            }
            if (call_descriptor->GetReturnType(old_index).representation() ==
                MachineRepresentation::kWord64) {
              Node* high_node = graph()->NewNode(
                  common()->Projection(new_index + 1), node, graph()->start());
              ReplaceNode(use_node, use_node, high_node);
              ++new_index;
            }
          }
        }
      }
      break;
    }
    case IrOpcode::kWord64And: {
      DCHECK_EQ(2, node->InputCount());
      Node* left = node->InputAt(0);
      Node* right = node->InputAt(1);

      Node* low_node =
          graph()->NewNode(machine()->Word32And(), GetReplacementLow(left),
                           GetReplacementLow(right));
      Node* high_node =
          graph()->NewNode(machine()->Word32And(), GetReplacementHigh(left),
                           GetReplacementHigh(right));
      ReplaceNode(node, low_node, high_node);
      break;
    }
    case IrOpcode::kTruncateInt64ToInt32: {
      DCHECK_EQ(1, node->InputCount());
      Node* input = node->InputAt(0);
      ReplaceNode(node, GetReplacementLow(input), nullptr);
      node->NullAllInputs();
      break;
    }
    case IrOpcode::kInt64Add: {
      DCHECK_EQ(2, node->InputCount());

      Node* right = node->InputAt(1);
      node->ReplaceInput(1, GetReplacementLow(right));
      node->AppendInput(zone(), GetReplacementHigh(right));

      Node* left = node->InputAt(0);
      node->ReplaceInput(0, GetReplacementLow(left));
      node->InsertInput(zone(), 1, GetReplacementHigh(left));

      NodeProperties::ChangeOp(node, machine()->Int32PairAdd());
      // We access the additional return values through projections.
      ReplaceNodeWithProjections(node);
      break;
    }
    case IrOpcode::kInt64Sub: {
      DCHECK_EQ(2, node->InputCount());

      Node* right = node->InputAt(1);
      node->ReplaceInput(1, GetReplacementLow(right));
      node->AppendInput(zone(), GetReplacementHigh(right));

      Node* left = node->InputAt(0);
      node->ReplaceInput(0, GetReplacementLow(left));
      node->InsertInput(zone(), 1, GetReplacementHigh(left));

      NodeProperties::ChangeOp(node, machine()->Int32PairSub());
      // We access the additional return values through projections.
      ReplaceNodeWithProjections(node);
      break;
    }
    case IrOpcode::kInt64Mul: {
      DCHECK_EQ(2, node->InputCount());

      Node* right = node->InputAt(1);
      node->ReplaceInput(1, GetReplacementLow(right));
      node->AppendInput(zone(), GetReplacementHigh(right));

      Node* left = node->InputAt(0);
      node->ReplaceInput(0, GetReplacementLow(left));
      node->InsertInput(zone(), 1, GetReplacementHigh(left));

      NodeProperties::ChangeOp(node, machine()->Int32PairMul());
      // We access the additional return values through projections.
      ReplaceNodeWithProjections(node);
      break;
    }
    case IrOpcode::kWord64Or: {
      DCHECK_EQ(2, node->InputCount());
      Node* left = node->InputAt(0);
      Node* right = node->InputAt(1);

      Node* low_node =
          graph()->NewNode(machine()->Word32Or(), GetReplacementLow(left),
                           GetReplacementLow(right));
      Node* high_node =
          graph()->NewNode(machine()->Word32Or(), GetReplacementHigh(left),
                           GetReplacementHigh(right));
      ReplaceNode(node, low_node, high_node);
      break;
    }
    case IrOpcode::kWord64Xor: {
      DCHECK_EQ(2, node->InputCount());
      Node* left = node->InputAt(0);
      Node* right = node->InputAt(1);

      Node* low_node =
          graph()->NewNode(machine()->Word32Xor(), GetReplacementLow(left),
                           GetReplacementLow(right));
      Node* high_node =
          graph()->NewNode(machine()->Word32Xor(), GetReplacementHigh(left),
                           GetReplacementHigh(right));
      ReplaceNode(node, low_node, high_node);
      break;
    }
    case IrOpcode::kWord64Shl: {
      // TODO(turbofan): if the shift count >= 32, then we can set the low word
      // of the output to 0 and just calculate the high word.
      DCHECK_EQ(2, node->InputCount());
      Node* shift = node->InputAt(1);
      if (HasReplacementLow(shift)) {
        // We do not have to care about the high word replacement, because
        // the shift can only be between 0 and 63 anyways.
        node->ReplaceInput(1, GetReplacementLow(shift));
      }

      Node* value = node->InputAt(0);
      node->ReplaceInput(0, GetReplacementLow(value));
      node->InsertInput(zone(), 1, GetReplacementHigh(value));

      NodeProperties::ChangeOp(node, machine()->Word32PairShl());
      // We access the additional return values through projections.
      ReplaceNodeWithProjections(node);
      break;
    }
    case IrOpcode::kWord64Shr: {
      // TODO(turbofan): if the shift count >= 32, then we can set the low word
      // of the output to 0 and just calculate the high word.
      DCHECK_EQ(2, node->InputCount());
      Node* shift = node->InputAt(1);
      if (HasReplacementLow(shift)) {
        // We do not have to care about the high word replacement, because
        // the shift can only be between 0 and 63 anyways.
        node->ReplaceInput(1, GetReplacementLow(shift));
      }

      Node* value = node->InputAt(0);
      node->ReplaceInput(0, GetReplacementLow(value));
      node->InsertInput(zone(), 1, GetReplacementHigh(value));

      NodeProperties::ChangeOp(node, machine()->Word32PairShr());
      // We access the additional return values through projections.
      ReplaceNodeWithProjections(node);
      break;
    }
    case IrOpcode::kWord64Sar: {
      // TODO(turbofan): if the shift count >= 32, then we can set the low word
      // of the output to 0 and just calculate the high word.
      DCHECK_EQ(2, node->InputCount());
      Node* shift = node->InputAt(1);
      if (HasReplacementLow(shift)) {
        // We do not have to care about the high word replacement, because
        // the shift can only be between 0 and 63 anyways.
        node->ReplaceInput(1, GetReplacementLow(shift));
      }

      Node* value = node->InputAt(0);
      node->ReplaceInput(0, GetReplacementLow(value));
      node->InsertInput(zone(), 1, GetReplacementHigh(value));

      NodeProperties::ChangeOp(node, machine()->Word32PairSar());
      // We access the additional return values through projections.
      ReplaceNodeWithProjections(node);
      break;
    }
    case IrOpcode::kWord64Equal: {
      DCHECK_EQ(2, node->InputCount());
      Node* left = node->InputAt(0);
      Node* right = node->InputAt(1);

      // TODO(wasm): Use explicit comparisons and && here?
      Node* replacement = graph()->NewNode(
          machine()->Word32Equal(),
          graph()->NewNode(
              machine()->Word32Or(),
              graph()->NewNode(machine()->Word32Xor(), GetReplacementLow(left),
                               GetReplacementLow(right)),
              graph()->NewNode(machine()->Word32Xor(), GetReplacementHigh(left),
                               GetReplacementHigh(right))),
          graph()->NewNode(common()->Int32Constant(0)));

      ReplaceNode(node, replacement, nullptr);
      break;
    }
    case IrOpcode::kInt64LessThan: {
      LowerComparison(node, machine()->Int32LessThan(),
                      machine()->Uint32LessThan());
      break;
    }
    case IrOpcode::kInt64LessThanOrEqual: {
      LowerComparison(node, machine()->Int32LessThan(),
                      machine()->Uint32LessThanOrEqual());
      break;
    }
    case IrOpcode::kUint64LessThan: {
      LowerComparison(node, machine()->Uint32LessThan(),
                      machine()->Uint32LessThan());
      break;
    }
    case IrOpcode::kUint64LessThanOrEqual: {
      LowerComparison(node, machine()->Uint32LessThan(),
                      machine()->Uint32LessThanOrEqual());
      break;
    }
    case IrOpcode::kSignExtendWord32ToInt64:
    case IrOpcode::kChangeInt32ToInt64: {
      DCHECK_EQ(1, node->InputCount());
      Node* input = node->InputAt(0);
      if (HasReplacementLow(input)) {
        input = GetReplacementLow(input);
      }
      // We use SAR to preserve the sign in the high word.
      ReplaceNode(
          node, input,
          graph()->NewNode(machine()->Word32Sar(), input,
                           graph()->NewNode(common()->Int32Constant(31))));
      node->NullAllInputs();
      break;
    }
    case IrOpcode::kChangeUint32ToUint64: {
      DCHECK_EQ(1, node->InputCount());
      Node* input = node->InputAt(0);
      if (HasReplacementLow(input)) {
        input = GetReplacementLow(input);
      }
      ReplaceNode(node, input, graph()->NewNode(common()->Int32Constant(0)));
      node->NullAllInputs();
      break;
    }
    case IrOpcode::kBitcastInt64ToFloat64: {
      DCHECK_EQ(1, node->InputCount());
      Node* input = node->InputAt(0);
      Node* stack_slot = graph()->NewNode(
          machine()->StackSlot(MachineRepresentation::kWord64));

      Node* store_high_word = graph()->NewNode(
          machine()->Store(
              StoreRepresentation(MachineRepresentation::kWord32,
                                  WriteBarrierKind::kNoWriteBarrier)),
          stack_slot,
          graph()->NewNode(
              common()->Int32Constant(kInt64UpperHalfMemoryOffset)),
          GetReplacementHigh(input), graph()->start(), graph()->start());

      Node* store_low_word = graph()->NewNode(
          machine()->Store(
              StoreRepresentation(MachineRepresentation::kWord32,
                                  WriteBarrierKind::kNoWriteBarrier)),
          stack_slot,
          graph()->NewNode(
              common()->Int32Constant(kInt64LowerHalfMemoryOffset)),
          GetReplacementLow(input), store_high_word, graph()->start());

      Node* load =
          graph()->NewNode(machine()->Load(MachineType::Float64()), stack_slot,
                           graph()->NewNode(common()->Int32Constant(0)),
                           store_low_word, graph()->start());

      ReplaceNode(node, load, nullptr);
      break;
    }
    case IrOpcode::kBitcastFloat64ToInt64: {
      DCHECK_EQ(1, node->InputCount());
      Node* input = node->InputAt(0);
      if (HasReplacementLow(input)) {
        input = GetReplacementLow(input);
      }
      Node* stack_slot = graph()->NewNode(
          machine()->StackSlot(MachineRepresentation::kWord64));
      Node* store = graph()->NewNode(
          machine()->Store(
              StoreRepresentation(MachineRepresentation::kFloat64,
                                  WriteBarrierKind::kNoWriteBarrier)),
          stack_slot, graph()->NewNode(common()->Int32Constant(0)), input,
          graph()->start(), graph()->start());

      Node* high_node = graph()->NewNode(
          machine()->Load(MachineType::Int32()), stack_slot,
          graph()->NewNode(
              common()->Int32Constant(kInt64UpperHalfMemoryOffset)),
          store, graph()->start());

      Node* low_node = graph()->NewNode(
          machine()->Load(MachineType::Int32()), stack_slot,
          graph()->NewNode(
              common()->Int32Constant(kInt64LowerHalfMemoryOffset)),
          store, graph()->start());
      ReplaceNode(node, low_node, high_node);
      break;
    }
    case IrOpcode::kWord64RolLowerable:
      DCHECK(machine()->Word32Rol().IsSupported());
      V8_FALLTHROUGH;
    case IrOpcode::kWord64RorLowerable: {
      DCHECK_EQ(3, node->InputCount());
      Node* input = node->InputAt(0);
      Node* shift = HasReplacementLow(node->InputAt(1))
                        ? GetReplacementLow(node->InputAt(1))
                        : node->InputAt(1);
      Int32Matcher m(shift);
      if (m.HasResolvedValue()) {
        // Precondition: 0 <= shift < 64.
        int32_t shift_value = m.ResolvedValue() & 0x3F;
        if (shift_value == 0) {
          ReplaceNode(node, GetReplacementLow(input),
                      GetReplacementHigh(input));
        } else if (shift_value == 32) {
          ReplaceNode(node, GetReplacementHigh(input),
                      GetReplacementLow(input));
        } else {
          Node* low_input;
          Node* high_input;
          if (shift_value < 32) {
            low_input = GetReplacementLow(input);
            high_input = GetReplacementHigh(input);
          } else {
            low_input = GetReplacementHigh(input);
            high_input = GetReplacementLow(input);
          }
          int32_t masked_shift_value = shift_value & 0x1F;
          Node* masked_shift =
              graph()->NewNode(common()->Int32Constant(masked_shift_value));
          Node* inv_shift = graph()->NewNode(
              common()->Int32Constant(32 - masked_shift_value));

          auto* op1 = machine()->Word32Shr();
          auto* op2 = machine()->Word32Shl();
          bool is_ror = node->opcode() == IrOpcode::kWord64RorLowerable;
          if (!is_ror) std::swap(op1, op2);

          Node* low_node =
              graph()->NewNode(machine()->Word32Or(),
                               graph()->NewNode(op1, low_input, masked_shift),
                               graph()->NewNode(op2, high_input, inv_shift));
          Node* high_node =
              graph()->NewNode(machine()->Word32Or(),
                               graph()->NewNode(op1, high_input, masked_shift),
                               graph()->NewNode(op2, low_input, inv_shift));
          ReplaceNode(node, low_node, high_node);
        }
      } else {
        Node* safe_shift = shift;
        if (!machine()->Word32ShiftIsSafe()) {
          safe_shift =
              graph()->NewNode(machine()->Word32And(), shift,
                               graph()->NewNode(common()->Int32Constant(0x1F)));
        }

        bool is_ror = node->opcode() == IrOpcode::kWord64RorLowerable;
        Node* inv_mask =
            is_ror ? graph()->NewNode(
                         machine()->Word32Xor(),
                         graph()->NewNode(
                             machine()->Word32Shr(),
                             graph()->NewNode(common()->Int32Constant(-1)),
                             safe_shift),
                         graph()->NewNode(common()->Int32Constant(-1)))
                   : graph()->NewNode(
                         machine()->Word32Shl(),
                         graph()->NewNode(common()->Int32Constant(-1)),
                         safe_shift);

        Node* bit_mask =
            graph()->NewNode(machine()->Word32Xor(), inv_mask,
                             graph()->NewNode(common()->Int32Constant(-1)));

        // We have to mask the shift value for this comparison. If
        // !machine()->Word32ShiftIsSafe() then the masking should already be
        // part of the graph.
        Node* masked_shift6 = shift;
        if (machine()->Word32ShiftIsSafe()) {
          masked_shift6 =
              graph()->NewNode(machine()->Word32And(), shift,
                               graph()->NewNode(common()->Int32Constant(0x3F)));
        }

        Diamond lt32(
            graph(), common(),
            graph()->NewNode(machine()->Int32LessThan(), masked_shift6,
                             graph()->NewNode(common()->Int32Constant(32))));
        lt32.Chain(NodeProperties::GetControlInput(node));

        // The low word and the high word can be swapped either at the input or
        // at the output. We swap the inputs so that shift does not have to be
        // kept for so long in a register.
        Node* input_low =
            lt32.Phi(MachineRepresentation::kWord32, GetReplacementLow(input),
                     GetReplacementHigh(input));
        Node* input_high =
            lt32.Phi(MachineRepresentation::kWord32, GetReplacementHigh(input),
                     GetReplacementLow(input));

        const Operator* oper =
            is_ror ? machine()->Word32Ror() : machine()->Word32Rol().op();

        Node* rotate_low = graph()->NewNode(oper, input_low, safe_shift);
        Node* rotate_high = graph()->NewNode(oper, input_high, safe_shift);

        auto* mask1 = bit_mask;
        auto* mask2 = inv_mask;
        if (!is_ror) std::swap(mask1, mask2);

        Node* low_node = graph()->NewNode(
            machine()->Word32Or(),
            graph()->NewNode(machine()->Word32And(), rotate_low, mask1),
            graph()->NewNode(machine()->Word32And(), rotate_high, mask2));
        Node* high_node = graph()->NewNode(
            machine()->Word32Or(),
            graph()->NewNode(machine()->Word32And(), rotate_high, mask1),
            graph()->NewNode(machine()->Word32And(), rotate_low, mask2));
        ReplaceNode(node, low_node, high_node);
      }
      break;
    }
    case IrOpcode::kWord64ClzLowerable: {
      DCHECK_EQ(2, node->InputCount());
      Node* input = node->InputAt(0);
      Diamond d(
          graph(), common(),
          graph()->NewNode(machine()->Word32Equal(), GetReplacementHigh(input),
                           graph()->NewNode(common()->Int32Constant(0))));
      d.Chain(NodeProperties::GetControlInput(node));

      Node* low_node = d.Phi(
          MachineRepresentation::kWord32,
          graph()->NewNode(machine()->Int32Add(),
                           graph()->NewNode(machine()->Word32Clz(),
                                            GetReplacementLow(input)),
                           graph()->NewNode(common()->Int32Constant(32))),
          graph()->NewNode(machine()->Word32Clz(), GetReplacementHigh(input)));
      ReplaceNode(node, low_node, graph()->NewNode(common()->Int32Constant(0)));
      break;
    }
    case IrOpcode::kWord64CtzLowerable: {
      DCHECK_EQ(2, node->InputCount());
      DCHECK(machine()->Word32Ctz().IsSupported());
      Node* input = node->InputAt(0);
      Diamond d(
          graph(), common(),
          graph()->NewNode(machine()->Word32Equal(), GetReplacementLow(input),
                           graph()->NewNode(common()->Int32Constant(0))));
      d.Chain(NodeProperties::GetControlInput(node));

      Node* low_node =
          d.Phi(MachineRepresentation::kWord32,
                graph()->NewNode(machine()->Int32Add(),
                                 graph()->NewNode(machine()->Word32Ctz().op(),
                                                  GetReplacementHigh(input)),
                                 graph()->NewNode(common()->Int32Constant(32))),
                graph()->NewNode(machine()->Word32Ctz().op(),
                                 GetReplacementLow(input)));
      ReplaceNode(node, low_node, graph()->NewNode(common()->Int32Constant(0)));
      break;
    }
    case IrOpcode::kWord64Ror:
    case IrOpcode::kWord64Rol:
    case IrOpcode::kWord64Ctz:
    case IrOpcode::kWord64Clz:
      FATAL("%s operator should not be used in 32-bit systems",
            node->op()->mnemonic());
    case IrOpcode::kWord64Popcnt: {
      DCHECK_EQ(1, node->InputCount());
      Node* input = node->InputAt(0);
      // We assume that a Word64Popcnt node only has been created if
      // Word32Popcnt is actually supported.
      DCHECK(machine()->Word32Popcnt().IsSupported());
      ReplaceNode(node, graph()->NewNode(
                            machine()->Int32Add(),
                            graph()->NewNode(machine()->Word32Popcnt().op(),
                                             GetReplacementLow(input)),
                            graph()->NewNode(machine()->Word32Popcnt().op(),
                                             GetReplacementHigh(input))),
                  graph()->NewNode(common()->Int32Constant(0)));
      break;
    }
    case IrOpcode::kPhi: {
      MachineRepresentation rep = PhiRepresentationOf(node->op());
      if (rep == MachineRepresentation::kWord64) {
        // The replacement nodes have already been created, we only have to
        // replace placeholder nodes.
        Node* low_node = GetReplacementLow(node);
        Node* high_node = GetReplacementHigh(node);
        for (int i = 0; i < node->op()->ValueInputCount(); i++) {
          low_node->ReplaceInput(i, GetReplacementLow(node->InputAt(i)));
          high_node->ReplaceInput(i, GetReplacementHigh(node->InputAt(i)));
        }
      } else {
        DefaultLowering(node);
      }
      break;
    }
    case IrOpcode::kLoopExitValue: {
      MachineRepresentation rep = LoopExitValueRepresentationOf(node->op());
      if (rep == MachineRepresentation::kWord64) {
        Node* low_node = graph()->NewNode(
            common()->LoopExitValue(MachineRepresentation::kWord32),
            GetReplacementLow(node->InputAt(0)), node->InputAt(1));
        Node* high_node = graph()->NewNode(
            common()->LoopExitValue(MachineRepresentation::kWord32),
            GetReplacementHigh(node->InputAt(0)), node->InputAt(1));
        ReplaceNode(node, low_node, high_node);
      } else {
        DefaultLowering(node);
      }
      break;
    }
    case IrOpcode::kWord64ReverseBytes: {
      Node* input = node->InputAt(0);
      ReplaceNode(node,
                  graph()->NewNode(machine()->Word32ReverseBytes(),
                                   GetReplacementHigh(input)),
                  graph()->NewNode(machine()->Word32ReverseBytes(),
                                   GetReplacementLow(input)));
      break;
    }
    case IrOpcode::kSignExtendWord8ToInt64: {
      DCHECK_EQ(1, node->InputCount());
      Node* input = node->InputAt(0);
      if (HasReplacementLow(input)) {
        input = GetReplacementLow(input);
      }
      // Sign extend low node to Int32
      input = graph()->NewNode(machine()->SignExtendWord8ToInt32(), input);

      // We use SAR to preserve the sign in the high word.
      ReplaceNode(
          node, input,
          graph()->NewNode(machine()->Word32Sar(), input,
                           graph()->NewNode(common()->Int32Constant(31))));
      node->NullAllInputs();
      break;
    }
    case IrOpcode::kSignExtendWord16ToInt64: {
      DCHECK_EQ(1, node->InputCount());
      Node* input = node->InputAt(0);
      if (HasReplacementLow(input)) {
        input = GetReplacementLow(input);
      }
      // Sign extend low node to Int32
      input = graph()->NewNode(machine()->SignExtendWord16ToInt32(), input);

      // We use SAR to preserve the sign in the high word.
      ReplaceNode(
          node, input,
          graph()->NewNode(machine()->Word32Sar(), input,
                           graph()->NewNode(common()->Int32Constant(31))));
      node->NullAllInputs();
      break;
    }
    case IrOpcode::kWord64AtomicLoad: {
      DCHECK_EQ(4, node->InputCount());
      AtomicLoadParameters params = AtomicLoadParametersOf(node->op());
      DefaultLowering(node, true);
      if (params.representation() == MachineType::Uint64()) {
        NodeProperties::ChangeOp(
            node, machine()->Word32AtomicPairLoad(params.order()));
        ReplaceNodeWithProjections(node);
      } else {
        NodeProperties::ChangeOp(node, machine()->Word32AtomicLoad(params));
        ReplaceNode(node, node, graph()->NewNode(common()->Int32Constant(0)));
      }
      break;
    }
    case IrOpcode::kWord64AtomicStore: {
      DCHECK_EQ(5, node->InputCount());
      AtomicStoreParameters params = AtomicStoreParametersOf(node->op());
      if (params.representation() == MachineRepresentation::kWord64) {
        LowerMemoryBaseAndIndex(node);
        Node* value = node->InputAt(2);
        node->ReplaceInput(2, GetReplacementLow(value));
        node->InsertInput(zone(), 3, GetReplacementHigh(value));
        NodeProperties::ChangeOp(
            node, machine()->Word32AtomicPairStore(params.order()));
      } else {
        DefaultLowering(node, true);
        NodeProperties::ChangeOp(node, machine()->Word32AtomicStore(params));
      }
      break;
    }
#define ATOMIC_CASE(name)                                                   \
  case IrOpcode::kWord64Atomic##name: {                                     \
    MachineType type = AtomicOpType(node->op());                            \
    if (type == MachineType::Uint64()) {                                    \
      LowerWord64AtomicBinop(node, machine()->Word32AtomicPair##name());    \
    } else {                                                                \
      LowerWord64AtomicNarrowOp(node, machine()->Word32Atomic##name(type)); \
    }                                                                       \
    break;                                                                  \
  }
      ATOMIC_CASE(Add)
      ATOMIC_CASE(Sub)
      ATOMIC_CASE(And)
      ATOMIC_CASE(Or)
      ATOMIC_CASE(Xor)
      ATOMIC_CASE(Exchange)
#undef ATOMIC_CASE
    case IrOpcode::kWord64AtomicCompareExchange: {
      MachineType type = AtomicOpType(node->op());
      if (type == MachineType::Uint64()) {
        LowerMemoryBaseAndIndex(node);
        Node* old_value = node->InputAt(2);
        Node* new_value = node->InputAt(3);
        node->ReplaceInput(2, GetReplacementLow(old_value));
        node->ReplaceInput(3, GetReplacementHigh(old_value));
        node->InsertInput(zone(), 4, GetReplacementLow(new_value));
        node->InsertInput(zone(), 5, GetReplacementHigh(new_value));
        NodeProperties::ChangeOp(node,
                                 machine()->Word32AtomicPairCompareExchange());
        ReplaceNodeWithProjections(node);
      } else {
        DCHECK(type == MachineType::Uint32() || type == MachineType::Uint16() ||
               type == MachineType::Uint8());
        DefaultLowering(node, true);
        NodeProperties::ChangeOp(node,
                                 machine()->Word32AtomicCompareExchange(type));
        ReplaceNode(node, node, graph()->NewNode(common()->Int32Constant(0)));
      }
      break;
    }
    case IrOpcode::kI64x2Splat: {
      DCHECK_EQ(1, node->InputCount());
      Node* input = node->InputAt(0);
      node->ReplaceInput(0, GetReplacementLow(input));
      node->AppendInput(zone(), GetReplacementHigh(input));
      NodeProperties::ChangeOp(node, machine()->I64x2SplatI32Pair());
      break;
    }
    case IrOpcode::kI64x2ExtractLane: {
      DCHECK_EQ(1, node->InputCount());
      Node* input = node->InputAt(0);
      int32_t lane = OpParameter<int32_t>(node->op());
      ReplaceNode(
          node, graph()->NewNode(machine()->I32x4ExtractLane(lane * 2), input),
          graph()->NewNode(machine()->I32x4ExtractLane(lane * 2 + 1), input));
      break;
    }
    case IrOpcode::kI64x2ReplaceLane: {
      DCHECK_EQ(2, node->InputCount());
      int32_t lane = OpParameter<int32_t>(node->op());
      Node* input = node->InputAt(1);
      node->ReplaceInput(1, GetReplacementLow(input));
      node->AppendInput(zone(), GetReplacementHigh(input));
      NodeProperties::ChangeOp(node, machine()->I64x2ReplaceLaneI32Pair(lane));
      break;
    }

    default: { DefaultLowering(node); }
  }
}

void Int64Lowering::LowerComparison(Node* node, const Operator* high_word_op,
                                    const Operator* low_word_op) {
  DCHECK_EQ(2, node->InputCount());
  Node* left = node->InputAt(0);
  Node* right = node->InputAt(1);
  Node* replacement = graph()->NewNode(
      machine()->Word32Or(),
      graph()->NewNode(high_word_op, GetReplacementHigh(left),
                       GetReplacementHigh(right)),
      graph()->NewNode(
          machine()->Word32And(),
          graph()->NewNode(machine()->Word32Equal(), GetReplacementHigh(left),
                           GetReplacementHigh(right)),
          graph()->NewNode(low_word_op, GetReplacementLow(left),
                           GetReplacementLow(right))));

  ReplaceNode(node, replacement, nullptr);
}

bool Int64Lowering::DefaultLowering(Node* node, bool low_word_only) {
  bool something_changed = false;
  for (int i = NodeProperties::PastValueIndex(node) - 1; i >= 0; i--) {
    Node* input = node->InputAt(i);
    if (HasReplacementLow(input)) {
      something_changed = true;
      node->ReplaceInput(i, GetReplacementLow(input));
    }
    if (!low_word_only && HasReplacementHigh(input)) {
      something_changed = true;
      node->InsertInput(zone(), i + 1, GetReplacementHigh(input));
    }
  }
  return something_changed;
}

const CallDescriptor* Int64Lowering::LowerCallDescriptor(
    const CallDescriptor* call_descriptor) {
  if (special_case_) {
    auto replacement = special_case_->replacements.find(call_descriptor);
    if (replacement != special_case_->replacements.end()) {
      return replacement->second;
    }
  }
  return GetI32WasmCallDescriptor(zone(), call_descriptor);
}

void Int64Lowering::ReplaceNode(Node* old, Node* new_low, Node* new_high) {
  // if new_low == nullptr, then also new_high == nullptr.
  DCHECK(new_low != nullptr || new_high == nullptr);
  replacements_[old->id()].low = new_low;
  replacements_[old->id()].high = new_high;
}

bool Int64Lowering::HasReplacementLow(Node* node) {
  return replacements_[node->id()].low != nullptr;
}

Node* Int64Lowering::GetReplacementLow(Node* node) {
  Node* result = replacements_[node->id()].low;
  DCHECK(result);
  return result;
}

bool Int64Lowering::HasReplacementHigh(Node* node) {
  return replacements_[node->id()].high != nullptr;
}

Node* Int64Lowering::GetReplacementHigh(Node* node) {
  Node* result = replacements_[node->id()].high;
  DCHECK(result);
  return result;
}

void Int64Lowering::PreparePhiReplacement(Node* phi) {
  MachineRepresentation rep = PhiRepresentationOf(phi->op());
  if (rep == MachineRepresentation::kWord64) {
    // We have to create the replacements for a phi node before we actually
    // lower the phi to break potential cycles in the graph. The replacements of
    // input nodes do not exist yet, so we use a placeholder node to pass the
    // graph verifier.
    int value_count = phi->op()->ValueInputCount();
    Node** inputs_low = zone()->NewArray<Node*>(value_count + 1);
    Node** inputs_high = zone()->NewArray<Node*>(value_count + 1);
    for (int i = 0; i < value_count; i++) {
      inputs_low[i] = placeholder_;
      inputs_high[i] = placeholder_;
    }
    inputs_low[value_count] = NodeProperties::GetControlInput(phi, 0);
    inputs_high[value_count] = NodeProperties::GetControlInput(phi, 0);
    ReplaceNode(phi,
                graph()->NewNode(
                    common()->Phi(MachineRepresentation::kWord32, value_count),
                    value_count + 1, inputs_low, false),
                graph()->NewNode(
                    common()->Phi(MachineRepresentation::kWord32, value_count),
                    value_count + 1, inputs_high, false));
  }
}

void Int64Lowering::ReplaceNodeWithProjections(Node* node) {
  DCHECK(node != nullptr);
  Node* low_node =
      graph()->NewNode(common()->Projection(0), node, graph()->start());
  Node* high_node =
      graph()->NewNode(common()->Projection(1), node, graph()->start());
  ReplaceNode(node, low_node, high_node);
}

void Int64Lowering::LowerMemoryBaseAndIndex(Node* node) {
  DCHECK(node != nullptr);
  // Low word only replacements for memory operands for 32-bit address space.
  Node* base = node->InputAt(0);
  Node* index = node->InputAt(1);
  if (HasReplacementLow(base)) {
    node->ReplaceInput(0, GetReplacementLow(base));
  }
  if (HasReplacementLow(index)) {
    node->ReplaceInput(1, GetReplacementLow(index));
  }
}

}  // namespace compiler
}  // namespace internal
}  // namespace v8