1// Copyright 2014 the V8 project authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "src/compiler/int64-lowering.h"
6
7#include "src/compiler/common-operator.h"
8#include "src/compiler/diamond.h"
9#include "src/compiler/graph.h"
10#include "src/compiler/linkage.h"
11#include "src/compiler/machine-operator.h"
12#include "src/compiler/node-matchers.h"
13#include "src/compiler/node-properties.h"
14#include "src/compiler/node.h"
15#include "src/compiler/wasm-compiler.h"
16// TODO(wasm): Remove this include.
17#include "src/wasm/wasm-linkage.h"
18#include "src/zone/zone.h"
19
20namespace v8 {
21namespace internal {
22namespace compiler {
23
24Int64Lowering::Int64Lowering(
25    Graph* graph, MachineOperatorBuilder* machine,
26    CommonOperatorBuilder* common, SimplifiedOperatorBuilder* simplified,
27    Zone* zone, Signature<MachineRepresentation>* signature,
28    std::unique_ptr<Int64LoweringSpecialCase> special_case)
29    : zone_(zone),
30      graph_(graph),
31      machine_(machine),
32      common_(common),
33      simplified_(simplified),
34      state_(graph->NodeCount(), State::kUnvisited),
35      stack_(zone),
36      replacements_(nullptr),
37      signature_(signature),
38      placeholder_(graph->NewNode(common->Dead())),
39      special_case_(std::move(special_case)) {
40  DCHECK_NOT_NULL(graph);
41  DCHECK_NOT_NULL(graph->end());
42  replacements_ = zone->NewArray<Replacement>(graph->NodeCount());
43  memset(replacements_, 0, sizeof(Replacement) * graph->NodeCount());
44}
45
46void Int64Lowering::LowerGraph() {
47  if (!machine()->Is32()) {
48    return;
49  }
50  stack_.push_back({graph()->end(), 0});
51  state_[graph()->end()->id()] = State::kOnStack;
52
53  while (!stack_.empty()) {
54    NodeState& top = stack_.back();
55    if (top.input_index == top.node->InputCount()) {
56      // All inputs of top have already been lowered, now lower top.
57      stack_.pop_back();
58      state_[top.node->id()] = State::kVisited;
59      LowerNode(top.node);
60    } else {
61      // Push the next input onto the stack.
62      Node* input = top.node->InputAt(top.input_index++);
63      if (state_[input->id()] == State::kUnvisited) {
64        if (input->opcode() == IrOpcode::kPhi) {
65          // To break cycles with phi nodes we push phis on a separate stack so
66          // that they are processed after all other nodes.
67          PreparePhiReplacement(input);
68          stack_.push_front({input, 0});
69        } else if (input->opcode() == IrOpcode::kEffectPhi ||
70                   input->opcode() == IrOpcode::kLoop) {
71          stack_.push_front({input, 0});
72        } else {
73          stack_.push_back({input, 0});
74        }
75        state_[input->id()] = State::kOnStack;
76      }
77    }
78  }
79}
80
81namespace {
82
83int GetReturnIndexAfterLowering(const CallDescriptor* call_descriptor,
84                                int old_index) {
85  int result = old_index;
86  for (int i = 0; i < old_index; i++) {
87    if (call_descriptor->GetReturnType(i).representation() ==
88        MachineRepresentation::kWord64) {
89      result++;
90    }
91  }
92  return result;
93}
94
95int GetReturnCountAfterLowering(const CallDescriptor* call_descriptor) {
96  return GetReturnIndexAfterLowering(
97      call_descriptor, static_cast<int>(call_descriptor->ReturnCount()));
98}
99
100int GetParameterIndexAfterLowering(
101    Signature<MachineRepresentation>* signature, int old_index) {
102  int result = old_index;
103  // Be robust towards special indexes (>= param count).
104  int max_to_check =
105      std::min(old_index, static_cast<int>(signature->parameter_count()));
106  for (int i = 0; i < max_to_check; i++) {
107    if (signature->GetParam(i) == MachineRepresentation::kWord64) {
108      result++;
109    }
110  }
111  return result;
112}
113
114int GetReturnCountAfterLowering(Signature<MachineRepresentation>* signature) {
115  int result = static_cast<int>(signature->return_count());
116  for (int i = 0; i < static_cast<int>(signature->return_count()); i++) {
117    if (signature->GetReturn(i) == MachineRepresentation::kWord64) {
118      result++;
119    }
120  }
121  return result;
122}
123
124}  // namespace
125
126void Int64Lowering::LowerWord64AtomicBinop(Node* node, const Operator* op) {
127  DCHECK_EQ(5, node->InputCount());
128  LowerMemoryBaseAndIndex(node);
129  Node* value = node->InputAt(2);
130  node->ReplaceInput(2, GetReplacementLow(value));
131  node->InsertInput(zone(), 3, GetReplacementHigh(value));
132  NodeProperties::ChangeOp(node, op);
133  ReplaceNodeWithProjections(node);
134}
135
136void Int64Lowering::LowerWord64AtomicNarrowOp(Node* node, const Operator* op) {
137  DefaultLowering(node, true);
138  NodeProperties::ChangeOp(node, op);
139  ReplaceNode(node, node, graph()->NewNode(common()->Int32Constant(0)));
140}
141
142// static
143int Int64Lowering::GetParameterCountAfterLowering(
144    Signature<MachineRepresentation>* signature) {
145  // GetParameterIndexAfterLowering(parameter_count) returns the parameter count
146  // after lowering.
147  return GetParameterIndexAfterLowering(
148      signature, static_cast<int>(signature->parameter_count()));
149}
150
151void Int64Lowering::GetIndexNodes(Node* index, Node** index_low,
152                                  Node** index_high) {
153#if defined(V8_TARGET_LITTLE_ENDIAN)
154  *index_low = index;
155  *index_high = graph()->NewNode(machine()->Int32Add(), index,
156                                 graph()->NewNode(common()->Int32Constant(4)));
157#elif defined(V8_TARGET_BIG_ENDIAN)
158  *index_low = graph()->NewNode(machine()->Int32Add(), index,
159                                graph()->NewNode(common()->Int32Constant(4)));
160  *index_high = index;
161#endif
162}
163
164void Int64Lowering::LowerLoadOperator(Node* node, MachineRepresentation rep,
165                                      const Operator* load_op) {
166  if (rep == MachineRepresentation::kWord64) {
167    LowerMemoryBaseAndIndex(node);
168    Node* base = node->InputAt(0);
169    Node* index = node->InputAt(1);
170    Node* index_low;
171    Node* index_high;
172    GetIndexNodes(index, &index_low, &index_high);
173    Node* high_node;
174    if (node->InputCount() > 2) {
175      Node* effect_high = node->InputAt(2);
176      Node* control_high = node->InputAt(3);
177      high_node = graph()->NewNode(load_op, base, index_high, effect_high,
178                                   control_high);
179      // change the effect change from old_node --> old_effect to
180      // old_node --> high_node --> old_effect.
181      node->ReplaceInput(2, high_node);
182    } else {
183      high_node = graph()->NewNode(load_op, base, index_high);
184    }
185    node->ReplaceInput(1, index_low);
186    NodeProperties::ChangeOp(node, load_op);
187    ReplaceNode(node, node, high_node);
188  } else {
189    DefaultLowering(node);
190  }
191}
192
193void Int64Lowering::LowerStoreOperator(Node* node, MachineRepresentation rep,
194                                       const Operator* store_op) {
195  if (rep == MachineRepresentation::kWord64) {
196    // We change the original store node to store the low word, and create
197    // a new store node to store the high word. The effect and control edges
198    // are copied from the original store to the new store node, the effect
199    // edge of the original store is redirected to the new store.
200    LowerMemoryBaseAndIndex(node);
201    Node* base = node->InputAt(0);
202    Node* index = node->InputAt(1);
203    Node* index_low;
204    Node* index_high;
205    GetIndexNodes(index, &index_low, &index_high);
206    Node* value = node->InputAt(2);
207    DCHECK(HasReplacementLow(value));
208    DCHECK(HasReplacementHigh(value));
209
210    Node* high_node;
211    if (node->InputCount() > 3) {
212      Node* effect_high = node->InputAt(3);
213      Node* control_high = node->InputAt(4);
214      high_node = graph()->NewNode(store_op, base, index_high,
215                                   GetReplacementHigh(value), effect_high,
216                                   control_high);
217      node->ReplaceInput(3, high_node);
218
219    } else {
220      high_node = graph()->NewNode(store_op, base, index_high,
221                                   GetReplacementHigh(value));
222    }
223
224    node->ReplaceInput(1, index_low);
225    node->ReplaceInput(2, GetReplacementLow(value));
226    NodeProperties::ChangeOp(node, store_op);
227    ReplaceNode(node, node, high_node);
228  } else {
229    DefaultLowering(node, true);
230  }
231}
232
233void Int64Lowering::LowerNode(Node* node) {
234  switch (node->opcode()) {
235    case IrOpcode::kInt64Constant: {
236      int64_t value = OpParameter<int64_t>(node->op());
237      Node* low_node = graph()->NewNode(
238          common()->Int32Constant(static_cast<int32_t>(value & 0xFFFFFFFF)));
239      Node* high_node = graph()->NewNode(
240          common()->Int32Constant(static_cast<int32_t>(value >> 32)));
241      ReplaceNode(node, low_node, high_node);
242      break;
243    }
244    case IrOpcode::kLoad: {
245      MachineRepresentation rep =
246          LoadRepresentationOf(node->op()).representation();
247      LowerLoadOperator(node, rep, machine()->Load(MachineType::Int32()));
248      break;
249    }
250    case IrOpcode::kUnalignedLoad: {
251      MachineRepresentation rep =
252          LoadRepresentationOf(node->op()).representation();
253      LowerLoadOperator(node, rep,
254                        machine()->UnalignedLoad(MachineType::Int32()));
255      break;
256    }
257    case IrOpcode::kLoadImmutable: {
258      MachineRepresentation rep =
259          LoadRepresentationOf(node->op()).representation();
260      LowerLoadOperator(node, rep,
261                        machine()->LoadImmutable(MachineType::Int32()));
262      break;
263    }
264    case IrOpcode::kLoadFromObject: {
265      ObjectAccess access = ObjectAccessOf(node->op());
266      LowerLoadOperator(node, access.machine_type.representation(),
267                        simplified()->LoadFromObject(ObjectAccess(
268                            MachineType::Int32(), access.write_barrier_kind)));
269      break;
270    }
271    case IrOpcode::kLoadImmutableFromObject: {
272      ObjectAccess access = ObjectAccessOf(node->op());
273      LowerLoadOperator(node, access.machine_type.representation(),
274                        simplified()->LoadImmutableFromObject(ObjectAccess(
275                            MachineType::Int32(), access.write_barrier_kind)));
276      break;
277    }
278    case IrOpcode::kStore: {
279      StoreRepresentation store_rep = StoreRepresentationOf(node->op());
280      LowerStoreOperator(
281          node, store_rep.representation(),
282          machine()->Store(StoreRepresentation(
283              MachineRepresentation::kWord32, store_rep.write_barrier_kind())));
284      break;
285    }
286    case IrOpcode::kUnalignedStore: {
287      UnalignedStoreRepresentation store_rep =
288          UnalignedStoreRepresentationOf(node->op());
289      LowerStoreOperator(
290          node, store_rep,
291          machine()->UnalignedStore(MachineRepresentation::kWord32));
292      break;
293    }
294    case IrOpcode::kStoreToObject: {
295      ObjectAccess access = ObjectAccessOf(node->op());
296      LowerStoreOperator(node, access.machine_type.representation(),
297                         simplified()->StoreToObject(ObjectAccess(
298                             MachineType::Int32(), access.write_barrier_kind)));
299      break;
300    }
301    case IrOpcode::kInitializeImmutableInObject: {
302      ObjectAccess access = ObjectAccessOf(node->op());
303      LowerStoreOperator(node, access.machine_type.representation(),
304                         simplified()->InitializeImmutableInObject(ObjectAccess(
305                             MachineType::Int32(), access.write_barrier_kind)));
306      break;
307    }
308    case IrOpcode::kStart: {
309      int parameter_count = GetParameterCountAfterLowering(signature());
310      // Only exchange the node if the parameter count actually changed.
311      if (parameter_count != static_cast<int>(signature()->parameter_count())) {
312        int delta =
313            parameter_count - static_cast<int>(signature()->parameter_count());
314        int new_output_count = node->op()->ValueOutputCount() + delta;
315        NodeProperties::ChangeOp(node, common()->Start(new_output_count));
316      }
317      break;
318    }
319    case IrOpcode::kParameter: {
320      DCHECK_EQ(1, node->InputCount());
321      int param_count = static_cast<int>(signature()->parameter_count());
322      // Only exchange the node if the parameter count actually changed. We do
323      // not even have to do the default lowering because the the start node,
324      // the only input of a parameter node, only changes if the parameter count
325      // changes.
326      if (GetParameterCountAfterLowering(signature()) != param_count) {
327        int old_index = ParameterIndexOf(node->op());
328        // Adjust old_index to be compliant with the signature.
329        --old_index;
330        int new_index = GetParameterIndexAfterLowering(signature(), old_index);
331        // Adjust new_index to consider the instance parameter.
332        ++new_index;
333        NodeProperties::ChangeOp(node, common()->Parameter(new_index));
334
335        if (old_index < 0 || old_index >= param_count) {
336          // Special parameters (JS closure/context) don't have kWord64
337          // representation anyway.
338          break;
339        }
340
341        if (signature()->GetParam(old_index) ==
342            MachineRepresentation::kWord64) {
343          Node* high_node = graph()->NewNode(common()->Parameter(new_index + 1),
344                                             graph()->start());
345          ReplaceNode(node, node, high_node);
346        }
347      }
348      break;
349    }
350    case IrOpcode::kReturn: {
351      int input_count = node->InputCount();
352      DefaultLowering(node);
353      if (input_count != node->InputCount()) {
354        int new_return_count = GetReturnCountAfterLowering(signature());
355        if (static_cast<int>(signature()->return_count()) != new_return_count) {
356          NodeProperties::ChangeOp(node, common()->Return(new_return_count));
357        }
358      }
359      break;
360    }
361    case IrOpcode::kTailCall: {
362      auto call_descriptor =
363          const_cast<CallDescriptor*>(CallDescriptorOf(node->op()));
364      bool returns_require_lowering =
365          GetReturnCountAfterLowering(call_descriptor) !=
366          static_cast<int>(call_descriptor->ReturnCount());
367      if (DefaultLowering(node) || returns_require_lowering) {
368        // Tail calls do not have return values, so adjusting the call
369        // descriptor is enough.
370        NodeProperties::ChangeOp(
371            node, common()->TailCall(LowerCallDescriptor(call_descriptor)));
372      }
373      break;
374    }
375    case IrOpcode::kCall: {
376      auto call_descriptor = CallDescriptorOf(node->op());
377
378      bool returns_require_lowering =
379          GetReturnCountAfterLowering(call_descriptor) !=
380          static_cast<int>(call_descriptor->ReturnCount());
381      if (DefaultLowering(node) || returns_require_lowering) {
382        // We have to adjust the call descriptor.
383        NodeProperties::ChangeOp(
384            node, common()->Call(LowerCallDescriptor(call_descriptor)));
385      }
386      if (returns_require_lowering) {
387        size_t return_arity = call_descriptor->ReturnCount();
388        if (return_arity == 1) {
389          // We access the additional return values through projections.
390          ReplaceNodeWithProjections(node);
391        } else {
392          ZoneVector<Node*> projections(return_arity, zone());
393          NodeProperties::CollectValueProjections(node, projections.data(),
394                                                  return_arity);
395          for (size_t old_index = 0, new_index = 0; old_index < return_arity;
396               ++old_index, ++new_index) {
397            Node* use_node = projections[old_index];
398            DCHECK_EQ(ProjectionIndexOf(use_node->op()), old_index);
399            DCHECK_EQ(GetReturnIndexAfterLowering(call_descriptor,
400                                                  static_cast<int>(old_index)),
401                      static_cast<int>(new_index));
402            if (new_index != old_index) {
403              NodeProperties::ChangeOp(
404                  use_node, common()->Projection(new_index));
405            }
406            if (call_descriptor->GetReturnType(old_index).representation() ==
407                MachineRepresentation::kWord64) {
408              Node* high_node = graph()->NewNode(
409                  common()->Projection(new_index + 1), node, graph()->start());
410              ReplaceNode(use_node, use_node, high_node);
411              ++new_index;
412            }
413          }
414        }
415      }
416      break;
417    }
418    case IrOpcode::kWord64And: {
419      DCHECK_EQ(2, node->InputCount());
420      Node* left = node->InputAt(0);
421      Node* right = node->InputAt(1);
422
423      Node* low_node =
424          graph()->NewNode(machine()->Word32And(), GetReplacementLow(left),
425                           GetReplacementLow(right));
426      Node* high_node =
427          graph()->NewNode(machine()->Word32And(), GetReplacementHigh(left),
428                           GetReplacementHigh(right));
429      ReplaceNode(node, low_node, high_node);
430      break;
431    }
432    case IrOpcode::kTruncateInt64ToInt32: {
433      DCHECK_EQ(1, node->InputCount());
434      Node* input = node->InputAt(0);
435      ReplaceNode(node, GetReplacementLow(input), nullptr);
436      node->NullAllInputs();
437      break;
438    }
439    case IrOpcode::kInt64Add: {
440      DCHECK_EQ(2, node->InputCount());
441
442      Node* right = node->InputAt(1);
443      node->ReplaceInput(1, GetReplacementLow(right));
444      node->AppendInput(zone(), GetReplacementHigh(right));
445
446      Node* left = node->InputAt(0);
447      node->ReplaceInput(0, GetReplacementLow(left));
448      node->InsertInput(zone(), 1, GetReplacementHigh(left));
449
450      NodeProperties::ChangeOp(node, machine()->Int32PairAdd());
451      // We access the additional return values through projections.
452      ReplaceNodeWithProjections(node);
453      break;
454    }
455    case IrOpcode::kInt64Sub: {
456      DCHECK_EQ(2, node->InputCount());
457
458      Node* right = node->InputAt(1);
459      node->ReplaceInput(1, GetReplacementLow(right));
460      node->AppendInput(zone(), GetReplacementHigh(right));
461
462      Node* left = node->InputAt(0);
463      node->ReplaceInput(0, GetReplacementLow(left));
464      node->InsertInput(zone(), 1, GetReplacementHigh(left));
465
466      NodeProperties::ChangeOp(node, machine()->Int32PairSub());
467      // We access the additional return values through projections.
468      ReplaceNodeWithProjections(node);
469      break;
470    }
471    case IrOpcode::kInt64Mul: {
472      DCHECK_EQ(2, node->InputCount());
473
474      Node* right = node->InputAt(1);
475      node->ReplaceInput(1, GetReplacementLow(right));
476      node->AppendInput(zone(), GetReplacementHigh(right));
477
478      Node* left = node->InputAt(0);
479      node->ReplaceInput(0, GetReplacementLow(left));
480      node->InsertInput(zone(), 1, GetReplacementHigh(left));
481
482      NodeProperties::ChangeOp(node, machine()->Int32PairMul());
483      // We access the additional return values through projections.
484      ReplaceNodeWithProjections(node);
485      break;
486    }
487    case IrOpcode::kWord64Or: {
488      DCHECK_EQ(2, node->InputCount());
489      Node* left = node->InputAt(0);
490      Node* right = node->InputAt(1);
491
492      Node* low_node =
493          graph()->NewNode(machine()->Word32Or(), GetReplacementLow(left),
494                           GetReplacementLow(right));
495      Node* high_node =
496          graph()->NewNode(machine()->Word32Or(), GetReplacementHigh(left),
497                           GetReplacementHigh(right));
498      ReplaceNode(node, low_node, high_node);
499      break;
500    }
501    case IrOpcode::kWord64Xor: {
502      DCHECK_EQ(2, node->InputCount());
503      Node* left = node->InputAt(0);
504      Node* right = node->InputAt(1);
505
506      Node* low_node =
507          graph()->NewNode(machine()->Word32Xor(), GetReplacementLow(left),
508                           GetReplacementLow(right));
509      Node* high_node =
510          graph()->NewNode(machine()->Word32Xor(), GetReplacementHigh(left),
511                           GetReplacementHigh(right));
512      ReplaceNode(node, low_node, high_node);
513      break;
514    }
515    case IrOpcode::kWord64Shl: {
516      // TODO(turbofan): if the shift count >= 32, then we can set the low word
517      // of the output to 0 and just calculate the high word.
518      DCHECK_EQ(2, node->InputCount());
519      Node* shift = node->InputAt(1);
520      if (HasReplacementLow(shift)) {
521        // We do not have to care about the high word replacement, because
522        // the shift can only be between 0 and 63 anyways.
523        node->ReplaceInput(1, GetReplacementLow(shift));
524      }
525
526      Node* value = node->InputAt(0);
527      node->ReplaceInput(0, GetReplacementLow(value));
528      node->InsertInput(zone(), 1, GetReplacementHigh(value));
529
530      NodeProperties::ChangeOp(node, machine()->Word32PairShl());
531      // We access the additional return values through projections.
532      ReplaceNodeWithProjections(node);
533      break;
534    }
535    case IrOpcode::kWord64Shr: {
536      // TODO(turbofan): if the shift count >= 32, then we can set the low word
537      // of the output to 0 and just calculate the high word.
538      DCHECK_EQ(2, node->InputCount());
539      Node* shift = node->InputAt(1);
540      if (HasReplacementLow(shift)) {
541        // We do not have to care about the high word replacement, because
542        // the shift can only be between 0 and 63 anyways.
543        node->ReplaceInput(1, GetReplacementLow(shift));
544      }
545
546      Node* value = node->InputAt(0);
547      node->ReplaceInput(0, GetReplacementLow(value));
548      node->InsertInput(zone(), 1, GetReplacementHigh(value));
549
550      NodeProperties::ChangeOp(node, machine()->Word32PairShr());
551      // We access the additional return values through projections.
552      ReplaceNodeWithProjections(node);
553      break;
554    }
555    case IrOpcode::kWord64Sar: {
556      // TODO(turbofan): if the shift count >= 32, then we can set the low word
557      // of the output to 0 and just calculate the high word.
558      DCHECK_EQ(2, node->InputCount());
559      Node* shift = node->InputAt(1);
560      if (HasReplacementLow(shift)) {
561        // We do not have to care about the high word replacement, because
562        // the shift can only be between 0 and 63 anyways.
563        node->ReplaceInput(1, GetReplacementLow(shift));
564      }
565
566      Node* value = node->InputAt(0);
567      node->ReplaceInput(0, GetReplacementLow(value));
568      node->InsertInput(zone(), 1, GetReplacementHigh(value));
569
570      NodeProperties::ChangeOp(node, machine()->Word32PairSar());
571      // We access the additional return values through projections.
572      ReplaceNodeWithProjections(node);
573      break;
574    }
575    case IrOpcode::kWord64Equal: {
576      DCHECK_EQ(2, node->InputCount());
577      Node* left = node->InputAt(0);
578      Node* right = node->InputAt(1);
579
580      // TODO(wasm): Use explicit comparisons and && here?
581      Node* replacement = graph()->NewNode(
582          machine()->Word32Equal(),
583          graph()->NewNode(
584              machine()->Word32Or(),
585              graph()->NewNode(machine()->Word32Xor(), GetReplacementLow(left),
586                               GetReplacementLow(right)),
587              graph()->NewNode(machine()->Word32Xor(), GetReplacementHigh(left),
588                               GetReplacementHigh(right))),
589          graph()->NewNode(common()->Int32Constant(0)));
590
591      ReplaceNode(node, replacement, nullptr);
592      break;
593    }
594    case IrOpcode::kInt64LessThan: {
595      LowerComparison(node, machine()->Int32LessThan(),
596                      machine()->Uint32LessThan());
597      break;
598    }
599    case IrOpcode::kInt64LessThanOrEqual: {
600      LowerComparison(node, machine()->Int32LessThan(),
601                      machine()->Uint32LessThanOrEqual());
602      break;
603    }
604    case IrOpcode::kUint64LessThan: {
605      LowerComparison(node, machine()->Uint32LessThan(),
606                      machine()->Uint32LessThan());
607      break;
608    }
609    case IrOpcode::kUint64LessThanOrEqual: {
610      LowerComparison(node, machine()->Uint32LessThan(),
611                      machine()->Uint32LessThanOrEqual());
612      break;
613    }
614    case IrOpcode::kSignExtendWord32ToInt64:
615    case IrOpcode::kChangeInt32ToInt64: {
616      DCHECK_EQ(1, node->InputCount());
617      Node* input = node->InputAt(0);
618      if (HasReplacementLow(input)) {
619        input = GetReplacementLow(input);
620      }
621      // We use SAR to preserve the sign in the high word.
622      ReplaceNode(
623          node, input,
624          graph()->NewNode(machine()->Word32Sar(), input,
625                           graph()->NewNode(common()->Int32Constant(31))));
626      node->NullAllInputs();
627      break;
628    }
629    case IrOpcode::kChangeUint32ToUint64: {
630      DCHECK_EQ(1, node->InputCount());
631      Node* input = node->InputAt(0);
632      if (HasReplacementLow(input)) {
633        input = GetReplacementLow(input);
634      }
635      ReplaceNode(node, input, graph()->NewNode(common()->Int32Constant(0)));
636      node->NullAllInputs();
637      break;
638    }
639    case IrOpcode::kBitcastInt64ToFloat64: {
640      DCHECK_EQ(1, node->InputCount());
641      Node* input = node->InputAt(0);
642      Node* stack_slot = graph()->NewNode(
643          machine()->StackSlot(MachineRepresentation::kWord64));
644
645      Node* store_high_word = graph()->NewNode(
646          machine()->Store(
647              StoreRepresentation(MachineRepresentation::kWord32,
648                                  WriteBarrierKind::kNoWriteBarrier)),
649          stack_slot,
650          graph()->NewNode(
651              common()->Int32Constant(kInt64UpperHalfMemoryOffset)),
652          GetReplacementHigh(input), graph()->start(), graph()->start());
653
654      Node* store_low_word = graph()->NewNode(
655          machine()->Store(
656              StoreRepresentation(MachineRepresentation::kWord32,
657                                  WriteBarrierKind::kNoWriteBarrier)),
658          stack_slot,
659          graph()->NewNode(
660              common()->Int32Constant(kInt64LowerHalfMemoryOffset)),
661          GetReplacementLow(input), store_high_word, graph()->start());
662
663      Node* load =
664          graph()->NewNode(machine()->Load(MachineType::Float64()), stack_slot,
665                           graph()->NewNode(common()->Int32Constant(0)),
666                           store_low_word, graph()->start());
667
668      ReplaceNode(node, load, nullptr);
669      break;
670    }
671    case IrOpcode::kBitcastFloat64ToInt64: {
672      DCHECK_EQ(1, node->InputCount());
673      Node* input = node->InputAt(0);
674      if (HasReplacementLow(input)) {
675        input = GetReplacementLow(input);
676      }
677      Node* stack_slot = graph()->NewNode(
678          machine()->StackSlot(MachineRepresentation::kWord64));
679      Node* store = graph()->NewNode(
680          machine()->Store(
681              StoreRepresentation(MachineRepresentation::kFloat64,
682                                  WriteBarrierKind::kNoWriteBarrier)),
683          stack_slot, graph()->NewNode(common()->Int32Constant(0)), input,
684          graph()->start(), graph()->start());
685
686      Node* high_node = graph()->NewNode(
687          machine()->Load(MachineType::Int32()), stack_slot,
688          graph()->NewNode(
689              common()->Int32Constant(kInt64UpperHalfMemoryOffset)),
690          store, graph()->start());
691
692      Node* low_node = graph()->NewNode(
693          machine()->Load(MachineType::Int32()), stack_slot,
694          graph()->NewNode(
695              common()->Int32Constant(kInt64LowerHalfMemoryOffset)),
696          store, graph()->start());
697      ReplaceNode(node, low_node, high_node);
698      break;
699    }
700    case IrOpcode::kWord64RolLowerable:
701      DCHECK(machine()->Word32Rol().IsSupported());
702      V8_FALLTHROUGH;
703    case IrOpcode::kWord64RorLowerable: {
704      DCHECK_EQ(3, node->InputCount());
705      Node* input = node->InputAt(0);
706      Node* shift = HasReplacementLow(node->InputAt(1))
707                        ? GetReplacementLow(node->InputAt(1))
708                        : node->InputAt(1);
709      Int32Matcher m(shift);
710      if (m.HasResolvedValue()) {
711        // Precondition: 0 <= shift < 64.
712        int32_t shift_value = m.ResolvedValue() & 0x3F;
713        if (shift_value == 0) {
714          ReplaceNode(node, GetReplacementLow(input),
715                      GetReplacementHigh(input));
716        } else if (shift_value == 32) {
717          ReplaceNode(node, GetReplacementHigh(input),
718                      GetReplacementLow(input));
719        } else {
720          Node* low_input;
721          Node* high_input;
722          if (shift_value < 32) {
723            low_input = GetReplacementLow(input);
724            high_input = GetReplacementHigh(input);
725          } else {
726            low_input = GetReplacementHigh(input);
727            high_input = GetReplacementLow(input);
728          }
729          int32_t masked_shift_value = shift_value & 0x1F;
730          Node* masked_shift =
731              graph()->NewNode(common()->Int32Constant(masked_shift_value));
732          Node* inv_shift = graph()->NewNode(
733              common()->Int32Constant(32 - masked_shift_value));
734
735          auto* op1 = machine()->Word32Shr();
736          auto* op2 = machine()->Word32Shl();
737          bool is_ror = node->opcode() == IrOpcode::kWord64RorLowerable;
738          if (!is_ror) std::swap(op1, op2);
739
740          Node* low_node =
741              graph()->NewNode(machine()->Word32Or(),
742                               graph()->NewNode(op1, low_input, masked_shift),
743                               graph()->NewNode(op2, high_input, inv_shift));
744          Node* high_node =
745              graph()->NewNode(machine()->Word32Or(),
746                               graph()->NewNode(op1, high_input, masked_shift),
747                               graph()->NewNode(op2, low_input, inv_shift));
748          ReplaceNode(node, low_node, high_node);
749        }
750      } else {
751        Node* safe_shift = shift;
752        if (!machine()->Word32ShiftIsSafe()) {
753          safe_shift =
754              graph()->NewNode(machine()->Word32And(), shift,
755                               graph()->NewNode(common()->Int32Constant(0x1F)));
756        }
757
758        bool is_ror = node->opcode() == IrOpcode::kWord64RorLowerable;
759        Node* inv_mask =
760            is_ror ? graph()->NewNode(
761                         machine()->Word32Xor(),
762                         graph()->NewNode(
763                             machine()->Word32Shr(),
764                             graph()->NewNode(common()->Int32Constant(-1)),
765                             safe_shift),
766                         graph()->NewNode(common()->Int32Constant(-1)))
767                   : graph()->NewNode(
768                         machine()->Word32Shl(),
769                         graph()->NewNode(common()->Int32Constant(-1)),
770                         safe_shift);
771
772        Node* bit_mask =
773            graph()->NewNode(machine()->Word32Xor(), inv_mask,
774                             graph()->NewNode(common()->Int32Constant(-1)));
775
776        // We have to mask the shift value for this comparison. If
777        // !machine()->Word32ShiftIsSafe() then the masking should already be
778        // part of the graph.
779        Node* masked_shift6 = shift;
780        if (machine()->Word32ShiftIsSafe()) {
781          masked_shift6 =
782              graph()->NewNode(machine()->Word32And(), shift,
783                               graph()->NewNode(common()->Int32Constant(0x3F)));
784        }
785
786        Diamond lt32(
787            graph(), common(),
788            graph()->NewNode(machine()->Int32LessThan(), masked_shift6,
789                             graph()->NewNode(common()->Int32Constant(32))));
790        lt32.Chain(NodeProperties::GetControlInput(node));
791
792        // The low word and the high word can be swapped either at the input or
793        // at the output. We swap the inputs so that shift does not have to be
794        // kept for so long in a register.
795        Node* input_low =
796            lt32.Phi(MachineRepresentation::kWord32, GetReplacementLow(input),
797                     GetReplacementHigh(input));
798        Node* input_high =
799            lt32.Phi(MachineRepresentation::kWord32, GetReplacementHigh(input),
800                     GetReplacementLow(input));
801
802        const Operator* oper =
803            is_ror ? machine()->Word32Ror() : machine()->Word32Rol().op();
804
805        Node* rotate_low = graph()->NewNode(oper, input_low, safe_shift);
806        Node* rotate_high = graph()->NewNode(oper, input_high, safe_shift);
807
808        auto* mask1 = bit_mask;
809        auto* mask2 = inv_mask;
810        if (!is_ror) std::swap(mask1, mask2);
811
812        Node* low_node = graph()->NewNode(
813            machine()->Word32Or(),
814            graph()->NewNode(machine()->Word32And(), rotate_low, mask1),
815            graph()->NewNode(machine()->Word32And(), rotate_high, mask2));
816        Node* high_node = graph()->NewNode(
817            machine()->Word32Or(),
818            graph()->NewNode(machine()->Word32And(), rotate_high, mask1),
819            graph()->NewNode(machine()->Word32And(), rotate_low, mask2));
820        ReplaceNode(node, low_node, high_node);
821      }
822      break;
823    }
824    case IrOpcode::kWord64ClzLowerable: {
825      DCHECK_EQ(2, node->InputCount());
826      Node* input = node->InputAt(0);
827      Diamond d(
828          graph(), common(),
829          graph()->NewNode(machine()->Word32Equal(), GetReplacementHigh(input),
830                           graph()->NewNode(common()->Int32Constant(0))));
831      d.Chain(NodeProperties::GetControlInput(node));
832
833      Node* low_node = d.Phi(
834          MachineRepresentation::kWord32,
835          graph()->NewNode(machine()->Int32Add(),
836                           graph()->NewNode(machine()->Word32Clz(),
837                                            GetReplacementLow(input)),
838                           graph()->NewNode(common()->Int32Constant(32))),
839          graph()->NewNode(machine()->Word32Clz(), GetReplacementHigh(input)));
840      ReplaceNode(node, low_node, graph()->NewNode(common()->Int32Constant(0)));
841      break;
842    }
843    case IrOpcode::kWord64CtzLowerable: {
844      DCHECK_EQ(2, node->InputCount());
845      DCHECK(machine()->Word32Ctz().IsSupported());
846      Node* input = node->InputAt(0);
847      Diamond d(
848          graph(), common(),
849          graph()->NewNode(machine()->Word32Equal(), GetReplacementLow(input),
850                           graph()->NewNode(common()->Int32Constant(0))));
851      d.Chain(NodeProperties::GetControlInput(node));
852
853      Node* low_node =
854          d.Phi(MachineRepresentation::kWord32,
855                graph()->NewNode(machine()->Int32Add(),
856                                 graph()->NewNode(machine()->Word32Ctz().op(),
857                                                  GetReplacementHigh(input)),
858                                 graph()->NewNode(common()->Int32Constant(32))),
859                graph()->NewNode(machine()->Word32Ctz().op(),
860                                 GetReplacementLow(input)));
861      ReplaceNode(node, low_node, graph()->NewNode(common()->Int32Constant(0)));
862      break;
863    }
864    case IrOpcode::kWord64Ror:
865    case IrOpcode::kWord64Rol:
866    case IrOpcode::kWord64Ctz:
867    case IrOpcode::kWord64Clz:
868      FATAL("%s operator should not be used in 32-bit systems",
869            node->op()->mnemonic());
870    case IrOpcode::kWord64Popcnt: {
871      DCHECK_EQ(1, node->InputCount());
872      Node* input = node->InputAt(0);
873      // We assume that a Word64Popcnt node only has been created if
874      // Word32Popcnt is actually supported.
875      DCHECK(machine()->Word32Popcnt().IsSupported());
876      ReplaceNode(node, graph()->NewNode(
877                            machine()->Int32Add(),
878                            graph()->NewNode(machine()->Word32Popcnt().op(),
879                                             GetReplacementLow(input)),
880                            graph()->NewNode(machine()->Word32Popcnt().op(),
881                                             GetReplacementHigh(input))),
882                  graph()->NewNode(common()->Int32Constant(0)));
883      break;
884    }
885    case IrOpcode::kPhi: {
886      MachineRepresentation rep = PhiRepresentationOf(node->op());
887      if (rep == MachineRepresentation::kWord64) {
888        // The replacement nodes have already been created, we only have to
889        // replace placeholder nodes.
890        Node* low_node = GetReplacementLow(node);
891        Node* high_node = GetReplacementHigh(node);
892        for (int i = 0; i < node->op()->ValueInputCount(); i++) {
893          low_node->ReplaceInput(i, GetReplacementLow(node->InputAt(i)));
894          high_node->ReplaceInput(i, GetReplacementHigh(node->InputAt(i)));
895        }
896      } else {
897        DefaultLowering(node);
898      }
899      break;
900    }
901    case IrOpcode::kLoopExitValue: {
902      MachineRepresentation rep = LoopExitValueRepresentationOf(node->op());
903      if (rep == MachineRepresentation::kWord64) {
904        Node* low_node = graph()->NewNode(
905            common()->LoopExitValue(MachineRepresentation::kWord32),
906            GetReplacementLow(node->InputAt(0)), node->InputAt(1));
907        Node* high_node = graph()->NewNode(
908            common()->LoopExitValue(MachineRepresentation::kWord32),
909            GetReplacementHigh(node->InputAt(0)), node->InputAt(1));
910        ReplaceNode(node, low_node, high_node);
911      } else {
912        DefaultLowering(node);
913      }
914      break;
915    }
916    case IrOpcode::kWord64ReverseBytes: {
917      Node* input = node->InputAt(0);
918      ReplaceNode(node,
919                  graph()->NewNode(machine()->Word32ReverseBytes(),
920                                   GetReplacementHigh(input)),
921                  graph()->NewNode(machine()->Word32ReverseBytes(),
922                                   GetReplacementLow(input)));
923      break;
924    }
925    case IrOpcode::kSignExtendWord8ToInt64: {
926      DCHECK_EQ(1, node->InputCount());
927      Node* input = node->InputAt(0);
928      if (HasReplacementLow(input)) {
929        input = GetReplacementLow(input);
930      }
931      // Sign extend low node to Int32
932      input = graph()->NewNode(machine()->SignExtendWord8ToInt32(), input);
933
934      // We use SAR to preserve the sign in the high word.
935      ReplaceNode(
936          node, input,
937          graph()->NewNode(machine()->Word32Sar(), input,
938                           graph()->NewNode(common()->Int32Constant(31))));
939      node->NullAllInputs();
940      break;
941    }
942    case IrOpcode::kSignExtendWord16ToInt64: {
943      DCHECK_EQ(1, node->InputCount());
944      Node* input = node->InputAt(0);
945      if (HasReplacementLow(input)) {
946        input = GetReplacementLow(input);
947      }
948      // Sign extend low node to Int32
949      input = graph()->NewNode(machine()->SignExtendWord16ToInt32(), input);
950
951      // We use SAR to preserve the sign in the high word.
952      ReplaceNode(
953          node, input,
954          graph()->NewNode(machine()->Word32Sar(), input,
955                           graph()->NewNode(common()->Int32Constant(31))));
956      node->NullAllInputs();
957      break;
958    }
959    case IrOpcode::kWord64AtomicLoad: {
960      DCHECK_EQ(4, node->InputCount());
961      AtomicLoadParameters params = AtomicLoadParametersOf(node->op());
962      DefaultLowering(node, true);
963      if (params.representation() == MachineType::Uint64()) {
964        NodeProperties::ChangeOp(
965            node, machine()->Word32AtomicPairLoad(params.order()));
966        ReplaceNodeWithProjections(node);
967      } else {
968        NodeProperties::ChangeOp(node, machine()->Word32AtomicLoad(params));
969        ReplaceNode(node, node, graph()->NewNode(common()->Int32Constant(0)));
970      }
971      break;
972    }
973    case IrOpcode::kWord64AtomicStore: {
974      DCHECK_EQ(5, node->InputCount());
975      AtomicStoreParameters params = AtomicStoreParametersOf(node->op());
976      if (params.representation() == MachineRepresentation::kWord64) {
977        LowerMemoryBaseAndIndex(node);
978        Node* value = node->InputAt(2);
979        node->ReplaceInput(2, GetReplacementLow(value));
980        node->InsertInput(zone(), 3, GetReplacementHigh(value));
981        NodeProperties::ChangeOp(
982            node, machine()->Word32AtomicPairStore(params.order()));
983      } else {
984        DefaultLowering(node, true);
985        NodeProperties::ChangeOp(node, machine()->Word32AtomicStore(params));
986      }
987      break;
988    }
989#define ATOMIC_CASE(name)                                                   \
990  case IrOpcode::kWord64Atomic##name: {                                     \
991    MachineType type = AtomicOpType(node->op());                            \
992    if (type == MachineType::Uint64()) {                                    \
993      LowerWord64AtomicBinop(node, machine()->Word32AtomicPair##name());    \
994    } else {                                                                \
995      LowerWord64AtomicNarrowOp(node, machine()->Word32Atomic##name(type)); \
996    }                                                                       \
997    break;                                                                  \
998  }
999      ATOMIC_CASE(Add)
1000      ATOMIC_CASE(Sub)
1001      ATOMIC_CASE(And)
1002      ATOMIC_CASE(Or)
1003      ATOMIC_CASE(Xor)
1004      ATOMIC_CASE(Exchange)
1005#undef ATOMIC_CASE
1006    case IrOpcode::kWord64AtomicCompareExchange: {
1007      MachineType type = AtomicOpType(node->op());
1008      if (type == MachineType::Uint64()) {
1009        LowerMemoryBaseAndIndex(node);
1010        Node* old_value = node->InputAt(2);
1011        Node* new_value = node->InputAt(3);
1012        node->ReplaceInput(2, GetReplacementLow(old_value));
1013        node->ReplaceInput(3, GetReplacementHigh(old_value));
1014        node->InsertInput(zone(), 4, GetReplacementLow(new_value));
1015        node->InsertInput(zone(), 5, GetReplacementHigh(new_value));
1016        NodeProperties::ChangeOp(node,
1017                                 machine()->Word32AtomicPairCompareExchange());
1018        ReplaceNodeWithProjections(node);
1019      } else {
1020        DCHECK(type == MachineType::Uint32() || type == MachineType::Uint16() ||
1021               type == MachineType::Uint8());
1022        DefaultLowering(node, true);
1023        NodeProperties::ChangeOp(node,
1024                                 machine()->Word32AtomicCompareExchange(type));
1025        ReplaceNode(node, node, graph()->NewNode(common()->Int32Constant(0)));
1026      }
1027      break;
1028    }
1029    case IrOpcode::kI64x2Splat: {
1030      DCHECK_EQ(1, node->InputCount());
1031      Node* input = node->InputAt(0);
1032      node->ReplaceInput(0, GetReplacementLow(input));
1033      node->AppendInput(zone(), GetReplacementHigh(input));
1034      NodeProperties::ChangeOp(node, machine()->I64x2SplatI32Pair());
1035      break;
1036    }
1037    case IrOpcode::kI64x2ExtractLane: {
1038      DCHECK_EQ(1, node->InputCount());
1039      Node* input = node->InputAt(0);
1040      int32_t lane = OpParameter<int32_t>(node->op());
1041      ReplaceNode(
1042          node, graph()->NewNode(machine()->I32x4ExtractLane(lane * 2), input),
1043          graph()->NewNode(machine()->I32x4ExtractLane(lane * 2 + 1), input));
1044      break;
1045    }
1046    case IrOpcode::kI64x2ReplaceLane: {
1047      DCHECK_EQ(2, node->InputCount());
1048      int32_t lane = OpParameter<int32_t>(node->op());
1049      Node* input = node->InputAt(1);
1050      node->ReplaceInput(1, GetReplacementLow(input));
1051      node->AppendInput(zone(), GetReplacementHigh(input));
1052      NodeProperties::ChangeOp(node, machine()->I64x2ReplaceLaneI32Pair(lane));
1053      break;
1054    }
1055
1056    default: { DefaultLowering(node); }
1057  }
1058}
1059
1060void Int64Lowering::LowerComparison(Node* node, const Operator* high_word_op,
1061                                    const Operator* low_word_op) {
1062  DCHECK_EQ(2, node->InputCount());
1063  Node* left = node->InputAt(0);
1064  Node* right = node->InputAt(1);
1065  Node* replacement = graph()->NewNode(
1066      machine()->Word32Or(),
1067      graph()->NewNode(high_word_op, GetReplacementHigh(left),
1068                       GetReplacementHigh(right)),
1069      graph()->NewNode(
1070          machine()->Word32And(),
1071          graph()->NewNode(machine()->Word32Equal(), GetReplacementHigh(left),
1072                           GetReplacementHigh(right)),
1073          graph()->NewNode(low_word_op, GetReplacementLow(left),
1074                           GetReplacementLow(right))));
1075
1076  ReplaceNode(node, replacement, nullptr);
1077}
1078
1079bool Int64Lowering::DefaultLowering(Node* node, bool low_word_only) {
1080  bool something_changed = false;
1081  for (int i = NodeProperties::PastValueIndex(node) - 1; i >= 0; i--) {
1082    Node* input = node->InputAt(i);
1083    if (HasReplacementLow(input)) {
1084      something_changed = true;
1085      node->ReplaceInput(i, GetReplacementLow(input));
1086    }
1087    if (!low_word_only && HasReplacementHigh(input)) {
1088      something_changed = true;
1089      node->InsertInput(zone(), i + 1, GetReplacementHigh(input));
1090    }
1091  }
1092  return something_changed;
1093}
1094
1095const CallDescriptor* Int64Lowering::LowerCallDescriptor(
1096    const CallDescriptor* call_descriptor) {
1097  if (special_case_) {
1098    auto replacement = special_case_->replacements.find(call_descriptor);
1099    if (replacement != special_case_->replacements.end()) {
1100      return replacement->second;
1101    }
1102  }
1103  return GetI32WasmCallDescriptor(zone(), call_descriptor);
1104}
1105
1106void Int64Lowering::ReplaceNode(Node* old, Node* new_low, Node* new_high) {
1107  // if new_low == nullptr, then also new_high == nullptr.
1108  DCHECK(new_low != nullptr || new_high == nullptr);
1109  replacements_[old->id()].low = new_low;
1110  replacements_[old->id()].high = new_high;
1111}
1112
1113bool Int64Lowering::HasReplacementLow(Node* node) {
1114  return replacements_[node->id()].low != nullptr;
1115}
1116
1117Node* Int64Lowering::GetReplacementLow(Node* node) {
1118  Node* result = replacements_[node->id()].low;
1119  DCHECK(result);
1120  return result;
1121}
1122
1123bool Int64Lowering::HasReplacementHigh(Node* node) {
1124  return replacements_[node->id()].high != nullptr;
1125}
1126
1127Node* Int64Lowering::GetReplacementHigh(Node* node) {
1128  Node* result = replacements_[node->id()].high;
1129  DCHECK(result);
1130  return result;
1131}
1132
1133void Int64Lowering::PreparePhiReplacement(Node* phi) {
1134  MachineRepresentation rep = PhiRepresentationOf(phi->op());
1135  if (rep == MachineRepresentation::kWord64) {
1136    // We have to create the replacements for a phi node before we actually
1137    // lower the phi to break potential cycles in the graph. The replacements of
1138    // input nodes do not exist yet, so we use a placeholder node to pass the
1139    // graph verifier.
1140    int value_count = phi->op()->ValueInputCount();
1141    Node** inputs_low = zone()->NewArray<Node*>(value_count + 1);
1142    Node** inputs_high = zone()->NewArray<Node*>(value_count + 1);
1143    for (int i = 0; i < value_count; i++) {
1144      inputs_low[i] = placeholder_;
1145      inputs_high[i] = placeholder_;
1146    }
1147    inputs_low[value_count] = NodeProperties::GetControlInput(phi, 0);
1148    inputs_high[value_count] = NodeProperties::GetControlInput(phi, 0);
1149    ReplaceNode(phi,
1150                graph()->NewNode(
1151                    common()->Phi(MachineRepresentation::kWord32, value_count),
1152                    value_count + 1, inputs_low, false),
1153                graph()->NewNode(
1154                    common()->Phi(MachineRepresentation::kWord32, value_count),
1155                    value_count + 1, inputs_high, false));
1156  }
1157}
1158
1159void Int64Lowering::ReplaceNodeWithProjections(Node* node) {
1160  DCHECK(node != nullptr);
1161  Node* low_node =
1162      graph()->NewNode(common()->Projection(0), node, graph()->start());
1163  Node* high_node =
1164      graph()->NewNode(common()->Projection(1), node, graph()->start());
1165  ReplaceNode(node, low_node, high_node);
1166}
1167
1168void Int64Lowering::LowerMemoryBaseAndIndex(Node* node) {
1169  DCHECK(node != nullptr);
1170  // Low word only replacements for memory operands for 32-bit address space.
1171  Node* base = node->InputAt(0);
1172  Node* index = node->InputAt(1);
1173  if (HasReplacementLow(base)) {
1174    node->ReplaceInput(0, GetReplacementLow(base));
1175  }
1176  if (HasReplacementLow(index)) {
1177    node->ReplaceInput(1, GetReplacementLow(index));
1178  }
1179}
1180
1181}  // namespace compiler
1182}  // namespace internal
1183}  // namespace v8
1184