1// Copyright 2019 the V8 project authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "src/compiler/memory-lowering.h"
6
7#include "src/codegen/interface-descriptors-inl.h"
8#include "src/compiler/access-builder.h"
9#include "src/compiler/js-graph.h"
10#include "src/compiler/linkage.h"
11#include "src/compiler/node-matchers.h"
12#include "src/compiler/node-properties.h"
13#include "src/compiler/node.h"
14#include "src/compiler/simplified-operator.h"
15#include "src/roots/roots-inl.h"
16#include "src/sandbox/external-pointer.h"
17
18#if V8_ENABLE_WEBASSEMBLY
19#include "src/wasm/wasm-linkage.h"
20#include "src/wasm/wasm-objects.h"
21#endif
22namespace v8 {
23namespace internal {
24namespace compiler {
25
26// An allocation group represents a set of allocations that have been folded
27// together.
28class MemoryLowering::AllocationGroup final : public ZoneObject {
29 public:
30  AllocationGroup(Node* node, AllocationType allocation, Zone* zone);
31  AllocationGroup(Node* node, AllocationType allocation, Node* size,
32                  Zone* zone);
33  ~AllocationGroup() = default;
34
35  void Add(Node* object);
36  bool Contains(Node* object) const;
37  bool IsYoungGenerationAllocation() const {
38    return allocation() == AllocationType::kYoung;
39  }
40
41  AllocationType allocation() const { return allocation_; }
42  Node* size() const { return size_; }
43
44 private:
45  ZoneSet<NodeId> node_ids_;
46  AllocationType const allocation_;
47  Node* const size_;
48
49  static inline AllocationType CheckAllocationType(AllocationType allocation) {
50    // For non-generational heap, all young allocations are redirected to old
51    // space.
52    if (FLAG_single_generation && allocation == AllocationType::kYoung) {
53      return AllocationType::kOld;
54    }
55    return allocation;
56  }
57
58  DISALLOW_IMPLICIT_CONSTRUCTORS(AllocationGroup);
59};
60
61MemoryLowering::MemoryLowering(JSGraph* jsgraph, Zone* zone,
62                               JSGraphAssembler* graph_assembler,
63                               AllocationFolding allocation_folding,
64                               WriteBarrierAssertFailedCallback callback,
65                               const char* function_debug_name)
66    : isolate_(jsgraph->isolate()),
67      zone_(zone),
68      graph_(jsgraph->graph()),
69      common_(jsgraph->common()),
70      machine_(jsgraph->machine()),
71      graph_assembler_(graph_assembler),
72      allocation_folding_(allocation_folding),
73      write_barrier_assert_failed_(callback),
74      function_debug_name_(function_debug_name) {}
75
76Zone* MemoryLowering::graph_zone() const { return graph()->zone(); }
77
78Reduction MemoryLowering::Reduce(Node* node) {
79  switch (node->opcode()) {
80    case IrOpcode::kAllocate:
81      // Allocate nodes were purged from the graph in effect-control
82      // linearization.
83      UNREACHABLE();
84    case IrOpcode::kAllocateRaw:
85      return ReduceAllocateRaw(node);
86    case IrOpcode::kLoadFromObject:
87    case IrOpcode::kLoadImmutableFromObject:
88      return ReduceLoadFromObject(node);
89    case IrOpcode::kLoadElement:
90      return ReduceLoadElement(node);
91    case IrOpcode::kLoadField:
92      return ReduceLoadField(node);
93    case IrOpcode::kStoreToObject:
94    case IrOpcode::kInitializeImmutableInObject:
95      return ReduceStoreToObject(node);
96    case IrOpcode::kStoreElement:
97      return ReduceStoreElement(node);
98    case IrOpcode::kStoreField:
99      return ReduceStoreField(node);
100    case IrOpcode::kStore:
101      return ReduceStore(node);
102    default:
103      return NoChange();
104  }
105}
106
107void MemoryLowering::EnsureAllocateOperator() {
108  if (allocate_operator_.is_set()) return;
109
110  auto descriptor = AllocateDescriptor{};
111  StubCallMode mode = isolate_ != nullptr ? StubCallMode::kCallCodeObject
112                                          : StubCallMode::kCallBuiltinPointer;
113  auto call_descriptor = Linkage::GetStubCallDescriptor(
114      graph_zone(), descriptor, descriptor.GetStackParameterCount(),
115      CallDescriptor::kCanUseRoots, Operator::kNoThrow, mode);
116  allocate_operator_.set(common()->Call(call_descriptor));
117}
118
119#if V8_ENABLE_WEBASSEMBLY
120Node* MemoryLowering::GetWasmInstanceNode() {
121  if (wasm_instance_node_.is_set()) return wasm_instance_node_.get();
122  for (Node* use : graph()->start()->uses()) {
123    if (use->opcode() == IrOpcode::kParameter &&
124        ParameterIndexOf(use->op()) == wasm::kWasmInstanceParameterIndex) {
125      wasm_instance_node_.set(use);
126      return use;
127    }
128  }
129  UNREACHABLE();  // The instance node must have been created before.
130}
131#endif  // V8_ENABLE_WEBASSEMBLY
132
133#define __ gasm()->
134
135Reduction MemoryLowering::ReduceAllocateRaw(
136    Node* node, AllocationType allocation_type,
137    AllowLargeObjects allow_large_objects, AllocationState const** state_ptr) {
138  DCHECK_EQ(IrOpcode::kAllocateRaw, node->opcode());
139  DCHECK_IMPLIES(allocation_folding_ == AllocationFolding::kDoAllocationFolding,
140                 state_ptr != nullptr);
141  if (FLAG_single_generation && allocation_type == AllocationType::kYoung) {
142    allocation_type = AllocationType::kOld;
143  }
144  // Code objects may have a maximum size smaller than kMaxHeapObjectSize due to
145  // guard pages. If we need to support allocating code here we would need to
146  // call MemoryChunkLayout::MaxRegularCodeObjectSize() at runtime.
147  DCHECK_NE(allocation_type, AllocationType::kCode);
148  Node* value;
149  Node* size = node->InputAt(0);
150  Node* effect = node->InputAt(1);
151  Node* control = node->InputAt(2);
152
153  gasm()->InitializeEffectControl(effect, control);
154
155  Node* allocate_builtin;
156  if (isolate_ != nullptr) {
157    if (allocation_type == AllocationType::kYoung) {
158      if (allow_large_objects == AllowLargeObjects::kTrue) {
159        allocate_builtin = __ AllocateInYoungGenerationStubConstant();
160      } else {
161        allocate_builtin = __ AllocateRegularInYoungGenerationStubConstant();
162      }
163    } else {
164      if (allow_large_objects == AllowLargeObjects::kTrue) {
165        allocate_builtin = __ AllocateInOldGenerationStubConstant();
166      } else {
167        allocate_builtin = __ AllocateRegularInOldGenerationStubConstant();
168      }
169    }
170  } else {
171    // This lowering is used by Wasm, where we compile isolate-independent
172    // code. Builtin calls simply encode the target builtin ID, which will
173    // be patched to the builtin's address later.
174#if V8_ENABLE_WEBASSEMBLY
175    Builtin builtin;
176    if (allocation_type == AllocationType::kYoung) {
177      if (allow_large_objects == AllowLargeObjects::kTrue) {
178        builtin = Builtin::kAllocateInYoungGeneration;
179      } else {
180        builtin = Builtin::kAllocateRegularInYoungGeneration;
181      }
182    } else {
183      if (allow_large_objects == AllowLargeObjects::kTrue) {
184        builtin = Builtin::kAllocateInOldGeneration;
185      } else {
186        builtin = Builtin::kAllocateRegularInOldGeneration;
187      }
188    }
189    static_assert(std::is_same<Smi, BuiltinPtr>(), "BuiltinPtr must be Smi");
190    allocate_builtin =
191        graph()->NewNode(common()->NumberConstant(static_cast<int>(builtin)));
192#else
193    UNREACHABLE();
194#endif
195  }
196
197  // Determine the top/limit addresses.
198  Node* top_address;
199  Node* limit_address;
200  if (isolate_ != nullptr) {
201    top_address = __ ExternalConstant(
202        allocation_type == AllocationType::kYoung
203            ? ExternalReference::new_space_allocation_top_address(isolate())
204            : ExternalReference::old_space_allocation_top_address(isolate()));
205    limit_address = __ ExternalConstant(
206        allocation_type == AllocationType::kYoung
207            ? ExternalReference::new_space_allocation_limit_address(isolate())
208            : ExternalReference::old_space_allocation_limit_address(isolate()));
209  } else {
210    // Wasm mode: producing isolate-independent code, loading the isolate
211    // address at runtime.
212#if V8_ENABLE_WEBASSEMBLY
213    Node* instance_node = GetWasmInstanceNode();
214    int top_address_offset =
215        allocation_type == AllocationType::kYoung
216            ? WasmInstanceObject::kNewAllocationTopAddressOffset
217            : WasmInstanceObject::kOldAllocationTopAddressOffset;
218    int limit_address_offset =
219        allocation_type == AllocationType::kYoung
220            ? WasmInstanceObject::kNewAllocationLimitAddressOffset
221            : WasmInstanceObject::kOldAllocationLimitAddressOffset;
222    top_address =
223        __ Load(MachineType::Pointer(), instance_node,
224                __ IntPtrConstant(top_address_offset - kHeapObjectTag));
225    limit_address =
226        __ Load(MachineType::Pointer(), instance_node,
227                __ IntPtrConstant(limit_address_offset - kHeapObjectTag));
228#else
229    UNREACHABLE();
230#endif  // V8_ENABLE_WEBASSEMBLY
231  }
232
233  // Check if we can fold this allocation into a previous allocation represented
234  // by the incoming {state}.
235  IntPtrMatcher m(size);
236  if (m.IsInRange(0, kMaxRegularHeapObjectSize) && FLAG_inline_new &&
237      allocation_folding_ == AllocationFolding::kDoAllocationFolding) {
238    intptr_t const object_size = m.ResolvedValue();
239    AllocationState const* state = *state_ptr;
240    if (state->size() <= kMaxRegularHeapObjectSize - object_size &&
241        state->group()->allocation() == allocation_type) {
242      // We can fold this Allocate {node} into the allocation {group}
243      // represented by the given {state}. Compute the upper bound for
244      // the new {state}.
245      intptr_t const state_size = state->size() + object_size;
246
247      // Update the reservation check to the actual maximum upper bound.
248      AllocationGroup* const group = state->group();
249      if (machine()->Is64()) {
250        if (OpParameter<int64_t>(group->size()->op()) < state_size) {
251          NodeProperties::ChangeOp(group->size(),
252                                   common()->Int64Constant(state_size));
253        }
254      } else {
255        if (OpParameter<int32_t>(group->size()->op()) < state_size) {
256          NodeProperties::ChangeOp(
257              group->size(),
258              common()->Int32Constant(static_cast<int32_t>(state_size)));
259        }
260      }
261
262      // Update the allocation top with the new object allocation.
263      // TODO(bmeurer): Defer writing back top as much as possible.
264      Node* top = __ IntAdd(state->top(), size);
265      __ Store(StoreRepresentation(MachineType::PointerRepresentation(),
266                                   kNoWriteBarrier),
267               top_address, __ IntPtrConstant(0), top);
268
269      // Compute the effective inner allocated address.
270      value = __ BitcastWordToTagged(
271          __ IntAdd(state->top(), __ IntPtrConstant(kHeapObjectTag)));
272      effect = gasm()->effect();
273      control = gasm()->control();
274
275      // Extend the allocation {group}.
276      group->Add(value);
277      *state_ptr =
278          AllocationState::Open(group, state_size, top, effect, zone());
279    } else {
280      auto call_runtime = __ MakeDeferredLabel();
281      auto done = __ MakeLabel(MachineType::PointerRepresentation());
282
283      // Setup a mutable reservation size node; will be patched as we fold
284      // additional allocations into this new group.
285      Node* reservation_size = __ UniqueIntPtrConstant(object_size);
286
287      // Load allocation top and limit.
288      Node* top =
289          __ Load(MachineType::Pointer(), top_address, __ IntPtrConstant(0));
290      Node* limit =
291          __ Load(MachineType::Pointer(), limit_address, __ IntPtrConstant(0));
292
293      // Check if we need to collect garbage before we can start bump pointer
294      // allocation (always done for folded allocations).
295      Node* check = __ UintLessThan(__ IntAdd(top, reservation_size), limit);
296
297      __ GotoIfNot(check, &call_runtime);
298      __ Goto(&done, top);
299
300      __ Bind(&call_runtime);
301      {
302        EnsureAllocateOperator();
303        Node* vfalse = __ BitcastTaggedToWord(__ Call(
304            allocate_operator_.get(), allocate_builtin, reservation_size));
305        vfalse = __ IntSub(vfalse, __ IntPtrConstant(kHeapObjectTag));
306        __ Goto(&done, vfalse);
307      }
308
309      __ Bind(&done);
310
311      // Compute the new top and write it back.
312      top = __ IntAdd(done.PhiAt(0), __ IntPtrConstant(object_size));
313      __ Store(StoreRepresentation(MachineType::PointerRepresentation(),
314                                   kNoWriteBarrier),
315               top_address, __ IntPtrConstant(0), top);
316
317      // Compute the initial object address.
318      value = __ BitcastWordToTagged(
319          __ IntAdd(done.PhiAt(0), __ IntPtrConstant(kHeapObjectTag)));
320      effect = gasm()->effect();
321      control = gasm()->control();
322
323      // Start a new allocation group.
324      AllocationGroup* group = zone()->New<AllocationGroup>(
325          value, allocation_type, reservation_size, zone());
326      *state_ptr =
327          AllocationState::Open(group, object_size, top, effect, zone());
328    }
329  } else {
330    auto call_runtime = __ MakeDeferredLabel();
331    auto done = __ MakeLabel(MachineRepresentation::kTaggedPointer);
332
333    // Load allocation top and limit.
334    Node* top =
335        __ Load(MachineType::Pointer(), top_address, __ IntPtrConstant(0));
336    Node* limit =
337        __ Load(MachineType::Pointer(), limit_address, __ IntPtrConstant(0));
338
339    // Compute the new top.
340    Node* new_top = __ IntAdd(top, size);
341
342    // Check if we can do bump pointer allocation here.
343    Node* check = __ UintLessThan(new_top, limit);
344    __ GotoIfNot(check, &call_runtime);
345    if (allow_large_objects == AllowLargeObjects::kTrue) {
346      __ GotoIfNot(
347          __ UintLessThan(size, __ IntPtrConstant(kMaxRegularHeapObjectSize)),
348          &call_runtime);
349    }
350    __ Store(StoreRepresentation(MachineType::PointerRepresentation(),
351                                 kNoWriteBarrier),
352             top_address, __ IntPtrConstant(0), new_top);
353    __ Goto(&done, __ BitcastWordToTagged(
354                       __ IntAdd(top, __ IntPtrConstant(kHeapObjectTag))));
355
356    __ Bind(&call_runtime);
357    EnsureAllocateOperator();
358    __ Goto(&done, __ Call(allocate_operator_.get(), allocate_builtin, size));
359
360    __ Bind(&done);
361    value = done.PhiAt(0);
362    effect = gasm()->effect();
363    control = gasm()->control();
364
365    if (state_ptr) {
366      // Create an unfoldable allocation group.
367      AllocationGroup* group =
368          zone()->New<AllocationGroup>(value, allocation_type, zone());
369      *state_ptr = AllocationState::Closed(group, effect, zone());
370    }
371  }
372
373  return Replace(value);
374}
375
376Reduction MemoryLowering::ReduceLoadFromObject(Node* node) {
377  DCHECK(node->opcode() == IrOpcode::kLoadFromObject ||
378         node->opcode() == IrOpcode::kLoadImmutableFromObject);
379  ObjectAccess const& access = ObjectAccessOf(node->op());
380
381  MachineType machine_type = access.machine_type;
382
383  if (machine_type.IsMapWord()) {
384    CHECK_EQ(machine_type.semantic(), MachineSemantic::kAny);
385    return ReduceLoadMap(node);
386  }
387
388  MachineRepresentation rep = machine_type.representation();
389  const Operator* load_op =
390      ElementSizeInBytes(rep) > kTaggedSize &&
391              !machine()->UnalignedLoadSupported(machine_type.representation())
392          ? machine()->UnalignedLoad(machine_type)
393          : machine()->Load(machine_type);
394  NodeProperties::ChangeOp(node, load_op);
395  return Changed(node);
396}
397
398Reduction MemoryLowering::ReduceLoadElement(Node* node) {
399  DCHECK_EQ(IrOpcode::kLoadElement, node->opcode());
400  ElementAccess const& access = ElementAccessOf(node->op());
401  Node* index = node->InputAt(1);
402  node->ReplaceInput(1, ComputeIndex(access, index));
403  MachineType type = access.machine_type;
404  DCHECK(!type.IsMapWord());
405  NodeProperties::ChangeOp(node, machine()->Load(type));
406  return Changed(node);
407}
408
409Node* MemoryLowering::DecodeExternalPointer(
410    Node* node, ExternalPointerTag external_pointer_tag) {
411#ifdef V8_SANDBOXED_EXTERNAL_POINTERS
412  DCHECK(V8_SANDBOXED_EXTERNAL_POINTERS_BOOL);
413  DCHECK(node->opcode() == IrOpcode::kLoad);
414  DCHECK_EQ(kExternalPointerSize, kUInt32Size);
415  DCHECK_NE(kExternalPointerNullTag, external_pointer_tag);
416  Node* effect = NodeProperties::GetEffectInput(node);
417  Node* control = NodeProperties::GetControlInput(node);
418  __ InitializeEffectControl(effect, control);
419
420  // Clone the load node and put it here.
421  // TODO(turbofan): consider adding GraphAssembler::Clone() suitable for
422  // cloning nodes from arbitrary locaions in effect/control chains.
423  STATIC_ASSERT(kExternalPointerIndexShift > kSystemPointerSizeLog2);
424  Node* shifted_index = __ AddNode(graph()->CloneNode(node));
425  Node* shift_amount =
426      __ Int32Constant(kExternalPointerIndexShift - kSystemPointerSizeLog2);
427  Node* offset = __ Word32Shr(shifted_index, shift_amount);
428
429  // Uncomment this to generate a breakpoint for debugging purposes.
430  // __ DebugBreak();
431
432  // Decode loaded external pointer.
433  //
434  // Here we access the external pointer table through an ExternalReference.
435  // Alternatively, we could also hardcode the address of the table since it is
436  // never reallocated. However, in that case we must be able to guarantee that
437  // the generated code is never executed under a different Isolate, as that
438  // would allow access to external objects from different Isolates. It also
439  // would break if the code is serialized/deserialized at some point.
440  Node* table_address = __ ExternalConstant(
441      ExternalReference::external_pointer_table_address(isolate()));
442  Node* table = __ Load(MachineType::Pointer(), table_address,
443                        Internals::kExternalPointerTableBufferOffset);
444  Node* decoded_ptr =
445      __ Load(MachineType::Pointer(), table, __ ChangeUint32ToUint64(offset));
446  Node* tag = __ IntPtrConstant(~external_pointer_tag);
447  decoded_ptr = __ WordAnd(decoded_ptr, tag);
448  return decoded_ptr;
449#else
450  return node;
451#endif  // V8_SANDBOXED_EXTERNAL_POINTERS
452}
453
454Reduction MemoryLowering::ReduceLoadMap(Node* node) {
455#ifdef V8_MAP_PACKING
456  NodeProperties::ChangeOp(node, machine()->Load(MachineType::AnyTagged()));
457
458  Node* effect = NodeProperties::GetEffectInput(node);
459  Node* control = NodeProperties::GetControlInput(node);
460  __ InitializeEffectControl(effect, control);
461
462  node = __ AddNode(graph()->CloneNode(node));
463  return Replace(__ UnpackMapWord(node));
464#else
465  NodeProperties::ChangeOp(node, machine()->Load(MachineType::TaggedPointer()));
466  return Changed(node);
467#endif
468}
469
470Reduction MemoryLowering::ReduceLoadField(Node* node) {
471  DCHECK_EQ(IrOpcode::kLoadField, node->opcode());
472  FieldAccess const& access = FieldAccessOf(node->op());
473  Node* offset = __ IntPtrConstant(access.offset - access.tag());
474  node->InsertInput(graph_zone(), 1, offset);
475  MachineType type = access.machine_type;
476  if (V8_SANDBOXED_EXTERNAL_POINTERS_BOOL &&
477      access.type.Is(Type::ExternalPointer())) {
478    // External pointer table indices are stored as 32-bit numbers
479    type = MachineType::Uint32();
480  }
481
482  if (type.IsMapWord()) {
483    DCHECK(!access.type.Is(Type::ExternalPointer()));
484    return ReduceLoadMap(node);
485  }
486
487  NodeProperties::ChangeOp(node, machine()->Load(type));
488
489#ifdef V8_SANDBOXED_EXTERNAL_POINTERS
490  if (access.type.Is(Type::ExternalPointer())) {
491    ExternalPointerTag tag = access.external_pointer_tag;
492    DCHECK_NE(kExternalPointerNullTag, tag);
493    node = DecodeExternalPointer(node, tag);
494    return Replace(node);
495  }
496#endif
497
498  return Changed(node);
499}
500
501Reduction MemoryLowering::ReduceStoreToObject(Node* node,
502                                              AllocationState const* state) {
503  DCHECK(node->opcode() == IrOpcode::kStoreToObject ||
504         node->opcode() == IrOpcode::kInitializeImmutableInObject);
505  ObjectAccess const& access = ObjectAccessOf(node->op());
506  Node* object = node->InputAt(0);
507  Node* value = node->InputAt(2);
508
509  WriteBarrierKind write_barrier_kind = ComputeWriteBarrierKind(
510      node, object, value, state, access.write_barrier_kind);
511  DCHECK(!access.machine_type.IsMapWord());
512  MachineRepresentation rep = access.machine_type.representation();
513  StoreRepresentation store_rep(rep, write_barrier_kind);
514  const Operator* store_op = ElementSizeInBytes(rep) > kTaggedSize &&
515                                     !machine()->UnalignedStoreSupported(rep)
516                                 ? machine()->UnalignedStore(rep)
517                                 : machine()->Store(store_rep);
518  NodeProperties::ChangeOp(node, store_op);
519  return Changed(node);
520}
521
522Reduction MemoryLowering::ReduceStoreElement(Node* node,
523                                             AllocationState const* state) {
524  DCHECK_EQ(IrOpcode::kStoreElement, node->opcode());
525  ElementAccess const& access = ElementAccessOf(node->op());
526  Node* object = node->InputAt(0);
527  Node* index = node->InputAt(1);
528  Node* value = node->InputAt(2);
529  node->ReplaceInput(1, ComputeIndex(access, index));
530  WriteBarrierKind write_barrier_kind = ComputeWriteBarrierKind(
531      node, object, value, state, access.write_barrier_kind);
532  NodeProperties::ChangeOp(
533      node, machine()->Store(StoreRepresentation(
534                access.machine_type.representation(), write_barrier_kind)));
535  return Changed(node);
536}
537
538Reduction MemoryLowering::ReduceStoreField(Node* node,
539                                           AllocationState const* state) {
540  DCHECK_EQ(IrOpcode::kStoreField, node->opcode());
541  FieldAccess const& access = FieldAccessOf(node->op());
542  // External pointer must never be stored by optimized code.
543  DCHECK_IMPLIES(V8_SANDBOXED_EXTERNAL_POINTERS_BOOL,
544                 !access.type.Is(Type::ExternalPointer()));
545  // SandboxedPointers are not currently stored by optimized code.
546  DCHECK(!access.type.Is(Type::SandboxedPointer()));
547  MachineType machine_type = access.machine_type;
548  Node* object = node->InputAt(0);
549  Node* value = node->InputAt(1);
550
551  Node* effect = NodeProperties::GetEffectInput(node);
552  Node* control = NodeProperties::GetControlInput(node);
553  __ InitializeEffectControl(effect, control);
554
555  WriteBarrierKind write_barrier_kind = ComputeWriteBarrierKind(
556      node, object, value, state, access.write_barrier_kind);
557  Node* offset = __ IntPtrConstant(access.offset - access.tag());
558  node->InsertInput(graph_zone(), 1, offset);
559
560  if (machine_type.IsMapWord()) {
561    machine_type = MachineType::TaggedPointer();
562#ifdef V8_MAP_PACKING
563    Node* mapword = __ PackMapWord(TNode<Map>::UncheckedCast(value));
564    node->ReplaceInput(2, mapword);
565#endif
566  }
567  NodeProperties::ChangeOp(
568      node, machine()->Store(StoreRepresentation(machine_type.representation(),
569                                                 write_barrier_kind)));
570  return Changed(node);
571}
572
573Reduction MemoryLowering::ReduceStore(Node* node,
574                                      AllocationState const* state) {
575  DCHECK_EQ(IrOpcode::kStore, node->opcode());
576  StoreRepresentation representation = StoreRepresentationOf(node->op());
577  Node* object = node->InputAt(0);
578  Node* value = node->InputAt(2);
579  WriteBarrierKind write_barrier_kind = ComputeWriteBarrierKind(
580      node, object, value, state, representation.write_barrier_kind());
581  if (write_barrier_kind != representation.write_barrier_kind()) {
582    NodeProperties::ChangeOp(
583        node, machine()->Store(StoreRepresentation(
584                  representation.representation(), write_barrier_kind)));
585    return Changed(node);
586  }
587  return NoChange();
588}
589
590Node* MemoryLowering::ComputeIndex(ElementAccess const& access, Node* index) {
591  int const element_size_shift =
592      ElementSizeLog2Of(access.machine_type.representation());
593  if (element_size_shift) {
594    index = __ WordShl(index, __ IntPtrConstant(element_size_shift));
595  }
596  int const fixed_offset = access.header_size - access.tag();
597  if (fixed_offset) {
598    index = __ IntAdd(index, __ IntPtrConstant(fixed_offset));
599  }
600  return index;
601}
602
603#undef __
604
605namespace {
606
607bool ValueNeedsWriteBarrier(Node* value, Isolate* isolate) {
608  while (true) {
609    switch (value->opcode()) {
610      case IrOpcode::kBitcastWordToTaggedSigned:
611        return false;
612      case IrOpcode::kHeapConstant: {
613        RootIndex root_index;
614        if (isolate->roots_table().IsRootHandle(HeapConstantOf(value->op()),
615                                                &root_index) &&
616            RootsTable::IsImmortalImmovable(root_index)) {
617          return false;
618        }
619        break;
620      }
621      default:
622        break;
623    }
624    return true;
625  }
626}
627
628}  // namespace
629
630Reduction MemoryLowering::ReduceAllocateRaw(Node* node) {
631  DCHECK_EQ(IrOpcode::kAllocateRaw, node->opcode());
632  const AllocateParameters& allocation = AllocateParametersOf(node->op());
633  return ReduceAllocateRaw(node, allocation.allocation_type(),
634                           allocation.allow_large_objects(), nullptr);
635}
636
637WriteBarrierKind MemoryLowering::ComputeWriteBarrierKind(
638    Node* node, Node* object, Node* value, AllocationState const* state,
639    WriteBarrierKind write_barrier_kind) {
640  if (state && state->IsYoungGenerationAllocation() &&
641      state->group()->Contains(object)) {
642    write_barrier_kind = kNoWriteBarrier;
643  }
644  if (!ValueNeedsWriteBarrier(value, isolate())) {
645    write_barrier_kind = kNoWriteBarrier;
646  }
647  if (FLAG_disable_write_barriers) {
648    write_barrier_kind = kNoWriteBarrier;
649  }
650  if (write_barrier_kind == WriteBarrierKind::kAssertNoWriteBarrier) {
651    write_barrier_assert_failed_(node, object, function_debug_name_, zone());
652  }
653  return write_barrier_kind;
654}
655
656MemoryLowering::AllocationGroup::AllocationGroup(Node* node,
657                                                 AllocationType allocation,
658                                                 Zone* zone)
659    : node_ids_(zone),
660      allocation_(CheckAllocationType(allocation)),
661      size_(nullptr) {
662  node_ids_.insert(node->id());
663}
664
665MemoryLowering::AllocationGroup::AllocationGroup(Node* node,
666                                                 AllocationType allocation,
667                                                 Node* size, Zone* zone)
668    : node_ids_(zone),
669      allocation_(CheckAllocationType(allocation)),
670      size_(size) {
671  node_ids_.insert(node->id());
672}
673
674void MemoryLowering::AllocationGroup::Add(Node* node) {
675  node_ids_.insert(node->id());
676}
677
678bool MemoryLowering::AllocationGroup::Contains(Node* node) const {
679  // Additions should stay within the same allocated object, so it's safe to
680  // ignore them.
681  while (node_ids_.find(node->id()) == node_ids_.end()) {
682    switch (node->opcode()) {
683      case IrOpcode::kBitcastTaggedToWord:
684      case IrOpcode::kBitcastWordToTagged:
685      case IrOpcode::kInt32Add:
686      case IrOpcode::kInt64Add:
687        node = NodeProperties::GetValueInput(node, 0);
688        break;
689      default:
690        return false;
691    }
692  }
693  return true;
694}
695
696MemoryLowering::AllocationState::AllocationState()
697    : group_(nullptr),
698      size_(std::numeric_limits<int>::max()),
699      top_(nullptr),
700      effect_(nullptr) {}
701
702MemoryLowering::AllocationState::AllocationState(AllocationGroup* group,
703                                                 Node* effect)
704    : group_(group),
705      size_(std::numeric_limits<int>::max()),
706      top_(nullptr),
707      effect_(effect) {}
708
709MemoryLowering::AllocationState::AllocationState(AllocationGroup* group,
710                                                 intptr_t size, Node* top,
711                                                 Node* effect)
712    : group_(group), size_(size), top_(top), effect_(effect) {}
713
714bool MemoryLowering::AllocationState::IsYoungGenerationAllocation() const {
715  return group() && group()->IsYoungGenerationAllocation();
716}
717
718}  // namespace compiler
719}  // namespace internal
720}  // namespace v8
721