1// Copyright 2017 the V8 project authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "src/wasm/wasm-serialization.h"
6
7#include "src/base/platform/wrappers.h"
8#include "src/codegen/assembler-inl.h"
9#include "src/codegen/external-reference-table.h"
10#include "src/objects/objects-inl.h"
11#include "src/objects/objects.h"
12#include "src/runtime/runtime.h"
13#include "src/snapshot/code-serializer.h"
14#include "src/utils/ostreams.h"
15#include "src/utils/utils.h"
16#include "src/utils/version.h"
17#include "src/wasm/code-space-access.h"
18#include "src/wasm/function-compiler.h"
19#include "src/wasm/module-compiler.h"
20#include "src/wasm/module-decoder.h"
21#include "src/wasm/wasm-code-manager.h"
22#include "src/wasm/wasm-engine.h"
23#include "src/wasm/wasm-module.h"
24#include "src/wasm/wasm-objects-inl.h"
25#include "src/wasm/wasm-objects.h"
26#include "src/wasm/wasm-result.h"
27
28namespace v8 {
29namespace internal {
30namespace wasm {
31
32namespace {
33constexpr uint8_t kLazyFunction = 2;
34constexpr uint8_t kLiftoffFunction = 3;
35constexpr uint8_t kTurboFanFunction = 4;
36
37// TODO(bbudge) Try to unify the various implementations of readers and writers
38// in Wasm, e.g. StreamProcessor and ZoneBuffer, with these.
39class Writer {
40 public:
41  explicit Writer(base::Vector<byte> buffer)
42      : start_(buffer.begin()), end_(buffer.end()), pos_(buffer.begin()) {}
43
44  size_t bytes_written() const { return pos_ - start_; }
45  byte* current_location() const { return pos_; }
46  size_t current_size() const { return end_ - pos_; }
47  base::Vector<byte> current_buffer() const {
48    return {current_location(), current_size()};
49  }
50
51  template <typename T>
52  void Write(const T& value) {
53    DCHECK_GE(current_size(), sizeof(T));
54    WriteUnalignedValue(reinterpret_cast<Address>(current_location()), value);
55    pos_ += sizeof(T);
56    if (FLAG_trace_wasm_serialization) {
57      StdoutStream{} << "wrote: " << static_cast<size_t>(value)
58                     << " sized: " << sizeof(T) << std::endl;
59    }
60  }
61
62  void WriteVector(const base::Vector<const byte> v) {
63    DCHECK_GE(current_size(), v.size());
64    if (v.size() > 0) {
65      memcpy(current_location(), v.begin(), v.size());
66      pos_ += v.size();
67    }
68    if (FLAG_trace_wasm_serialization) {
69      StdoutStream{} << "wrote vector of " << v.size() << " elements"
70                     << std::endl;
71    }
72  }
73
74  void Skip(size_t size) { pos_ += size; }
75
76 private:
77  byte* const start_;
78  byte* const end_;
79  byte* pos_;
80};
81
82class Reader {
83 public:
84  explicit Reader(base::Vector<const byte> buffer)
85      : start_(buffer.begin()), end_(buffer.end()), pos_(buffer.begin()) {}
86
87  size_t bytes_read() const { return pos_ - start_; }
88  const byte* current_location() const { return pos_; }
89  size_t current_size() const { return end_ - pos_; }
90  base::Vector<const byte> current_buffer() const {
91    return {current_location(), current_size()};
92  }
93
94  template <typename T>
95  T Read() {
96    DCHECK_GE(current_size(), sizeof(T));
97    T value =
98        ReadUnalignedValue<T>(reinterpret_cast<Address>(current_location()));
99    pos_ += sizeof(T);
100    if (FLAG_trace_wasm_serialization) {
101      StdoutStream{} << "read: " << static_cast<size_t>(value)
102                     << " sized: " << sizeof(T) << std::endl;
103    }
104    return value;
105  }
106
107  template <typename T>
108  base::Vector<const T> ReadVector(size_t size) {
109    DCHECK_GE(current_size(), size);
110    base::Vector<const byte> bytes{pos_, size * sizeof(T)};
111    pos_ += size * sizeof(T);
112    if (FLAG_trace_wasm_serialization) {
113      StdoutStream{} << "read vector of " << size << " elements of size "
114                     << sizeof(T) << " (total size " << size * sizeof(T) << ")"
115                     << std::endl;
116    }
117    return base::Vector<const T>::cast(bytes);
118  }
119
120  void Skip(size_t size) { pos_ += size; }
121
122 private:
123  const byte* const start_;
124  const byte* const end_;
125  const byte* pos_;
126};
127
128void WriteHeader(Writer* writer) {
129  writer->Write(SerializedData::kMagicNumber);
130  writer->Write(Version::Hash());
131  writer->Write(static_cast<uint32_t>(CpuFeatures::SupportedFeatures()));
132  writer->Write(FlagList::Hash());
133  DCHECK_EQ(WasmSerializer::kHeaderSize, writer->bytes_written());
134}
135
136// On Intel, call sites are encoded as a displacement. For linking and for
137// serialization/deserialization, we want to store/retrieve a tag (the function
138// index). On Intel, that means accessing the raw displacement.
139// On ARM64, call sites are encoded as either a literal load or a direct branch.
140// Other platforms simply require accessing the target address.
141void SetWasmCalleeTag(RelocInfo* rinfo, uint32_t tag) {
142#if V8_TARGET_ARCH_X64 || V8_TARGET_ARCH_IA32
143  DCHECK(rinfo->HasTargetAddressAddress());
144  DCHECK(!RelocInfo::IsCompressedEmbeddedObject(rinfo->rmode()));
145  WriteUnalignedValue(rinfo->target_address_address(), tag);
146#elif V8_TARGET_ARCH_ARM64
147  Instruction* instr = reinterpret_cast<Instruction*>(rinfo->pc());
148  if (instr->IsLdrLiteralX()) {
149    WriteUnalignedValue(rinfo->constant_pool_entry_address(),
150                        static_cast<Address>(tag));
151  } else {
152    DCHECK(instr->IsBranchAndLink() || instr->IsUnconditionalBranch());
153    instr->SetBranchImmTarget(
154        reinterpret_cast<Instruction*>(rinfo->pc() + tag * kInstrSize));
155  }
156#else
157  Address addr = static_cast<Address>(tag);
158  if (rinfo->rmode() == RelocInfo::EXTERNAL_REFERENCE) {
159    rinfo->set_target_external_reference(addr, SKIP_ICACHE_FLUSH);
160  } else if (rinfo->rmode() == RelocInfo::WASM_STUB_CALL) {
161    rinfo->set_wasm_stub_call_address(addr, SKIP_ICACHE_FLUSH);
162  } else {
163    rinfo->set_target_address(addr, SKIP_WRITE_BARRIER, SKIP_ICACHE_FLUSH);
164  }
165#endif
166}
167
168uint32_t GetWasmCalleeTag(RelocInfo* rinfo) {
169#if V8_TARGET_ARCH_X64 || V8_TARGET_ARCH_IA32
170  DCHECK(!RelocInfo::IsCompressedEmbeddedObject(rinfo->rmode()));
171  return ReadUnalignedValue<uint32_t>(rinfo->target_address_address());
172#elif V8_TARGET_ARCH_ARM64
173  Instruction* instr = reinterpret_cast<Instruction*>(rinfo->pc());
174  if (instr->IsLdrLiteralX()) {
175    return ReadUnalignedValue<uint32_t>(rinfo->constant_pool_entry_address());
176  } else {
177    DCHECK(instr->IsBranchAndLink() || instr->IsUnconditionalBranch());
178    return static_cast<uint32_t>(instr->ImmPCOffset() / kInstrSize);
179  }
180#else
181  Address addr;
182  if (rinfo->rmode() == RelocInfo::EXTERNAL_REFERENCE) {
183    addr = rinfo->target_external_reference();
184  } else if (rinfo->rmode() == RelocInfo::WASM_STUB_CALL) {
185    addr = rinfo->wasm_stub_call_address();
186  } else {
187    addr = rinfo->target_address();
188  }
189  return static_cast<uint32_t>(addr);
190#endif
191}
192
193constexpr size_t kHeaderSize = sizeof(size_t);  // total code size
194
195constexpr size_t kCodeHeaderSize = sizeof(uint8_t) +  // code kind
196                                   sizeof(int) +      // offset of constant pool
197                                   sizeof(int) +  // offset of safepoint table
198                                   sizeof(int) +  // offset of handler table
199                                   sizeof(int) +  // offset of code comments
200                                   sizeof(int) +  // unpadded binary size
201                                   sizeof(int) +  // stack slots
202                                   sizeof(int) +  // tagged parameter slots
203                                   sizeof(int) +  // code size
204                                   sizeof(int) +  // reloc size
205                                   sizeof(int) +  // source positions size
206                                   sizeof(int) +  // protected instructions size
207                                   sizeof(WasmCode::Kind) +  // code kind
208                                   sizeof(ExecutionTier);    // tier
209
210// A List of all isolate-independent external references. This is used to create
211// a tag from the Address of an external reference and vice versa.
212class ExternalReferenceList {
213 public:
214  ExternalReferenceList(const ExternalReferenceList&) = delete;
215  ExternalReferenceList& operator=(const ExternalReferenceList&) = delete;
216
217  uint32_t tag_from_address(Address ext_ref_address) const {
218    auto tag_addr_less_than = [this](uint32_t tag, Address searched_addr) {
219      return external_reference_by_tag_[tag] < searched_addr;
220    };
221    auto it = std::lower_bound(std::begin(tags_ordered_by_address_),
222                               std::end(tags_ordered_by_address_),
223                               ext_ref_address, tag_addr_less_than);
224    DCHECK_NE(std::end(tags_ordered_by_address_), it);
225    uint32_t tag = *it;
226    DCHECK_EQ(address_from_tag(tag), ext_ref_address);
227    return tag;
228  }
229
230  Address address_from_tag(uint32_t tag) const {
231    DCHECK_GT(kNumExternalReferences, tag);
232    return external_reference_by_tag_[tag];
233  }
234
235  static const ExternalReferenceList& Get() {
236    static ExternalReferenceList list;  // Lazily initialized.
237    return list;
238  }
239
240 private:
241  // Private constructor. There will only be a single instance of this object.
242  ExternalReferenceList() {
243    for (uint32_t i = 0; i < kNumExternalReferences; ++i) {
244      tags_ordered_by_address_[i] = i;
245    }
246    auto addr_by_tag_less_than = [this](uint32_t a, uint32_t b) {
247      return external_reference_by_tag_[a] < external_reference_by_tag_[b];
248    };
249    std::sort(std::begin(tags_ordered_by_address_),
250              std::end(tags_ordered_by_address_), addr_by_tag_less_than);
251  }
252
253#define COUNT_EXTERNAL_REFERENCE(name, ...) +1
254  static constexpr uint32_t kNumExternalReferencesList =
255      EXTERNAL_REFERENCE_LIST(COUNT_EXTERNAL_REFERENCE);
256  static constexpr uint32_t kNumExternalReferencesIntrinsics =
257      FOR_EACH_INTRINSIC(COUNT_EXTERNAL_REFERENCE);
258  static constexpr uint32_t kNumExternalReferences =
259      kNumExternalReferencesList + kNumExternalReferencesIntrinsics;
260#undef COUNT_EXTERNAL_REFERENCE
261
262  Address external_reference_by_tag_[kNumExternalReferences] = {
263#define EXT_REF_ADDR(name, desc) ExternalReference::name().address(),
264      EXTERNAL_REFERENCE_LIST(EXT_REF_ADDR)
265#undef EXT_REF_ADDR
266#define RUNTIME_ADDR(name, ...) \
267  ExternalReference::Create(Runtime::k##name).address(),
268          FOR_EACH_INTRINSIC(RUNTIME_ADDR)
269#undef RUNTIME_ADDR
270  };
271  uint32_t tags_ordered_by_address_[kNumExternalReferences];
272};
273
274static_assert(std::is_trivially_destructible<ExternalReferenceList>::value,
275              "static destructors not allowed");
276
277}  // namespace
278
279class V8_EXPORT_PRIVATE NativeModuleSerializer {
280 public:
281  NativeModuleSerializer(const NativeModule*, base::Vector<WasmCode* const>);
282  NativeModuleSerializer(const NativeModuleSerializer&) = delete;
283  NativeModuleSerializer& operator=(const NativeModuleSerializer&) = delete;
284
285  size_t Measure() const;
286  bool Write(Writer* writer);
287
288 private:
289  size_t MeasureCode(const WasmCode*) const;
290  void WriteHeader(Writer*, size_t total_code_size);
291  void WriteCode(const WasmCode*, Writer*);
292
293  const NativeModule* const native_module_;
294  const base::Vector<WasmCode* const> code_table_;
295  bool write_called_ = false;
296  size_t total_written_code_ = 0;
297  int num_turbofan_functions_ = 0;
298};
299
300NativeModuleSerializer::NativeModuleSerializer(
301    const NativeModule* module, base::Vector<WasmCode* const> code_table)
302    : native_module_(module), code_table_(code_table) {
303  DCHECK_NOT_NULL(native_module_);
304  // TODO(mtrofin): persist the export wrappers. Ideally, we'd only persist
305  // the unique ones, i.e. the cache.
306}
307
308size_t NativeModuleSerializer::MeasureCode(const WasmCode* code) const {
309  if (code == nullptr) return sizeof(uint8_t);
310  DCHECK_EQ(WasmCode::kWasmFunction, code->kind());
311  if (code->tier() != ExecutionTier::kTurbofan) {
312    return sizeof(uint8_t);
313  }
314  return kCodeHeaderSize + code->instructions().size() +
315         code->reloc_info().size() + code->source_positions().size() +
316         code->protected_instructions_data().size();
317}
318
319size_t NativeModuleSerializer::Measure() const {
320  size_t size = kHeaderSize;
321  for (WasmCode* code : code_table_) {
322    size += MeasureCode(code);
323  }
324  return size;
325}
326
327void NativeModuleSerializer::WriteHeader(Writer* writer,
328                                         size_t total_code_size) {
329  // TODO(eholk): We need to properly preserve the flag whether the trap
330  // handler was used or not when serializing.
331
332  writer->Write(total_code_size);
333}
334
335void NativeModuleSerializer::WriteCode(const WasmCode* code, Writer* writer) {
336  if (code == nullptr) {
337    writer->Write(kLazyFunction);
338    return;
339  }
340
341  DCHECK_EQ(WasmCode::kWasmFunction, code->kind());
342  // Only serialize TurboFan code, as Liftoff code can contain breakpoints or
343  // non-relocatable constants.
344  if (code->tier() != ExecutionTier::kTurbofan) {
345    // We check if the function has been executed already. If so, we serialize
346    // it as {kLiftoffFunction} so that upon deserialization the function will
347    // get compiled with Liftoff eagerly. If the function has not been executed
348    // yet, we serialize it as {kLazyFunction}, and the function will not get
349    // compiled upon deserialization.
350    NativeModule* native_module = code->native_module();
351    uint32_t budget =
352        native_module->tiering_budget_array()[declared_function_index(
353            native_module->module(), code->index())];
354    writer->Write(budget == static_cast<uint32_t>(FLAG_wasm_tiering_budget)
355                      ? kLazyFunction
356                      : kLiftoffFunction);
357    return;
358  }
359
360  ++num_turbofan_functions_;
361  writer->Write(kTurboFanFunction);
362  // Write the size of the entire code section, followed by the code header.
363  writer->Write(code->constant_pool_offset());
364  writer->Write(code->safepoint_table_offset());
365  writer->Write(code->handler_table_offset());
366  writer->Write(code->code_comments_offset());
367  writer->Write(code->unpadded_binary_size());
368  writer->Write(code->stack_slots());
369  writer->Write(code->raw_tagged_parameter_slots_for_serialization());
370  writer->Write(code->instructions().length());
371  writer->Write(code->reloc_info().length());
372  writer->Write(code->source_positions().length());
373  writer->Write(code->protected_instructions_data().length());
374  writer->Write(code->kind());
375  writer->Write(code->tier());
376
377  // Get a pointer to the destination buffer, to hold relocated code.
378  byte* serialized_code_start = writer->current_buffer().begin();
379  byte* code_start = serialized_code_start;
380  size_t code_size = code->instructions().size();
381  writer->Skip(code_size);
382  // Write the reloc info, source positions, and protected code.
383  writer->WriteVector(code->reloc_info());
384  writer->WriteVector(code->source_positions());
385  writer->WriteVector(code->protected_instructions_data());
386#if V8_TARGET_ARCH_MIPS || V8_TARGET_ARCH_MIPS64 || V8_TARGET_ARCH_ARM || \
387    V8_TARGET_ARCH_PPC || V8_TARGET_ARCH_PPC64 || V8_TARGET_ARCH_S390X || \
388    V8_TARGET_ARCH_RISCV64
389  // On platforms that don't support misaligned word stores, copy to an aligned
390  // buffer if necessary so we can relocate the serialized code.
391  std::unique_ptr<byte[]> aligned_buffer;
392  if (!IsAligned(reinterpret_cast<Address>(serialized_code_start),
393                 kSystemPointerSize)) {
394    // 'byte' does not guarantee an alignment but seems to work well enough in
395    // practice.
396    aligned_buffer.reset(new byte[code_size]);
397    code_start = aligned_buffer.get();
398  }
399#endif
400  memcpy(code_start, code->instructions().begin(), code_size);
401  // Relocate the code.
402  int mask = RelocInfo::ModeMask(RelocInfo::WASM_CALL) |
403             RelocInfo::ModeMask(RelocInfo::WASM_STUB_CALL) |
404             RelocInfo::ModeMask(RelocInfo::EXTERNAL_REFERENCE) |
405             RelocInfo::ModeMask(RelocInfo::INTERNAL_REFERENCE) |
406             RelocInfo::ModeMask(RelocInfo::INTERNAL_REFERENCE_ENCODED);
407  RelocIterator orig_iter(code->instructions(), code->reloc_info(),
408                          code->constant_pool(), mask);
409  for (RelocIterator iter(
410           {code_start, code->instructions().size()}, code->reloc_info(),
411           reinterpret_cast<Address>(code_start) + code->constant_pool_offset(),
412           mask);
413       !iter.done(); iter.next(), orig_iter.next()) {
414    RelocInfo::Mode mode = orig_iter.rinfo()->rmode();
415    switch (mode) {
416      case RelocInfo::WASM_CALL: {
417        Address orig_target = orig_iter.rinfo()->wasm_call_address();
418        uint32_t tag =
419            native_module_->GetFunctionIndexFromJumpTableSlot(orig_target);
420        SetWasmCalleeTag(iter.rinfo(), tag);
421      } break;
422      case RelocInfo::WASM_STUB_CALL: {
423        Address target = orig_iter.rinfo()->wasm_stub_call_address();
424        uint32_t tag = native_module_->GetRuntimeStubId(target);
425        DCHECK_GT(WasmCode::kRuntimeStubCount, tag);
426        SetWasmCalleeTag(iter.rinfo(), tag);
427      } break;
428      case RelocInfo::EXTERNAL_REFERENCE: {
429        Address orig_target = orig_iter.rinfo()->target_external_reference();
430        uint32_t ext_ref_tag =
431            ExternalReferenceList::Get().tag_from_address(orig_target);
432        SetWasmCalleeTag(iter.rinfo(), ext_ref_tag);
433      } break;
434      case RelocInfo::INTERNAL_REFERENCE:
435      case RelocInfo::INTERNAL_REFERENCE_ENCODED: {
436        Address orig_target = orig_iter.rinfo()->target_internal_reference();
437        Address offset = orig_target - code->instruction_start();
438        Assembler::deserialization_set_target_internal_reference_at(
439            iter.rinfo()->pc(), offset, mode);
440      } break;
441      default:
442        UNREACHABLE();
443    }
444  }
445  // If we copied to an aligned buffer, copy code into serialized buffer.
446  if (code_start != serialized_code_start) {
447    memcpy(serialized_code_start, code_start, code_size);
448  }
449  total_written_code_ += code_size;
450}
451
452bool NativeModuleSerializer::Write(Writer* writer) {
453  DCHECK(!write_called_);
454  write_called_ = true;
455
456  size_t total_code_size = 0;
457  for (WasmCode* code : code_table_) {
458    if (code && code->tier() == ExecutionTier::kTurbofan) {
459      DCHECK(IsAligned(code->instructions().size(), kCodeAlignment));
460      total_code_size += code->instructions().size();
461    }
462  }
463  WriteHeader(writer, total_code_size);
464
465  for (WasmCode* code : code_table_) {
466    WriteCode(code, writer);
467  }
468  // If not a single function was written, serialization was not successful.
469  if (num_turbofan_functions_ == 0) return false;
470
471  // Make sure that the serialized total code size was correct.
472  CHECK_EQ(total_written_code_, total_code_size);
473
474  return true;
475}
476
477WasmSerializer::WasmSerializer(NativeModule* native_module)
478    : native_module_(native_module),
479      code_table_(native_module->SnapshotCodeTable()) {}
480
481size_t WasmSerializer::GetSerializedNativeModuleSize() const {
482  NativeModuleSerializer serializer(native_module_,
483                                    base::VectorOf(code_table_));
484  return kHeaderSize + serializer.Measure();
485}
486
487bool WasmSerializer::SerializeNativeModule(base::Vector<byte> buffer) const {
488  NativeModuleSerializer serializer(native_module_,
489                                    base::VectorOf(code_table_));
490  size_t measured_size = kHeaderSize + serializer.Measure();
491  if (buffer.size() < measured_size) return false;
492
493  Writer writer(buffer);
494  WriteHeader(&writer);
495
496  if (!serializer.Write(&writer)) return false;
497  DCHECK_EQ(measured_size, writer.bytes_written());
498  return true;
499}
500
501struct DeserializationUnit {
502  base::Vector<const byte> src_code_buffer;
503  std::unique_ptr<WasmCode> code;
504  NativeModule::JumpTablesRef jump_tables;
505};
506
507class DeserializationQueue {
508 public:
509  void Add(std::vector<DeserializationUnit> batch) {
510    DCHECK(!batch.empty());
511    base::MutexGuard guard(&mutex_);
512    queue_.emplace(std::move(batch));
513  }
514
515  std::vector<DeserializationUnit> Pop() {
516    base::MutexGuard guard(&mutex_);
517    if (queue_.empty()) return {};
518    auto batch = std::move(queue_.front());
519    queue_.pop();
520    return batch;
521  }
522
523  std::vector<DeserializationUnit> PopAll() {
524    base::MutexGuard guard(&mutex_);
525    if (queue_.empty()) return {};
526    auto units = std::move(queue_.front());
527    queue_.pop();
528    while (!queue_.empty()) {
529      units.insert(units.end(), std::make_move_iterator(queue_.front().begin()),
530                   std::make_move_iterator(queue_.front().end()));
531      queue_.pop();
532    }
533    return units;
534  }
535
536  size_t NumBatches() const {
537    base::MutexGuard guard(&mutex_);
538    return queue_.size();
539  }
540
541 private:
542  mutable base::Mutex mutex_;
543  std::queue<std::vector<DeserializationUnit>> queue_;
544};
545
546class V8_EXPORT_PRIVATE NativeModuleDeserializer {
547 public:
548  explicit NativeModuleDeserializer(NativeModule*);
549  NativeModuleDeserializer(const NativeModuleDeserializer&) = delete;
550  NativeModuleDeserializer& operator=(const NativeModuleDeserializer&) = delete;
551
552  bool Read(Reader* reader);
553
554  base::Vector<const int> lazy_functions() {
555    return base::VectorOf(lazy_functions_);
556  }
557
558  base::Vector<const int> liftoff_functions() {
559    return base::VectorOf(liftoff_functions_);
560  }
561
562 private:
563  friend class DeserializeCodeTask;
564
565  void ReadHeader(Reader* reader);
566  DeserializationUnit ReadCode(int fn_index, Reader* reader);
567  void CopyAndRelocate(const DeserializationUnit& unit);
568  void Publish(std::vector<DeserializationUnit> batch);
569
570  NativeModule* const native_module_;
571#ifdef DEBUG
572  bool read_called_ = false;
573#endif
574
575  // Updated in {ReadCode}.
576  size_t remaining_code_size_ = 0;
577  base::Vector<byte> current_code_space_;
578  NativeModule::JumpTablesRef current_jump_tables_;
579  std::vector<int> lazy_functions_;
580  std::vector<int> liftoff_functions_;
581};
582
583class DeserializeCodeTask : public JobTask {
584 public:
585  DeserializeCodeTask(NativeModuleDeserializer* deserializer,
586                      DeserializationQueue* reloc_queue)
587      : deserializer_(deserializer), reloc_queue_(reloc_queue) {}
588
589  void Run(JobDelegate* delegate) override {
590    CodeSpaceWriteScope code_space_write_scope(deserializer_->native_module_);
591    do {
592      // Repeatedly publish everything that was copied already.
593      TryPublishing(delegate);
594
595      auto batch = reloc_queue_->Pop();
596      if (batch.empty()) break;
597      for (const auto& unit : batch) {
598        deserializer_->CopyAndRelocate(unit);
599      }
600      publish_queue_.Add(std::move(batch));
601      delegate->NotifyConcurrencyIncrease();
602    } while (!delegate->ShouldYield());
603  }
604
605  size_t GetMaxConcurrency(size_t /* worker_count */) const override {
606    // Number of copy&reloc batches, plus 1 if there is also something to
607    // publish.
608    bool publish = publishing_.load(std::memory_order_relaxed) == false &&
609                   publish_queue_.NumBatches() > 0;
610    return reloc_queue_->NumBatches() + (publish ? 1 : 0);
611  }
612
613 private:
614  void TryPublishing(JobDelegate* delegate) {
615    // Publishing is sequential, so only start publishing if no one else is.
616    if (publishing_.exchange(true, std::memory_order_relaxed)) return;
617
618    WasmCodeRefScope code_scope;
619    while (true) {
620      bool yield = false;
621      while (!yield) {
622        auto to_publish = publish_queue_.PopAll();
623        if (to_publish.empty()) break;
624        deserializer_->Publish(std::move(to_publish));
625        yield = delegate->ShouldYield();
626      }
627      publishing_.store(false, std::memory_order_relaxed);
628      if (yield) break;
629      // After finishing publishing, check again if new work arrived in the mean
630      // time. If so, continue publishing.
631      if (publish_queue_.NumBatches() == 0) break;
632      if (publishing_.exchange(true, std::memory_order_relaxed)) break;
633      // We successfully reset {publishing_} from {false} to {true}.
634    }
635  }
636
637  NativeModuleDeserializer* const deserializer_;
638  DeserializationQueue* const reloc_queue_;
639  DeserializationQueue publish_queue_;
640  std::atomic<bool> publishing_{false};
641};
642
643NativeModuleDeserializer::NativeModuleDeserializer(NativeModule* native_module)
644    : native_module_(native_module) {}
645
646bool NativeModuleDeserializer::Read(Reader* reader) {
647  DCHECK(!read_called_);
648#ifdef DEBUG
649  read_called_ = true;
650#endif
651
652  ReadHeader(reader);
653  uint32_t total_fns = native_module_->num_functions();
654  uint32_t first_wasm_fn = native_module_->num_imported_functions();
655
656  WasmCodeRefScope wasm_code_ref_scope;
657
658  DeserializationQueue reloc_queue;
659
660  std::unique_ptr<JobHandle> job_handle = V8::GetCurrentPlatform()->PostJob(
661      TaskPriority::kUserVisible,
662      std::make_unique<DeserializeCodeTask>(this, &reloc_queue));
663
664  // Choose a batch size such that we do not create too small batches (>=100k
665  // code bytes), but also not too many (<=100 batches).
666  constexpr size_t kMinBatchSizeInBytes = 100000;
667  size_t batch_limit =
668      std::max(kMinBatchSizeInBytes, remaining_code_size_ / 100);
669
670  std::vector<DeserializationUnit> batch;
671  size_t batch_size = 0;
672  CodeSpaceWriteScope code_space_write_scope(native_module_);
673  for (uint32_t i = first_wasm_fn; i < total_fns; ++i) {
674    DeserializationUnit unit = ReadCode(i, reader);
675    if (!unit.code) continue;
676    batch_size += unit.code->instructions().size();
677    batch.emplace_back(std::move(unit));
678    if (batch_size >= batch_limit) {
679      reloc_queue.Add(std::move(batch));
680      DCHECK(batch.empty());
681      batch_size = 0;
682      job_handle->NotifyConcurrencyIncrease();
683    }
684  }
685
686  // We should have read the expected amount of code now, and should have fully
687  // utilized the allocated code space.
688  DCHECK_EQ(0, remaining_code_size_);
689  DCHECK_EQ(0, current_code_space_.size());
690
691  if (!batch.empty()) {
692    reloc_queue.Add(std::move(batch));
693    job_handle->NotifyConcurrencyIncrease();
694  }
695
696  // Wait for all tasks to finish, while participating in their work.
697  job_handle->Join();
698
699  return reader->current_size() == 0;
700}
701
702void NativeModuleDeserializer::ReadHeader(Reader* reader) {
703  remaining_code_size_ = reader->Read<size_t>();
704}
705
706DeserializationUnit NativeModuleDeserializer::ReadCode(int fn_index,
707                                                       Reader* reader) {
708  uint8_t code_kind = reader->Read<uint8_t>();
709  if (code_kind == kLazyFunction) {
710    lazy_functions_.push_back(fn_index);
711    return {};
712  }
713  if (code_kind == kLiftoffFunction) {
714    liftoff_functions_.push_back(fn_index);
715    return {};
716  }
717
718  int constant_pool_offset = reader->Read<int>();
719  int safepoint_table_offset = reader->Read<int>();
720  int handler_table_offset = reader->Read<int>();
721  int code_comment_offset = reader->Read<int>();
722  int unpadded_binary_size = reader->Read<int>();
723  int stack_slot_count = reader->Read<int>();
724  uint32_t tagged_parameter_slots = reader->Read<uint32_t>();
725  int code_size = reader->Read<int>();
726  int reloc_size = reader->Read<int>();
727  int source_position_size = reader->Read<int>();
728  int protected_instructions_size = reader->Read<int>();
729  WasmCode::Kind kind = reader->Read<WasmCode::Kind>();
730  ExecutionTier tier = reader->Read<ExecutionTier>();
731
732  DCHECK(IsAligned(code_size, kCodeAlignment));
733  DCHECK_GE(remaining_code_size_, code_size);
734  if (current_code_space_.size() < static_cast<size_t>(code_size)) {
735    // Allocate the next code space. Don't allocate more than 90% of
736    // {kMaxCodeSpaceSize}, to leave some space for jump tables.
737    constexpr size_t kMaxReservation =
738        RoundUp<kCodeAlignment>(WasmCodeAllocator::kMaxCodeSpaceSize * 9 / 10);
739    size_t code_space_size = std::min(kMaxReservation, remaining_code_size_);
740    std::tie(current_code_space_, current_jump_tables_) =
741        native_module_->AllocateForDeserializedCode(code_space_size);
742    DCHECK_EQ(current_code_space_.size(), code_space_size);
743    DCHECK(current_jump_tables_.is_valid());
744  }
745
746  DeserializationUnit unit;
747  unit.src_code_buffer = reader->ReadVector<byte>(code_size);
748  auto reloc_info = reader->ReadVector<byte>(reloc_size);
749  auto source_pos = reader->ReadVector<byte>(source_position_size);
750  auto protected_instructions =
751      reader->ReadVector<byte>(protected_instructions_size);
752
753  base::Vector<uint8_t> instructions =
754      current_code_space_.SubVector(0, code_size);
755  current_code_space_ += code_size;
756  remaining_code_size_ -= code_size;
757
758  unit.code = native_module_->AddDeserializedCode(
759      fn_index, instructions, stack_slot_count, tagged_parameter_slots,
760      safepoint_table_offset, handler_table_offset, constant_pool_offset,
761      code_comment_offset, unpadded_binary_size, protected_instructions,
762      reloc_info, source_pos, kind, tier);
763  unit.jump_tables = current_jump_tables_;
764  return unit;
765}
766
767void NativeModuleDeserializer::CopyAndRelocate(
768    const DeserializationUnit& unit) {
769  memcpy(unit.code->instructions().begin(), unit.src_code_buffer.begin(),
770         unit.src_code_buffer.size());
771
772  // Relocate the code.
773  int mask = RelocInfo::ModeMask(RelocInfo::WASM_CALL) |
774             RelocInfo::ModeMask(RelocInfo::WASM_STUB_CALL) |
775             RelocInfo::ModeMask(RelocInfo::EXTERNAL_REFERENCE) |
776             RelocInfo::ModeMask(RelocInfo::INTERNAL_REFERENCE) |
777             RelocInfo::ModeMask(RelocInfo::INTERNAL_REFERENCE_ENCODED);
778  for (RelocIterator iter(unit.code->instructions(), unit.code->reloc_info(),
779                          unit.code->constant_pool(), mask);
780       !iter.done(); iter.next()) {
781    RelocInfo::Mode mode = iter.rinfo()->rmode();
782    switch (mode) {
783      case RelocInfo::WASM_CALL: {
784        uint32_t tag = GetWasmCalleeTag(iter.rinfo());
785        Address target =
786            native_module_->GetNearCallTargetForFunction(tag, unit.jump_tables);
787        iter.rinfo()->set_wasm_call_address(target, SKIP_ICACHE_FLUSH);
788        break;
789      }
790      case RelocInfo::WASM_STUB_CALL: {
791        uint32_t tag = GetWasmCalleeTag(iter.rinfo());
792        DCHECK_LT(tag, WasmCode::kRuntimeStubCount);
793        Address target = native_module_->GetNearRuntimeStubEntry(
794            static_cast<WasmCode::RuntimeStubId>(tag), unit.jump_tables);
795        iter.rinfo()->set_wasm_stub_call_address(target, SKIP_ICACHE_FLUSH);
796        break;
797      }
798      case RelocInfo::EXTERNAL_REFERENCE: {
799        uint32_t tag = GetWasmCalleeTag(iter.rinfo());
800        Address address = ExternalReferenceList::Get().address_from_tag(tag);
801        iter.rinfo()->set_target_external_reference(address, SKIP_ICACHE_FLUSH);
802        break;
803      }
804      case RelocInfo::INTERNAL_REFERENCE:
805      case RelocInfo::INTERNAL_REFERENCE_ENCODED: {
806        Address offset = iter.rinfo()->target_internal_reference();
807        Address target = unit.code->instruction_start() + offset;
808        Assembler::deserialization_set_target_internal_reference_at(
809            iter.rinfo()->pc(), target, mode);
810        break;
811      }
812      default:
813        UNREACHABLE();
814    }
815  }
816
817  // Finally, flush the icache for that code.
818  FlushInstructionCache(unit.code->instructions().begin(),
819                        unit.code->instructions().size());
820}
821
822void NativeModuleDeserializer::Publish(std::vector<DeserializationUnit> batch) {
823  DCHECK(!batch.empty());
824  std::vector<std::unique_ptr<WasmCode>> codes;
825  codes.reserve(batch.size());
826  for (auto& unit : batch) {
827    codes.emplace_back(std::move(unit).code);
828  }
829  auto published_codes = native_module_->PublishCode(base::VectorOf(codes));
830  for (auto* wasm_code : published_codes) {
831    wasm_code->MaybePrint();
832    wasm_code->Validate();
833  }
834}
835
836bool IsSupportedVersion(base::Vector<const byte> header) {
837  if (header.size() < WasmSerializer::kHeaderSize) return false;
838  byte current_version[WasmSerializer::kHeaderSize];
839  Writer writer({current_version, WasmSerializer::kHeaderSize});
840  WriteHeader(&writer);
841  return memcmp(header.begin(), current_version, WasmSerializer::kHeaderSize) ==
842         0;
843}
844
845MaybeHandle<WasmModuleObject> DeserializeNativeModule(
846    Isolate* isolate, base::Vector<const byte> data,
847    base::Vector<const byte> wire_bytes_vec,
848    base::Vector<const char> source_url) {
849  if (!IsWasmCodegenAllowed(isolate, isolate->native_context())) return {};
850  if (!IsSupportedVersion(data)) return {};
851
852  // Make the copy of the wire bytes early, so we use the same memory for
853  // decoding, lookup in the native module cache, and insertion into the cache.
854  auto owned_wire_bytes = base::OwnedVector<uint8_t>::Of(wire_bytes_vec);
855
856  // TODO(titzer): module features should be part of the serialization format.
857  WasmEngine* wasm_engine = GetWasmEngine();
858  WasmFeatures enabled_features = WasmFeatures::FromIsolate(isolate);
859  ModuleResult decode_result = DecodeWasmModule(
860      enabled_features, owned_wire_bytes.start(), owned_wire_bytes.end(), false,
861      i::wasm::kWasmOrigin, isolate->counters(), isolate->metrics_recorder(),
862      isolate->GetOrRegisterRecorderContextId(isolate->native_context()),
863      DecodingMethod::kDeserialize, wasm_engine->allocator());
864  if (decode_result.failed()) return {};
865  std::shared_ptr<WasmModule> module = std::move(decode_result).value();
866  CHECK_NOT_NULL(module);
867
868  auto shared_native_module = wasm_engine->MaybeGetNativeModule(
869      module->origin, owned_wire_bytes.as_vector(), isolate);
870  if (shared_native_module == nullptr) {
871    DynamicTiering dynamic_tiering = isolate->IsWasmDynamicTieringEnabled()
872                                         ? DynamicTiering::kEnabled
873                                         : DynamicTiering::kDisabled;
874    const bool kIncludeLiftoff = dynamic_tiering == DynamicTiering::kDisabled;
875    size_t code_size_estimate =
876        wasm::WasmCodeManager::EstimateNativeModuleCodeSize(
877            module.get(), kIncludeLiftoff, dynamic_tiering);
878    shared_native_module = wasm_engine->NewNativeModule(
879        isolate, enabled_features, std::move(module), code_size_estimate);
880    // We have to assign a compilation ID here, as it is required for a
881    // potential re-compilation, e.g. triggered by
882    // {TierDownAllModulesPerIsolate}. The value is -2 so that it is different
883    // than the compilation ID of actual compilations, and also different than
884    // the sentinel value of the CompilationState.
885    shared_native_module->compilation_state()->set_compilation_id(-2);
886    shared_native_module->SetWireBytes(std::move(owned_wire_bytes));
887
888    NativeModuleDeserializer deserializer(shared_native_module.get());
889    Reader reader(data + WasmSerializer::kHeaderSize);
890    bool error = !deserializer.Read(&reader);
891    if (error) {
892      wasm_engine->UpdateNativeModuleCache(error, &shared_native_module,
893                                           isolate);
894      return {};
895    }
896    shared_native_module->compilation_state()->InitializeAfterDeserialization(
897        deserializer.lazy_functions(), deserializer.liftoff_functions());
898    wasm_engine->UpdateNativeModuleCache(error, &shared_native_module, isolate);
899  }
900
901  Handle<FixedArray> export_wrappers;
902  CompileJsToWasmWrappers(isolate, shared_native_module->module(),
903                          &export_wrappers);
904
905  Handle<Script> script =
906      wasm_engine->GetOrCreateScript(isolate, shared_native_module, source_url);
907  Handle<WasmModuleObject> module_object = WasmModuleObject::New(
908      isolate, shared_native_module, script, export_wrappers);
909
910  // Finish the Wasm script now and make it public to the debugger.
911  isolate->debug()->OnAfterCompile(script);
912
913  // Log the code within the generated module for profiling.
914  shared_native_module->LogWasmCodes(isolate, *script);
915
916  return module_object;
917}
918
919}  // namespace wasm
920}  // namespace internal
921}  // namespace v8
922