1 // Copyright 2014 the V8 project authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "src/base/numbers/double.h"
6 #include "src/codegen/assembler-inl.h"
7 #include "src/codegen/callable.h"
8 #include "src/codegen/macro-assembler.h"
9 #include "src/codegen/optimized-compilation-info.h"
10 #include "src/compiler/backend/code-generator-impl.h"
11 #include "src/compiler/backend/code-generator.h"
12 #include "src/compiler/backend/gap-resolver.h"
13 #include "src/compiler/node-matchers.h"
14 #include "src/compiler/osr.h"
15 #include "src/heap/memory-chunk.h"
16
17 #if V8_ENABLE_WEBASSEMBLY
18 #include "src/wasm/wasm-code-manager.h"
19 #include "src/wasm/wasm-objects.h"
20 #endif // V8_ENABLE_WEBASSEMBLY
21
22 namespace v8 {
23 namespace internal {
24 namespace compiler {
25
26 #define __ tasm()->
27
28 #define kScratchReg r11
29
30 // Adds PPC-specific methods to convert InstructionOperands.
31 class PPCOperandConverter final : public InstructionOperandConverter {
32 public:
PPCOperandConverter(CodeGenerator* gen, Instruction* instr)33 PPCOperandConverter(CodeGenerator* gen, Instruction* instr)
34 : InstructionOperandConverter(gen, instr) {}
35
OutputCount()36 size_t OutputCount() { return instr_->OutputCount(); }
37
OutputRCBit() const38 RCBit OutputRCBit() const {
39 switch (instr_->flags_mode()) {
40 case kFlags_branch:
41 case kFlags_deoptimize:
42 case kFlags_set:
43 case kFlags_trap:
44 case kFlags_select:
45 return SetRC;
46 case kFlags_none:
47 return LeaveRC;
48 }
49 UNREACHABLE();
50 }
51
CompareLogical() const52 bool CompareLogical() const {
53 switch (instr_->flags_condition()) {
54 case kUnsignedLessThan:
55 case kUnsignedGreaterThanOrEqual:
56 case kUnsignedLessThanOrEqual:
57 case kUnsignedGreaterThan:
58 return true;
59 default:
60 return false;
61 }
62 UNREACHABLE();
63 }
64
InputImmediate(size_t index)65 Operand InputImmediate(size_t index) {
66 Constant constant = ToConstant(instr_->InputAt(index));
67 switch (constant.type()) {
68 case Constant::kInt32:
69 return Operand(constant.ToInt32());
70 case Constant::kFloat32:
71 return Operand::EmbeddedNumber(constant.ToFloat32());
72 case Constant::kFloat64:
73 return Operand::EmbeddedNumber(constant.ToFloat64().value());
74 case Constant::kInt64:
75 #if V8_TARGET_ARCH_PPC64
76 return Operand(constant.ToInt64());
77 #endif
78 case Constant::kExternalReference:
79 return Operand(constant.ToExternalReference());
80 case Constant::kDelayedStringConstant:
81 return Operand::EmbeddedStringConstant(
82 constant.ToDelayedStringConstant());
83 case Constant::kCompressedHeapObject:
84 case Constant::kHeapObject:
85 case Constant::kRpoNumber:
86 break;
87 }
88 UNREACHABLE();
89 }
90
MemoryOperand(AddressingMode* mode, size_t* first_index)91 MemOperand MemoryOperand(AddressingMode* mode, size_t* first_index) {
92 const size_t index = *first_index;
93 AddressingMode addr_mode = AddressingModeField::decode(instr_->opcode());
94 if (mode) *mode = addr_mode;
95 switch (addr_mode) {
96 case kMode_None:
97 break;
98 case kMode_MRI:
99 *first_index += 2;
100 return MemOperand(InputRegister(index + 0), InputInt64(index + 1));
101 case kMode_MRR:
102 *first_index += 2;
103 return MemOperand(InputRegister(index + 0), InputRegister(index + 1));
104 }
105 UNREACHABLE();
106 }
107
MemoryOperand(AddressingMode* mode = NULL, size_t first_index = 0)108 MemOperand MemoryOperand(AddressingMode* mode = NULL,
109 size_t first_index = 0) {
110 return MemoryOperand(mode, &first_index);
111 }
112
ToMemOperand(InstructionOperand* op) const113 MemOperand ToMemOperand(InstructionOperand* op) const {
114 DCHECK_NOT_NULL(op);
115 DCHECK(op->IsStackSlot() || op->IsFPStackSlot());
116 return SlotToMemOperand(AllocatedOperand::cast(op)->index());
117 }
118
SlotToMemOperand(int slot) const119 MemOperand SlotToMemOperand(int slot) const {
120 FrameOffset offset = frame_access_state()->GetFrameOffset(slot);
121 return MemOperand(offset.from_stack_pointer() ? sp : fp, offset.offset());
122 }
123 };
124
HasRegisterInput(Instruction* instr, size_t index)125 static inline bool HasRegisterInput(Instruction* instr, size_t index) {
126 return instr->InputAt(index)->IsRegister();
127 }
128
129 namespace {
130
131 class OutOfLineRecordWrite final : public OutOfLineCode {
132 public:
OutOfLineRecordWrite(CodeGenerator* gen, Register object, Register offset, Register value, Register scratch0, Register scratch1, RecordWriteMode mode, StubCallMode stub_mode, UnwindingInfoWriter* unwinding_info_writer)133 OutOfLineRecordWrite(CodeGenerator* gen, Register object, Register offset,
134 Register value, Register scratch0, Register scratch1,
135 RecordWriteMode mode, StubCallMode stub_mode,
136 UnwindingInfoWriter* unwinding_info_writer)
137 : OutOfLineCode(gen),
138 object_(object),
139 offset_(offset),
140 offset_immediate_(0),
141 value_(value),
142 scratch0_(scratch0),
143 scratch1_(scratch1),
144 mode_(mode),
145 #if V8_ENABLE_WEBASSEMBLY
146 stub_mode_(stub_mode),
147 #endif // V8_ENABLE_WEBASSEMBLY
148 must_save_lr_(!gen->frame_access_state()->has_frame()),
149 unwinding_info_writer_(unwinding_info_writer),
150 zone_(gen->zone()) {
151 DCHECK(!AreAliased(object, offset, scratch0, scratch1));
152 DCHECK(!AreAliased(value, offset, scratch0, scratch1));
153 }
154
OutOfLineRecordWrite(CodeGenerator* gen, Register object, int32_t offset, Register value, Register scratch0, Register scratch1, RecordWriteMode mode, StubCallMode stub_mode, UnwindingInfoWriter* unwinding_info_writer)155 OutOfLineRecordWrite(CodeGenerator* gen, Register object, int32_t offset,
156 Register value, Register scratch0, Register scratch1,
157 RecordWriteMode mode, StubCallMode stub_mode,
158 UnwindingInfoWriter* unwinding_info_writer)
159 : OutOfLineCode(gen),
160 object_(object),
161 offset_(no_reg),
162 offset_immediate_(offset),
163 value_(value),
164 scratch0_(scratch0),
165 scratch1_(scratch1),
166 mode_(mode),
167 #if V8_ENABLE_WEBASSEMBLY
168 stub_mode_(stub_mode),
169 #endif // V8_ENABLE_WEBASSEMBLY
170 must_save_lr_(!gen->frame_access_state()->has_frame()),
171 unwinding_info_writer_(unwinding_info_writer),
172 zone_(gen->zone()) {
173 }
174
175 void Generate() final {
176 ConstantPoolUnavailableScope constant_pool_unavailable(tasm());
177 if (COMPRESS_POINTERS_BOOL) {
178 __ DecompressTaggedPointer(value_, value_);
179 }
180 __ CheckPageFlag(value_, scratch0_,
181 MemoryChunk::kPointersToHereAreInterestingMask, eq,
182 exit());
183 if (offset_ == no_reg) {
184 __ addi(scratch1_, object_, Operand(offset_immediate_));
185 } else {
186 DCHECK_EQ(0, offset_immediate_);
187 __ add(scratch1_, object_, offset_);
188 }
189 RememberedSetAction const remembered_set_action =
190 mode_ > RecordWriteMode::kValueIsMap ||
191 FLAG_use_full_record_write_builtin
192 ? RememberedSetAction::kEmit
193 : RememberedSetAction::kOmit;
194 SaveFPRegsMode const save_fp_mode = frame()->DidAllocateDoubleRegisters()
195 ? SaveFPRegsMode::kSave
196 : SaveFPRegsMode::kIgnore;
197 if (must_save_lr_) {
198 // We need to save and restore lr if the frame was elided.
199 __ mflr(scratch0_);
200 __ Push(scratch0_);
201 unwinding_info_writer_->MarkLinkRegisterOnTopOfStack(__ pc_offset());
202 }
203 if (mode_ == RecordWriteMode::kValueIsEphemeronKey) {
204 __ CallEphemeronKeyBarrier(object_, scratch1_, save_fp_mode);
205 #if V8_ENABLE_WEBASSEMBLY
206 } else if (stub_mode_ == StubCallMode::kCallWasmRuntimeStub) {
207 __ CallRecordWriteStubSaveRegisters(object_, scratch1_,
208 remembered_set_action, save_fp_mode,
209 StubCallMode::kCallWasmRuntimeStub);
210 #endif // V8_ENABLE_WEBASSEMBLY
211 } else {
212 __ CallRecordWriteStubSaveRegisters(object_, scratch1_,
213 remembered_set_action, save_fp_mode);
214 }
215 if (must_save_lr_) {
216 // We need to save and restore lr if the frame was elided.
217 __ Pop(scratch0_);
218 __ mtlr(scratch0_);
219 unwinding_info_writer_->MarkPopLinkRegisterFromTopOfStack(__ pc_offset());
220 }
221 }
222
223 private:
224 Register const object_;
225 Register const offset_;
226 int32_t const offset_immediate_; // Valid if offset_ == no_reg.
227 Register const value_;
228 Register const scratch0_;
229 Register const scratch1_;
230 RecordWriteMode const mode_;
231 #if V8_ENABLE_WEBASSEMBLY
232 StubCallMode stub_mode_;
233 #endif // V8_ENABLE_WEBASSEMBLY
234 bool must_save_lr_;
235 UnwindingInfoWriter* const unwinding_info_writer_;
236 Zone* zone_;
237 };
238
FlagsConditionToCondition(FlagsCondition condition, ArchOpcode op)239 Condition FlagsConditionToCondition(FlagsCondition condition, ArchOpcode op) {
240 switch (condition) {
241 case kEqual:
242 return eq;
243 case kNotEqual:
244 return ne;
245 case kSignedLessThan:
246 case kUnsignedLessThan:
247 return lt;
248 case kSignedGreaterThanOrEqual:
249 case kUnsignedGreaterThanOrEqual:
250 return ge;
251 case kSignedLessThanOrEqual:
252 case kUnsignedLessThanOrEqual:
253 return le;
254 case kSignedGreaterThan:
255 case kUnsignedGreaterThan:
256 return gt;
257 case kOverflow:
258 // Overflow checked for add/sub only.
259 switch (op) {
260 #if V8_TARGET_ARCH_PPC64
261 case kPPC_Add32:
262 case kPPC_Add64:
263 case kPPC_Sub:
264 #endif
265 case kPPC_AddWithOverflow32:
266 case kPPC_SubWithOverflow32:
267 return lt;
268 default:
269 break;
270 }
271 break;
272 case kNotOverflow:
273 switch (op) {
274 #if V8_TARGET_ARCH_PPC64
275 case kPPC_Add32:
276 case kPPC_Add64:
277 case kPPC_Sub:
278 #endif
279 case kPPC_AddWithOverflow32:
280 case kPPC_SubWithOverflow32:
281 return ge;
282 default:
283 break;
284 }
285 break;
286 default:
287 break;
288 }
289 UNREACHABLE();
290 }
291
292 } // namespace
293
294 #define ASSEMBLE_FLOAT_UNOP_RC(asm_instr, round) \
295 do { \
296 __ asm_instr(i.OutputDoubleRegister(), i.InputDoubleRegister(0), \
297 i.OutputRCBit()); \
298 if (round) { \
299 __ frsp(i.OutputDoubleRegister(), i.OutputDoubleRegister()); \
300 } \
301 } while (0)
302
303 #define ASSEMBLE_FLOAT_BINOP_RC(asm_instr, round) \
304 do { \
305 __ asm_instr(i.OutputDoubleRegister(), i.InputDoubleRegister(0), \
306 i.InputDoubleRegister(1), i.OutputRCBit()); \
307 if (round) { \
308 __ frsp(i.OutputDoubleRegister(), i.OutputDoubleRegister()); \
309 } \
310 } while (0)
311
312 #define ASSEMBLE_BINOP(asm_instr_reg, asm_instr_imm) \
313 do { \
314 if (HasRegisterInput(instr, 1)) { \
315 __ asm_instr_reg(i.OutputRegister(), i.InputRegister(0), \
316 i.InputRegister(1)); \
317 } else { \
318 __ asm_instr_imm(i.OutputRegister(), i.InputRegister(0), \
319 i.InputImmediate(1)); \
320 } \
321 } while (0)
322
323 #define ASSEMBLE_BINOP_RC(asm_instr_reg, asm_instr_imm) \
324 do { \
325 if (HasRegisterInput(instr, 1)) { \
326 __ asm_instr_reg(i.OutputRegister(), i.InputRegister(0), \
327 i.InputRegister(1), i.OutputRCBit()); \
328 } else { \
329 __ asm_instr_imm(i.OutputRegister(), i.InputRegister(0), \
330 i.InputImmediate(1), i.OutputRCBit()); \
331 } \
332 } while (0)
333
334 #define ASSEMBLE_BINOP_INT_RC(asm_instr_reg, asm_instr_imm) \
335 do { \
336 if (HasRegisterInput(instr, 1)) { \
337 __ asm_instr_reg(i.OutputRegister(), i.InputRegister(0), \
338 i.InputRegister(1), i.OutputRCBit()); \
339 } else { \
340 __ asm_instr_imm(i.OutputRegister(), i.InputRegister(0), \
341 i.InputImmediate(1), i.OutputRCBit()); \
342 } \
343 } while (0)
344
345 #define ASSEMBLE_ADD_WITH_OVERFLOW() \
346 do { \
347 if (HasRegisterInput(instr, 1)) { \
348 __ AddAndCheckForOverflow(i.OutputRegister(), i.InputRegister(0), \
349 i.InputRegister(1), kScratchReg, r0); \
350 } else { \
351 __ AddAndCheckForOverflow(i.OutputRegister(), i.InputRegister(0), \
352 i.InputInt32(1), kScratchReg, r0); \
353 } \
354 } while (0)
355
356 #define ASSEMBLE_SUB_WITH_OVERFLOW() \
357 do { \
358 if (HasRegisterInput(instr, 1)) { \
359 __ SubAndCheckForOverflow(i.OutputRegister(), i.InputRegister(0), \
360 i.InputRegister(1), kScratchReg, r0); \
361 } else { \
362 __ AddAndCheckForOverflow(i.OutputRegister(), i.InputRegister(0), \
363 -i.InputInt32(1), kScratchReg, r0); \
364 } \
365 } while (0)
366
367 #if V8_TARGET_ARCH_PPC64
368 #define ASSEMBLE_ADD_WITH_OVERFLOW32() \
369 do { \
370 ASSEMBLE_ADD_WITH_OVERFLOW(); \
371 __ extsw(kScratchReg, kScratchReg, SetRC); \
372 } while (0)
373
374 #define ASSEMBLE_SUB_WITH_OVERFLOW32() \
375 do { \
376 ASSEMBLE_SUB_WITH_OVERFLOW(); \
377 __ extsw(kScratchReg, kScratchReg, SetRC); \
378 } while (0)
379 #else
380 #define ASSEMBLE_ADD_WITH_OVERFLOW32 ASSEMBLE_ADD_WITH_OVERFLOW
381 #define ASSEMBLE_SUB_WITH_OVERFLOW32 ASSEMBLE_SUB_WITH_OVERFLOW
382 #endif
383
384 #define ASSEMBLE_COMPARE(cmp_instr, cmpl_instr) \
385 do { \
386 const CRegister cr = cr0; \
387 if (HasRegisterInput(instr, 1)) { \
388 if (i.CompareLogical()) { \
389 __ cmpl_instr(i.InputRegister(0), i.InputRegister(1), cr); \
390 } else { \
391 __ cmp_instr(i.InputRegister(0), i.InputRegister(1), cr); \
392 } \
393 } else { \
394 if (i.CompareLogical()) { \
395 __ cmpl_instr##i(i.InputRegister(0), i.InputImmediate(1), cr); \
396 } else { \
397 __ cmp_instr##i(i.InputRegister(0), i.InputImmediate(1), cr); \
398 } \
399 } \
400 DCHECK_EQ(SetRC, i.OutputRCBit()); \
401 } while (0)
402
403 #define ASSEMBLE_FLOAT_COMPARE(cmp_instr) \
404 do { \
405 const CRegister cr = cr0; \
406 __ cmp_instr(i.InputDoubleRegister(0), i.InputDoubleRegister(1), cr); \
407 DCHECK_EQ(SetRC, i.OutputRCBit()); \
408 } while (0)
409
410 #define ASSEMBLE_MODULO(div_instr, mul_instr) \
411 do { \
412 const Register scratch = kScratchReg; \
413 __ div_instr(scratch, i.InputRegister(0), i.InputRegister(1)); \
414 __ mul_instr(scratch, scratch, i.InputRegister(1)); \
415 __ sub(i.OutputRegister(), i.InputRegister(0), scratch, LeaveOE, \
416 i.OutputRCBit()); \
417 } while (0)
418
419 #define ASSEMBLE_FLOAT_MODULO() \
420 do { \
421 FrameScope scope(tasm(), StackFrame::MANUAL); \
422 __ PrepareCallCFunction(0, 2, kScratchReg); \
423 __ MovToFloatParameters(i.InputDoubleRegister(0), \
424 i.InputDoubleRegister(1)); \
425 __ CallCFunction(ExternalReference::mod_two_doubles_operation(), 0, 2); \
426 __ MovFromFloatResult(i.OutputDoubleRegister()); \
427 DCHECK_EQ(LeaveRC, i.OutputRCBit()); \
428 } while (0)
429
430 #define ASSEMBLE_IEEE754_UNOP(name) \
431 do { \
432 /* TODO(bmeurer): We should really get rid of this special instruction, */ \
433 /* and generate a CallAddress instruction instead. */ \
434 FrameScope scope(tasm(), StackFrame::MANUAL); \
435 __ PrepareCallCFunction(0, 1, kScratchReg); \
436 __ MovToFloatParameter(i.InputDoubleRegister(0)); \
437 __ CallCFunction(ExternalReference::ieee754_##name##_function(), 0, 1); \
438 /* Move the result in the double result register. */ \
439 __ MovFromFloatResult(i.OutputDoubleRegister()); \
440 DCHECK_EQ(LeaveRC, i.OutputRCBit()); \
441 } while (0)
442
443 #define ASSEMBLE_IEEE754_BINOP(name) \
444 do { \
445 /* TODO(bmeurer): We should really get rid of this special instruction, */ \
446 /* and generate a CallAddress instruction instead. */ \
447 FrameScope scope(tasm(), StackFrame::MANUAL); \
448 __ PrepareCallCFunction(0, 2, kScratchReg); \
449 __ MovToFloatParameters(i.InputDoubleRegister(0), \
450 i.InputDoubleRegister(1)); \
451 __ CallCFunction(ExternalReference::ieee754_##name##_function(), 0, 2); \
452 /* Move the result in the double result register. */ \
453 __ MovFromFloatResult(i.OutputDoubleRegister()); \
454 DCHECK_EQ(LeaveRC, i.OutputRCBit()); \
455 } while (0)
456
457 #define ASSEMBLE_LOAD_FLOAT(asm_instr, asm_instrp, asm_instrx) \
458 do { \
459 DoubleRegister result = i.OutputDoubleRegister(); \
460 AddressingMode mode = kMode_None; \
461 MemOperand operand = i.MemoryOperand(&mode); \
462 bool is_atomic = i.InputInt32(2); \
463 if (mode == kMode_MRI) { \
464 if (CpuFeatures::IsSupported(PPC_10_PLUS)) { \
465 __ asm_instrp(result, operand); \
466 } else { \
467 __ asm_instr(result, operand); \
468 } \
469 } else { \
470 __ asm_instrx(result, operand); \
471 } \
472 if (is_atomic) __ lwsync(); \
473 DCHECK_EQ(LeaveRC, i.OutputRCBit()); \
474 } while (0)
475
476 #define ASSEMBLE_LOAD_INTEGER(asm_instr, asm_instrp, asm_instrx) \
477 do { \
478 Register result = i.OutputRegister(); \
479 AddressingMode mode = kMode_None; \
480 MemOperand operand = i.MemoryOperand(&mode); \
481 bool is_atomic = i.InputInt32(2); \
482 if (mode == kMode_MRI) { \
483 if (CpuFeatures::IsSupported(PPC_10_PLUS)) { \
484 __ asm_instrp(result, operand); \
485 } else { \
486 __ asm_instr(result, operand); \
487 } \
488 } else { \
489 __ asm_instrx(result, operand); \
490 } \
491 if (is_atomic) __ lwsync(); \
492 DCHECK_EQ(LeaveRC, i.OutputRCBit()); \
493 } while (0)
494
495 #define ASSEMBLE_LOAD_INTEGER_RR(asm_instr) \
496 do { \
497 Register result = i.OutputRegister(); \
498 AddressingMode mode = kMode_None; \
499 MemOperand operand = i.MemoryOperand(&mode); \
500 DCHECK_EQ(mode, kMode_MRR); \
501 bool is_atomic = i.InputInt32(2); \
502 __ asm_instr(result, operand); \
503 if (is_atomic) __ lwsync(); \
504 DCHECK_EQ(LeaveRC, i.OutputRCBit()); \
505 } while (0)
506
507 #define ASSEMBLE_STORE_FLOAT(asm_instr, asm_instrx) \
508 do { \
509 size_t index = 0; \
510 AddressingMode mode = kMode_None; \
511 MemOperand operand = i.MemoryOperand(&mode, &index); \
512 DoubleRegister value = i.InputDoubleRegister(index); \
513 bool is_atomic = i.InputInt32(3); \
514 if (is_atomic) __ lwsync(); \
515 /* removed frsp as instruction-selector checked */ \
516 /* value to be kFloat32 */ \
517 if (mode == kMode_MRI) { \
518 __ asm_instr(value, operand); \
519 } else { \
520 __ asm_instrx(value, operand); \
521 } \
522 if (is_atomic) __ sync(); \
523 DCHECK_EQ(LeaveRC, i.OutputRCBit()); \
524 } while (0)
525
526 #define ASSEMBLE_STORE_INTEGER(asm_instr, asm_instrx) \
527 do { \
528 size_t index = 0; \
529 AddressingMode mode = kMode_None; \
530 MemOperand operand = i.MemoryOperand(&mode, &index); \
531 Register value = i.InputRegister(index); \
532 bool is_atomic = i.InputInt32(3); \
533 if (is_atomic) __ lwsync(); \
534 if (mode == kMode_MRI) { \
535 __ asm_instr(value, operand); \
536 } else { \
537 __ asm_instrx(value, operand); \
538 } \
539 if (is_atomic) __ sync(); \
540 DCHECK_EQ(LeaveRC, i.OutputRCBit()); \
541 } while (0)
542
543 #define ASSEMBLE_STORE_INTEGER_RR(asm_instr) \
544 do { \
545 size_t index = 0; \
546 AddressingMode mode = kMode_None; \
547 MemOperand operand = i.MemoryOperand(&mode, &index); \
548 DCHECK_EQ(mode, kMode_MRR); \
549 Register value = i.InputRegister(index); \
550 bool is_atomic = i.InputInt32(3); \
551 if (is_atomic) __ lwsync(); \
552 __ asm_instr(value, operand); \
553 if (is_atomic) __ sync(); \
554 DCHECK_EQ(LeaveRC, i.OutputRCBit()); \
555 } while (0)
556
557 #if V8_TARGET_ARCH_PPC64
558 // TODO(mbrandy): fix paths that produce garbage in offset's upper 32-bits.
559 #define CleanUInt32(x) __ ClearLeftImm(x, x, Operand(32))
560 #else
561 #define CleanUInt32(x)
562 #endif
563
is_wasm_on_be(bool IsWasm)564 static inline bool is_wasm_on_be(bool IsWasm) {
565 #if V8_TARGET_BIG_ENDIAN
566 return IsWasm;
567 #else
568 return false;
569 #endif
570 }
571
572 #define MAYBE_REVERSE_IF_WASM(dst, src, op, reset) \
573 if (is_wasm_on_be(info()->IsWasm())) { \
574 __ op(dst, src, kScratchReg); \
575 if (reset) src = dst; \
576 }
577
578 #define ASSEMBLE_ATOMIC_EXCHANGE(_type, reverse_op) \
579 do { \
580 Register val = i.InputRegister(2); \
581 Register dst = i.OutputRegister(); \
582 MAYBE_REVERSE_IF_WASM(ip, val, reverse_op, true); \
583 __ AtomicExchange<_type>( \
584 MemOperand(i.InputRegister(0), i.InputRegister(1)), val, dst); \
585 MAYBE_REVERSE_IF_WASM(dst, dst, reverse_op, false); \
586 } while (false)
587
588 #define ASSEMBLE_ATOMIC_COMPARE_EXCHANGE(_type, reverse_op) \
589 do { \
590 Register expected_val = i.InputRegister(2); \
591 Register new_val = i.InputRegister(3); \
592 Register dst = i.OutputRegister(); \
593 MAYBE_REVERSE_IF_WASM(ip, expected_val, reverse_op, true); \
594 MAYBE_REVERSE_IF_WASM(r0, new_val, reverse_op, true); \
595 __ AtomicCompareExchange<_type>( \
596 MemOperand(i.InputRegister(0), i.InputRegister(1)), expected_val, \
597 new_val, dst, kScratchReg); \
598 MAYBE_REVERSE_IF_WASM(dst, dst, reverse_op, false); \
599 } while (false)
600
601 #define ASSEMBLE_ATOMIC_BINOP_BYTE(bin_inst, _type) \
602 do { \
603 auto bin_op = [&](Register dst, Register lhs, Register rhs) { \
604 if (std::is_signed<_type>::value) { \
605 __ extsb(dst, lhs); \
606 __ bin_inst(dst, dst, rhs); \
607 } else { \
608 __ bin_inst(dst, lhs, rhs); \
609 } \
610 }; \
611 MemOperand dst_operand = \
612 MemOperand(i.InputRegister(0), i.InputRegister(1)); \
613 __ AtomicOps<_type>(dst_operand, i.InputRegister(2), i.OutputRegister(), \
614 kScratchReg, bin_op); \
615 break; \
616 } while (false)
617
618 #define ASSEMBLE_ATOMIC_BINOP(bin_inst, _type, reverse_op, scratch) \
619 do { \
620 auto bin_op = [&](Register dst, Register lhs, Register rhs) { \
621 Register _lhs = lhs; \
622 if (is_wasm_on_be(info()->IsWasm())) { \
623 __ reverse_op(dst, lhs, scratch); \
624 _lhs = dst; \
625 } \
626 if (std::is_signed<_type>::value) { \
627 switch (sizeof(_type)) { \
628 case 1: \
629 UNREACHABLE(); \
630 break; \
631 case 2: \
632 __ extsh(dst, _lhs); \
633 break; \
634 case 4: \
635 __ extsw(dst, _lhs); \
636 break; \
637 case 8: \
638 break; \
639 default: \
640 UNREACHABLE(); \
641 } \
642 } \
643 __ bin_inst(dst, _lhs, rhs); \
644 if (is_wasm_on_be(info()->IsWasm())) { \
645 __ reverse_op(dst, dst, scratch); \
646 } \
647 }; \
648 MemOperand dst_operand = \
649 MemOperand(i.InputRegister(0), i.InputRegister(1)); \
650 __ AtomicOps<_type>(dst_operand, i.InputRegister(2), i.OutputRegister(), \
651 kScratchReg, bin_op); \
652 if (is_wasm_on_be(info()->IsWasm())) { \
653 __ reverse_op(i.OutputRegister(), i.OutputRegister(), scratch); \
654 } \
655 break; \
656 } while (false)
657
AssembleDeconstructFrame()658 void CodeGenerator::AssembleDeconstructFrame() {
659 __ LeaveFrame(StackFrame::MANUAL);
660 unwinding_info_writer_.MarkFrameDeconstructed(__ pc_offset());
661 }
662
AssemblePrepareTailCall()663 void CodeGenerator::AssemblePrepareTailCall() {
664 if (frame_access_state()->has_frame()) {
665 __ RestoreFrameStateForTailCall();
666 }
667 frame_access_state()->SetFrameAccessToSP();
668 }
669
670 namespace {
671
FlushPendingPushRegisters(TurboAssembler* tasm, FrameAccessState* frame_access_state, ZoneVector<Register>* pending_pushes)672 void FlushPendingPushRegisters(TurboAssembler* tasm,
673 FrameAccessState* frame_access_state,
674 ZoneVector<Register>* pending_pushes) {
675 switch (pending_pushes->size()) {
676 case 0:
677 break;
678 case 1:
679 tasm->Push((*pending_pushes)[0]);
680 break;
681 case 2:
682 tasm->Push((*pending_pushes)[0], (*pending_pushes)[1]);
683 break;
684 case 3:
685 tasm->Push((*pending_pushes)[0], (*pending_pushes)[1],
686 (*pending_pushes)[2]);
687 break;
688 default:
689 UNREACHABLE();
690 }
691 frame_access_state->IncreaseSPDelta(pending_pushes->size());
692 pending_pushes->clear();
693 }
694
AdjustStackPointerForTailCall( TurboAssembler* tasm, FrameAccessState* state, int new_slot_above_sp, ZoneVector<Register>* pending_pushes = nullptr, bool allow_shrinkage = true)695 void AdjustStackPointerForTailCall(
696 TurboAssembler* tasm, FrameAccessState* state, int new_slot_above_sp,
697 ZoneVector<Register>* pending_pushes = nullptr,
698 bool allow_shrinkage = true) {
699 int current_sp_offset = state->GetSPToFPSlotCount() +
700 StandardFrameConstants::kFixedSlotCountAboveFp;
701 int stack_slot_delta = new_slot_above_sp - current_sp_offset;
702 if (stack_slot_delta > 0) {
703 if (pending_pushes != nullptr) {
704 FlushPendingPushRegisters(tasm, state, pending_pushes);
705 }
706 tasm->AddS64(sp, sp, Operand(-stack_slot_delta * kSystemPointerSize), r0);
707 state->IncreaseSPDelta(stack_slot_delta);
708 } else if (allow_shrinkage && stack_slot_delta < 0) {
709 if (pending_pushes != nullptr) {
710 FlushPendingPushRegisters(tasm, state, pending_pushes);
711 }
712 tasm->AddS64(sp, sp, Operand(-stack_slot_delta * kSystemPointerSize), r0);
713 state->IncreaseSPDelta(stack_slot_delta);
714 }
715 }
716
717 } // namespace
718
AssembleTailCallBeforeGap(Instruction* instr, int first_unused_slot_offset)719 void CodeGenerator::AssembleTailCallBeforeGap(Instruction* instr,
720 int first_unused_slot_offset) {
721 ZoneVector<MoveOperands*> pushes(zone());
722 GetPushCompatibleMoves(instr, kRegisterPush, &pushes);
723
724 if (!pushes.empty() &&
725 (LocationOperand::cast(pushes.back()->destination()).index() + 1 ==
726 first_unused_slot_offset)) {
727 PPCOperandConverter g(this, instr);
728 ZoneVector<Register> pending_pushes(zone());
729 for (auto move : pushes) {
730 LocationOperand destination_location(
731 LocationOperand::cast(move->destination()));
732 InstructionOperand source(move->source());
733 AdjustStackPointerForTailCall(
734 tasm(), frame_access_state(),
735 destination_location.index() - pending_pushes.size(),
736 &pending_pushes);
737 // Pushes of non-register data types are not supported.
738 DCHECK(source.IsRegister());
739 LocationOperand source_location(LocationOperand::cast(source));
740 pending_pushes.push_back(source_location.GetRegister());
741 // TODO(arm): We can push more than 3 registers at once. Add support in
742 // the macro-assembler for pushing a list of registers.
743 if (pending_pushes.size() == 3) {
744 FlushPendingPushRegisters(tasm(), frame_access_state(),
745 &pending_pushes);
746 }
747 move->Eliminate();
748 }
749 FlushPendingPushRegisters(tasm(), frame_access_state(), &pending_pushes);
750 }
751 AdjustStackPointerForTailCall(tasm(), frame_access_state(),
752 first_unused_slot_offset, nullptr, false);
753 }
754
AssembleTailCallAfterGap(Instruction* instr, int first_unused_slot_offset)755 void CodeGenerator::AssembleTailCallAfterGap(Instruction* instr,
756 int first_unused_slot_offset) {
757 AdjustStackPointerForTailCall(tasm(), frame_access_state(),
758 first_unused_slot_offset);
759 }
760
761 // Check that {kJavaScriptCallCodeStartRegister} is correct.
AssembleCodeStartRegisterCheck()762 void CodeGenerator::AssembleCodeStartRegisterCheck() {
763 Register scratch = kScratchReg;
764 __ ComputeCodeStartAddress(scratch);
765 __ CmpS64(scratch, kJavaScriptCallCodeStartRegister);
766 __ Assert(eq, AbortReason::kWrongFunctionCodeStart);
767 }
768
769 // Check if the code object is marked for deoptimization. If it is, then it
770 // jumps to the CompileLazyDeoptimizedCode builtin. In order to do this we need
771 // to:
772 // 1. read from memory the word that contains that bit, which can be found in
773 // the flags in the referenced {CodeDataContainer} object;
774 // 2. test kMarkedForDeoptimizationBit in those flags; and
775 // 3. if it is not zero then it jumps to the builtin.
BailoutIfDeoptimized()776 void CodeGenerator::BailoutIfDeoptimized() {
777 if (FLAG_debug_code) {
778 // Check that {kJavaScriptCallCodeStartRegister} is correct.
779 __ ComputeCodeStartAddress(ip);
780 __ CmpS64(ip, kJavaScriptCallCodeStartRegister);
781 __ Assert(eq, AbortReason::kWrongFunctionCodeStart);
782 }
783
784 int offset = Code::kCodeDataContainerOffset - Code::kHeaderSize;
785 __ LoadTaggedPointerField(
786 r11, MemOperand(kJavaScriptCallCodeStartRegister, offset), r0);
787 __ LoadS32(r11,
788 FieldMemOperand(r11, CodeDataContainer::kKindSpecificFlagsOffset),
789 r0);
790 __ TestBit(r11, Code::kMarkedForDeoptimizationBit);
791 __ Jump(BUILTIN_CODE(isolate(), CompileLazyDeoptimizedCode),
792 RelocInfo::CODE_TARGET, ne, cr0);
793 }
794
795 // Assembles an instruction after register allocation, producing machine code.
AssembleArchInstruction( Instruction* instr)796 CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction(
797 Instruction* instr) {
798 PPCOperandConverter i(this, instr);
799 ArchOpcode opcode = ArchOpcodeField::decode(instr->opcode());
800
801 switch (opcode) {
802 case kArchCallCodeObject: {
803 v8::internal::Assembler::BlockTrampolinePoolScope block_trampoline_pool(
804 tasm());
805 if (HasRegisterInput(instr, 0)) {
806 Register reg = i.InputRegister(0);
807 DCHECK_IMPLIES(
808 instr->HasCallDescriptorFlag(CallDescriptor::kFixedTargetRegister),
809 reg == kJavaScriptCallCodeStartRegister);
810 __ CallCodeObject(reg);
811 } else {
812 __ Call(i.InputCode(0), RelocInfo::CODE_TARGET);
813 }
814 RecordCallPosition(instr);
815 DCHECK_EQ(LeaveRC, i.OutputRCBit());
816 frame_access_state()->ClearSPDelta();
817 break;
818 }
819 case kArchCallBuiltinPointer: {
820 DCHECK(!instr->InputAt(0)->IsImmediate());
821 Register builtin_index = i.InputRegister(0);
822 __ CallBuiltinByIndex(builtin_index);
823 RecordCallPosition(instr);
824 frame_access_state()->ClearSPDelta();
825 break;
826 }
827 #if V8_ENABLE_WEBASSEMBLY
828 case kArchCallWasmFunction: {
829 // We must not share code targets for calls to builtins for wasm code, as
830 // they might need to be patched individually.
831 if (instr->InputAt(0)->IsImmediate()) {
832 Constant constant = i.ToConstant(instr->InputAt(0));
833 #ifdef V8_TARGET_ARCH_PPC64
834 Address wasm_code = static_cast<Address>(constant.ToInt64());
835 #else
836 Address wasm_code = static_cast<Address>(constant.ToInt32());
837 #endif
838 __ Call(wasm_code, constant.rmode());
839 } else {
840 __ Call(i.InputRegister(0));
841 }
842 RecordCallPosition(instr);
843 DCHECK_EQ(LeaveRC, i.OutputRCBit());
844 frame_access_state()->ClearSPDelta();
845 break;
846 }
847 case kArchTailCallWasm: {
848 // We must not share code targets for calls to builtins for wasm code, as
849 // they might need to be patched individually.
850 if (instr->InputAt(0)->IsImmediate()) {
851 Constant constant = i.ToConstant(instr->InputAt(0));
852 #ifdef V8_TARGET_ARCH_PPC64
853 Address wasm_code = static_cast<Address>(constant.ToInt64());
854 #else
855 Address wasm_code = static_cast<Address>(constant.ToInt32());
856 #endif
857 __ Jump(wasm_code, constant.rmode());
858 } else {
859 __ Jump(i.InputRegister(0));
860 }
861 DCHECK_EQ(LeaveRC, i.OutputRCBit());
862 frame_access_state()->ClearSPDelta();
863 frame_access_state()->SetFrameAccessToDefault();
864 break;
865 }
866 #endif // V8_ENABLE_WEBASSEMBLY
867 case kArchTailCallCodeObject: {
868 if (HasRegisterInput(instr, 0)) {
869 Register reg = i.InputRegister(0);
870 DCHECK_IMPLIES(
871 instr->HasCallDescriptorFlag(CallDescriptor::kFixedTargetRegister),
872 reg == kJavaScriptCallCodeStartRegister);
873 __ JumpCodeObject(reg);
874 } else {
875 // We cannot use the constant pool to load the target since
876 // we've already restored the caller's frame.
877 ConstantPoolUnavailableScope constant_pool_unavailable(tasm());
878 __ Jump(i.InputCode(0), RelocInfo::CODE_TARGET);
879 }
880 DCHECK_EQ(LeaveRC, i.OutputRCBit());
881 frame_access_state()->ClearSPDelta();
882 frame_access_state()->SetFrameAccessToDefault();
883 break;
884 }
885 case kArchTailCallAddress: {
886 CHECK(!instr->InputAt(0)->IsImmediate());
887 Register reg = i.InputRegister(0);
888 DCHECK_IMPLIES(
889 instr->HasCallDescriptorFlag(CallDescriptor::kFixedTargetRegister),
890 reg == kJavaScriptCallCodeStartRegister);
891 __ Jump(reg);
892 frame_access_state()->ClearSPDelta();
893 frame_access_state()->SetFrameAccessToDefault();
894 break;
895 }
896 case kArchCallJSFunction: {
897 v8::internal::Assembler::BlockTrampolinePoolScope block_trampoline_pool(
898 tasm());
899 Register func = i.InputRegister(0);
900 if (FLAG_debug_code) {
901 // Check the function's context matches the context argument.
902 __ LoadTaggedPointerField(
903 kScratchReg, FieldMemOperand(func, JSFunction::kContextOffset), r0);
904 __ CmpS64(cp, kScratchReg);
905 __ Assert(eq, AbortReason::kWrongFunctionContext);
906 }
907 static_assert(kJavaScriptCallCodeStartRegister == r5, "ABI mismatch");
908 __ LoadTaggedPointerField(
909 r5, FieldMemOperand(func, JSFunction::kCodeOffset), r0);
910 __ CallCodeObject(r5);
911 RecordCallPosition(instr);
912 DCHECK_EQ(LeaveRC, i.OutputRCBit());
913 frame_access_state()->ClearSPDelta();
914 break;
915 }
916 case kArchPrepareCallCFunction: {
917 int const num_gp_parameters = ParamField::decode(instr->opcode());
918 int const num_fp_parameters = FPParamField::decode(instr->opcode());
919 __ PrepareCallCFunction(num_gp_parameters + num_fp_parameters,
920 kScratchReg);
921 // Frame alignment requires using FP-relative frame addressing.
922 frame_access_state()->SetFrameAccessToFP();
923 break;
924 }
925 case kArchSaveCallerRegisters: {
926 fp_mode_ =
927 static_cast<SaveFPRegsMode>(MiscField::decode(instr->opcode()));
928 DCHECK(fp_mode_ == SaveFPRegsMode::kIgnore ||
929 fp_mode_ == SaveFPRegsMode::kSave);
930 // kReturnRegister0 should have been saved before entering the stub.
931 int bytes = __ PushCallerSaved(fp_mode_, kReturnRegister0);
932 DCHECK(IsAligned(bytes, kSystemPointerSize));
933 DCHECK_EQ(0, frame_access_state()->sp_delta());
934 frame_access_state()->IncreaseSPDelta(bytes / kSystemPointerSize);
935 DCHECK(!caller_registers_saved_);
936 caller_registers_saved_ = true;
937 break;
938 }
939 case kArchRestoreCallerRegisters: {
940 DCHECK(fp_mode_ ==
941 static_cast<SaveFPRegsMode>(MiscField::decode(instr->opcode())));
942 DCHECK(fp_mode_ == SaveFPRegsMode::kIgnore ||
943 fp_mode_ == SaveFPRegsMode::kSave);
944 // Don't overwrite the returned value.
945 int bytes = __ PopCallerSaved(fp_mode_, kReturnRegister0);
946 frame_access_state()->IncreaseSPDelta(-(bytes / kSystemPointerSize));
947 DCHECK_EQ(0, frame_access_state()->sp_delta());
948 DCHECK(caller_registers_saved_);
949 caller_registers_saved_ = false;
950 break;
951 }
952 case kArchPrepareTailCall:
953 AssemblePrepareTailCall();
954 break;
955 case kArchComment:
956 #ifdef V8_TARGET_ARCH_PPC64
957 __ RecordComment(reinterpret_cast<const char*>(i.InputInt64(0)));
958 #else
959 __ RecordComment(reinterpret_cast<const char*>(i.InputInt32(0)));
960 #endif
961 break;
962 case kArchCallCFunction: {
963 int const num_gp_parameters = ParamField::decode(instr->opcode());
964 int const fp_param_field = FPParamField::decode(instr->opcode());
965 int num_fp_parameters = fp_param_field;
966 bool has_function_descriptor = false;
967 #if ABI_USES_FUNCTION_DESCRIPTORS
968 // AIX/PPC64BE Linux uses a function descriptor
969 int kNumFPParametersMask = kHasFunctionDescriptorBitMask - 1;
970 num_fp_parameters = kNumFPParametersMask & fp_param_field;
971 has_function_descriptor =
972 (fp_param_field & kHasFunctionDescriptorBitMask) != 0;
973 #endif
974 #if V8_ENABLE_WEBASSEMBLY
975 Label start_call;
976 int start_pc_offset = 0;
977 bool isWasmCapiFunction =
978 linkage()->GetIncomingDescriptor()->IsWasmCapiFunction();
979 if (isWasmCapiFunction) {
980 __ mflr(r0);
981 __ LoadPC(kScratchReg);
982 __ bind(&start_call);
983 start_pc_offset = __ pc_offset();
984 // We are going to patch this instruction after emitting
985 // CallCFunction, using a zero offset here as placeholder for now.
986 // patch_wasm_cpi_return_address assumes `addi` is used here to
987 // add the offset to pc.
988 __ addi(kScratchReg, kScratchReg, Operand::Zero());
989 __ StoreU64(kScratchReg,
990 MemOperand(fp, WasmExitFrameConstants::kCallingPCOffset));
991 __ mtlr(r0);
992 }
993 #endif // V8_ENABLE_WEBASSEMBLY
994 if (instr->InputAt(0)->IsImmediate()) {
995 ExternalReference ref = i.InputExternalReference(0);
996 __ CallCFunction(ref, num_gp_parameters, num_fp_parameters,
997 has_function_descriptor);
998 } else {
999 Register func = i.InputRegister(0);
1000 __ CallCFunction(func, num_gp_parameters, num_fp_parameters,
1001 has_function_descriptor);
1002 }
1003 #if V8_ENABLE_WEBASSEMBLY
1004 if (isWasmCapiFunction) {
1005 int offset_since_start_call = __ SizeOfCodeGeneratedSince(&start_call);
1006 // Here we are going to patch the `addi` instruction above to use the
1007 // correct offset.
1008 // LoadPC emits two instructions and pc is the address of its
1009 // second emitted instruction therefore there is one more instruction to
1010 // count.
1011 offset_since_start_call += kInstrSize;
1012 __ patch_wasm_cpi_return_address(kScratchReg, start_pc_offset,
1013 offset_since_start_call);
1014 RecordSafepoint(instr->reference_map());
1015 }
1016 #endif // V8_ENABLE_WEBASSEMBLY
1017 frame_access_state()->SetFrameAccessToDefault();
1018 // Ideally, we should decrement SP delta to match the change of stack
1019 // pointer in CallCFunction. However, for certain architectures (e.g.
1020 // ARM), there may be more strict alignment requirement, causing old SP
1021 // to be saved on the stack. In those cases, we can not calculate the SP
1022 // delta statically.
1023 frame_access_state()->ClearSPDelta();
1024 if (caller_registers_saved_) {
1025 // Need to re-sync SP delta introduced in kArchSaveCallerRegisters.
1026 // Here, we assume the sequence to be:
1027 // kArchSaveCallerRegisters;
1028 // kArchCallCFunction;
1029 // kArchRestoreCallerRegisters;
1030 int bytes =
1031 __ RequiredStackSizeForCallerSaved(fp_mode_, kReturnRegister0);
1032 frame_access_state()->IncreaseSPDelta(bytes / kSystemPointerSize);
1033 }
1034 break;
1035 }
1036 case kArchJmp:
1037 AssembleArchJump(i.InputRpo(0));
1038 DCHECK_EQ(LeaveRC, i.OutputRCBit());
1039 break;
1040 case kArchBinarySearchSwitch:
1041 AssembleArchBinarySearchSwitch(instr);
1042 break;
1043 case kArchTableSwitch:
1044 AssembleArchTableSwitch(instr);
1045 DCHECK_EQ(LeaveRC, i.OutputRCBit());
1046 break;
1047 case kArchAbortCSADcheck:
1048 DCHECK(i.InputRegister(0) == r4);
1049 {
1050 // We don't actually want to generate a pile of code for this, so just
1051 // claim there is a stack frame, without generating one.
1052 FrameScope scope(tasm(), StackFrame::NO_FRAME_TYPE);
1053 __ Call(isolate()->builtins()->code_handle(Builtin::kAbortCSADcheck),
1054 RelocInfo::CODE_TARGET);
1055 }
1056 __ stop();
1057 break;
1058 case kArchDebugBreak:
1059 __ DebugBreak();
1060 break;
1061 case kArchNop:
1062 case kArchThrowTerminator:
1063 // don't emit code for nops.
1064 DCHECK_EQ(LeaveRC, i.OutputRCBit());
1065 break;
1066 case kArchDeoptimize: {
1067 DeoptimizationExit* exit =
1068 BuildTranslation(instr, -1, 0, 0, OutputFrameStateCombine::Ignore());
1069 __ b(exit->label());
1070 break;
1071 }
1072 case kArchRet:
1073 AssembleReturn(instr->InputAt(0));
1074 DCHECK_EQ(LeaveRC, i.OutputRCBit());
1075 break;
1076 case kArchFramePointer:
1077 __ mr(i.OutputRegister(), fp);
1078 DCHECK_EQ(LeaveRC, i.OutputRCBit());
1079 break;
1080 case kArchParentFramePointer:
1081 if (frame_access_state()->has_frame()) {
1082 __ LoadU64(i.OutputRegister(), MemOperand(fp, 0));
1083 } else {
1084 __ mr(i.OutputRegister(), fp);
1085 }
1086 break;
1087 case kArchStackPointerGreaterThan: {
1088 // Potentially apply an offset to the current stack pointer before the
1089 // comparison to consider the size difference of an optimized frame versus
1090 // the contained unoptimized frames.
1091
1092 Register lhs_register = sp;
1093 uint32_t offset;
1094
1095 if (ShouldApplyOffsetToStackCheck(instr, &offset)) {
1096 lhs_register = i.TempRegister(0);
1097 __ SubS64(lhs_register, sp, Operand(offset), kScratchReg);
1098 }
1099
1100 constexpr size_t kValueIndex = 0;
1101 DCHECK(instr->InputAt(kValueIndex)->IsRegister());
1102 __ CmpU64(lhs_register, i.InputRegister(kValueIndex), cr0);
1103 break;
1104 }
1105 case kArchStackCheckOffset:
1106 __ LoadSmiLiteral(i.OutputRegister(),
1107 Smi::FromInt(GetStackCheckOffset()));
1108 break;
1109 case kArchTruncateDoubleToI:
1110 __ TruncateDoubleToI(isolate(), zone(), i.OutputRegister(),
1111 i.InputDoubleRegister(0), DetermineStubCallMode());
1112 DCHECK_EQ(LeaveRC, i.OutputRCBit());
1113 break;
1114 case kArchStoreWithWriteBarrier: {
1115 RecordWriteMode mode =
1116 static_cast<RecordWriteMode>(MiscField::decode(instr->opcode()));
1117 Register object = i.InputRegister(0);
1118 Register value = i.InputRegister(2);
1119 Register scratch0 = i.TempRegister(0);
1120 Register scratch1 = i.TempRegister(1);
1121 OutOfLineRecordWrite* ool;
1122
1123 if (FLAG_debug_code) {
1124 // Checking that |value| is not a cleared weakref: our write barrier
1125 // does not support that for now.
1126 __ CmpS64(value, Operand(kClearedWeakHeapObjectLower32), kScratchReg);
1127 __ Check(ne, AbortReason::kOperandIsCleared);
1128 }
1129
1130 AddressingMode addressing_mode =
1131 AddressingModeField::decode(instr->opcode());
1132 if (addressing_mode == kMode_MRI) {
1133 int32_t offset = i.InputInt32(1);
1134 ool = zone()->New<OutOfLineRecordWrite>(
1135 this, object, offset, value, scratch0, scratch1, mode,
1136 DetermineStubCallMode(), &unwinding_info_writer_);
1137 __ StoreTaggedField(value, MemOperand(object, offset), r0);
1138 } else {
1139 DCHECK_EQ(kMode_MRR, addressing_mode);
1140 Register offset(i.InputRegister(1));
1141 ool = zone()->New<OutOfLineRecordWrite>(
1142 this, object, offset, value, scratch0, scratch1, mode,
1143 DetermineStubCallMode(), &unwinding_info_writer_);
1144 __ StoreTaggedField(value, MemOperand(object, offset), r0);
1145 }
1146 if (mode > RecordWriteMode::kValueIsPointer) {
1147 __ JumpIfSmi(value, ool->exit());
1148 }
1149 __ CheckPageFlag(object, scratch0,
1150 MemoryChunk::kPointersFromHereAreInterestingMask, ne,
1151 ool->entry());
1152 __ bind(ool->exit());
1153 break;
1154 }
1155 case kArchStackSlot: {
1156 FrameOffset offset =
1157 frame_access_state()->GetFrameOffset(i.InputInt32(0));
1158 __ AddS64(i.OutputRegister(), offset.from_stack_pointer() ? sp : fp,
1159 Operand(offset.offset()), r0);
1160 break;
1161 }
1162 case kPPC_Peek: {
1163 int reverse_slot = i.InputInt32(0);
1164 int offset =
1165 FrameSlotToFPOffset(frame()->GetTotalFrameSlotCount() - reverse_slot);
1166 if (instr->OutputAt(0)->IsFPRegister()) {
1167 LocationOperand* op = LocationOperand::cast(instr->OutputAt(0));
1168 if (op->representation() == MachineRepresentation::kFloat64) {
1169 __ LoadF64(i.OutputDoubleRegister(), MemOperand(fp, offset), r0);
1170 } else if (op->representation() == MachineRepresentation::kFloat32) {
1171 __ LoadF32(i.OutputFloatRegister(), MemOperand(fp, offset), r0);
1172 } else {
1173 DCHECK_EQ(MachineRepresentation::kSimd128, op->representation());
1174 __ mov(ip, Operand(offset));
1175 __ LoadSimd128(i.OutputSimd128Register(), MemOperand(fp, ip));
1176 }
1177 } else {
1178 __ LoadU64(i.OutputRegister(), MemOperand(fp, offset), r0);
1179 }
1180 break;
1181 }
1182 case kPPC_Sync: {
1183 __ sync();
1184 break;
1185 }
1186 case kPPC_And:
1187 if (HasRegisterInput(instr, 1)) {
1188 __ and_(i.OutputRegister(), i.InputRegister(0), i.InputRegister(1),
1189 i.OutputRCBit());
1190 } else {
1191 __ andi(i.OutputRegister(), i.InputRegister(0), i.InputImmediate(1));
1192 }
1193 break;
1194 case kPPC_AndComplement:
1195 __ andc(i.OutputRegister(), i.InputRegister(0), i.InputRegister(1),
1196 i.OutputRCBit());
1197 break;
1198 case kPPC_Or:
1199 if (HasRegisterInput(instr, 1)) {
1200 __ orx(i.OutputRegister(), i.InputRegister(0), i.InputRegister(1),
1201 i.OutputRCBit());
1202 } else {
1203 __ ori(i.OutputRegister(), i.InputRegister(0), i.InputImmediate(1));
1204 DCHECK_EQ(LeaveRC, i.OutputRCBit());
1205 }
1206 break;
1207 case kPPC_OrComplement:
1208 __ orc(i.OutputRegister(), i.InputRegister(0), i.InputRegister(1),
1209 i.OutputRCBit());
1210 break;
1211 case kPPC_Xor:
1212 if (HasRegisterInput(instr, 1)) {
1213 __ xor_(i.OutputRegister(), i.InputRegister(0), i.InputRegister(1),
1214 i.OutputRCBit());
1215 } else {
1216 __ xori(i.OutputRegister(), i.InputRegister(0), i.InputImmediate(1));
1217 DCHECK_EQ(LeaveRC, i.OutputRCBit());
1218 }
1219 break;
1220 case kPPC_ShiftLeft32:
1221 ASSEMBLE_BINOP_RC(ShiftLeftU32, ShiftLeftU32);
1222 break;
1223 case kPPC_ShiftLeft64:
1224 ASSEMBLE_BINOP_RC(ShiftLeftU64, ShiftLeftU64);
1225 break;
1226 case kPPC_ShiftRight32:
1227 ASSEMBLE_BINOP_RC(ShiftRightU32, ShiftRightU32);
1228 break;
1229 case kPPC_ShiftRight64:
1230 ASSEMBLE_BINOP_RC(ShiftRightU64, ShiftRightU64);
1231 break;
1232 case kPPC_ShiftRightAlg32:
1233 ASSEMBLE_BINOP_INT_RC(ShiftRightS32, ShiftRightS32);
1234 break;
1235 case kPPC_ShiftRightAlg64:
1236 ASSEMBLE_BINOP_INT_RC(ShiftRightS64, ShiftRightS64);
1237 break;
1238 #if !V8_TARGET_ARCH_PPC64
1239 case kPPC_AddPair:
1240 // i.InputRegister(0) ... left low word.
1241 // i.InputRegister(1) ... left high word.
1242 // i.InputRegister(2) ... right low word.
1243 // i.InputRegister(3) ... right high word.
1244 __ addc(i.OutputRegister(0), i.InputRegister(0), i.InputRegister(2));
1245 __ adde(i.OutputRegister(1), i.InputRegister(1), i.InputRegister(3));
1246 DCHECK_EQ(LeaveRC, i.OutputRCBit());
1247 break;
1248 case kPPC_SubPair:
1249 // i.InputRegister(0) ... left low word.
1250 // i.InputRegister(1) ... left high word.
1251 // i.InputRegister(2) ... right low word.
1252 // i.InputRegister(3) ... right high word.
1253 __ subc(i.OutputRegister(0), i.InputRegister(0), i.InputRegister(2));
1254 __ sube(i.OutputRegister(1), i.InputRegister(1), i.InputRegister(3));
1255 DCHECK_EQ(LeaveRC, i.OutputRCBit());
1256 break;
1257 case kPPC_MulPair:
1258 // i.InputRegister(0) ... left low word.
1259 // i.InputRegister(1) ... left high word.
1260 // i.InputRegister(2) ... right low word.
1261 // i.InputRegister(3) ... right high word.
1262 __ mullw(i.TempRegister(0), i.InputRegister(0), i.InputRegister(3));
1263 __ mullw(i.TempRegister(1), i.InputRegister(2), i.InputRegister(1));
1264 __ add(i.TempRegister(0), i.TempRegister(0), i.TempRegister(1));
1265 __ mullw(i.OutputRegister(0), i.InputRegister(0), i.InputRegister(2));
1266 __ mulhwu(i.OutputRegister(1), i.InputRegister(0), i.InputRegister(2));
1267 __ add(i.OutputRegister(1), i.OutputRegister(1), i.TempRegister(0));
1268 break;
1269 case kPPC_ShiftLeftPair: {
1270 Register second_output =
1271 instr->OutputCount() >= 2 ? i.OutputRegister(1) : i.TempRegister(0);
1272 if (instr->InputAt(2)->IsImmediate()) {
1273 __ ShiftLeftPair(i.OutputRegister(0), second_output, i.InputRegister(0),
1274 i.InputRegister(1), i.InputInt32(2));
1275 } else {
1276 __ ShiftLeftPair(i.OutputRegister(0), second_output, i.InputRegister(0),
1277 i.InputRegister(1), kScratchReg, i.InputRegister(2));
1278 }
1279 break;
1280 }
1281 case kPPC_ShiftRightPair: {
1282 Register second_output =
1283 instr->OutputCount() >= 2 ? i.OutputRegister(1) : i.TempRegister(0);
1284 if (instr->InputAt(2)->IsImmediate()) {
1285 __ ShiftRightPair(i.OutputRegister(0), second_output,
1286 i.InputRegister(0), i.InputRegister(1),
1287 i.InputInt32(2));
1288 } else {
1289 __ ShiftRightPair(i.OutputRegister(0), second_output,
1290 i.InputRegister(0), i.InputRegister(1), kScratchReg,
1291 i.InputRegister(2));
1292 }
1293 break;
1294 }
1295 case kPPC_ShiftRightAlgPair: {
1296 Register second_output =
1297 instr->OutputCount() >= 2 ? i.OutputRegister(1) : i.TempRegister(0);
1298 if (instr->InputAt(2)->IsImmediate()) {
1299 __ ShiftRightAlgPair(i.OutputRegister(0), second_output,
1300 i.InputRegister(0), i.InputRegister(1),
1301 i.InputInt32(2));
1302 } else {
1303 __ ShiftRightAlgPair(i.OutputRegister(0), second_output,
1304 i.InputRegister(0), i.InputRegister(1),
1305 kScratchReg, i.InputRegister(2));
1306 }
1307 break;
1308 }
1309 #endif
1310 case kPPC_RotRight32:
1311 if (HasRegisterInput(instr, 1)) {
1312 __ subfic(kScratchReg, i.InputRegister(1), Operand(32));
1313 __ rotlw(i.OutputRegister(), i.InputRegister(0), kScratchReg,
1314 i.OutputRCBit());
1315 } else {
1316 int sh = i.InputInt32(1);
1317 __ rotrwi(i.OutputRegister(), i.InputRegister(0), sh, i.OutputRCBit());
1318 }
1319 break;
1320 #if V8_TARGET_ARCH_PPC64
1321 case kPPC_RotRight64:
1322 if (HasRegisterInput(instr, 1)) {
1323 __ subfic(kScratchReg, i.InputRegister(1), Operand(64));
1324 __ rotld(i.OutputRegister(), i.InputRegister(0), kScratchReg,
1325 i.OutputRCBit());
1326 } else {
1327 int sh = i.InputInt32(1);
1328 __ rotrdi(i.OutputRegister(), i.InputRegister(0), sh, i.OutputRCBit());
1329 }
1330 break;
1331 #endif
1332 case kPPC_Not:
1333 __ notx(i.OutputRegister(), i.InputRegister(0), i.OutputRCBit());
1334 break;
1335 case kPPC_RotLeftAndMask32:
1336 __ rlwinm(i.OutputRegister(), i.InputRegister(0), i.InputInt32(1),
1337 31 - i.InputInt32(2), 31 - i.InputInt32(3), i.OutputRCBit());
1338 break;
1339 #if V8_TARGET_ARCH_PPC64
1340 case kPPC_RotLeftAndClear64:
1341 __ rldic(i.OutputRegister(), i.InputRegister(0), i.InputInt32(1),
1342 63 - i.InputInt32(2), i.OutputRCBit());
1343 break;
1344 case kPPC_RotLeftAndClearLeft64:
1345 __ rldicl(i.OutputRegister(), i.InputRegister(0), i.InputInt32(1),
1346 63 - i.InputInt32(2), i.OutputRCBit());
1347 break;
1348 case kPPC_RotLeftAndClearRight64:
1349 __ rldicr(i.OutputRegister(), i.InputRegister(0), i.InputInt32(1),
1350 63 - i.InputInt32(2), i.OutputRCBit());
1351 break;
1352 #endif
1353 case kPPC_Add32:
1354 #if V8_TARGET_ARCH_PPC64
1355 if (FlagsModeField::decode(instr->opcode()) != kFlags_none) {
1356 ASSEMBLE_ADD_WITH_OVERFLOW();
1357 } else {
1358 #endif
1359 if (HasRegisterInput(instr, 1)) {
1360 __ add(i.OutputRegister(), i.InputRegister(0), i.InputRegister(1),
1361 LeaveOE, i.OutputRCBit());
1362 } else {
1363 __ AddS64(i.OutputRegister(), i.InputRegister(0), i.InputImmediate(1),
1364 r0, LeaveOE, i.OutputRCBit());
1365 }
1366 __ extsw(i.OutputRegister(), i.OutputRegister());
1367 #if V8_TARGET_ARCH_PPC64
1368 }
1369 #endif
1370 break;
1371 #if V8_TARGET_ARCH_PPC64
1372 case kPPC_Add64:
1373 if (FlagsModeField::decode(instr->opcode()) != kFlags_none) {
1374 ASSEMBLE_ADD_WITH_OVERFLOW();
1375 } else {
1376 if (HasRegisterInput(instr, 1)) {
1377 __ add(i.OutputRegister(), i.InputRegister(0), i.InputRegister(1),
1378 LeaveOE, i.OutputRCBit());
1379 } else {
1380 __ AddS64(i.OutputRegister(), i.InputRegister(0), i.InputImmediate(1),
1381 r0, LeaveOE, i.OutputRCBit());
1382 }
1383 }
1384 break;
1385 #endif
1386 case kPPC_AddWithOverflow32:
1387 ASSEMBLE_ADD_WITH_OVERFLOW32();
1388 break;
1389 case kPPC_AddDouble:
1390 ASSEMBLE_FLOAT_BINOP_RC(fadd, MiscField::decode(instr->opcode()));
1391 break;
1392 case kPPC_Sub:
1393 #if V8_TARGET_ARCH_PPC64
1394 if (FlagsModeField::decode(instr->opcode()) != kFlags_none) {
1395 ASSEMBLE_SUB_WITH_OVERFLOW();
1396 } else {
1397 #endif
1398 if (HasRegisterInput(instr, 1)) {
1399 __ sub(i.OutputRegister(), i.InputRegister(0), i.InputRegister(1),
1400 LeaveOE, i.OutputRCBit());
1401 } else {
1402 __ SubS64(i.OutputRegister(), i.InputRegister(0), i.InputImmediate(1),
1403 r0, LeaveOE, i.OutputRCBit());
1404 }
1405 #if V8_TARGET_ARCH_PPC64
1406 }
1407 #endif
1408 break;
1409 case kPPC_SubWithOverflow32:
1410 ASSEMBLE_SUB_WITH_OVERFLOW32();
1411 break;
1412 case kPPC_SubDouble:
1413 ASSEMBLE_FLOAT_BINOP_RC(fsub, MiscField::decode(instr->opcode()));
1414 break;
1415 case kPPC_Mul32:
1416 __ mullw(i.OutputRegister(), i.InputRegister(0), i.InputRegister(1),
1417 LeaveOE, i.OutputRCBit());
1418 break;
1419 #if V8_TARGET_ARCH_PPC64
1420 case kPPC_Mul64:
1421 __ mulld(i.OutputRegister(), i.InputRegister(0), i.InputRegister(1),
1422 LeaveOE, i.OutputRCBit());
1423 break;
1424 #endif
1425
1426 case kPPC_Mul32WithHigh32:
1427 if (i.OutputRegister(0) == i.InputRegister(0) ||
1428 i.OutputRegister(0) == i.InputRegister(1) ||
1429 i.OutputRegister(1) == i.InputRegister(0) ||
1430 i.OutputRegister(1) == i.InputRegister(1)) {
1431 __ mullw(kScratchReg, i.InputRegister(0), i.InputRegister(1)); // low
1432 __ mulhw(i.OutputRegister(1), i.InputRegister(0),
1433 i.InputRegister(1)); // high
1434 __ mr(i.OutputRegister(0), kScratchReg);
1435 } else {
1436 __ mullw(i.OutputRegister(0), i.InputRegister(0),
1437 i.InputRegister(1)); // low
1438 __ mulhw(i.OutputRegister(1), i.InputRegister(0),
1439 i.InputRegister(1)); // high
1440 }
1441 break;
1442 case kPPC_MulHigh32:
1443 __ mulhw(i.OutputRegister(), i.InputRegister(0), i.InputRegister(1),
1444 i.OutputRCBit());
1445 // High 32 bits are undefined and need to be cleared.
1446 CleanUInt32(i.OutputRegister());
1447 break;
1448 case kPPC_MulHighU32:
1449 __ mulhwu(i.OutputRegister(), i.InputRegister(0), i.InputRegister(1),
1450 i.OutputRCBit());
1451 // High 32 bits are undefined and need to be cleared.
1452 CleanUInt32(i.OutputRegister());
1453 break;
1454 case kPPC_MulDouble:
1455 ASSEMBLE_FLOAT_BINOP_RC(fmul, MiscField::decode(instr->opcode()));
1456 break;
1457 case kPPC_Div32:
1458 __ divw(i.OutputRegister(), i.InputRegister(0), i.InputRegister(1));
1459 DCHECK_EQ(LeaveRC, i.OutputRCBit());
1460 break;
1461 #if V8_TARGET_ARCH_PPC64
1462 case kPPC_Div64:
1463 __ divd(i.OutputRegister(), i.InputRegister(0), i.InputRegister(1));
1464 DCHECK_EQ(LeaveRC, i.OutputRCBit());
1465 break;
1466 #endif
1467 case kPPC_DivU32:
1468 __ divwu(i.OutputRegister(), i.InputRegister(0), i.InputRegister(1));
1469 DCHECK_EQ(LeaveRC, i.OutputRCBit());
1470 break;
1471 #if V8_TARGET_ARCH_PPC64
1472 case kPPC_DivU64:
1473 __ divdu(i.OutputRegister(), i.InputRegister(0), i.InputRegister(1));
1474 DCHECK_EQ(LeaveRC, i.OutputRCBit());
1475 break;
1476 #endif
1477 case kPPC_DivDouble:
1478 ASSEMBLE_FLOAT_BINOP_RC(fdiv, MiscField::decode(instr->opcode()));
1479 break;
1480 case kPPC_Mod32:
1481 if (CpuFeatures::IsSupported(PPC_9_PLUS)) {
1482 __ modsw(i.OutputRegister(), i.InputRegister(0), i.InputRegister(1));
1483 } else {
1484 ASSEMBLE_MODULO(divw, mullw);
1485 }
1486 break;
1487 #if V8_TARGET_ARCH_PPC64
1488 case kPPC_Mod64:
1489 if (CpuFeatures::IsSupported(PPC_9_PLUS)) {
1490 __ modsd(i.OutputRegister(), i.InputRegister(0), i.InputRegister(1));
1491 } else {
1492 ASSEMBLE_MODULO(divd, mulld);
1493 }
1494 break;
1495 #endif
1496 case kPPC_ModU32:
1497 if (CpuFeatures::IsSupported(PPC_9_PLUS)) {
1498 __ moduw(i.OutputRegister(), i.InputRegister(0), i.InputRegister(1));
1499 } else {
1500 ASSEMBLE_MODULO(divwu, mullw);
1501 }
1502 break;
1503 #if V8_TARGET_ARCH_PPC64
1504 case kPPC_ModU64:
1505 if (CpuFeatures::IsSupported(PPC_9_PLUS)) {
1506 __ modud(i.OutputRegister(), i.InputRegister(0), i.InputRegister(1));
1507 } else {
1508 ASSEMBLE_MODULO(divdu, mulld);
1509 }
1510 break;
1511 #endif
1512 case kPPC_ModDouble:
1513 // TODO(bmeurer): We should really get rid of this special instruction,
1514 // and generate a CallAddress instruction instead.
1515 ASSEMBLE_FLOAT_MODULO();
1516 break;
1517 case kIeee754Float64Acos:
1518 ASSEMBLE_IEEE754_UNOP(acos);
1519 break;
1520 case kIeee754Float64Acosh:
1521 ASSEMBLE_IEEE754_UNOP(acosh);
1522 break;
1523 case kIeee754Float64Asin:
1524 ASSEMBLE_IEEE754_UNOP(asin);
1525 break;
1526 case kIeee754Float64Asinh:
1527 ASSEMBLE_IEEE754_UNOP(asinh);
1528 break;
1529 case kIeee754Float64Atan:
1530 ASSEMBLE_IEEE754_UNOP(atan);
1531 break;
1532 case kIeee754Float64Atan2:
1533 ASSEMBLE_IEEE754_BINOP(atan2);
1534 break;
1535 case kIeee754Float64Atanh:
1536 ASSEMBLE_IEEE754_UNOP(atanh);
1537 break;
1538 case kIeee754Float64Tan:
1539 ASSEMBLE_IEEE754_UNOP(tan);
1540 break;
1541 case kIeee754Float64Tanh:
1542 ASSEMBLE_IEEE754_UNOP(tanh);
1543 break;
1544 case kIeee754Float64Cbrt:
1545 ASSEMBLE_IEEE754_UNOP(cbrt);
1546 break;
1547 case kIeee754Float64Sin:
1548 ASSEMBLE_IEEE754_UNOP(sin);
1549 break;
1550 case kIeee754Float64Sinh:
1551 ASSEMBLE_IEEE754_UNOP(sinh);
1552 break;
1553 case kIeee754Float64Cos:
1554 ASSEMBLE_IEEE754_UNOP(cos);
1555 break;
1556 case kIeee754Float64Cosh:
1557 ASSEMBLE_IEEE754_UNOP(cosh);
1558 break;
1559 case kIeee754Float64Exp:
1560 ASSEMBLE_IEEE754_UNOP(exp);
1561 break;
1562 case kIeee754Float64Expm1:
1563 ASSEMBLE_IEEE754_UNOP(expm1);
1564 break;
1565 case kIeee754Float64Log:
1566 ASSEMBLE_IEEE754_UNOP(log);
1567 break;
1568 case kIeee754Float64Log1p:
1569 ASSEMBLE_IEEE754_UNOP(log1p);
1570 break;
1571 case kIeee754Float64Log2:
1572 ASSEMBLE_IEEE754_UNOP(log2);
1573 break;
1574 case kIeee754Float64Log10:
1575 ASSEMBLE_IEEE754_UNOP(log10);
1576 break;
1577 case kIeee754Float64Pow:
1578 ASSEMBLE_IEEE754_BINOP(pow);
1579 break;
1580 case kPPC_Neg:
1581 __ neg(i.OutputRegister(), i.InputRegister(0), LeaveOE, i.OutputRCBit());
1582 break;
1583 case kPPC_MaxDouble:
1584 __ MaxF64(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
1585 i.InputDoubleRegister(1), kScratchDoubleReg);
1586 break;
1587 case kPPC_MinDouble:
1588 __ MinF64(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
1589 i.InputDoubleRegister(1), kScratchDoubleReg);
1590 break;
1591 case kPPC_AbsDouble:
1592 ASSEMBLE_FLOAT_UNOP_RC(fabs, 0);
1593 break;
1594 case kPPC_SqrtDouble:
1595 ASSEMBLE_FLOAT_UNOP_RC(fsqrt, MiscField::decode(instr->opcode()));
1596 break;
1597 case kPPC_FloorDouble:
1598 ASSEMBLE_FLOAT_UNOP_RC(frim, MiscField::decode(instr->opcode()));
1599 break;
1600 case kPPC_CeilDouble:
1601 ASSEMBLE_FLOAT_UNOP_RC(frip, MiscField::decode(instr->opcode()));
1602 break;
1603 case kPPC_TruncateDouble:
1604 ASSEMBLE_FLOAT_UNOP_RC(friz, MiscField::decode(instr->opcode()));
1605 break;
1606 case kPPC_RoundDouble:
1607 ASSEMBLE_FLOAT_UNOP_RC(frin, MiscField::decode(instr->opcode()));
1608 break;
1609 case kPPC_NegDouble:
1610 ASSEMBLE_FLOAT_UNOP_RC(fneg, 0);
1611 break;
1612 case kPPC_Cntlz32:
1613 __ cntlzw(i.OutputRegister(), i.InputRegister(0));
1614 DCHECK_EQ(LeaveRC, i.OutputRCBit());
1615 break;
1616 #if V8_TARGET_ARCH_PPC64
1617 case kPPC_Cntlz64:
1618 __ cntlzd(i.OutputRegister(), i.InputRegister(0));
1619 DCHECK_EQ(LeaveRC, i.OutputRCBit());
1620 break;
1621 #endif
1622 case kPPC_Popcnt32:
1623 __ Popcnt32(i.OutputRegister(), i.InputRegister(0));
1624 DCHECK_EQ(LeaveRC, i.OutputRCBit());
1625 break;
1626 #if V8_TARGET_ARCH_PPC64
1627 case kPPC_Popcnt64:
1628 __ Popcnt64(i.OutputRegister(), i.InputRegister(0));
1629 DCHECK_EQ(LeaveRC, i.OutputRCBit());
1630 break;
1631 #endif
1632 case kPPC_Cmp32:
1633 ASSEMBLE_COMPARE(cmpw, cmplw);
1634 break;
1635 #if V8_TARGET_ARCH_PPC64
1636 case kPPC_Cmp64:
1637 ASSEMBLE_COMPARE(cmp, cmpl);
1638 break;
1639 #endif
1640 case kPPC_CmpDouble:
1641 ASSEMBLE_FLOAT_COMPARE(fcmpu);
1642 break;
1643 case kPPC_Tst32:
1644 if (HasRegisterInput(instr, 1)) {
1645 __ and_(r0, i.InputRegister(0), i.InputRegister(1), i.OutputRCBit());
1646 } else {
1647 __ andi(r0, i.InputRegister(0), i.InputImmediate(1));
1648 }
1649 #if V8_TARGET_ARCH_PPC64
1650 __ extsw(r0, r0, i.OutputRCBit());
1651 #endif
1652 DCHECK_EQ(SetRC, i.OutputRCBit());
1653 break;
1654 #if V8_TARGET_ARCH_PPC64
1655 case kPPC_Tst64:
1656 if (HasRegisterInput(instr, 1)) {
1657 __ and_(r0, i.InputRegister(0), i.InputRegister(1), i.OutputRCBit());
1658 } else {
1659 __ andi(r0, i.InputRegister(0), i.InputImmediate(1));
1660 }
1661 DCHECK_EQ(SetRC, i.OutputRCBit());
1662 break;
1663 #endif
1664 case kPPC_Float64SilenceNaN: {
1665 DoubleRegister value = i.InputDoubleRegister(0);
1666 DoubleRegister result = i.OutputDoubleRegister();
1667 __ CanonicalizeNaN(result, value);
1668 break;
1669 }
1670 case kPPC_Push: {
1671 int stack_decrement = i.InputInt32(0);
1672 int slots = stack_decrement / kSystemPointerSize;
1673 LocationOperand* op = LocationOperand::cast(instr->InputAt(1));
1674 MachineRepresentation rep = op->representation();
1675 int pushed_slots = ElementSizeInPointers(rep);
1676 // Slot-sized arguments are never padded but there may be a gap if
1677 // the slot allocator reclaimed other padding slots. Adjust the stack
1678 // here to skip any gap.
1679 __ AllocateStackSpace((slots - pushed_slots) * kSystemPointerSize);
1680 switch (rep) {
1681 case MachineRepresentation::kFloat32:
1682 __ StoreF32WithUpdate(i.InputDoubleRegister(1),
1683 MemOperand(sp, -kSystemPointerSize), r0);
1684 break;
1685 case MachineRepresentation::kFloat64:
1686 __ StoreF64WithUpdate(i.InputDoubleRegister(1),
1687 MemOperand(sp, -kDoubleSize), r0);
1688 break;
1689 case MachineRepresentation::kSimd128:
1690 __ addi(sp, sp, Operand(-kSimd128Size));
1691 __ StoreSimd128(i.InputSimd128Register(1), MemOperand(r0, sp));
1692 break;
1693 default:
1694 __ StoreU64WithUpdate(i.InputRegister(1),
1695 MemOperand(sp, -kSystemPointerSize), r0);
1696 break;
1697 }
1698 frame_access_state()->IncreaseSPDelta(slots);
1699 DCHECK_EQ(LeaveRC, i.OutputRCBit());
1700 break;
1701 }
1702 case kPPC_PushFrame: {
1703 int num_slots = i.InputInt32(1);
1704 if (instr->InputAt(0)->IsFPRegister()) {
1705 LocationOperand* op = LocationOperand::cast(instr->InputAt(0));
1706 if (op->representation() == MachineRepresentation::kFloat64) {
1707 __ StoreF64WithUpdate(i.InputDoubleRegister(0),
1708 MemOperand(sp, -num_slots * kSystemPointerSize),
1709 r0);
1710 } else {
1711 DCHECK_EQ(MachineRepresentation::kFloat32, op->representation());
1712 __ StoreF32WithUpdate(i.InputDoubleRegister(0),
1713 MemOperand(sp, -num_slots * kSystemPointerSize),
1714 r0);
1715 }
1716 } else {
1717 __ StoreU64WithUpdate(i.InputRegister(0),
1718 MemOperand(sp, -num_slots * kSystemPointerSize),
1719 r0);
1720 }
1721 break;
1722 }
1723 case kPPC_StoreToStackSlot: {
1724 int slot = i.InputInt32(1);
1725 if (instr->InputAt(0)->IsFPRegister()) {
1726 LocationOperand* op = LocationOperand::cast(instr->InputAt(0));
1727 if (op->representation() == MachineRepresentation::kFloat64) {
1728 __ StoreF64(i.InputDoubleRegister(0),
1729 MemOperand(sp, slot * kSystemPointerSize), r0);
1730 } else if (op->representation() == MachineRepresentation::kFloat32) {
1731 __ StoreF32(i.InputDoubleRegister(0),
1732 MemOperand(sp, slot * kSystemPointerSize), r0);
1733 } else {
1734 DCHECK_EQ(MachineRepresentation::kSimd128, op->representation());
1735 __ mov(ip, Operand(slot * kSystemPointerSize));
1736 __ StoreSimd128(i.InputSimd128Register(0), MemOperand(ip, sp));
1737 }
1738 } else {
1739 __ StoreU64(i.InputRegister(0),
1740 MemOperand(sp, slot * kSystemPointerSize), r0);
1741 }
1742 break;
1743 }
1744 case kPPC_ExtendSignWord8:
1745 __ extsb(i.OutputRegister(), i.InputRegister(0));
1746 DCHECK_EQ(LeaveRC, i.OutputRCBit());
1747 break;
1748 case kPPC_ExtendSignWord16:
1749 __ extsh(i.OutputRegister(), i.InputRegister(0));
1750 DCHECK_EQ(LeaveRC, i.OutputRCBit());
1751 break;
1752 #if V8_TARGET_ARCH_PPC64
1753 case kPPC_ExtendSignWord32:
1754 __ extsw(i.OutputRegister(), i.InputRegister(0));
1755 DCHECK_EQ(LeaveRC, i.OutputRCBit());
1756 break;
1757 case kPPC_Uint32ToUint64:
1758 // Zero extend
1759 __ clrldi(i.OutputRegister(), i.InputRegister(0), Operand(32));
1760 DCHECK_EQ(LeaveRC, i.OutputRCBit());
1761 break;
1762 case kPPC_Int64ToInt32:
1763 __ extsw(i.OutputRegister(), i.InputRegister(0));
1764 DCHECK_EQ(LeaveRC, i.OutputRCBit());
1765 break;
1766 case kPPC_Int64ToFloat32:
1767 __ ConvertInt64ToFloat(i.InputRegister(0), i.OutputDoubleRegister());
1768 DCHECK_EQ(LeaveRC, i.OutputRCBit());
1769 break;
1770 case kPPC_Int64ToDouble:
1771 __ ConvertInt64ToDouble(i.InputRegister(0), i.OutputDoubleRegister());
1772 DCHECK_EQ(LeaveRC, i.OutputRCBit());
1773 break;
1774 case kPPC_Uint64ToFloat32:
1775 __ ConvertUnsignedInt64ToFloat(i.InputRegister(0),
1776 i.OutputDoubleRegister());
1777 DCHECK_EQ(LeaveRC, i.OutputRCBit());
1778 break;
1779 case kPPC_Uint64ToDouble:
1780 __ ConvertUnsignedInt64ToDouble(i.InputRegister(0),
1781 i.OutputDoubleRegister());
1782 DCHECK_EQ(LeaveRC, i.OutputRCBit());
1783 break;
1784 #endif
1785 case kPPC_Int32ToFloat32:
1786 __ ConvertIntToFloat(i.InputRegister(0), i.OutputDoubleRegister());
1787 DCHECK_EQ(LeaveRC, i.OutputRCBit());
1788 break;
1789 case kPPC_Int32ToDouble:
1790 __ ConvertIntToDouble(i.InputRegister(0), i.OutputDoubleRegister());
1791 DCHECK_EQ(LeaveRC, i.OutputRCBit());
1792 break;
1793 case kPPC_Uint32ToFloat32:
1794 __ ConvertUnsignedIntToFloat(i.InputRegister(0),
1795 i.OutputDoubleRegister());
1796 DCHECK_EQ(LeaveRC, i.OutputRCBit());
1797 break;
1798 case kPPC_Uint32ToDouble:
1799 __ ConvertUnsignedIntToDouble(i.InputRegister(0),
1800 i.OutputDoubleRegister());
1801 DCHECK_EQ(LeaveRC, i.OutputRCBit());
1802 break;
1803 case kPPC_Float32ToInt32: {
1804 bool set_overflow_to_min_i32 = MiscField::decode(instr->opcode());
1805 if (set_overflow_to_min_i32) {
1806 __ mtfsb0(VXCVI); // clear FPSCR:VXCVI bit
1807 }
1808 __ fctiwz(kScratchDoubleReg, i.InputDoubleRegister(0));
1809 __ MovDoubleLowToInt(i.OutputRegister(), kScratchDoubleReg);
1810 if (set_overflow_to_min_i32) {
1811 // Avoid INT32_MAX as an overflow indicator and use INT32_MIN instead,
1812 // because INT32_MIN allows easier out-of-bounds detection.
1813 CRegister cr = cr7;
1814 int crbit = v8::internal::Assembler::encode_crbit(
1815 cr, static_cast<CRBit>(VXCVI % CRWIDTH));
1816 __ mcrfs(cr, VXCVI); // extract FPSCR field containing VXCVI into cr7
1817 __ li(kScratchReg, Operand(1));
1818 __ ShiftLeftU64(kScratchReg, kScratchReg,
1819 Operand(31)); // generate INT32_MIN.
1820 __ isel(i.OutputRegister(0), kScratchReg, i.OutputRegister(0), crbit);
1821 }
1822 break;
1823 }
1824 case kPPC_Float32ToUint32: {
1825 bool set_overflow_to_min_u32 = MiscField::decode(instr->opcode());
1826 if (set_overflow_to_min_u32) {
1827 __ mtfsb0(VXCVI); // clear FPSCR:VXCVI bit
1828 }
1829 __ fctiwuz(kScratchDoubleReg, i.InputDoubleRegister(0));
1830 __ MovDoubleLowToInt(i.OutputRegister(), kScratchDoubleReg);
1831 if (set_overflow_to_min_u32) {
1832 // Avoid UINT32_MAX as an overflow indicator and use 0 instead,
1833 // because 0 allows easier out-of-bounds detection.
1834 CRegister cr = cr7;
1835 int crbit = v8::internal::Assembler::encode_crbit(
1836 cr, static_cast<CRBit>(VXCVI % CRWIDTH));
1837 __ mcrfs(cr, VXCVI); // extract FPSCR field containing VXCVI into cr7
1838 __ li(kScratchReg, Operand::Zero());
1839 __ isel(i.OutputRegister(0), kScratchReg, i.OutputRegister(0), crbit);
1840 }
1841 break;
1842 }
1843 case kPPC_DoubleToInt32:
1844 case kPPC_DoubleToUint32:
1845 case kPPC_DoubleToInt64: {
1846 #if V8_TARGET_ARCH_PPC64
1847 bool check_conversion =
1848 (opcode == kPPC_DoubleToInt64 && i.OutputCount() > 1);
1849 __ mtfsb0(VXCVI); // clear FPSCR:VXCVI bit
1850 #endif
1851 __ ConvertDoubleToInt64(i.InputDoubleRegister(0),
1852 #if !V8_TARGET_ARCH_PPC64
1853 kScratchReg,
1854 #endif
1855 i.OutputRegister(0), kScratchDoubleReg);
1856 #if V8_TARGET_ARCH_PPC64
1857 CRegister cr = cr7;
1858 int crbit = v8::internal::Assembler::encode_crbit(
1859 cr, static_cast<CRBit>(VXCVI % CRWIDTH));
1860 __ mcrfs(cr, VXCVI); // extract FPSCR field containing VXCVI into cr7
1861 // Handle conversion failures (such as overflow).
1862 if (CpuFeatures::IsSupported(PPC_7_PLUS)) {
1863 if (check_conversion) {
1864 __ li(i.OutputRegister(1), Operand(1));
1865 __ isel(i.OutputRegister(1), r0, i.OutputRegister(1), crbit);
1866 } else {
1867 __ isel(i.OutputRegister(0), r0, i.OutputRegister(0), crbit);
1868 }
1869 } else {
1870 if (check_conversion) {
1871 __ li(i.OutputRegister(1), Operand::Zero());
1872 __ bc(v8::internal::kInstrSize * 2, BT, crbit);
1873 __ li(i.OutputRegister(1), Operand(1));
1874 } else {
1875 __ mr(ip, i.OutputRegister(0));
1876 __ li(i.OutputRegister(0), Operand::Zero());
1877 __ bc(v8::internal::kInstrSize * 2, BT, crbit);
1878 __ mr(i.OutputRegister(0), ip);
1879 }
1880 }
1881 #endif
1882 DCHECK_EQ(LeaveRC, i.OutputRCBit());
1883 break;
1884 }
1885 #if V8_TARGET_ARCH_PPC64
1886 case kPPC_DoubleToUint64: {
1887 bool check_conversion = (i.OutputCount() > 1);
1888 if (check_conversion) {
1889 __ mtfsb0(VXCVI); // clear FPSCR:VXCVI bit
1890 }
1891 __ ConvertDoubleToUnsignedInt64(i.InputDoubleRegister(0),
1892 i.OutputRegister(0), kScratchDoubleReg);
1893 if (check_conversion) {
1894 // Set 2nd output to zero if conversion fails.
1895 CRegister cr = cr7;
1896 int crbit = v8::internal::Assembler::encode_crbit(
1897 cr, static_cast<CRBit>(VXCVI % CRWIDTH));
1898 __ mcrfs(cr, VXCVI); // extract FPSCR field containing VXCVI into cr7
1899 if (CpuFeatures::IsSupported(PPC_7_PLUS)) {
1900 __ li(i.OutputRegister(1), Operand(1));
1901 __ isel(i.OutputRegister(1), r0, i.OutputRegister(1), crbit);
1902 } else {
1903 __ li(i.OutputRegister(1), Operand::Zero());
1904 __ bc(v8::internal::kInstrSize * 2, BT, crbit);
1905 __ li(i.OutputRegister(1), Operand(1));
1906 }
1907 }
1908 DCHECK_EQ(LeaveRC, i.OutputRCBit());
1909 break;
1910 }
1911 #endif
1912 case kPPC_DoubleToFloat32:
1913 ASSEMBLE_FLOAT_UNOP_RC(frsp, 0);
1914 break;
1915 case kPPC_Float32ToDouble:
1916 // Nothing to do.
1917 __ Move(i.OutputDoubleRegister(), i.InputDoubleRegister(0));
1918 DCHECK_EQ(LeaveRC, i.OutputRCBit());
1919 break;
1920 case kPPC_DoubleExtractLowWord32:
1921 __ MovDoubleLowToInt(i.OutputRegister(), i.InputDoubleRegister(0));
1922 DCHECK_EQ(LeaveRC, i.OutputRCBit());
1923 break;
1924 case kPPC_DoubleExtractHighWord32:
1925 __ MovDoubleHighToInt(i.OutputRegister(), i.InputDoubleRegister(0));
1926 DCHECK_EQ(LeaveRC, i.OutputRCBit());
1927 break;
1928 case kPPC_DoubleInsertLowWord32:
1929 __ InsertDoubleLow(i.OutputDoubleRegister(), i.InputRegister(1), r0);
1930 DCHECK_EQ(LeaveRC, i.OutputRCBit());
1931 break;
1932 case kPPC_DoubleInsertHighWord32:
1933 __ InsertDoubleHigh(i.OutputDoubleRegister(), i.InputRegister(1), r0);
1934 DCHECK_EQ(LeaveRC, i.OutputRCBit());
1935 break;
1936 case kPPC_DoubleConstruct:
1937 #if V8_TARGET_ARCH_PPC64
1938 __ MovInt64ComponentsToDouble(i.OutputDoubleRegister(),
1939 i.InputRegister(0), i.InputRegister(1), r0);
1940 #else
1941 __ MovInt64ToDouble(i.OutputDoubleRegister(), i.InputRegister(0),
1942 i.InputRegister(1));
1943 #endif
1944 DCHECK_EQ(LeaveRC, i.OutputRCBit());
1945 break;
1946 case kPPC_BitcastFloat32ToInt32:
1947 __ MovFloatToInt(i.OutputRegister(), i.InputDoubleRegister(0),
1948 kScratchDoubleReg);
1949 break;
1950 case kPPC_BitcastInt32ToFloat32:
1951 __ MovIntToFloat(i.OutputDoubleRegister(), i.InputRegister(0), ip);
1952 break;
1953 #if V8_TARGET_ARCH_PPC64
1954 case kPPC_BitcastDoubleToInt64:
1955 __ MovDoubleToInt64(i.OutputRegister(), i.InputDoubleRegister(0));
1956 break;
1957 case kPPC_BitcastInt64ToDouble:
1958 __ MovInt64ToDouble(i.OutputDoubleRegister(), i.InputRegister(0));
1959 break;
1960 #endif
1961 case kPPC_LoadWordU8:
1962 ASSEMBLE_LOAD_INTEGER(lbz, plbz, lbzx);
1963 break;
1964 case kPPC_LoadWordS8:
1965 ASSEMBLE_LOAD_INTEGER(lbz, plbz, lbzx);
1966 __ extsb(i.OutputRegister(), i.OutputRegister());
1967 break;
1968 case kPPC_LoadWordU16:
1969 ASSEMBLE_LOAD_INTEGER(lhz, plhz, lhzx);
1970 break;
1971 case kPPC_LoadWordS16:
1972 ASSEMBLE_LOAD_INTEGER(lha, plha, lhax);
1973 break;
1974 case kPPC_LoadWordU32:
1975 ASSEMBLE_LOAD_INTEGER(lwz, plwz, lwzx);
1976 break;
1977 case kPPC_LoadWordS32:
1978 ASSEMBLE_LOAD_INTEGER(lwa, plwa, lwax);
1979 break;
1980 #if V8_TARGET_ARCH_PPC64
1981 case kPPC_LoadWord64:
1982 ASSEMBLE_LOAD_INTEGER(ld, pld, ldx);
1983 break;
1984 #endif
1985 case kPPC_LoadFloat32:
1986 ASSEMBLE_LOAD_FLOAT(lfs, plfs, lfsx);
1987 break;
1988 case kPPC_LoadDouble:
1989 ASSEMBLE_LOAD_FLOAT(lfd, plfd, lfdx);
1990 break;
1991 case kPPC_LoadSimd128: {
1992 Simd128Register result = i.OutputSimd128Register();
1993 AddressingMode mode = kMode_None;
1994 MemOperand operand = i.MemoryOperand(&mode);
1995 bool is_atomic = i.InputInt32(2);
1996 DCHECK_EQ(mode, kMode_MRR);
1997 __ LoadSimd128(result, operand);
1998 if (is_atomic) __ lwsync();
1999 DCHECK_EQ(LeaveRC, i.OutputRCBit());
2000 break;
2001 }
2002 case kPPC_LoadReverseSimd128RR: {
2003 __ xxbrq(i.OutputSimd128Register(), i.InputSimd128Register(0));
2004 break;
2005 }
2006 case kPPC_StoreWord8:
2007 ASSEMBLE_STORE_INTEGER(stb, stbx);
2008 break;
2009 case kPPC_StoreWord16:
2010 ASSEMBLE_STORE_INTEGER(sth, sthx);
2011 break;
2012 case kPPC_StoreWord32:
2013 ASSEMBLE_STORE_INTEGER(stw, stwx);
2014 break;
2015 #if V8_TARGET_ARCH_PPC64
2016 case kPPC_StoreWord64:
2017 ASSEMBLE_STORE_INTEGER(std, stdx);
2018 break;
2019 #endif
2020 case kPPC_StoreFloat32:
2021 ASSEMBLE_STORE_FLOAT(stfs, stfsx);
2022 break;
2023 case kPPC_StoreDouble:
2024 ASSEMBLE_STORE_FLOAT(stfd, stfdx);
2025 break;
2026 case kPPC_StoreSimd128: {
2027 size_t index = 0;
2028 AddressingMode mode = kMode_None;
2029 MemOperand operand = i.MemoryOperand(&mode, &index);
2030 Simd128Register value = i.InputSimd128Register(index);
2031 bool is_atomic = i.InputInt32(3);
2032 if (is_atomic) __ lwsync();
2033 DCHECK_EQ(mode, kMode_MRR);
2034 __ StoreSimd128(value, operand);
2035 if (is_atomic) __ sync();
2036 DCHECK_EQ(LeaveRC, i.OutputRCBit());
2037 break;
2038 }
2039 case kAtomicLoadInt8:
2040 case kAtomicLoadInt16:
2041 UNREACHABLE();
2042 case kAtomicExchangeInt8:
2043 __ AtomicExchange<int8_t>(
2044 MemOperand(i.InputRegister(0), i.InputRegister(1)),
2045 i.InputRegister(2), i.OutputRegister());
2046 break;
2047 case kPPC_AtomicExchangeUint8:
2048 __ AtomicExchange<uint8_t>(
2049 MemOperand(i.InputRegister(0), i.InputRegister(1)),
2050 i.InputRegister(2), i.OutputRegister());
2051 break;
2052 case kAtomicExchangeInt16: {
2053 ASSEMBLE_ATOMIC_EXCHANGE(int16_t, ByteReverseU16);
2054 __ extsh(i.OutputRegister(), i.OutputRegister());
2055 break;
2056 }
2057 case kPPC_AtomicExchangeUint16: {
2058 ASSEMBLE_ATOMIC_EXCHANGE(uint16_t, ByteReverseU16);
2059 break;
2060 }
2061 case kPPC_AtomicExchangeWord32: {
2062 ASSEMBLE_ATOMIC_EXCHANGE(uint32_t, ByteReverseU32);
2063 break;
2064 }
2065 case kPPC_AtomicExchangeWord64: {
2066 ASSEMBLE_ATOMIC_EXCHANGE(uint64_t, ByteReverseU64);
2067 break;
2068 }
2069 case kAtomicCompareExchangeInt8:
2070 __ AtomicCompareExchange<int8_t>(
2071 MemOperand(i.InputRegister(0), i.InputRegister(1)),
2072 i.InputRegister(2), i.InputRegister(3), i.OutputRegister(),
2073 kScratchReg);
2074 break;
2075 case kPPC_AtomicCompareExchangeUint8:
2076 __ AtomicCompareExchange<uint8_t>(
2077 MemOperand(i.InputRegister(0), i.InputRegister(1)),
2078 i.InputRegister(2), i.InputRegister(3), i.OutputRegister(),
2079 kScratchReg);
2080 break;
2081 case kAtomicCompareExchangeInt16: {
2082 ASSEMBLE_ATOMIC_COMPARE_EXCHANGE(int16_t, ByteReverseU16);
2083 __ extsh(i.OutputRegister(), i.OutputRegister());
2084 break;
2085 }
2086 case kPPC_AtomicCompareExchangeUint16: {
2087 ASSEMBLE_ATOMIC_COMPARE_EXCHANGE(uint16_t, ByteReverseU16);
2088 break;
2089 }
2090 case kPPC_AtomicCompareExchangeWord32: {
2091 ASSEMBLE_ATOMIC_COMPARE_EXCHANGE(uint32_t, ByteReverseU32);
2092 break;
2093 }
2094 case kPPC_AtomicCompareExchangeWord64: {
2095 ASSEMBLE_ATOMIC_COMPARE_EXCHANGE(uint64_t, ByteReverseU64);
2096 } break;
2097
2098 #define ATOMIC_BINOP_CASE(op, inst) \
2099 case kPPC_Atomic##op##Int8: \
2100 ASSEMBLE_ATOMIC_BINOP_BYTE(inst, int8_t); \
2101 __ extsb(i.OutputRegister(), i.OutputRegister()); \
2102 break; \
2103 case kPPC_Atomic##op##Uint8: \
2104 ASSEMBLE_ATOMIC_BINOP_BYTE(inst, uint8_t); \
2105 break; \
2106 case kPPC_Atomic##op##Int16: \
2107 ASSEMBLE_ATOMIC_BINOP(inst, int16_t, ByteReverseU16, r0); \
2108 __ extsh(i.OutputRegister(), i.OutputRegister()); \
2109 break; \
2110 case kPPC_Atomic##op##Uint16: \
2111 ASSEMBLE_ATOMIC_BINOP(inst, uint16_t, ByteReverseU16, r0); \
2112 break; \
2113 case kPPC_Atomic##op##Int32: \
2114 ASSEMBLE_ATOMIC_BINOP(inst, int32_t, ByteReverseU32, r0); \
2115 __ extsw(i.OutputRegister(), i.OutputRegister()); \
2116 break; \
2117 case kPPC_Atomic##op##Uint32: \
2118 ASSEMBLE_ATOMIC_BINOP(inst, uint32_t, ByteReverseU32, r0); \
2119 break; \
2120 case kPPC_Atomic##op##Int64: \
2121 case kPPC_Atomic##op##Uint64: \
2122 ASSEMBLE_ATOMIC_BINOP(inst, uint64_t, ByteReverseU64, r0); \
2123 break;
2124 ATOMIC_BINOP_CASE(Add, add)
2125 ATOMIC_BINOP_CASE(Sub, sub)
2126 ATOMIC_BINOP_CASE(And, and_)
2127 ATOMIC_BINOP_CASE(Or, orx)
2128 ATOMIC_BINOP_CASE(Xor, xor_)
2129 #undef ATOMIC_BINOP_CASE
2130
2131 case kPPC_ByteRev32: {
2132 Register input = i.InputRegister(0);
2133 Register output = i.OutputRegister();
2134 Register temp1 = r0;
2135 if (CpuFeatures::IsSupported(PPC_10_PLUS)) {
2136 __ brw(output, input);
2137 __ extsw(output, output);
2138 break;
2139 }
2140 __ rotlwi(temp1, input, 8);
2141 __ rlwimi(temp1, input, 24, 0, 7);
2142 __ rlwimi(temp1, input, 24, 16, 23);
2143 __ extsw(output, temp1);
2144 break;
2145 }
2146 case kPPC_LoadByteRev32: {
2147 ASSEMBLE_LOAD_INTEGER_RR(lwbrx);
2148 break;
2149 }
2150 case kPPC_StoreByteRev32: {
2151 ASSEMBLE_STORE_INTEGER_RR(stwbrx);
2152 break;
2153 }
2154 case kPPC_ByteRev64: {
2155 Register input = i.InputRegister(0);
2156 Register output = i.OutputRegister();
2157 Register temp1 = r0;
2158 Register temp2 = kScratchReg;
2159 Register temp3 = i.TempRegister(0);
2160 if (CpuFeatures::IsSupported(PPC_10_PLUS)) {
2161 __ brd(output, input);
2162 break;
2163 }
2164 __ rldicl(temp1, input, 32, 32);
2165 __ rotlwi(temp2, input, 8);
2166 __ rlwimi(temp2, input, 24, 0, 7);
2167 __ rotlwi(temp3, temp1, 8);
2168 __ rlwimi(temp2, input, 24, 16, 23);
2169 __ rlwimi(temp3, temp1, 24, 0, 7);
2170 __ rlwimi(temp3, temp1, 24, 16, 23);
2171 __ rldicr(temp2, temp2, 32, 31);
2172 __ orx(output, temp2, temp3);
2173 break;
2174 }
2175 case kPPC_LoadByteRev64: {
2176 ASSEMBLE_LOAD_INTEGER_RR(ldbrx);
2177 break;
2178 }
2179 case kPPC_StoreByteRev64: {
2180 ASSEMBLE_STORE_INTEGER_RR(stdbrx);
2181 break;
2182 }
2183 case kPPC_F64x2Splat: {
2184 constexpr int lane_width_in_bytes = 8;
2185 Simd128Register dst = i.OutputSimd128Register();
2186 __ MovDoubleToInt64(kScratchReg, i.InputDoubleRegister(0));
2187 __ mtvsrd(dst, kScratchReg);
2188 __ vinsertd(dst, dst, Operand(1 * lane_width_in_bytes));
2189 break;
2190 }
2191 case kPPC_F32x4Splat: {
2192 Simd128Register dst = i.OutputSimd128Register();
2193 __ MovFloatToInt(kScratchReg, i.InputDoubleRegister(0),
2194 kScratchDoubleReg);
2195 __ mtvsrd(dst, kScratchReg);
2196 __ vspltw(dst, dst, Operand(1));
2197 break;
2198 }
2199 case kPPC_I64x2Splat: {
2200 constexpr int lane_width_in_bytes = 8;
2201 Simd128Register dst = i.OutputSimd128Register();
2202 __ mtvsrd(dst, i.InputRegister(0));
2203 __ vinsertd(dst, dst, Operand(1 * lane_width_in_bytes));
2204 break;
2205 }
2206 case kPPC_I32x4Splat: {
2207 Simd128Register dst = i.OutputSimd128Register();
2208 __ mtvsrd(dst, i.InputRegister(0));
2209 __ vspltw(dst, dst, Operand(1));
2210 break;
2211 }
2212 case kPPC_I16x8Splat: {
2213 Simd128Register dst = i.OutputSimd128Register();
2214 __ mtvsrd(dst, i.InputRegister(0));
2215 __ vsplth(dst, dst, Operand(3));
2216 break;
2217 }
2218 case kPPC_I8x16Splat: {
2219 Simd128Register dst = i.OutputSimd128Register();
2220 __ mtvsrd(dst, i.InputRegister(0));
2221 __ vspltb(dst, dst, Operand(7));
2222 break;
2223 }
2224 case kPPC_F64x2ExtractLane: {
2225 constexpr int lane_width_in_bytes = 8;
2226 __ vextractd(kScratchSimd128Reg, i.InputSimd128Register(0),
2227 Operand((1 - i.InputInt8(1)) * lane_width_in_bytes));
2228 __ mfvsrd(kScratchReg, kScratchSimd128Reg);
2229 __ MovInt64ToDouble(i.OutputDoubleRegister(), kScratchReg);
2230 break;
2231 }
2232 case kPPC_F32x4ExtractLane: {
2233 constexpr int lane_width_in_bytes = 4;
2234 __ vextractuw(kScratchSimd128Reg, i.InputSimd128Register(0),
2235 Operand((3 - i.InputInt8(1)) * lane_width_in_bytes));
2236 __ mfvsrd(kScratchReg, kScratchSimd128Reg);
2237 __ MovIntToFloat(i.OutputDoubleRegister(), kScratchReg, ip);
2238 break;
2239 }
2240 case kPPC_I64x2ExtractLane: {
2241 constexpr int lane_width_in_bytes = 8;
2242 __ vextractd(kScratchSimd128Reg, i.InputSimd128Register(0),
2243 Operand((1 - i.InputInt8(1)) * lane_width_in_bytes));
2244 __ mfvsrd(i.OutputRegister(), kScratchSimd128Reg);
2245 break;
2246 }
2247 case kPPC_I32x4ExtractLane: {
2248 constexpr int lane_width_in_bytes = 4;
2249 __ vextractuw(kScratchSimd128Reg, i.InputSimd128Register(0),
2250 Operand((3 - i.InputInt8(1)) * lane_width_in_bytes));
2251 __ mfvsrd(i.OutputRegister(), kScratchSimd128Reg);
2252 break;
2253 }
2254 case kPPC_I16x8ExtractLaneU: {
2255 constexpr int lane_width_in_bytes = 2;
2256 __ vextractuh(kScratchSimd128Reg, i.InputSimd128Register(0),
2257 Operand((7 - i.InputInt8(1)) * lane_width_in_bytes));
2258 __ mfvsrd(i.OutputRegister(), kScratchSimd128Reg);
2259 break;
2260 }
2261 case kPPC_I16x8ExtractLaneS: {
2262 constexpr int lane_width_in_bytes = 2;
2263 __ vextractuh(kScratchSimd128Reg, i.InputSimd128Register(0),
2264 Operand((7 - i.InputInt8(1)) * lane_width_in_bytes));
2265 __ mfvsrd(kScratchReg, kScratchSimd128Reg);
2266 __ extsh(i.OutputRegister(), kScratchReg);
2267 break;
2268 }
2269 case kPPC_I8x16ExtractLaneU: {
2270 __ vextractub(kScratchSimd128Reg, i.InputSimd128Register(0),
2271 Operand(15 - i.InputInt8(1)));
2272 __ mfvsrd(i.OutputRegister(), kScratchSimd128Reg);
2273 break;
2274 }
2275 case kPPC_I8x16ExtractLaneS: {
2276 __ vextractub(kScratchSimd128Reg, i.InputSimd128Register(0),
2277 Operand(15 - i.InputInt8(1)));
2278 __ mfvsrd(kScratchReg, kScratchSimd128Reg);
2279 __ extsb(i.OutputRegister(), kScratchReg);
2280 break;
2281 }
2282 case kPPC_F64x2ReplaceLane: {
2283 DCHECK_EQ(i.OutputSimd128Register(), i.InputSimd128Register(0));
2284 constexpr int lane_width_in_bytes = 8;
2285 Simd128Register dst = i.OutputSimd128Register();
2286 __ MovDoubleToInt64(r0, i.InputDoubleRegister(2));
2287 if (CpuFeatures::IsSupported(PPC_10_PLUS)) {
2288 __ vinsd(dst, r0, Operand((1 - i.InputInt8(1)) * lane_width_in_bytes));
2289 } else {
2290 __ mtvsrd(kScratchSimd128Reg, r0);
2291 __ vinsertd(dst, kScratchSimd128Reg,
2292 Operand((1 - i.InputInt8(1)) * lane_width_in_bytes));
2293 }
2294 break;
2295 }
2296 case kPPC_F32x4ReplaceLane: {
2297 DCHECK_EQ(i.OutputSimd128Register(), i.InputSimd128Register(0));
2298 constexpr int lane_width_in_bytes = 4;
2299 Simd128Register dst = i.OutputSimd128Register();
2300 __ MovFloatToInt(r0, i.InputDoubleRegister(2), kScratchDoubleReg);
2301 if (CpuFeatures::IsSupported(PPC_10_PLUS)) {
2302 __ vinsw(dst, r0, Operand((3 - i.InputInt8(1)) * lane_width_in_bytes));
2303 } else {
2304 __ mtvsrd(kScratchSimd128Reg, r0);
2305 __ vinsertw(dst, kScratchSimd128Reg,
2306 Operand((3 - i.InputInt8(1)) * lane_width_in_bytes));
2307 }
2308 break;
2309 }
2310 case kPPC_I64x2ReplaceLane: {
2311 DCHECK_EQ(i.OutputSimd128Register(), i.InputSimd128Register(0));
2312 constexpr int lane_width_in_bytes = 8;
2313 Simd128Register dst = i.OutputSimd128Register();
2314 if (CpuFeatures::IsSupported(PPC_10_PLUS)) {
2315 __ vinsd(dst, i.InputRegister(2),
2316 Operand((1 - i.InputInt8(1)) * lane_width_in_bytes));
2317 } else {
2318 __ mtvsrd(kScratchSimd128Reg, i.InputRegister(2));
2319 __ vinsertd(dst, kScratchSimd128Reg,
2320 Operand((1 - i.InputInt8(1)) * lane_width_in_bytes));
2321 }
2322 break;
2323 }
2324 case kPPC_I32x4ReplaceLane: {
2325 DCHECK_EQ(i.OutputSimd128Register(), i.InputSimd128Register(0));
2326 constexpr int lane_width_in_bytes = 4;
2327 Simd128Register dst = i.OutputSimd128Register();
2328 if (CpuFeatures::IsSupported(PPC_10_PLUS)) {
2329 __ vinsw(dst, i.InputRegister(2),
2330 Operand((3 - i.InputInt8(1)) * lane_width_in_bytes));
2331 } else {
2332 __ mtvsrd(kScratchSimd128Reg, i.InputRegister(2));
2333 __ vinsertw(dst, kScratchSimd128Reg,
2334 Operand((3 - i.InputInt8(1)) * lane_width_in_bytes));
2335 }
2336 break;
2337 }
2338 case kPPC_I16x8ReplaceLane: {
2339 DCHECK_EQ(i.OutputSimd128Register(), i.InputSimd128Register(0));
2340 constexpr int lane_width_in_bytes = 2;
2341 Simd128Register dst = i.OutputSimd128Register();
2342 __ mtvsrd(kScratchSimd128Reg, i.InputRegister(2));
2343 __ vinserth(dst, kScratchSimd128Reg,
2344 Operand((7 - i.InputInt8(1)) * lane_width_in_bytes));
2345 break;
2346 }
2347 case kPPC_I8x16ReplaceLane: {
2348 DCHECK_EQ(i.OutputSimd128Register(), i.InputSimd128Register(0));
2349 Simd128Register dst = i.OutputSimd128Register();
2350 __ mtvsrd(kScratchSimd128Reg, i.InputRegister(2));
2351 __ vinsertb(dst, kScratchSimd128Reg, Operand(15 - i.InputInt8(1)));
2352 break;
2353 }
2354 case kPPC_F64x2Add: {
2355 __ xvadddp(i.OutputSimd128Register(), i.InputSimd128Register(0),
2356 i.InputSimd128Register(1));
2357 break;
2358 }
2359 case kPPC_F64x2Sub: {
2360 __ xvsubdp(i.OutputSimd128Register(), i.InputSimd128Register(0),
2361 i.InputSimd128Register(1));
2362 break;
2363 }
2364 case kPPC_F64x2Mul: {
2365 __ xvmuldp(i.OutputSimd128Register(), i.InputSimd128Register(0),
2366 i.InputSimd128Register(1));
2367 break;
2368 }
2369 case kPPC_F32x4Add: {
2370 __ vaddfp(i.OutputSimd128Register(), i.InputSimd128Register(0),
2371 i.InputSimd128Register(1));
2372 break;
2373 }
2374 case kPPC_F32x4Sub: {
2375 __ vsubfp(i.OutputSimd128Register(), i.InputSimd128Register(0),
2376 i.InputSimd128Register(1));
2377 break;
2378 }
2379 case kPPC_F32x4Mul: {
2380 __ xvmulsp(i.OutputSimd128Register(), i.InputSimd128Register(0),
2381 i.InputSimd128Register(1));
2382 break;
2383 }
2384 case kPPC_I64x2Add: {
2385 __ vaddudm(i.OutputSimd128Register(), i.InputSimd128Register(0),
2386 i.InputSimd128Register(1));
2387 break;
2388 }
2389 case kPPC_I64x2Sub: {
2390 __ vsubudm(i.OutputSimd128Register(), i.InputSimd128Register(0),
2391 i.InputSimd128Register(1));
2392 break;
2393 }
2394 case kPPC_I64x2Mul: {
2395 constexpr int lane_width_in_bytes = 8;
2396 Simd128Register src0 = i.InputSimd128Register(0);
2397 Simd128Register src1 = i.InputSimd128Register(1);
2398 Simd128Register tempFPReg0 = i.ToSimd128Register(instr->TempAt(0));
2399 Register tempReg1 = i.ToRegister(instr->TempAt(2));
2400 Register scratch_0 = ip;
2401 Register scratch_1 = r0;
2402 Simd128Register dst = i.OutputSimd128Register();
2403 if (CpuFeatures::IsSupported(PPC_10_PLUS)) {
2404 __ vmulld(dst, src0, src1);
2405 } else {
2406 for (int i = 0; i < 2; i++) {
2407 if (i > 0) {
2408 __ vextractd(kScratchSimd128Reg, src0,
2409 Operand(1 * lane_width_in_bytes));
2410 __ vextractd(tempFPReg0, src1, Operand(1 * lane_width_in_bytes));
2411 src0 = kScratchSimd128Reg;
2412 src1 = tempFPReg0;
2413 }
2414 __ mfvsrd(scratch_0, src0);
2415 __ mfvsrd(scratch_1, src1);
2416 __ mulld(scratch_0, scratch_0, scratch_1);
2417 scratch_0 = r0;
2418 scratch_1 = tempReg1;
2419 }
2420 __ mtvsrdd(dst, ip, r0);
2421 }
2422 break;
2423 }
2424 case kPPC_I32x4Add: {
2425 __ vadduwm(i.OutputSimd128Register(), i.InputSimd128Register(0),
2426 i.InputSimd128Register(1));
2427 break;
2428 }
2429 case kPPC_I32x4Sub: {
2430 __ vsubuwm(i.OutputSimd128Register(), i.InputSimd128Register(0),
2431 i.InputSimd128Register(1));
2432 break;
2433 }
2434 case kPPC_I32x4Mul: {
2435 __ vmuluwm(i.OutputSimd128Register(), i.InputSimd128Register(0),
2436 i.InputSimd128Register(1));
2437 break;
2438 }
2439 case kPPC_I16x8Add: {
2440 __ vadduhm(i.OutputSimd128Register(), i.InputSimd128Register(0),
2441 i.InputSimd128Register(1));
2442 break;
2443 }
2444 case kPPC_I16x8Sub: {
2445 __ vsubuhm(i.OutputSimd128Register(), i.InputSimd128Register(0),
2446 i.InputSimd128Register(1));
2447 break;
2448 }
2449 case kPPC_I16x8Mul: {
2450 Simd128Register src0 = i.InputSimd128Register(0);
2451 Simd128Register src1 = i.InputSimd128Register(1);
2452 Simd128Register dst = i.OutputSimd128Register();
2453 __ vxor(kScratchSimd128Reg, kScratchSimd128Reg, kScratchSimd128Reg);
2454 __ vmladduhm(dst, src0, src1, kScratchSimd128Reg);
2455 break;
2456 }
2457 case kPPC_I8x16Add: {
2458 __ vaddubm(i.OutputSimd128Register(), i.InputSimd128Register(0),
2459 i.InputSimd128Register(1));
2460 break;
2461 }
2462 case kPPC_I8x16Sub: {
2463 __ vsububm(i.OutputSimd128Register(), i.InputSimd128Register(0),
2464 i.InputSimd128Register(1));
2465 break;
2466 }
2467 case kPPC_I32x4MinS: {
2468 __ vminsw(i.OutputSimd128Register(), i.InputSimd128Register(0),
2469 i.InputSimd128Register(1));
2470 break;
2471 }
2472 case kPPC_I32x4MinU: {
2473 __ vminuw(i.OutputSimd128Register(), i.InputSimd128Register(0),
2474 i.InputSimd128Register(1));
2475 break;
2476 }
2477 case kPPC_I16x8MinS: {
2478 __ vminsh(i.OutputSimd128Register(), i.InputSimd128Register(0),
2479 i.InputSimd128Register(1));
2480 break;
2481 }
2482 case kPPC_I16x8MinU: {
2483 __ vminuh(i.OutputSimd128Register(), i.InputSimd128Register(0),
2484 i.InputSimd128Register(1));
2485 break;
2486 }
2487 case kPPC_I8x16MinS: {
2488 __ vminsb(i.OutputSimd128Register(), i.InputSimd128Register(0),
2489 i.InputSimd128Register(1));
2490 break;
2491 }
2492 case kPPC_I8x16MinU: {
2493 __ vminub(i.OutputSimd128Register(), i.InputSimd128Register(0),
2494 i.InputSimd128Register(1));
2495 break;
2496 }
2497 case kPPC_I32x4MaxS: {
2498 __ vmaxsw(i.OutputSimd128Register(), i.InputSimd128Register(0),
2499 i.InputSimd128Register(1));
2500 break;
2501 }
2502 case kPPC_I32x4MaxU: {
2503 __ vmaxuw(i.OutputSimd128Register(), i.InputSimd128Register(0),
2504 i.InputSimd128Register(1));
2505 break;
2506 }
2507 case kPPC_I16x8MaxS: {
2508 __ vmaxsh(i.OutputSimd128Register(), i.InputSimd128Register(0),
2509 i.InputSimd128Register(1));
2510 break;
2511 }
2512 case kPPC_I16x8MaxU: {
2513 __ vmaxuh(i.OutputSimd128Register(), i.InputSimd128Register(0),
2514 i.InputSimd128Register(1));
2515 break;
2516 }
2517 case kPPC_I8x16MaxS: {
2518 __ vmaxsb(i.OutputSimd128Register(), i.InputSimd128Register(0),
2519 i.InputSimd128Register(1));
2520 break;
2521 }
2522 case kPPC_I8x16MaxU: {
2523 __ vmaxub(i.OutputSimd128Register(), i.InputSimd128Register(0),
2524 i.InputSimd128Register(1));
2525 break;
2526 }
2527 case kPPC_F64x2Eq: {
2528 __ xvcmpeqdp(i.OutputSimd128Register(), i.InputSimd128Register(0),
2529 i.InputSimd128Register(1));
2530 break;
2531 }
2532 case kPPC_F64x2Ne: {
2533 __ xvcmpeqdp(kScratchSimd128Reg, i.InputSimd128Register(0),
2534 i.InputSimd128Register(1));
2535 __ vnor(i.OutputSimd128Register(), kScratchSimd128Reg,
2536 kScratchSimd128Reg);
2537 break;
2538 }
2539 case kPPC_F64x2Le: {
2540 __ xvcmpgedp(i.OutputSimd128Register(), i.InputSimd128Register(1),
2541 i.InputSimd128Register(0));
2542 break;
2543 }
2544 case kPPC_F64x2Lt: {
2545 __ xvcmpgtdp(i.OutputSimd128Register(), i.InputSimd128Register(1),
2546 i.InputSimd128Register(0));
2547 break;
2548 }
2549 case kPPC_F32x4Eq: {
2550 __ xvcmpeqsp(i.OutputSimd128Register(), i.InputSimd128Register(0),
2551 i.InputSimd128Register(1));
2552 break;
2553 }
2554 case kPPC_I64x2Eq: {
2555 __ vcmpequd(i.OutputSimd128Register(), i.InputSimd128Register(0),
2556 i.InputSimd128Register(1));
2557 break;
2558 }
2559 case kPPC_I32x4Eq: {
2560 __ vcmpequw(i.OutputSimd128Register(), i.InputSimd128Register(0),
2561 i.InputSimd128Register(1));
2562 break;
2563 }
2564 case kPPC_I16x8Eq: {
2565 __ vcmpequh(i.OutputSimd128Register(), i.InputSimd128Register(0),
2566 i.InputSimd128Register(1));
2567 break;
2568 }
2569 case kPPC_I8x16Eq: {
2570 __ vcmpequb(i.OutputSimd128Register(), i.InputSimd128Register(0),
2571 i.InputSimd128Register(1));
2572 break;
2573 }
2574 case kPPC_F32x4Ne: {
2575 __ xvcmpeqsp(kScratchSimd128Reg, i.InputSimd128Register(0),
2576 i.InputSimd128Register(1));
2577 __ vnor(i.OutputSimd128Register(), kScratchSimd128Reg,
2578 kScratchSimd128Reg);
2579 break;
2580 }
2581 case kPPC_I64x2Ne: {
2582 __ vcmpequd(kScratchSimd128Reg, i.InputSimd128Register(0),
2583 i.InputSimd128Register(1));
2584 __ vnor(i.OutputSimd128Register(), kScratchSimd128Reg,
2585 kScratchSimd128Reg);
2586 break;
2587 }
2588 case kPPC_I32x4Ne: {
2589 __ vcmpequw(kScratchSimd128Reg, i.InputSimd128Register(0),
2590 i.InputSimd128Register(1));
2591 __ vnor(i.OutputSimd128Register(), kScratchSimd128Reg,
2592 kScratchSimd128Reg);
2593 break;
2594 }
2595 case kPPC_I16x8Ne: {
2596 __ vcmpequh(kScratchSimd128Reg, i.InputSimd128Register(0),
2597 i.InputSimd128Register(1));
2598 __ vnor(i.OutputSimd128Register(), kScratchSimd128Reg,
2599 kScratchSimd128Reg);
2600 break;
2601 }
2602 case kPPC_I8x16Ne: {
2603 __ vcmpequb(kScratchSimd128Reg, i.InputSimd128Register(0),
2604 i.InputSimd128Register(1));
2605 __ vnor(i.OutputSimd128Register(), kScratchSimd128Reg,
2606 kScratchSimd128Reg);
2607 break;
2608 }
2609 case kPPC_F32x4Lt: {
2610 __ xvcmpgtsp(i.OutputSimd128Register(), i.InputSimd128Register(1),
2611 i.InputSimd128Register(0));
2612 break;
2613 }
2614 case kPPC_F32x4Le: {
2615 __ xvcmpgesp(i.OutputSimd128Register(), i.InputSimd128Register(1),
2616 i.InputSimd128Register(0));
2617 break;
2618 }
2619 case kPPC_I64x2GtS: {
2620 __ vcmpgtsd(i.OutputSimd128Register(), i.InputSimd128Register(0),
2621 i.InputSimd128Register(1));
2622 break;
2623 }
2624 case kPPC_I32x4GtS: {
2625 __ vcmpgtsw(i.OutputSimd128Register(), i.InputSimd128Register(0),
2626 i.InputSimd128Register(1));
2627 break;
2628 }
2629 case kPPC_I64x2GeS: {
2630 __ vcmpgtsd(kScratchSimd128Reg, i.InputSimd128Register(1),
2631 i.InputSimd128Register(0));
2632 __ vnor(i.OutputSimd128Register(), kScratchSimd128Reg,
2633 kScratchSimd128Reg);
2634 break;
2635 }
2636 case kPPC_I32x4GeS: {
2637 __ vcmpgtsw(kScratchSimd128Reg, i.InputSimd128Register(1),
2638 i.InputSimd128Register(0));
2639 __ vnor(i.OutputSimd128Register(), kScratchSimd128Reg,
2640 kScratchSimd128Reg);
2641 break;
2642 }
2643 case kPPC_I32x4GtU: {
2644 __ vcmpgtuw(i.OutputSimd128Register(), i.InputSimd128Register(0),
2645 i.InputSimd128Register(1));
2646
2647 break;
2648 }
2649 case kPPC_I32x4GeU: {
2650 __ vcmpequw(kScratchSimd128Reg, i.InputSimd128Register(0),
2651 i.InputSimd128Register(1));
2652 __ vcmpgtuw(i.OutputSimd128Register(), i.InputSimd128Register(0),
2653 i.InputSimd128Register(1));
2654 __ vor(i.OutputSimd128Register(), i.OutputSimd128Register(),
2655 kScratchSimd128Reg);
2656 break;
2657 }
2658 case kPPC_I16x8GtS: {
2659 __ vcmpgtsh(i.OutputSimd128Register(), i.InputSimd128Register(0),
2660 i.InputSimd128Register(1));
2661 break;
2662 }
2663 case kPPC_I16x8GeS: {
2664 __ vcmpgtsh(kScratchSimd128Reg, i.InputSimd128Register(1),
2665 i.InputSimd128Register(0));
2666 __ vnor(i.OutputSimd128Register(), kScratchSimd128Reg,
2667 kScratchSimd128Reg);
2668 break;
2669 }
2670 case kPPC_I16x8GtU: {
2671 __ vcmpgtuh(i.OutputSimd128Register(), i.InputSimd128Register(0),
2672 i.InputSimd128Register(1));
2673 break;
2674 }
2675 case kPPC_I16x8GeU: {
2676 __ vcmpequh(kScratchSimd128Reg, i.InputSimd128Register(0),
2677 i.InputSimd128Register(1));
2678 __ vcmpgtuh(i.OutputSimd128Register(), i.InputSimd128Register(0),
2679 i.InputSimd128Register(1));
2680 __ vor(i.OutputSimd128Register(), i.OutputSimd128Register(),
2681 kScratchSimd128Reg);
2682 break;
2683 }
2684 case kPPC_I8x16GtS: {
2685 __ vcmpgtsb(i.OutputSimd128Register(), i.InputSimd128Register(0),
2686 i.InputSimd128Register(1));
2687 break;
2688 }
2689 case kPPC_I8x16GeS: {
2690 __ vcmpgtsb(kScratchSimd128Reg, i.InputSimd128Register(1),
2691 i.InputSimd128Register(0));
2692 __ vnor(i.OutputSimd128Register(), kScratchSimd128Reg,
2693 kScratchSimd128Reg);
2694 break;
2695 }
2696 case kPPC_I8x16GtU: {
2697 __ vcmpgtub(i.OutputSimd128Register(), i.InputSimd128Register(0),
2698 i.InputSimd128Register(1));
2699 break;
2700 }
2701 case kPPC_I8x16GeU: {
2702 __ vcmpequb(kScratchSimd128Reg, i.InputSimd128Register(0),
2703 i.InputSimd128Register(1));
2704 __ vcmpgtub(i.OutputSimd128Register(), i.InputSimd128Register(0),
2705 i.InputSimd128Register(1));
2706 __ vor(i.OutputSimd128Register(), i.OutputSimd128Register(),
2707 kScratchSimd128Reg);
2708 break;
2709 }
2710 #define VECTOR_SHIFT(op) \
2711 { \
2712 __ mtvsrd(kScratchSimd128Reg, i.InputRegister(1)); \
2713 __ vspltb(kScratchSimd128Reg, kScratchSimd128Reg, Operand(7)); \
2714 __ op(i.OutputSimd128Register(), i.InputSimd128Register(0), \
2715 kScratchSimd128Reg); \
2716 }
2717 case kPPC_I64x2Shl: {
2718 VECTOR_SHIFT(vsld)
2719 break;
2720 }
2721 case kPPC_I64x2ShrS: {
2722 VECTOR_SHIFT(vsrad)
2723 break;
2724 }
2725 case kPPC_I64x2ShrU: {
2726 VECTOR_SHIFT(vsrd)
2727 break;
2728 }
2729 case kPPC_I32x4Shl: {
2730 VECTOR_SHIFT(vslw)
2731 break;
2732 }
2733 case kPPC_I32x4ShrS: {
2734 VECTOR_SHIFT(vsraw)
2735 break;
2736 }
2737 case kPPC_I32x4ShrU: {
2738 VECTOR_SHIFT(vsrw)
2739 break;
2740 }
2741 case kPPC_I16x8Shl: {
2742 VECTOR_SHIFT(vslh)
2743 break;
2744 }
2745 case kPPC_I16x8ShrS: {
2746 VECTOR_SHIFT(vsrah)
2747 break;
2748 }
2749 case kPPC_I16x8ShrU: {
2750 VECTOR_SHIFT(vsrh)
2751 break;
2752 }
2753 case kPPC_I8x16Shl: {
2754 VECTOR_SHIFT(vslb)
2755 break;
2756 }
2757 case kPPC_I8x16ShrS: {
2758 VECTOR_SHIFT(vsrab)
2759 break;
2760 }
2761 case kPPC_I8x16ShrU: {
2762 VECTOR_SHIFT(vsrb)
2763 break;
2764 }
2765 #undef VECTOR_SHIFT
2766 case kPPC_S128And: {
2767 Simd128Register dst = i.OutputSimd128Register();
2768 Simd128Register src = i.InputSimd128Register(1);
2769 __ vand(dst, i.InputSimd128Register(0), src);
2770 break;
2771 }
2772 case kPPC_S128Or: {
2773 Simd128Register dst = i.OutputSimd128Register();
2774 Simd128Register src = i.InputSimd128Register(1);
2775 __ vor(dst, i.InputSimd128Register(0), src);
2776 break;
2777 }
2778 case kPPC_S128Xor: {
2779 Simd128Register dst = i.OutputSimd128Register();
2780 Simd128Register src = i.InputSimd128Register(1);
2781 __ vxor(dst, i.InputSimd128Register(0), src);
2782 break;
2783 }
2784 case kPPC_S128Const: {
2785 uint64_t low = make_uint64(i.InputUint32(1), i.InputUint32(0));
2786 uint64_t high = make_uint64(i.InputUint32(3), i.InputUint32(2));
2787 __ mov(r0, Operand(low));
2788 __ mov(ip, Operand(high));
2789 __ mtvsrdd(i.OutputSimd128Register(), ip, r0);
2790 break;
2791 }
2792 case kPPC_S128Zero: {
2793 Simd128Register dst = i.OutputSimd128Register();
2794 __ vxor(dst, dst, dst);
2795 break;
2796 }
2797 case kPPC_S128AllOnes: {
2798 Simd128Register dst = i.OutputSimd128Register();
2799 __ vcmpequb(dst, dst, dst);
2800 break;
2801 }
2802 case kPPC_S128Not: {
2803 Simd128Register dst = i.OutputSimd128Register();
2804 Simd128Register src = i.InputSimd128Register(0);
2805 __ vnor(dst, src, src);
2806 break;
2807 }
2808 case kPPC_S128Select: {
2809 Simd128Register dst = i.OutputSimd128Register();
2810 Simd128Register mask = i.InputSimd128Register(0);
2811 Simd128Register src1 = i.InputSimd128Register(1);
2812 Simd128Register src2 = i.InputSimd128Register(2);
2813 __ vsel(dst, src2, src1, mask);
2814 break;
2815 }
2816 case kPPC_F64x2Abs: {
2817 __ xvabsdp(i.OutputSimd128Register(), i.InputSimd128Register(0));
2818 break;
2819 }
2820 case kPPC_F64x2Neg: {
2821 __ xvnegdp(i.OutputSimd128Register(), i.InputSimd128Register(0));
2822 break;
2823 }
2824 case kPPC_F64x2Sqrt: {
2825 __ xvsqrtdp(i.OutputSimd128Register(), i.InputSimd128Register(0));
2826 break;
2827 }
2828 case kPPC_F32x4Abs: {
2829 __ xvabssp(i.OutputSimd128Register(), i.InputSimd128Register(0));
2830 break;
2831 }
2832 case kPPC_F32x4Neg: {
2833 __ xvnegsp(i.OutputSimd128Register(), i.InputSimd128Register(0));
2834 break;
2835 }
2836 case kPPC_F32x4RecipApprox: {
2837 __ xvresp(i.OutputSimd128Register(), i.InputSimd128Register(0));
2838 break;
2839 }
2840 case kPPC_F32x4RecipSqrtApprox: {
2841 __ xvrsqrtesp(i.OutputSimd128Register(), i.InputSimd128Register(0));
2842 break;
2843 }
2844 case kPPC_F32x4Sqrt: {
2845 __ xvsqrtsp(i.OutputSimd128Register(), i.InputSimd128Register(0));
2846 break;
2847 }
2848 case kPPC_I64x2Neg: {
2849 __ vnegd(i.OutputSimd128Register(), i.InputSimd128Register(0));
2850 break;
2851 }
2852 case kPPC_I32x4Neg: {
2853 __ vnegw(i.OutputSimd128Register(), i.InputSimd128Register(0));
2854 break;
2855 }
2856 case kPPC_I64x2Abs: {
2857 Simd128Register tempFPReg1 = i.ToSimd128Register(instr->TempAt(0));
2858 Simd128Register src = i.InputSimd128Register(0);
2859 constexpr int shift_bits = 63;
2860 __ xxspltib(kScratchSimd128Reg, Operand(shift_bits));
2861 __ vsrad(kScratchSimd128Reg, src, kScratchSimd128Reg);
2862 __ vxor(tempFPReg1, src, kScratchSimd128Reg);
2863 __ vsubudm(i.OutputSimd128Register(), tempFPReg1, kScratchSimd128Reg);
2864 break;
2865 }
2866 case kPPC_I32x4Abs: {
2867 Simd128Register tempFPReg1 = i.ToSimd128Register(instr->TempAt(0));
2868 Simd128Register src = i.InputSimd128Register(0);
2869 constexpr int shift_bits = 31;
2870 __ xxspltib(kScratchSimd128Reg, Operand(shift_bits));
2871 __ vsraw(kScratchSimd128Reg, src, kScratchSimd128Reg);
2872 __ vxor(tempFPReg1, src, kScratchSimd128Reg);
2873 __ vsubuwm(i.OutputSimd128Register(), tempFPReg1, kScratchSimd128Reg);
2874 break;
2875 }
2876 case kPPC_I16x8Neg: {
2877 Simd128Register tempFPReg1 = i.ToSimd128Register(instr->TempAt(0));
2878 __ vspltish(kScratchSimd128Reg, Operand(1));
2879 __ vnor(tempFPReg1, i.InputSimd128Register(0), i.InputSimd128Register(0));
2880 __ vadduhm(i.OutputSimd128Register(), kScratchSimd128Reg, tempFPReg1);
2881 break;
2882 }
2883 case kPPC_I16x8Abs: {
2884 Simd128Register tempFPReg1 = i.ToSimd128Register(instr->TempAt(0));
2885 Simd128Register src = i.InputSimd128Register(0);
2886 constexpr int shift_bits = 15;
2887 __ xxspltib(kScratchSimd128Reg, Operand(shift_bits));
2888 __ vsrah(kScratchSimd128Reg, src, kScratchSimd128Reg);
2889 __ vxor(tempFPReg1, src, kScratchSimd128Reg);
2890 __ vsubuhm(i.OutputSimd128Register(), tempFPReg1, kScratchSimd128Reg);
2891 break;
2892 }
2893 case kPPC_I8x16Neg: {
2894 Simd128Register tempFPReg1 = i.ToSimd128Register(instr->TempAt(0));
2895 __ xxspltib(kScratchSimd128Reg, Operand(1));
2896 __ vnor(tempFPReg1, i.InputSimd128Register(0), i.InputSimd128Register(0));
2897 __ vaddubm(i.OutputSimd128Register(), kScratchSimd128Reg, tempFPReg1);
2898 break;
2899 }
2900 case kPPC_I8x16Abs: {
2901 Simd128Register tempFPReg1 = i.ToSimd128Register(instr->TempAt(0));
2902 Simd128Register src = i.InputSimd128Register(0);
2903 constexpr int shift_bits = 7;
2904 __ xxspltib(kScratchSimd128Reg, Operand(shift_bits));
2905 __ vsrab(kScratchSimd128Reg, src, kScratchSimd128Reg);
2906 __ vxor(tempFPReg1, src, kScratchSimd128Reg);
2907 __ vsububm(i.OutputSimd128Register(), tempFPReg1, kScratchSimd128Reg);
2908 break;
2909 }
2910 case kPPC_V128AnyTrue: {
2911 Simd128Register src = i.InputSimd128Register(0);
2912 Register dst = i.OutputRegister();
2913 constexpr uint8_t fxm = 0x2; // field mask.
2914 constexpr int bit_number = 24;
2915 __ li(r0, Operand(0));
2916 __ li(ip, Operand(1));
2917 // Check if both lanes are 0, if so then return false.
2918 __ vxor(kScratchSimd128Reg, kScratchSimd128Reg, kScratchSimd128Reg);
2919 __ mtcrf(r0, fxm); // Clear cr6.
2920 __ vcmpequd(kScratchSimd128Reg, src, kScratchSimd128Reg, SetRC);
2921 __ isel(dst, r0, ip, bit_number);
2922 break;
2923 }
2924 #define SIMD_ALL_TRUE(opcode) \
2925 Simd128Register src = i.InputSimd128Register(0); \
2926 Register dst = i.OutputRegister(); \
2927 constexpr uint8_t fxm = 0x2; /* field mask. */ \
2928 constexpr int bit_number = 24; \
2929 __ li(r0, Operand(0)); \
2930 __ li(ip, Operand(1)); \
2931 /* Check if all lanes > 0, if not then return false.*/ \
2932 __ vxor(kScratchSimd128Reg, kScratchSimd128Reg, kScratchSimd128Reg); \
2933 __ mtcrf(r0, fxm); /* Clear cr6.*/ \
2934 __ opcode(kScratchSimd128Reg, src, kScratchSimd128Reg, SetRC); \
2935 __ isel(dst, ip, r0, bit_number);
2936 case kPPC_I64x2AllTrue: {
2937 SIMD_ALL_TRUE(vcmpgtud)
2938 break;
2939 }
2940 case kPPC_I32x4AllTrue: {
2941 SIMD_ALL_TRUE(vcmpgtuw)
2942 break;
2943 }
2944 case kPPC_I16x8AllTrue: {
2945 SIMD_ALL_TRUE(vcmpgtuh)
2946 break;
2947 }
2948 case kPPC_I8x16AllTrue: {
2949 SIMD_ALL_TRUE(vcmpgtub)
2950 break;
2951 }
2952 #undef SIMD_ALL_TRUE
2953 case kPPC_I32x4SConvertF32x4: {
2954 Simd128Register src = i.InputSimd128Register(0);
2955 // NaN to 0
2956 __ vor(kScratchSimd128Reg, src, src);
2957 __ xvcmpeqsp(kScratchSimd128Reg, kScratchSimd128Reg, kScratchSimd128Reg);
2958 __ vand(kScratchSimd128Reg, src, kScratchSimd128Reg);
2959 __ xvcvspsxws(i.OutputSimd128Register(), kScratchSimd128Reg);
2960 break;
2961 }
2962 case kPPC_I32x4UConvertF32x4: {
2963 __ xvcvspuxws(i.OutputSimd128Register(), i.InputSimd128Register(0));
2964 break;
2965 }
2966 case kPPC_F32x4SConvertI32x4: {
2967 __ xvcvsxwsp(i.OutputSimd128Register(), i.InputSimd128Register(0));
2968 break;
2969 }
2970 case kPPC_F32x4UConvertI32x4: {
2971 __ xvcvuxwsp(i.OutputSimd128Register(), i.InputSimd128Register(0));
2972 break;
2973 }
2974
2975 case kPPC_I64x2SConvertI32x4Low: {
2976 __ vupklsw(i.OutputSimd128Register(), i.InputSimd128Register(0));
2977 break;
2978 }
2979 case kPPC_I64x2SConvertI32x4High: {
2980 __ vupkhsw(i.OutputSimd128Register(), i.InputSimd128Register(0));
2981 break;
2982 }
2983 case kPPC_I64x2UConvertI32x4Low: {
2984 constexpr int lane_width_in_bytes = 8;
2985 __ vupklsw(i.OutputSimd128Register(), i.InputSimd128Register(0));
2986 // Zero extend.
2987 __ mov(ip, Operand(0xFFFFFFFF));
2988 __ mtvsrd(kScratchSimd128Reg, ip);
2989 __ vinsertd(kScratchSimd128Reg, kScratchSimd128Reg,
2990 Operand(1 * lane_width_in_bytes));
2991 __ vand(i.OutputSimd128Register(), kScratchSimd128Reg,
2992 i.OutputSimd128Register());
2993 break;
2994 }
2995 case kPPC_I64x2UConvertI32x4High: {
2996 constexpr int lane_width_in_bytes = 8;
2997 __ vupkhsw(i.OutputSimd128Register(), i.InputSimd128Register(0));
2998 // Zero extend.
2999 __ mov(ip, Operand(0xFFFFFFFF));
3000 __ mtvsrd(kScratchSimd128Reg, ip);
3001 __ vinsertd(kScratchSimd128Reg, kScratchSimd128Reg,
3002 Operand(1 * lane_width_in_bytes));
3003 __ vand(i.OutputSimd128Register(), kScratchSimd128Reg,
3004 i.OutputSimd128Register());
3005 break;
3006 }
3007
3008 case kPPC_I32x4SConvertI16x8Low: {
3009 __ vupklsh(i.OutputSimd128Register(), i.InputSimd128Register(0));
3010 break;
3011 }
3012 case kPPC_I32x4SConvertI16x8High: {
3013 __ vupkhsh(i.OutputSimd128Register(), i.InputSimd128Register(0));
3014 break;
3015 }
3016 case kPPC_I32x4UConvertI16x8Low: {
3017 __ vupklsh(i.OutputSimd128Register(), i.InputSimd128Register(0));
3018 // Zero extend.
3019 __ mov(ip, Operand(0xFFFF));
3020 __ mtvsrd(kScratchSimd128Reg, ip);
3021 __ vspltw(kScratchSimd128Reg, kScratchSimd128Reg, Operand(1));
3022 __ vand(i.OutputSimd128Register(), kScratchSimd128Reg,
3023 i.OutputSimd128Register());
3024 break;
3025 }
3026 case kPPC_I32x4UConvertI16x8High: {
3027 __ vupkhsh(i.OutputSimd128Register(), i.InputSimd128Register(0));
3028 // Zero extend.
3029 __ mov(ip, Operand(0xFFFF));
3030 __ mtvsrd(kScratchSimd128Reg, ip);
3031 __ vspltw(kScratchSimd128Reg, kScratchSimd128Reg, Operand(1));
3032 __ vand(i.OutputSimd128Register(), kScratchSimd128Reg,
3033 i.OutputSimd128Register());
3034 break;
3035 }
3036
3037 case kPPC_I16x8SConvertI8x16Low: {
3038 __ vupklsb(i.OutputSimd128Register(), i.InputSimd128Register(0));
3039 break;
3040 }
3041 case kPPC_I16x8SConvertI8x16High: {
3042 __ vupkhsb(i.OutputSimd128Register(), i.InputSimd128Register(0));
3043 break;
3044 }
3045 case kPPC_I16x8UConvertI8x16Low: {
3046 __ vupklsb(i.OutputSimd128Register(), i.InputSimd128Register(0));
3047 // Zero extend.
3048 __ li(ip, Operand(0xFF));
3049 __ mtvsrd(kScratchSimd128Reg, ip);
3050 __ vsplth(kScratchSimd128Reg, kScratchSimd128Reg, Operand(3));
3051 __ vand(i.OutputSimd128Register(), kScratchSimd128Reg,
3052 i.OutputSimd128Register());
3053 break;
3054 }
3055 case kPPC_I16x8UConvertI8x16High: {
3056 __ vupkhsb(i.OutputSimd128Register(), i.InputSimd128Register(0));
3057 // Zero extend.
3058 __ li(ip, Operand(0xFF));
3059 __ mtvsrd(kScratchSimd128Reg, ip);
3060 __ vsplth(kScratchSimd128Reg, kScratchSimd128Reg, Operand(3));
3061 __ vand(i.OutputSimd128Register(), kScratchSimd128Reg,
3062 i.OutputSimd128Register());
3063 break;
3064 }
3065 case kPPC_I16x8SConvertI32x4: {
3066 __ vpkswss(i.OutputSimd128Register(), i.InputSimd128Register(1),
3067 i.InputSimd128Register(0));
3068 break;
3069 }
3070 case kPPC_I16x8UConvertI32x4: {
3071 __ vpkswus(i.OutputSimd128Register(), i.InputSimd128Register(1),
3072 i.InputSimd128Register(0));
3073 break;
3074 }
3075 case kPPC_I8x16SConvertI16x8: {
3076 __ vpkshss(i.OutputSimd128Register(), i.InputSimd128Register(1),
3077 i.InputSimd128Register(0));
3078 break;
3079 }
3080 case kPPC_I8x16UConvertI16x8: {
3081 __ vpkshus(i.OutputSimd128Register(), i.InputSimd128Register(1),
3082 i.InputSimd128Register(0));
3083 break;
3084 }
3085 case kPPC_I8x16Shuffle: {
3086 Simd128Register dst = i.OutputSimd128Register(),
3087 src0 = i.InputSimd128Register(0),
3088 src1 = i.InputSimd128Register(1);
3089 uint64_t low = make_uint64(i.InputUint32(3), i.InputUint32(2));
3090 uint64_t high = make_uint64(i.InputUint32(5), i.InputUint32(4));
3091 __ mov(r0, Operand(low));
3092 __ mov(ip, Operand(high));
3093 __ mtvsrdd(dst, ip, r0);
3094 __ vperm(dst, src0, src1, dst);
3095 break;
3096 }
3097 case kPPC_I16x8AddSatS: {
3098 __ vaddshs(i.OutputSimd128Register(), i.InputSimd128Register(0),
3099 i.InputSimd128Register(1));
3100 break;
3101 }
3102 case kPPC_I16x8SubSatS: {
3103 __ vsubshs(i.OutputSimd128Register(), i.InputSimd128Register(0),
3104 i.InputSimd128Register(1));
3105 break;
3106 }
3107 case kPPC_I16x8AddSatU: {
3108 __ vadduhs(i.OutputSimd128Register(), i.InputSimd128Register(0),
3109 i.InputSimd128Register(1));
3110 break;
3111 }
3112 case kPPC_I16x8SubSatU: {
3113 __ vsubuhs(i.OutputSimd128Register(), i.InputSimd128Register(0),
3114 i.InputSimd128Register(1));
3115 break;
3116 }
3117 case kPPC_I8x16AddSatS: {
3118 __ vaddsbs(i.OutputSimd128Register(), i.InputSimd128Register(0),
3119 i.InputSimd128Register(1));
3120 break;
3121 }
3122 case kPPC_I8x16SubSatS: {
3123 __ vsubsbs(i.OutputSimd128Register(), i.InputSimd128Register(0),
3124 i.InputSimd128Register(1));
3125 break;
3126 }
3127 case kPPC_I8x16AddSatU: {
3128 __ vaddubs(i.OutputSimd128Register(), i.InputSimd128Register(0),
3129 i.InputSimd128Register(1));
3130 break;
3131 }
3132 case kPPC_I8x16SubSatU: {
3133 __ vsububs(i.OutputSimd128Register(), i.InputSimd128Register(0),
3134 i.InputSimd128Register(1));
3135 break;
3136 }
3137 case kPPC_I8x16Swizzle: {
3138 Simd128Register dst = i.OutputSimd128Register(),
3139 src0 = i.InputSimd128Register(0),
3140 src1 = i.InputSimd128Register(1),
3141 tempFPReg1 = i.ToSimd128Register(instr->TempAt(0));
3142 // Saturate the indices to 5 bits. Input indices more than 31 should
3143 // return 0.
3144 __ xxspltib(tempFPReg1, Operand(31));
3145 __ vminub(tempFPReg1, src1, tempFPReg1);
3146 // input needs to be reversed.
3147 __ xxbrq(dst, src0);
3148 __ vxor(kScratchSimd128Reg, kScratchSimd128Reg, kScratchSimd128Reg);
3149 __ vperm(dst, dst, kScratchSimd128Reg, tempFPReg1);
3150 break;
3151 }
3152 case kPPC_F64x2Qfma: {
3153 Simd128Register src0 = i.InputSimd128Register(0);
3154 Simd128Register src1 = i.InputSimd128Register(1);
3155 Simd128Register src2 = i.InputSimd128Register(2);
3156 Simd128Register dst = i.OutputSimd128Register();
3157 __ vor(kScratchSimd128Reg, src1, src1);
3158 __ xvmaddmdp(kScratchSimd128Reg, src2, src0);
3159 __ vor(dst, kScratchSimd128Reg, kScratchSimd128Reg);
3160 break;
3161 }
3162 case kPPC_F64x2Qfms: {
3163 Simd128Register src0 = i.InputSimd128Register(0);
3164 Simd128Register src1 = i.InputSimd128Register(1);
3165 Simd128Register src2 = i.InputSimd128Register(2);
3166 Simd128Register dst = i.OutputSimd128Register();
3167 __ vor(kScratchSimd128Reg, src1, src1);
3168 __ xvnmsubmdp(kScratchSimd128Reg, src2, src0);
3169 __ vor(dst, kScratchSimd128Reg, kScratchSimd128Reg);
3170 break;
3171 }
3172 case kPPC_F32x4Qfma: {
3173 Simd128Register src0 = i.InputSimd128Register(0);
3174 Simd128Register src1 = i.InputSimd128Register(1);
3175 Simd128Register src2 = i.InputSimd128Register(2);
3176 Simd128Register dst = i.OutputSimd128Register();
3177 __ vor(kScratchSimd128Reg, src1, src1);
3178 __ xvmaddmsp(kScratchSimd128Reg, src2, src0);
3179 __ vor(dst, kScratchSimd128Reg, kScratchSimd128Reg);
3180 break;
3181 }
3182 case kPPC_F32x4Qfms: {
3183 Simd128Register src0 = i.InputSimd128Register(0);
3184 Simd128Register src1 = i.InputSimd128Register(1);
3185 Simd128Register src2 = i.InputSimd128Register(2);
3186 Simd128Register dst = i.OutputSimd128Register();
3187 __ vor(kScratchSimd128Reg, src1, src1);
3188 __ xvnmsubmsp(kScratchSimd128Reg, src2, src0);
3189 __ vor(dst, kScratchSimd128Reg, kScratchSimd128Reg);
3190 break;
3191 }
3192 case kPPC_I16x8RoundingAverageU: {
3193 __ vavguh(i.OutputSimd128Register(), i.InputSimd128Register(0),
3194 i.InputSimd128Register(1));
3195 break;
3196 }
3197 case kPPC_I8x16RoundingAverageU: {
3198 __ vavgub(i.OutputSimd128Register(), i.InputSimd128Register(0),
3199 i.InputSimd128Register(1));
3200 break;
3201 }
3202 case kPPC_S128AndNot: {
3203 Simd128Register dst = i.OutputSimd128Register();
3204 Simd128Register src = i.InputSimd128Register(0);
3205 __ vandc(dst, src, i.InputSimd128Register(1));
3206 break;
3207 }
3208 case kPPC_F64x2Div: {
3209 __ xvdivdp(i.OutputSimd128Register(), i.InputSimd128Register(0),
3210 i.InputSimd128Register(1));
3211 break;
3212 }
3213 #define F64X2_MIN_MAX_NAN(result) \
3214 Simd128Register tempFPReg1 = i.ToSimd128Register(instr->TempAt(0)); \
3215 __ xvcmpeqdp(tempFPReg1, i.InputSimd128Register(0), \
3216 i.InputSimd128Register(0)); \
3217 __ vsel(result, i.InputSimd128Register(0), result, tempFPReg1); \
3218 __ xvcmpeqdp(tempFPReg1, i.InputSimd128Register(1), \
3219 i.InputSimd128Register(1)); \
3220 __ vsel(i.OutputSimd128Register(), i.InputSimd128Register(1), result, \
3221 tempFPReg1); \
3222 /* Use xvmindp to turn any selected SNANs to QNANs. */ \
3223 __ xvmindp(i.OutputSimd128Register(), i.OutputSimd128Register(), \
3224 i.OutputSimd128Register());
3225 case kPPC_F64x2Min: {
3226 __ xvmindp(kScratchSimd128Reg, i.InputSimd128Register(0),
3227 i.InputSimd128Register(1));
3228 // We need to check if an input is NAN and preserve it.
3229 F64X2_MIN_MAX_NAN(kScratchSimd128Reg)
3230 break;
3231 }
3232 case kPPC_F64x2Max: {
3233 __ xvmaxdp(kScratchSimd128Reg, i.InputSimd128Register(0),
3234 i.InputSimd128Register(1));
3235 // We need to check if an input is NAN and preserve it.
3236 F64X2_MIN_MAX_NAN(kScratchSimd128Reg)
3237 break;
3238 }
3239 #undef F64X2_MIN_MAX_NAN
3240 case kPPC_F32x4Div: {
3241 __ xvdivsp(i.OutputSimd128Register(), i.InputSimd128Register(0),
3242 i.InputSimd128Register(1));
3243 break;
3244 }
3245 case kPPC_F32x4Min: {
3246 __ vminfp(i.OutputSimd128Register(), i.InputSimd128Register(0),
3247 i.InputSimd128Register(1));
3248 break;
3249 }
3250 case kPPC_F32x4Max: {
3251 __ vmaxfp(i.OutputSimd128Register(), i.InputSimd128Register(0),
3252 i.InputSimd128Register(1));
3253 break;
3254 }
3255 case kPPC_F64x2Ceil: {
3256 __ xvrdpip(i.OutputSimd128Register(), i.InputSimd128Register(0));
3257 break;
3258 }
3259 case kPPC_F64x2Floor: {
3260 __ xvrdpim(i.OutputSimd128Register(), i.InputSimd128Register(0));
3261 break;
3262 }
3263 case kPPC_F64x2Trunc: {
3264 __ xvrdpiz(i.OutputSimd128Register(), i.InputSimd128Register(0));
3265 break;
3266 }
3267 case kPPC_F32x4Ceil: {
3268 __ xvrspip(i.OutputSimd128Register(), i.InputSimd128Register(0));
3269 break;
3270 }
3271 case kPPC_F32x4Floor: {
3272 __ xvrspim(i.OutputSimd128Register(), i.InputSimd128Register(0));
3273 break;
3274 }
3275 case kPPC_F32x4Trunc: {
3276 __ xvrspiz(i.OutputSimd128Register(), i.InputSimd128Register(0));
3277 break;
3278 }
3279 case kPPC_I64x2BitMask: {
3280 if (CpuFeatures::IsSupported(PPC_10_PLUS)) {
3281 __ vextractdm(i.OutputRegister(), i.InputSimd128Register(0));
3282 } else {
3283 __ mov(kScratchReg,
3284 Operand(0x8080808080800040)); // Select 0 for the high bits.
3285 __ mtvsrd(kScratchSimd128Reg, kScratchReg);
3286 __ vbpermq(kScratchSimd128Reg, i.InputSimd128Register(0),
3287 kScratchSimd128Reg);
3288 __ vextractub(kScratchSimd128Reg, kScratchSimd128Reg, Operand(6));
3289 __ mfvsrd(i.OutputRegister(), kScratchSimd128Reg);
3290 }
3291 break;
3292 }
3293 case kPPC_I32x4BitMask: {
3294 if (CpuFeatures::IsSupported(PPC_10_PLUS)) {
3295 __ vextractwm(i.OutputRegister(), i.InputSimd128Register(0));
3296 } else {
3297 __ mov(kScratchReg,
3298 Operand(0x8080808000204060)); // Select 0 for the high bits.
3299 __ mtvsrd(kScratchSimd128Reg, kScratchReg);
3300 __ vbpermq(kScratchSimd128Reg, i.InputSimd128Register(0),
3301 kScratchSimd128Reg);
3302 __ vextractub(kScratchSimd128Reg, kScratchSimd128Reg, Operand(6));
3303 __ mfvsrd(i.OutputRegister(), kScratchSimd128Reg);
3304 }
3305 break;
3306 }
3307 case kPPC_I16x8BitMask: {
3308 if (CpuFeatures::IsSupported(PPC_10_PLUS)) {
3309 __ vextracthm(i.OutputRegister(), i.InputSimd128Register(0));
3310 } else {
3311 __ mov(kScratchReg, Operand(0x10203040506070));
3312 __ mtvsrd(kScratchSimd128Reg, kScratchReg);
3313 __ vbpermq(kScratchSimd128Reg, i.InputSimd128Register(0),
3314 kScratchSimd128Reg);
3315 __ vextractub(kScratchSimd128Reg, kScratchSimd128Reg, Operand(6));
3316 __ mfvsrd(i.OutputRegister(), kScratchSimd128Reg);
3317 }
3318 break;
3319 }
3320 case kPPC_I8x16BitMask: {
3321 if (CpuFeatures::IsSupported(PPC_10_PLUS)) {
3322 __ vextractbm(i.OutputRegister(), i.InputSimd128Register(0));
3323 } else {
3324 Register temp = i.ToRegister(instr->TempAt(0));
3325 __ mov(temp, Operand(0x8101820283038));
3326 __ mov(ip, Operand(0x4048505860687078));
3327 __ mtvsrdd(kScratchSimd128Reg, temp, ip);
3328 __ vbpermq(kScratchSimd128Reg, i.InputSimd128Register(0),
3329 kScratchSimd128Reg);
3330 __ vextractuh(kScratchSimd128Reg, kScratchSimd128Reg, Operand(6));
3331 __ mfvsrd(i.OutputRegister(), kScratchSimd128Reg);
3332 }
3333 break;
3334 }
3335 case kPPC_I32x4DotI16x8S: {
3336 __ vxor(kScratchSimd128Reg, kScratchSimd128Reg, kScratchSimd128Reg);
3337 __ vmsumshm(i.OutputSimd128Register(), i.InputSimd128Register(0),
3338 i.InputSimd128Register(1), kScratchSimd128Reg);
3339 break;
3340 }
3341 case kPPC_F32x4Pmin: {
3342 Simd128Register dst = i.OutputSimd128Register(),
3343 src0 = i.InputSimd128Register(0),
3344 src1 = i.InputSimd128Register(1);
3345 __ xvcmpgtsp(kScratchSimd128Reg, src0, src1);
3346 __ vsel(dst, src0, src1, kScratchSimd128Reg);
3347 break;
3348 }
3349 case kPPC_F32x4Pmax: {
3350 Simd128Register dst = i.OutputSimd128Register(),
3351 src0 = i.InputSimd128Register(0),
3352 src1 = i.InputSimd128Register(1);
3353 __ xvcmpgtsp(kScratchSimd128Reg, src1, src0);
3354 __ vsel(dst, src0, src1, kScratchSimd128Reg);
3355 break;
3356 }
3357 case kPPC_F64x2Pmin: {
3358 Simd128Register dst = i.OutputSimd128Register(),
3359 src0 = i.InputSimd128Register(0),
3360 src1 = i.InputSimd128Register(1);
3361 __ xvcmpgtdp(kScratchSimd128Reg, src0, src1);
3362 __ vsel(dst, src0, src1, kScratchSimd128Reg);
3363 break;
3364 }
3365 case kPPC_F64x2Pmax: {
3366 Simd128Register dst = i.OutputSimd128Register(),
3367 src0 = i.InputSimd128Register(0),
3368 src1 = i.InputSimd128Register(1);
3369 __ xvcmpgtdp(kScratchSimd128Reg, src1, src0);
3370 __ vsel(dst, src0, src1, kScratchSimd128Reg);
3371 break;
3372 }
3373 #define ASSEMBLE_LOAD_TRANSFORM(scratch, load_instr) \
3374 AddressingMode mode = kMode_None; \
3375 MemOperand operand = i.MemoryOperand(&mode); \
3376 DCHECK_EQ(mode, kMode_MRR); \
3377 __ load_instr(scratch, operand);
3378 #if V8_TARGET_BIG_ENDIAN
3379 #define MAYBE_REVERSE_BYTES(reg, instr) __ instr(reg, reg);
3380 #else
3381 #define MAYBE_REVERSE_BYTES(reg, instr)
3382 #endif
3383 case kPPC_S128Load8Splat: {
3384 Simd128Register dst = i.OutputSimd128Register();
3385 ASSEMBLE_LOAD_TRANSFORM(kScratchSimd128Reg, lxsibzx)
3386 __ vspltb(dst, kScratchSimd128Reg, Operand(7));
3387 break;
3388 }
3389 case kPPC_S128Load16Splat: {
3390 Simd128Register dst = i.OutputSimd128Register();
3391 ASSEMBLE_LOAD_TRANSFORM(kScratchSimd128Reg, lxsihzx)
3392 MAYBE_REVERSE_BYTES(kScratchSimd128Reg, xxbrh)
3393 __ vsplth(dst, kScratchSimd128Reg, Operand(3));
3394 break;
3395 }
3396 case kPPC_S128Load32Splat: {
3397 Simd128Register dst = i.OutputSimd128Register();
3398 ASSEMBLE_LOAD_TRANSFORM(kScratchSimd128Reg, lxsiwzx)
3399 MAYBE_REVERSE_BYTES(kScratchSimd128Reg, xxbrw)
3400 __ vspltw(dst, kScratchSimd128Reg, Operand(1));
3401 break;
3402 }
3403 case kPPC_S128Load64Splat: {
3404 constexpr int lane_width_in_bytes = 8;
3405 Simd128Register dst = i.OutputSimd128Register();
3406 ASSEMBLE_LOAD_TRANSFORM(dst, lxsdx)
3407 MAYBE_REVERSE_BYTES(dst, xxbrd)
3408 __ vinsertd(dst, dst, Operand(1 * lane_width_in_bytes));
3409 break;
3410 }
3411 case kPPC_S128Load8x8S: {
3412 Simd128Register dst = i.OutputSimd128Register();
3413 ASSEMBLE_LOAD_TRANSFORM(kScratchSimd128Reg, lxsdx)
3414 MAYBE_REVERSE_BYTES(kScratchSimd128Reg, xxbrd)
3415 __ vupkhsb(dst, kScratchSimd128Reg);
3416 break;
3417 }
3418 case kPPC_S128Load8x8U: {
3419 Simd128Register dst = i.OutputSimd128Register();
3420 ASSEMBLE_LOAD_TRANSFORM(kScratchSimd128Reg, lxsdx)
3421 MAYBE_REVERSE_BYTES(kScratchSimd128Reg, xxbrd)
3422 __ vupkhsb(dst, kScratchSimd128Reg);
3423 // Zero extend.
3424 __ li(ip, Operand(0xFF));
3425 __ mtvsrd(kScratchSimd128Reg, ip);
3426 __ vsplth(kScratchSimd128Reg, kScratchSimd128Reg, Operand(3));
3427 __ vand(dst, kScratchSimd128Reg, dst);
3428 break;
3429 }
3430 case kPPC_S128Load16x4S: {
3431 Simd128Register dst = i.OutputSimd128Register();
3432 ASSEMBLE_LOAD_TRANSFORM(kScratchSimd128Reg, lxsdx)
3433 MAYBE_REVERSE_BYTES(kScratchSimd128Reg, xxbrd)
3434 __ vupkhsh(dst, kScratchSimd128Reg);
3435 break;
3436 }
3437 case kPPC_S128Load16x4U: {
3438 Simd128Register dst = i.OutputSimd128Register();
3439 ASSEMBLE_LOAD_TRANSFORM(kScratchSimd128Reg, lxsdx)
3440 MAYBE_REVERSE_BYTES(kScratchSimd128Reg, xxbrd)
3441 __ vupkhsh(dst, kScratchSimd128Reg);
3442 // Zero extend.
3443 __ mov(ip, Operand(0xFFFF));
3444 __ mtvsrd(kScratchSimd128Reg, ip);
3445 __ vspltw(kScratchSimd128Reg, kScratchSimd128Reg, Operand(1));
3446 __ vand(dst, kScratchSimd128Reg, dst);
3447
3448 break;
3449 }
3450 case kPPC_S128Load32x2S: {
3451 Simd128Register dst = i.OutputSimd128Register();
3452 ASSEMBLE_LOAD_TRANSFORM(kScratchSimd128Reg, lxsdx)
3453 MAYBE_REVERSE_BYTES(kScratchSimd128Reg, xxbrd)
3454 __ vupkhsw(dst, kScratchSimd128Reg);
3455 break;
3456 }
3457 case kPPC_S128Load32x2U: {
3458 constexpr int lane_width_in_bytes = 8;
3459 Simd128Register dst = i.OutputSimd128Register();
3460 ASSEMBLE_LOAD_TRANSFORM(kScratchSimd128Reg, lxsdx)
3461 MAYBE_REVERSE_BYTES(kScratchSimd128Reg, xxbrd)
3462 __ vupkhsw(dst, kScratchSimd128Reg);
3463 // Zero extend.
3464 __ mov(ip, Operand(0xFFFFFFFF));
3465 __ mtvsrd(kScratchSimd128Reg, ip);
3466 __ vinsertd(kScratchSimd128Reg, kScratchSimd128Reg,
3467 Operand(1 * lane_width_in_bytes));
3468 __ vand(dst, kScratchSimd128Reg, dst);
3469 break;
3470 }
3471 case kPPC_S128Load32Zero: {
3472 constexpr int lane_width_in_bytes = 4;
3473 Simd128Register dst = i.OutputSimd128Register();
3474 ASSEMBLE_LOAD_TRANSFORM(kScratchSimd128Reg, lxsiwzx)
3475 MAYBE_REVERSE_BYTES(kScratchSimd128Reg, xxbrw)
3476 __ vxor(dst, dst, dst);
3477 __ vinsertw(dst, kScratchSimd128Reg, Operand(3 * lane_width_in_bytes));
3478 break;
3479 }
3480 case kPPC_S128Load64Zero: {
3481 constexpr int lane_width_in_bytes = 8;
3482 Simd128Register dst = i.OutputSimd128Register();
3483 ASSEMBLE_LOAD_TRANSFORM(kScratchSimd128Reg, lxsdx)
3484 MAYBE_REVERSE_BYTES(kScratchSimd128Reg, xxbrd)
3485 __ vxor(dst, dst, dst);
3486 __ vinsertd(dst, kScratchSimd128Reg, Operand(1 * lane_width_in_bytes));
3487 break;
3488 }
3489 #undef ASSEMBLE_LOAD_TRANSFORM
3490 case kPPC_S128Load8Lane: {
3491 Simd128Register dst = i.OutputSimd128Register();
3492 DCHECK_EQ(dst, i.InputSimd128Register(0));
3493 AddressingMode mode = kMode_None;
3494 size_t index = 1;
3495 MemOperand operand = i.MemoryOperand(&mode, &index);
3496 DCHECK_EQ(mode, kMode_MRR);
3497 __ lxsibzx(kScratchSimd128Reg, operand);
3498 __ vinsertb(dst, kScratchSimd128Reg, Operand(15 - i.InputUint8(3)));
3499 break;
3500 }
3501 case kPPC_S128Load16Lane: {
3502 Simd128Register dst = i.OutputSimd128Register();
3503 DCHECK_EQ(dst, i.InputSimd128Register(0));
3504 constexpr int lane_width_in_bytes = 2;
3505 AddressingMode mode = kMode_None;
3506 size_t index = 1;
3507 MemOperand operand = i.MemoryOperand(&mode, &index);
3508 DCHECK_EQ(mode, kMode_MRR);
3509 __ lxsihzx(kScratchSimd128Reg, operand);
3510 MAYBE_REVERSE_BYTES(kScratchSimd128Reg, xxbrh)
3511 __ vinserth(dst, kScratchSimd128Reg,
3512 Operand((7 - i.InputUint8(3)) * lane_width_in_bytes));
3513 break;
3514 }
3515 case kPPC_S128Load32Lane: {
3516 Simd128Register dst = i.OutputSimd128Register();
3517 DCHECK_EQ(dst, i.InputSimd128Register(0));
3518 constexpr int lane_width_in_bytes = 4;
3519 AddressingMode mode = kMode_None;
3520 size_t index = 1;
3521 MemOperand operand = i.MemoryOperand(&mode, &index);
3522 DCHECK_EQ(mode, kMode_MRR);
3523 __ lxsiwzx(kScratchSimd128Reg, operand);
3524 MAYBE_REVERSE_BYTES(kScratchSimd128Reg, xxbrw)
3525 __ vinsertw(dst, kScratchSimd128Reg,
3526 Operand((3 - i.InputUint8(3)) * lane_width_in_bytes));
3527 break;
3528 }
3529 case kPPC_S128Load64Lane: {
3530 Simd128Register dst = i.OutputSimd128Register();
3531 DCHECK_EQ(dst, i.InputSimd128Register(0));
3532 constexpr int lane_width_in_bytes = 8;
3533 AddressingMode mode = kMode_None;
3534 size_t index = 1;
3535 MemOperand operand = i.MemoryOperand(&mode, &index);
3536 DCHECK_EQ(mode, kMode_MRR);
3537 __ lxsdx(kScratchSimd128Reg, operand);
3538 MAYBE_REVERSE_BYTES(kScratchSimd128Reg, xxbrd)
3539 __ vinsertd(dst, kScratchSimd128Reg,
3540 Operand((1 - i.InputUint8(3)) * lane_width_in_bytes));
3541 break;
3542 }
3543 case kPPC_S128Store8Lane: {
3544 AddressingMode mode = kMode_None;
3545 size_t index = 1;
3546 MemOperand operand = i.MemoryOperand(&mode, &index);
3547 DCHECK_EQ(mode, kMode_MRR);
3548 __ vextractub(kScratchSimd128Reg, i.InputSimd128Register(0),
3549 Operand(15 - i.InputUint8(3)));
3550 __ stxsibx(kScratchSimd128Reg, operand);
3551 break;
3552 }
3553 case kPPC_S128Store16Lane: {
3554 AddressingMode mode = kMode_None;
3555 constexpr int lane_width_in_bytes = 2;
3556 size_t index = 1;
3557 MemOperand operand = i.MemoryOperand(&mode, &index);
3558 DCHECK_EQ(mode, kMode_MRR);
3559 __ vextractuh(kScratchSimd128Reg, i.InputSimd128Register(0),
3560 Operand((7 - i.InputUint8(3)) * lane_width_in_bytes));
3561 MAYBE_REVERSE_BYTES(kScratchSimd128Reg, xxbrh)
3562 __ stxsihx(kScratchSimd128Reg, operand);
3563 break;
3564 }
3565 case kPPC_S128Store32Lane: {
3566 AddressingMode mode = kMode_None;
3567 constexpr int lane_width_in_bytes = 4;
3568 size_t index = 1;
3569 MemOperand operand = i.MemoryOperand(&mode, &index);
3570 DCHECK_EQ(mode, kMode_MRR);
3571 __ vextractuw(kScratchSimd128Reg, i.InputSimd128Register(0),
3572 Operand((3 - i.InputUint8(3)) * lane_width_in_bytes));
3573 MAYBE_REVERSE_BYTES(kScratchSimd128Reg, xxbrw)
3574 __ stxsiwx(kScratchSimd128Reg, operand);
3575 break;
3576 }
3577 case kPPC_S128Store64Lane: {
3578 AddressingMode mode = kMode_None;
3579 constexpr int lane_width_in_bytes = 8;
3580 size_t index = 1;
3581 MemOperand operand = i.MemoryOperand(&mode, &index);
3582 DCHECK_EQ(mode, kMode_MRR);
3583 __ vextractd(kScratchSimd128Reg, i.InputSimd128Register(0),
3584 Operand((1 - i.InputUint8(3)) * lane_width_in_bytes));
3585 MAYBE_REVERSE_BYTES(kScratchSimd128Reg, xxbrd)
3586 __ stxsdx(kScratchSimd128Reg, operand);
3587 break;
3588 }
3589 #undef MAYBE_REVERSE_BYTES
3590 #define EXT_ADD_PAIRWISE(mul_even, mul_odd, add) \
3591 __ mul_even(tempFPReg1, src, kScratchSimd128Reg); \
3592 __ mul_odd(kScratchSimd128Reg, src, kScratchSimd128Reg); \
3593 __ add(dst, tempFPReg1, kScratchSimd128Reg);
3594 case kPPC_I32x4ExtAddPairwiseI16x8S: {
3595 Simd128Register src = i.InputSimd128Register(0);
3596 Simd128Register dst = i.OutputSimd128Register();
3597 Simd128Register tempFPReg1 = i.ToSimd128Register(instr->TempAt(0));
3598 __ vspltish(kScratchSimd128Reg, Operand(1));
3599 EXT_ADD_PAIRWISE(vmulesh, vmulosh, vadduwm)
3600 break;
3601 }
3602 case kPPC_I32x4ExtAddPairwiseI16x8U: {
3603 Simd128Register src = i.InputSimd128Register(0);
3604 Simd128Register dst = i.OutputSimd128Register();
3605 Simd128Register tempFPReg1 = i.ToSimd128Register(instr->TempAt(0));
3606 __ vspltish(kScratchSimd128Reg, Operand(1));
3607 EXT_ADD_PAIRWISE(vmuleuh, vmulouh, vadduwm)
3608 break;
3609 }
3610
3611 case kPPC_I16x8ExtAddPairwiseI8x16S: {
3612 Simd128Register src = i.InputSimd128Register(0);
3613 Simd128Register dst = i.OutputSimd128Register();
3614 Simd128Register tempFPReg1 = i.ToSimd128Register(instr->TempAt(0));
3615 __ xxspltib(kScratchSimd128Reg, Operand(1));
3616 EXT_ADD_PAIRWISE(vmulesb, vmulosb, vadduhm)
3617 break;
3618 }
3619 case kPPC_I16x8ExtAddPairwiseI8x16U: {
3620 Simd128Register src = i.InputSimd128Register(0);
3621 Simd128Register dst = i.OutputSimd128Register();
3622 Simd128Register tempFPReg1 = i.ToSimd128Register(instr->TempAt(0));
3623 __ xxspltib(kScratchSimd128Reg, Operand(1));
3624 EXT_ADD_PAIRWISE(vmuleub, vmuloub, vadduhm)
3625 break;
3626 }
3627 #undef EXT_ADD_PAIRWISE
3628 case kPPC_I16x8Q15MulRSatS: {
3629 __ vxor(kScratchSimd128Reg, kScratchSimd128Reg, kScratchSimd128Reg);
3630 __ vmhraddshs(i.OutputSimd128Register(), i.InputSimd128Register(0),
3631 i.InputSimd128Register(1), kScratchSimd128Reg);
3632 break;
3633 }
3634 #define EXT_MUL(mul_even, mul_odd) \
3635 Simd128Register dst = i.OutputSimd128Register(), \
3636 src0 = i.InputSimd128Register(0), \
3637 src1 = i.InputSimd128Register(1); \
3638 __ mul_even(dst, src0, src1); \
3639 __ mul_odd(kScratchSimd128Reg, src0, src1);
3640 case kPPC_I64x2ExtMulLowI32x4S: {
3641 constexpr int lane_width_in_bytes = 8;
3642 EXT_MUL(vmulesw, vmulosw)
3643 __ vextractd(dst, dst, Operand(1 * lane_width_in_bytes));
3644 __ vextractd(kScratchSimd128Reg, kScratchSimd128Reg,
3645 Operand(1 * lane_width_in_bytes));
3646 __ vinsertd(dst, kScratchSimd128Reg, Operand(1 * lane_width_in_bytes));
3647 break;
3648 }
3649 case kPPC_I64x2ExtMulHighI32x4S: {
3650 constexpr int lane_width_in_bytes = 8;
3651 EXT_MUL(vmulesw, vmulosw)
3652 __ vinsertd(dst, kScratchSimd128Reg, Operand(1 * lane_width_in_bytes));
3653 break;
3654 }
3655 case kPPC_I64x2ExtMulLowI32x4U: {
3656 constexpr int lane_width_in_bytes = 8;
3657 EXT_MUL(vmuleuw, vmulouw)
3658 __ vextractd(dst, dst, Operand(1 * lane_width_in_bytes));
3659 __ vextractd(kScratchSimd128Reg, kScratchSimd128Reg,
3660 Operand(1 * lane_width_in_bytes));
3661 __ vinsertd(dst, kScratchSimd128Reg, Operand(1 * lane_width_in_bytes));
3662 break;
3663 }
3664 case kPPC_I64x2ExtMulHighI32x4U: {
3665 constexpr int lane_width_in_bytes = 8;
3666 EXT_MUL(vmuleuw, vmulouw)
3667 __ vinsertd(dst, kScratchSimd128Reg, Operand(1 * lane_width_in_bytes));
3668 break;
3669 }
3670 case kPPC_I32x4ExtMulLowI16x8S: {
3671 EXT_MUL(vmulesh, vmulosh)
3672 __ vmrglw(dst, dst, kScratchSimd128Reg);
3673 break;
3674 }
3675 case kPPC_I32x4ExtMulHighI16x8S: {
3676 EXT_MUL(vmulesh, vmulosh)
3677 __ vmrghw(dst, dst, kScratchSimd128Reg);
3678 break;
3679 }
3680 case kPPC_I32x4ExtMulLowI16x8U: {
3681 EXT_MUL(vmuleuh, vmulouh)
3682 __ vmrglw(dst, dst, kScratchSimd128Reg);
3683 break;
3684 }
3685 case kPPC_I32x4ExtMulHighI16x8U: {
3686 EXT_MUL(vmuleuh, vmulouh)
3687 __ vmrghw(dst, dst, kScratchSimd128Reg);
3688 break;
3689 }
3690 case kPPC_I16x8ExtMulLowI8x16S: {
3691 EXT_MUL(vmulesb, vmulosb)
3692 __ vmrglh(dst, dst, kScratchSimd128Reg);
3693 break;
3694 }
3695 case kPPC_I16x8ExtMulHighI8x16S: {
3696 EXT_MUL(vmulesb, vmulosb)
3697 __ vmrghh(dst, dst, kScratchSimd128Reg);
3698 break;
3699 }
3700 case kPPC_I16x8ExtMulLowI8x16U: {
3701 EXT_MUL(vmuleub, vmuloub)
3702 __ vmrglh(dst, dst, kScratchSimd128Reg);
3703 break;
3704 }
3705 case kPPC_I16x8ExtMulHighI8x16U: {
3706 EXT_MUL(vmuleub, vmuloub)
3707 __ vmrghh(dst, dst, kScratchSimd128Reg);
3708 break;
3709 }
3710 #undef EXT_MUL
3711 case kPPC_F64x2ConvertLowI32x4S: {
3712 __ vupklsw(kScratchSimd128Reg, i.InputSimd128Register(0));
3713 __ xvcvsxddp(i.OutputSimd128Register(), kScratchSimd128Reg);
3714 break;
3715 }
3716 case kPPC_F64x2ConvertLowI32x4U: {
3717 Simd128Register dst = i.OutputSimd128Register();
3718 constexpr int lane_width_in_bytes = 8;
3719 __ vupklsw(dst, i.InputSimd128Register(0));
3720 // Zero extend.
3721 __ mov(ip, Operand(0xFFFFFFFF));
3722 __ mtvsrd(kScratchSimd128Reg, ip);
3723 __ vinsertd(kScratchSimd128Reg, kScratchSimd128Reg,
3724 Operand(1 * lane_width_in_bytes));
3725 __ vand(dst, kScratchSimd128Reg, dst);
3726 __ xvcvuxddp(dst, dst);
3727 break;
3728 }
3729 case kPPC_F64x2PromoteLowF32x4: {
3730 constexpr int lane_number = 8;
3731 Simd128Register src = i.InputSimd128Register(0);
3732 Simd128Register dst = i.OutputSimd128Register();
3733 __ vextractd(kScratchSimd128Reg, src, Operand(lane_number));
3734 __ vinsertw(kScratchSimd128Reg, kScratchSimd128Reg, Operand(lane_number));
3735 __ xvcvspdp(dst, kScratchSimd128Reg);
3736 break;
3737 }
3738 case kPPC_F32x4DemoteF64x2Zero: {
3739 constexpr int lane_number = 8;
3740 Simd128Register src = i.InputSimd128Register(0);
3741 Simd128Register dst = i.OutputSimd128Register();
3742 __ xvcvdpsp(kScratchSimd128Reg, src);
3743 __ vextractuw(dst, kScratchSimd128Reg, Operand(lane_number));
3744 __ vinsertw(kScratchSimd128Reg, dst, Operand(4));
3745 __ vxor(dst, dst, dst);
3746 __ vinsertd(dst, kScratchSimd128Reg, Operand(lane_number));
3747 break;
3748 }
3749 case kPPC_I32x4TruncSatF64x2SZero: {
3750 constexpr int lane_number = 8;
3751 Simd128Register src = i.InputSimd128Register(0);
3752 Simd128Register dst = i.OutputSimd128Register();
3753 // NaN to 0.
3754 __ vor(kScratchSimd128Reg, src, src);
3755 __ xvcmpeqdp(kScratchSimd128Reg, kScratchSimd128Reg, kScratchSimd128Reg);
3756 __ vand(kScratchSimd128Reg, src, kScratchSimd128Reg);
3757 __ xvcvdpsxws(kScratchSimd128Reg, kScratchSimd128Reg);
3758 __ vextractuw(dst, kScratchSimd128Reg, Operand(lane_number));
3759 __ vinsertw(kScratchSimd128Reg, dst, Operand(4));
3760 __ vxor(dst, dst, dst);
3761 __ vinsertd(dst, kScratchSimd128Reg, Operand(lane_number));
3762 break;
3763 }
3764 case kPPC_I32x4TruncSatF64x2UZero: {
3765 constexpr int lane_number = 8;
3766 Simd128Register src = i.InputSimd128Register(0);
3767 Simd128Register dst = i.OutputSimd128Register();
3768 __ xvcvdpuxws(kScratchSimd128Reg, src);
3769 __ vextractuw(dst, kScratchSimd128Reg, Operand(lane_number));
3770 __ vinsertw(kScratchSimd128Reg, dst, Operand(4));
3771 __ vxor(dst, dst, dst);
3772 __ vinsertd(dst, kScratchSimd128Reg, Operand(lane_number));
3773 break;
3774 }
3775 case kPPC_I8x16Popcnt: {
3776 __ vpopcntb(i.OutputSimd128Register(), i.InputSimd128Register(0));
3777 break;
3778 }
3779 case kPPC_StoreCompressTagged: {
3780 ASSEMBLE_STORE_INTEGER(StoreTaggedField, StoreTaggedField);
3781 break;
3782 }
3783 case kPPC_LoadDecompressTaggedSigned: {
3784 CHECK(instr->HasOutput());
3785 ASSEMBLE_LOAD_INTEGER(lwz, plwz, lwzx);
3786 break;
3787 }
3788 case kPPC_LoadDecompressTaggedPointer: {
3789 CHECK(instr->HasOutput());
3790 ASSEMBLE_LOAD_INTEGER(lwz, plwz, lwzx);
3791 __ add(i.OutputRegister(), i.OutputRegister(), kRootRegister);
3792 break;
3793 }
3794 case kPPC_LoadDecompressAnyTagged: {
3795 CHECK(instr->HasOutput());
3796 ASSEMBLE_LOAD_INTEGER(lwz, plwz, lwzx);
3797 __ add(i.OutputRegister(), i.OutputRegister(), kRootRegister);
3798 break;
3799 }
3800 default:
3801 UNREACHABLE();
3802 }
3803 return kSuccess;
3804 }
3805
3806 // Assembles branches after an instruction.
AssembleArchBranch(Instruction* instr, BranchInfo* branch)3807 void CodeGenerator::AssembleArchBranch(Instruction* instr, BranchInfo* branch) {
3808 PPCOperandConverter i(this, instr);
3809 Label* tlabel = branch->true_label;
3810 Label* flabel = branch->false_label;
3811 ArchOpcode op = instr->arch_opcode();
3812 FlagsCondition condition = branch->condition;
3813 CRegister cr = cr0;
3814
3815 Condition cond = FlagsConditionToCondition(condition, op);
3816 if (op == kPPC_CmpDouble) {
3817 // check for unordered if necessary
3818 if (cond == le) {
3819 __ bunordered(flabel, cr);
3820 // Unnecessary for eq/lt since only FU bit will be set.
3821 } else if (cond == gt) {
3822 __ bunordered(tlabel, cr);
3823 // Unnecessary for ne/ge since only FU bit will be set.
3824 }
3825 }
3826 __ b(cond, tlabel, cr);
3827 if (!branch->fallthru) __ b(flabel); // no fallthru to flabel.
3828 }
3829
AssembleArchDeoptBranch(Instruction* instr, BranchInfo* branch)3830 void CodeGenerator::AssembleArchDeoptBranch(Instruction* instr,
3831 BranchInfo* branch) {
3832 AssembleArchBranch(instr, branch);
3833 }
3834
AssembleArchJumpRegardlessOfAssemblyOrder( RpoNumber target)3835 void CodeGenerator::AssembleArchJumpRegardlessOfAssemblyOrder(
3836 RpoNumber target) {
3837 __ b(GetLabel(target));
3838 }
3839
3840 #if V8_ENABLE_WEBASSEMBLY
AssembleArchTrap(Instruction* instr, FlagsCondition condition)3841 void CodeGenerator::AssembleArchTrap(Instruction* instr,
3842 FlagsCondition condition) {
3843 class OutOfLineTrap final : public OutOfLineCode {
3844 public:
3845 OutOfLineTrap(CodeGenerator* gen, Instruction* instr)
3846 : OutOfLineCode(gen), instr_(instr), gen_(gen) {}
3847
3848 void Generate() final {
3849 PPCOperandConverter i(gen_, instr_);
3850 TrapId trap_id =
3851 static_cast<TrapId>(i.InputInt32(instr_->InputCount() - 1));
3852 GenerateCallToTrap(trap_id);
3853 }
3854
3855 private:
3856 void GenerateCallToTrap(TrapId trap_id) {
3857 if (trap_id == TrapId::kInvalid) {
3858 // We cannot test calls to the runtime in cctest/test-run-wasm.
3859 // Therefore we emit a call to C here instead of a call to the runtime.
3860 // We use the context register as the scratch register, because we do
3861 // not have a context here.
3862 __ PrepareCallCFunction(0, 0, cp);
3863 __ CallCFunction(
3864 ExternalReference::wasm_call_trap_callback_for_testing(), 0);
3865 __ LeaveFrame(StackFrame::WASM);
3866 auto call_descriptor = gen_->linkage()->GetIncomingDescriptor();
3867 int pop_count = static_cast<int>(call_descriptor->ParameterSlotCount());
3868 __ Drop(pop_count);
3869 __ Ret();
3870 } else {
3871 gen_->AssembleSourcePosition(instr_);
3872 // A direct call to a wasm runtime stub defined in this module.
3873 // Just encode the stub index. This will be patched when the code
3874 // is added to the native module and copied into wasm code space.
3875 __ Call(static_cast<Address>(trap_id), RelocInfo::WASM_STUB_CALL);
3876 ReferenceMap* reference_map =
3877 gen_->zone()->New<ReferenceMap>(gen_->zone());
3878 gen_->RecordSafepoint(reference_map);
3879 if (FLAG_debug_code) {
3880 __ stop();
3881 }
3882 }
3883 }
3884
3885 Instruction* instr_;
3886 CodeGenerator* gen_;
3887 };
3888 auto ool = zone()->New<OutOfLineTrap>(this, instr);
3889 Label* tlabel = ool->entry();
3890 Label end;
3891
3892 ArchOpcode op = instr->arch_opcode();
3893 CRegister cr = cr0;
3894 Condition cond = FlagsConditionToCondition(condition, op);
3895 if (op == kPPC_CmpDouble) {
3896 // check for unordered if necessary
3897 if (cond == le) {
3898 __ bunordered(&end, cr);
3899 // Unnecessary for eq/lt since only FU bit will be set.
3900 } else if (cond == gt) {
3901 __ bunordered(tlabel, cr);
3902 // Unnecessary for ne/ge since only FU bit will be set.
3903 }
3904 }
3905 __ b(cond, tlabel, cr);
3906 __ bind(&end);
3907 }
3908 #endif // V8_ENABLE_WEBASSEMBLY
3909
3910 // Assembles boolean materializations after an instruction.
AssembleArchBoolean(Instruction* instr, FlagsCondition condition)3911 void CodeGenerator::AssembleArchBoolean(Instruction* instr,
3912 FlagsCondition condition) {
3913 PPCOperandConverter i(this, instr);
3914 Label done;
3915 ArchOpcode op = instr->arch_opcode();
3916 CRegister cr = cr0;
3917 int reg_value = -1;
3918
3919 // Materialize a full 32-bit 1 or 0 value. The result register is always the
3920 // last output of the instruction.
3921 DCHECK_NE(0u, instr->OutputCount());
3922 Register reg = i.OutputRegister(instr->OutputCount() - 1);
3923
3924 Condition cond = FlagsConditionToCondition(condition, op);
3925 if (op == kPPC_CmpDouble) {
3926 // check for unordered if necessary
3927 if (cond == le) {
3928 reg_value = 0;
3929 __ li(reg, Operand::Zero());
3930 __ bunordered(&done, cr);
3931 } else if (cond == gt) {
3932 reg_value = 1;
3933 __ li(reg, Operand(1));
3934 __ bunordered(&done, cr);
3935 }
3936 // Unnecessary for eq/lt & ne/ge since only FU bit will be set.
3937 }
3938
3939 if (CpuFeatures::IsSupported(PPC_7_PLUS)) {
3940 switch (cond) {
3941 case eq:
3942 case lt:
3943 case gt:
3944 if (reg_value != 1) __ li(reg, Operand(1));
3945 __ li(kScratchReg, Operand::Zero());
3946 __ isel(cond, reg, reg, kScratchReg, cr);
3947 break;
3948 case ne:
3949 case ge:
3950 case le:
3951 if (reg_value != 1) __ li(reg, Operand(1));
3952 // r0 implies logical zero in this form
3953 __ isel(NegateCondition(cond), reg, r0, reg, cr);
3954 break;
3955 default:
3956 UNREACHABLE();
3957 }
3958 } else {
3959 if (reg_value != 0) __ li(reg, Operand::Zero());
3960 __ b(NegateCondition(cond), &done, cr);
3961 __ li(reg, Operand(1));
3962 }
3963 __ bind(&done);
3964 }
3965
AssembleArchBinarySearchSwitch(Instruction* instr)3966 void CodeGenerator::AssembleArchBinarySearchSwitch(Instruction* instr) {
3967 PPCOperandConverter i(this, instr);
3968 Register input = i.InputRegister(0);
3969 std::vector<std::pair<int32_t, Label*>> cases;
3970 for (size_t index = 2; index < instr->InputCount(); index += 2) {
3971 cases.push_back({i.InputInt32(index + 0), GetLabel(i.InputRpo(index + 1))});
3972 }
3973 AssembleArchBinarySearchSwitchRange(input, i.InputRpo(1), cases.data(),
3974 cases.data() + cases.size());
3975 }
3976
AssembleArchTableSwitch(Instruction* instr)3977 void CodeGenerator::AssembleArchTableSwitch(Instruction* instr) {
3978 PPCOperandConverter i(this, instr);
3979 Register input = i.InputRegister(0);
3980 int32_t const case_count = static_cast<int32_t>(instr->InputCount() - 2);
3981 Label** cases = zone()->NewArray<Label*>(case_count);
3982 for (int32_t index = 0; index < case_count; ++index) {
3983 cases[index] = GetLabel(i.InputRpo(index + 2));
3984 }
3985 Label* const table = AddJumpTable(cases, case_count);
3986 __ CmpU64(input, Operand(case_count), r0);
3987 __ bge(GetLabel(i.InputRpo(1)));
3988 __ mov_label_addr(kScratchReg, table);
3989 __ ShiftLeftU64(r0, input, Operand(kSystemPointerSizeLog2));
3990 __ LoadU64(kScratchReg, MemOperand(kScratchReg, r0));
3991 __ Jump(kScratchReg);
3992 }
3993
AssembleArchSelect(Instruction* instr, FlagsCondition condition)3994 void CodeGenerator::AssembleArchSelect(Instruction* instr,
3995 FlagsCondition condition) {
3996 UNIMPLEMENTED();
3997 }
3998
FinishFrame(Frame* frame)3999 void CodeGenerator::FinishFrame(Frame* frame) {
4000 auto call_descriptor = linkage()->GetIncomingDescriptor();
4001 const DoubleRegList double_saves = call_descriptor->CalleeSavedFPRegisters();
4002
4003 // Save callee-saved Double registers.
4004 if (!double_saves.is_empty()) {
4005 frame->AlignSavedCalleeRegisterSlots();
4006 DCHECK_EQ(kNumCalleeSavedDoubles, double_saves.Count());
4007 frame->AllocateSavedCalleeRegisterSlots(kNumCalleeSavedDoubles *
4008 (kDoubleSize / kSystemPointerSize));
4009 }
4010 // Save callee-saved registers.
4011 const RegList saves =
4012 FLAG_enable_embedded_constant_pool
4013 ? call_descriptor->CalleeSavedRegisters() - kConstantPoolRegister
4014 : call_descriptor->CalleeSavedRegisters();
4015 if (!saves.is_empty()) {
4016 // register save area does not include the fp or constant pool pointer.
4017 const int num_saves =
4018 kNumCalleeSaved - 1 - (FLAG_enable_embedded_constant_pool ? 1 : 0);
4019 frame->AllocateSavedCalleeRegisterSlots(num_saves);
4020 }
4021 }
4022
AssembleConstructFrame()4023 void CodeGenerator::AssembleConstructFrame() {
4024 auto call_descriptor = linkage()->GetIncomingDescriptor();
4025 if (frame_access_state()->has_frame()) {
4026 if (call_descriptor->IsCFunctionCall()) {
4027 #if V8_ENABLE_WEBASSEMBLY
4028 if (info()->GetOutputStackFrameType() == StackFrame::C_WASM_ENTRY) {
4029 __ StubPrologue(StackFrame::C_WASM_ENTRY);
4030 // Reserve stack space for saving the c_entry_fp later.
4031 __ addi(sp, sp, Operand(-kSystemPointerSize));
4032 #else
4033 // For balance.
4034 if (false) {
4035 #endif // V8_ENABLE_WEBASSEMBLY
4036 } else {
4037 __ mflr(r0);
4038 if (FLAG_enable_embedded_constant_pool) {
4039 __ Push(r0, fp, kConstantPoolRegister);
4040 // Adjust FP to point to saved FP.
4041 __ SubS64(fp, sp,
4042 Operand(StandardFrameConstants::kConstantPoolOffset), r0);
4043 } else {
4044 __ Push(r0, fp);
4045 __ mr(fp, sp);
4046 }
4047 }
4048 } else if (call_descriptor->IsJSFunctionCall()) {
4049 __ Prologue();
4050 } else {
4051 StackFrame::Type type = info()->GetOutputStackFrameType();
4052 // TODO(mbrandy): Detect cases where ip is the entrypoint (for
4053 // efficient initialization of the constant pool pointer register).
4054 __ StubPrologue(type);
4055 #if V8_ENABLE_WEBASSEMBLY
4056 if (call_descriptor->IsWasmFunctionCall() ||
4057 call_descriptor->IsWasmImportWrapper() ||
4058 call_descriptor->IsWasmCapiFunction()) {
4059 __ Push(kWasmInstanceRegister);
4060 }
4061 if (call_descriptor->IsWasmCapiFunction()) {
4062 // Reserve space for saving the PC later.
4063 __ addi(sp, sp, Operand(-kSystemPointerSize));
4064 }
4065 #endif // V8_ENABLE_WEBASSEMBLY
4066 }
4067 unwinding_info_writer_.MarkFrameConstructed(__ pc_offset());
4068 }
4069
4070 int required_slots =
4071 frame()->GetTotalFrameSlotCount() - frame()->GetFixedSlotCount();
4072 if (info()->is_osr()) {
4073 // TurboFan OSR-compiled functions cannot be entered directly.
4074 __ Abort(AbortReason::kShouldNotDirectlyEnterOsrFunction);
4075
4076 // Unoptimized code jumps directly to this entrypoint while the unoptimized
4077 // frame is still on the stack. Optimized code uses OSR values directly from
4078 // the unoptimized frame. Thus, all that needs to be done is to allocate the
4079 // remaining stack slots.
4080 __ RecordComment("-- OSR entrypoint --");
4081 osr_pc_offset_ = __ pc_offset();
4082 required_slots -= osr_helper()->UnoptimizedFrameSlots();
4083 }
4084
4085 const DoubleRegList saves_fp = call_descriptor->CalleeSavedFPRegisters();
4086 const RegList saves =
4087 FLAG_enable_embedded_constant_pool
4088 ? call_descriptor->CalleeSavedRegisters() - kConstantPoolRegister
4089 : call_descriptor->CalleeSavedRegisters();
4090
4091 if (required_slots > 0) {
4092 #if V8_ENABLE_WEBASSEMBLY
4093 if (info()->IsWasm() && required_slots * kSystemPointerSize > 4 * KB) {
4094 // For WebAssembly functions with big frames we have to do the stack
4095 // overflow check before we construct the frame. Otherwise we may not
4096 // have enough space on the stack to call the runtime for the stack
4097 // overflow.
4098 Label done;
4099
4100 // If the frame is bigger than the stack, we throw the stack overflow
4101 // exception unconditionally. Thereby we can avoid the integer overflow
4102 // check in the condition code.
4103 if (required_slots * kSystemPointerSize < FLAG_stack_size * KB) {
4104 Register scratch = ip;
4105 __ LoadU64(
4106 scratch,
4107 FieldMemOperand(kWasmInstanceRegister,
4108 WasmInstanceObject::kRealStackLimitAddressOffset),
4109 r0);
4110 __ LoadU64(scratch, MemOperand(scratch), r0);
4111 __ AddS64(scratch, scratch,
4112 Operand(required_slots * kSystemPointerSize), r0);
4113 __ CmpU64(sp, scratch);
4114 __ bge(&done);
4115 }
4116
4117 __ Call(wasm::WasmCode::kWasmStackOverflow, RelocInfo::WASM_STUB_CALL);
4118 // The call does not return, hence we can ignore any references and just
4119 // define an empty safepoint.
4120 ReferenceMap* reference_map = zone()->New<ReferenceMap>(zone());
4121 RecordSafepoint(reference_map);
4122 if (FLAG_debug_code) __ stop();
4123
4124 __ bind(&done);
4125 }
4126 #endif // V8_ENABLE_WEBASSEMBLY
4127
4128 // Skip callee-saved and return slots, which are pushed below.
4129 required_slots -= saves.Count();
4130 required_slots -= frame()->GetReturnSlotCount();
4131 required_slots -= (kDoubleSize / kSystemPointerSize) * saves_fp.Count();
4132 __ AddS64(sp, sp, Operand(-required_slots * kSystemPointerSize), r0);
4133 }
4134
4135 // Save callee-saved Double registers.
4136 if (!saves_fp.is_empty()) {
4137 __ MultiPushDoubles(saves_fp);
4138 DCHECK_EQ(kNumCalleeSavedDoubles, saves_fp.Count());
4139 }
4140
4141 // Save callee-saved registers.
4142 if (!saves.is_empty()) {
4143 __ MultiPush(saves);
4144 // register save area does not include the fp or constant pool pointer.
4145 }
4146
4147 const int returns = frame()->GetReturnSlotCount();
4148 // Create space for returns.
4149 __ AllocateStackSpace(returns * kSystemPointerSize);
4150 }
4151
4152 void CodeGenerator::AssembleReturn(InstructionOperand* additional_pop_count) {
4153 auto call_descriptor = linkage()->GetIncomingDescriptor();
4154
4155 const int returns = frame()->GetReturnSlotCount();
4156 if (returns != 0) {
4157 // Create space for returns.
4158 __ AddS64(sp, sp, Operand(returns * kSystemPointerSize), r0);
4159 }
4160
4161 // Restore registers.
4162 const RegList saves =
4163 FLAG_enable_embedded_constant_pool
4164 ? call_descriptor->CalleeSavedRegisters() - kConstantPoolRegister
4165 : call_descriptor->CalleeSavedRegisters();
4166 if (!saves.is_empty()) {
4167 __ MultiPop(saves);
4168 }
4169
4170 // Restore double registers.
4171 const DoubleRegList double_saves = call_descriptor->CalleeSavedFPRegisters();
4172 if (!double_saves.is_empty()) {
4173 __ MultiPopDoubles(double_saves);
4174 }
4175
4176 unwinding_info_writer_.MarkBlockWillExit();
4177
4178 PPCOperandConverter g(this, nullptr);
4179 const int parameter_slots =
4180 static_cast<int>(call_descriptor->ParameterSlotCount());
4181
4182 // {aditional_pop_count} is only greater than zero if {parameter_slots = 0}.
4183 // Check RawMachineAssembler::PopAndReturn.
4184 if (parameter_slots != 0) {
4185 if (additional_pop_count->IsImmediate()) {
4186 DCHECK_EQ(g.ToConstant(additional_pop_count).ToInt32(), 0);
4187 } else if (FLAG_debug_code) {
4188 __ cmpi(g.ToRegister(additional_pop_count), Operand(0));
4189 __ Assert(eq, AbortReason::kUnexpectedAdditionalPopValue);
4190 }
4191 }
4192
4193 Register argc_reg = r6;
4194 // Functions with JS linkage have at least one parameter (the receiver).
4195 // If {parameter_slots} == 0, it means it is a builtin with
4196 // kDontAdaptArgumentsSentinel, which takes care of JS arguments popping
4197 // itself.
4198 const bool drop_jsargs = parameter_slots != 0 &&
4199 frame_access_state()->has_frame() &&
4200 call_descriptor->IsJSFunctionCall();
4201
4202 if (call_descriptor->IsCFunctionCall()) {
4203 AssembleDeconstructFrame();
4204 } else if (frame_access_state()->has_frame()) {
4205 // Canonicalize JSFunction return sites for now unless they have an variable
4206 // number of stack slot pops
4207 if (additional_pop_count->IsImmediate() &&
4208 g.ToConstant(additional_pop_count).ToInt32() == 0) {
4209 if (return_label_.is_bound()) {
4210 __ b(&return_label_);
4211 return;
4212 } else {
4213 __ bind(&return_label_);
4214 }
4215 }
4216 if (drop_jsargs) {
4217 // Get the actual argument count.
4218 DCHECK(!call_descriptor->CalleeSavedRegisters().has(argc_reg));
4219 __ LoadU64(argc_reg, MemOperand(fp, StandardFrameConstants::kArgCOffset));
4220 }
4221 AssembleDeconstructFrame();
4222 }
4223 // Constant pool is unavailable since the frame has been destructed
4224 ConstantPoolUnavailableScope constant_pool_unavailable(tasm());
4225 if (drop_jsargs) {
4226 // We must pop all arguments from the stack (including the receiver).
4227 // The number of arguments without the receiver is
4228 // max(argc_reg, parameter_slots-1), and the receiver is added in
4229 // DropArguments().
4230 DCHECK(!call_descriptor->CalleeSavedRegisters().has(argc_reg));
4231 if (parameter_slots > 1) {
4232 Label skip;
4233 __ CmpS64(argc_reg, Operand(parameter_slots), r0);
4234 __ bgt(&skip);
4235 __ mov(argc_reg, Operand(parameter_slots));
4236 __ bind(&skip);
4237 }
4238 __ DropArguments(argc_reg, TurboAssembler::kCountIsInteger,
4239 TurboAssembler::kCountIncludesReceiver);
4240 } else if (additional_pop_count->IsImmediate()) {
4241 int additional_count = g.ToConstant(additional_pop_count).ToInt32();
4242 __ Drop(parameter_slots + additional_count);
4243 } else if (parameter_slots == 0) {
4244 __ Drop(g.ToRegister(additional_pop_count));
4245 } else {
4246 // {additional_pop_count} is guaranteed to be zero if {parameter_slots !=
4247 // 0}. Check RawMachineAssembler::PopAndReturn.
4248 __ Drop(parameter_slots);
4249 }
4250 __ Ret();
4251 }
4252
4253 void CodeGenerator::FinishCode() {}
4254
4255 void CodeGenerator::PrepareForDeoptimizationExits(
4256 ZoneDeque<DeoptimizationExit*>* exits) {
4257 int total_size = 0;
4258 for (DeoptimizationExit* exit : deoptimization_exits_) {
4259 total_size += (exit->kind() == DeoptimizeKind::kLazy)
4260 ? Deoptimizer::kLazyDeoptExitSize
4261 : Deoptimizer::kEagerDeoptExitSize;
4262 }
4263
4264 __ CheckTrampolinePoolQuick(total_size);
4265 }
4266
4267 void CodeGenerator::AssembleMove(InstructionOperand* source,
4268 InstructionOperand* destination) {
4269 PPCOperandConverter g(this, nullptr);
4270 // Dispatch on the source and destination operand kinds. Not all
4271 // combinations are possible.
4272 if (source->IsRegister()) {
4273 DCHECK(destination->IsRegister() || destination->IsStackSlot());
4274 Register src = g.ToRegister(source);
4275 if (destination->IsRegister()) {
4276 __ Move(g.ToRegister(destination), src);
4277 } else {
4278 __ StoreU64(src, g.ToMemOperand(destination), r0);
4279 }
4280 } else if (source->IsStackSlot()) {
4281 DCHECK(destination->IsRegister() || destination->IsStackSlot());
4282 MemOperand src = g.ToMemOperand(source);
4283 if (destination->IsRegister()) {
4284 __ LoadU64(g.ToRegister(destination), src, r0);
4285 } else {
4286 Register temp = kScratchReg;
4287 __ LoadU64(temp, src, r0);
4288 __ StoreU64(temp, g.ToMemOperand(destination), r0);
4289 }
4290 } else if (source->IsConstant()) {
4291 Constant src = g.ToConstant(source);
4292 if (destination->IsRegister() || destination->IsStackSlot()) {
4293 Register dst =
4294 destination->IsRegister() ? g.ToRegister(destination) : kScratchReg;
4295 switch (src.type()) {
4296 case Constant::kInt32:
4297 #if V8_ENABLE_WEBASSEMBLY && !V8_TARGET_ARCH_PPC64
4298 if (RelocInfo::IsWasmReference(src.rmode())) {
4299 __ mov(dst, Operand(src.ToInt32(), src.rmode()));
4300 break;
4301 }
4302 #endif // V8_ENABLE_WEBASSEMBLY && !V8_TARGET_ARCH_PPC64
4303 __ mov(dst, Operand(src.ToInt32()));
4304 break;
4305 case Constant::kInt64:
4306 #if V8_ENABLE_WEBASSEMBLY && V8_TARGET_ARCH_PPC64
4307 if (RelocInfo::IsWasmReference(src.rmode())) {
4308 __ mov(dst, Operand(src.ToInt64(), src.rmode()));
4309 break;
4310 }
4311 #endif // V8_ENABLE_WEBASSEMBLY && V8_TARGET_ARCH_PPC64
4312 __ mov(dst, Operand(src.ToInt64()));
4313 break;
4314 case Constant::kFloat32:
4315 __ mov(dst, Operand::EmbeddedNumber(src.ToFloat32()));
4316 break;
4317 case Constant::kFloat64:
4318 __ mov(dst, Operand::EmbeddedNumber(src.ToFloat64().value()));
4319 break;
4320 case Constant::kExternalReference:
4321 __ Move(dst, src.ToExternalReference());
4322 break;
4323 case Constant::kDelayedStringConstant:
4324 __ mov(dst, Operand::EmbeddedStringConstant(
4325 src.ToDelayedStringConstant()));
4326 break;
4327 case Constant::kHeapObject: {
4328 Handle<HeapObject> src_object = src.ToHeapObject();
4329 RootIndex index;
4330 if (IsMaterializableFromRoot(src_object, &index)) {
4331 __ LoadRoot(dst, index);
4332 } else {
4333 __ Move(dst, src_object);
4334 }
4335 break;
4336 }
4337 case Constant::kCompressedHeapObject: {
4338 Handle<HeapObject> src_object = src.ToHeapObject();
4339 RootIndex index;
4340 if (IsMaterializableFromRoot(src_object, &index)) {
4341 __ LoadRoot(dst, index);
4342 } else {
4343 // TODO(v8:7703, jyan@ca.ibm.com): Turn into a
4344 // COMPRESSED_EMBEDDED_OBJECT when the constant pool entry size is
4345 // tagged size.
4346 __ Move(dst, src_object, RelocInfo::FULL_EMBEDDED_OBJECT);
4347 }
4348 break;
4349 }
4350 case Constant::kRpoNumber:
4351 UNREACHABLE(); // TODO(dcarney): loading RPO constants on PPC.
4352 }
4353 if (destination->IsStackSlot()) {
4354 __ StoreU64(dst, g.ToMemOperand(destination), r0);
4355 }
4356 } else {
4357 DoubleRegister dst = destination->IsFPRegister()
4358 ? g.ToDoubleRegister(destination)
4359 : kScratchDoubleReg;
4360 base::Double value;
4361 #if V8_HOST_ARCH_IA32 || V8_HOST_ARCH_X64
4362 // casting double precision snan to single precision
4363 // converts it to qnan on ia32/x64
4364 if (src.type() == Constant::kFloat32) {
4365 uint32_t val = src.ToFloat32AsInt();
4366 if ((val & 0x7F800000) == 0x7F800000) {
4367 uint64_t dval = static_cast<uint64_t>(val);
4368 dval = ((dval & 0xC0000000) << 32) | ((dval & 0x40000000) << 31) |
4369 ((dval & 0x40000000) << 30) | ((dval & 0x7FFFFFFF) << 29);
4370 value = base::Double(dval);
4371 } else {
4372 value = base::Double(static_cast<double>(src.ToFloat32()));
4373 }
4374 } else {
4375 value = base::Double(src.ToFloat64());
4376 }
4377 #else
4378 value = src.type() == Constant::kFloat32
4379 ? base::Double(static_cast<double>(src.ToFloat32()))
4380 : base::Double(src.ToFloat64());
4381 #endif
4382 __ LoadDoubleLiteral(dst, value, kScratchReg);
4383 if (destination->IsDoubleStackSlot()) {
4384 __ StoreF64(dst, g.ToMemOperand(destination), r0);
4385 } else if (destination->IsFloatStackSlot()) {
4386 __ StoreF32(dst, g.ToMemOperand(destination), r0);
4387 }
4388 }
4389 } else if (source->IsFPRegister()) {
4390 MachineRepresentation rep = LocationOperand::cast(source)->representation();
4391 if (rep == MachineRepresentation::kSimd128) {
4392 if (destination->IsSimd128Register()) {
4393 __ vor(g.ToSimd128Register(destination), g.ToSimd128Register(source),
4394 g.ToSimd128Register(source));
4395 } else {
4396 DCHECK(destination->IsSimd128StackSlot());
4397 MemOperand dst = g.ToMemOperand(destination);
4398 __ mov(ip, Operand(dst.offset()));
4399 __ StoreSimd128(g.ToSimd128Register(source), MemOperand(dst.ra(), ip));
4400 }
4401 } else {
4402 DoubleRegister src = g.ToDoubleRegister(source);
4403 if (destination->IsFPRegister()) {
4404 DoubleRegister dst = g.ToDoubleRegister(destination);
4405 __ Move(dst, src);
4406 } else {
4407 DCHECK(destination->IsFPStackSlot());
4408 LocationOperand* op = LocationOperand::cast(source);
4409 if (op->representation() == MachineRepresentation::kFloat64) {
4410 __ StoreF64(src, g.ToMemOperand(destination), r0);
4411 } else {
4412 __ StoreF32(src, g.ToMemOperand(destination), r0);
4413 }
4414 }
4415 }
4416 } else if (source->IsFPStackSlot()) {
4417 DCHECK(destination->IsFPRegister() || destination->IsFPStackSlot());
4418 MemOperand src = g.ToMemOperand(source);
4419 if (destination->IsFPRegister()) {
4420 LocationOperand* op = LocationOperand::cast(source);
4421 if (op->representation() == MachineRepresentation::kFloat64) {
4422 __ LoadF64(g.ToDoubleRegister(destination), src, r0);
4423 } else if (op->representation() == MachineRepresentation::kFloat32) {
4424 __ LoadF32(g.ToDoubleRegister(destination), src, r0);
4425 } else {
4426 DCHECK_EQ(MachineRepresentation::kSimd128, op->representation());
4427 MemOperand src = g.ToMemOperand(source);
4428 __ mov(ip, Operand(src.offset()));
4429 __ LoadSimd128(g.ToSimd128Register(destination),
4430 MemOperand(src.ra(), ip));
4431 }
4432 } else {
4433 LocationOperand* op = LocationOperand::cast(source);
4434 DoubleRegister temp = kScratchDoubleReg;
4435 if (op->representation() == MachineRepresentation::kFloat64) {
4436 __ LoadF64(temp, src, r0);
4437 __ StoreF64(temp, g.ToMemOperand(destination), r0);
4438 } else if (op->representation() == MachineRepresentation::kFloat32) {
4439 __ LoadF32(temp, src, r0);
4440 __ StoreF32(temp, g.ToMemOperand(destination), r0);
4441 } else {
4442 DCHECK_EQ(MachineRepresentation::kSimd128, op->representation());
4443 // push v0, to be used as scratch
4444 __ addi(sp, sp, Operand(-kSimd128Size));
4445 __ StoreSimd128(v0, MemOperand(r0, sp));
4446 MemOperand src = g.ToMemOperand(source);
4447 MemOperand dst = g.ToMemOperand(destination);
4448 __ mov(ip, Operand(src.offset()));
4449 __ LoadSimd128(v0, MemOperand(src.ra(), ip));
4450 __ mov(ip, Operand(dst.offset()));
4451 __ StoreSimd128(v0, MemOperand(dst.ra(), ip));
4452 // restore v0
4453 __ LoadSimd128(v0, MemOperand(r0, sp));
4454 __ addi(sp, sp, Operand(kSimd128Size));
4455 }
4456 }
4457 } else {
4458 UNREACHABLE();
4459 }
4460 }
4461
4462 // Swaping contents in source and destination.
4463 // source and destination could be:
4464 // Register,
4465 // FloatRegister,
4466 // DoubleRegister,
4467 // StackSlot,
4468 // FloatStackSlot,
4469 // or DoubleStackSlot
4470 void CodeGenerator::AssembleSwap(InstructionOperand* source,
4471 InstructionOperand* destination) {
4472 PPCOperandConverter g(this, nullptr);
4473 if (source->IsRegister()) {
4474 Register src = g.ToRegister(source);
4475 if (destination->IsRegister()) {
4476 __ SwapP(src, g.ToRegister(destination), kScratchReg);
4477 } else {
4478 DCHECK(destination->IsStackSlot());
4479 __ SwapP(src, g.ToMemOperand(destination), kScratchReg);
4480 }
4481 } else if (source->IsStackSlot()) {
4482 DCHECK(destination->IsStackSlot());
4483 __ SwapP(g.ToMemOperand(source), g.ToMemOperand(destination), kScratchReg,
4484 r0);
4485 } else if (source->IsFloatRegister()) {
4486 DoubleRegister src = g.ToDoubleRegister(source);
4487 if (destination->IsFloatRegister()) {
4488 __ SwapFloat32(src, g.ToDoubleRegister(destination), kScratchDoubleReg);
4489 } else {
4490 DCHECK(destination->IsFloatStackSlot());
4491 __ SwapFloat32(src, g.ToMemOperand(destination), kScratchDoubleReg);
4492 }
4493 } else if (source->IsDoubleRegister()) {
4494 DoubleRegister src = g.ToDoubleRegister(source);
4495 if (destination->IsDoubleRegister()) {
4496 __ SwapDouble(src, g.ToDoubleRegister(destination), kScratchDoubleReg);
4497 } else {
4498 DCHECK(destination->IsDoubleStackSlot());
4499 __ SwapDouble(src, g.ToMemOperand(destination), kScratchDoubleReg);
4500 }
4501 } else if (source->IsFloatStackSlot()) {
4502 DCHECK(destination->IsFloatStackSlot());
4503 __ SwapFloat32(g.ToMemOperand(source), g.ToMemOperand(destination),
4504 kScratchDoubleReg, d0);
4505 } else if (source->IsDoubleStackSlot()) {
4506 DCHECK(destination->IsDoubleStackSlot());
4507 __ SwapDouble(g.ToMemOperand(source), g.ToMemOperand(destination),
4508 kScratchDoubleReg, d0);
4509
4510 } else if (source->IsSimd128Register()) {
4511 Simd128Register src = g.ToSimd128Register(source);
4512 if (destination->IsSimd128Register()) {
4513 __ SwapSimd128(src, g.ToSimd128Register(destination), kScratchSimd128Reg);
4514 } else {
4515 DCHECK(destination->IsSimd128StackSlot());
4516 __ SwapSimd128(src, g.ToMemOperand(destination), kScratchSimd128Reg);
4517 }
4518 } else if (source->IsSimd128StackSlot()) {
4519 DCHECK(destination->IsSimd128StackSlot());
4520 __ SwapSimd128(g.ToMemOperand(source), g.ToMemOperand(destination),
4521 kScratchSimd128Reg);
4522
4523 } else {
4524 UNREACHABLE();
4525 }
4526
4527 return;
4528 }
4529
4530 void CodeGenerator::AssembleJumpTable(Label** targets, size_t target_count) {
4531 for (size_t index = 0; index < target_count; ++index) {
4532 __ emit_label_addr(targets[index]);
4533 }
4534 }
4535
4536 #undef __
4537
4538 } // namespace compiler
4539 } // namespace internal
4540 } // namespace v8
4541