LLVM 18.0.0git
WebAssemblyISelLowering.cpp
Go to the documentation of this file.
1//=- WebAssemblyISelLowering.cpp - WebAssembly DAG Lowering Implementation -==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// This file implements the WebAssemblyTargetLowering class.
11///
12//===----------------------------------------------------------------------===//
13
32#include "llvm/IR/Function.h"
33#include "llvm/IR/Intrinsics.h"
34#include "llvm/IR/IntrinsicsWebAssembly.h"
36#include "llvm/Support/Debug.h"
42using namespace llvm;
43
44#define DEBUG_TYPE "wasm-lower"
45
47 const TargetMachine &TM, const WebAssemblySubtarget &STI)
48 : TargetLowering(TM), Subtarget(&STI) {
49 auto MVTPtr = Subtarget->hasAddr64() ? MVT::i64 : MVT::i32;
50
51 // Booleans always contain 0 or 1.
53 // Except in SIMD vectors
55 // We don't know the microarchitecture here, so just reduce register pressure.
57 // Tell ISel that we have a stack pointer.
59 Subtarget->hasAddr64() ? WebAssembly::SP64 : WebAssembly::SP32);
60 // Set up the register classes.
61 addRegisterClass(MVT::i32, &WebAssembly::I32RegClass);
62 addRegisterClass(MVT::i64, &WebAssembly::I64RegClass);
63 addRegisterClass(MVT::f32, &WebAssembly::F32RegClass);
64 addRegisterClass(MVT::f64, &WebAssembly::F64RegClass);
65 if (Subtarget->hasSIMD128()) {
66 addRegisterClass(MVT::v16i8, &WebAssembly::V128RegClass);
67 addRegisterClass(MVT::v8i16, &WebAssembly::V128RegClass);
68 addRegisterClass(MVT::v4i32, &WebAssembly::V128RegClass);
69 addRegisterClass(MVT::v4f32, &WebAssembly::V128RegClass);
70 addRegisterClass(MVT::v2i64, &WebAssembly::V128RegClass);
71 addRegisterClass(MVT::v2f64, &WebAssembly::V128RegClass);
72 }
73 if (Subtarget->hasReferenceTypes()) {
74 addRegisterClass(MVT::externref, &WebAssembly::EXTERNREFRegClass);
75 addRegisterClass(MVT::funcref, &WebAssembly::FUNCREFRegClass);
76 }
77 // Compute derived properties from the register classes.
79
80 // Transform loads and stores to pointers in address space 1 to loads and
81 // stores to WebAssembly global variables, outside linear memory.
82 for (auto T : {MVT::i32, MVT::i64, MVT::f32, MVT::f64}) {
85 }
86 if (Subtarget->hasSIMD128()) {
87 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32, MVT::v2i64,
88 MVT::v2f64}) {
91 }
92 }
93 if (Subtarget->hasReferenceTypes()) {
94 // We need custom load and store lowering for both externref, funcref and
95 // Other. The MVT::Other here represents tables of reference types.
96 for (auto T : {MVT::externref, MVT::funcref, MVT::Other}) {
99 }
100 }
101
108
109 // Take the default expansion for va_arg, va_copy, and va_end. There is no
110 // default action for va_start, so we do that custom.
115
116 for (auto T : {MVT::f32, MVT::f64, MVT::v4f32, MVT::v2f64}) {
117 // Don't expand the floating-point types to constant pools.
119 // Expand floating-point comparisons.
123 // Expand floating-point library function operators.
124 for (auto Op :
127 // Note supported floating-point library function operators that otherwise
128 // default to expand.
132 // Support minimum and maximum, which otherwise default to expand.
135 // WebAssembly currently has no builtin f16 support.
139 setTruncStoreAction(T, MVT::f16, Expand);
140 }
141
142 // Expand unavailable integer operations.
143 for (auto Op :
147 for (auto T : {MVT::i32, MVT::i64})
149 if (Subtarget->hasSIMD128())
150 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64})
152 }
153
154 if (Subtarget->hasNontrappingFPToInt())
156 for (auto T : {MVT::i32, MVT::i64})
158
159 // SIMD-specific configuration
160 if (Subtarget->hasSIMD128()) {
161 // Combine vector mask reductions into alltrue/anytrue
163
164 // Convert vector to integer bitcasts to bitmask
166
167 // Hoist bitcasts out of shuffles
169
170 // Combine extends of extract_subvectors into widening ops
172
173 // Combine int_to_fp or fp_extend of extract_vectors and vice versa into
174 // conversions ops
177
178 // Combine fp_to_{s,u}int_sat or fp_round of concat_vectors or vice versa
179 // into conversion ops
182
184
185 // Support saturating add for i8x16 and i16x8
186 for (auto Op : {ISD::SADDSAT, ISD::UADDSAT})
187 for (auto T : {MVT::v16i8, MVT::v8i16})
189
190 // Support integer abs
191 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64})
193
194 // Custom lower BUILD_VECTORs to minimize number of replace_lanes
195 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32, MVT::v2i64,
196 MVT::v2f64})
198
199 // We have custom shuffle lowering to expose the shuffle mask
200 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32, MVT::v2i64,
201 MVT::v2f64})
203
204 // Support splatting
205 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32, MVT::v2i64,
206 MVT::v2f64})
208
209 // Custom lowering since wasm shifts must have a scalar shift amount
210 for (auto Op : {ISD::SHL, ISD::SRA, ISD::SRL})
211 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64})
213
214 // Custom lower lane accesses to expand out variable indices
216 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32, MVT::v2i64,
217 MVT::v2f64})
219
220 // There is no i8x16.mul instruction
221 setOperationAction(ISD::MUL, MVT::v16i8, Expand);
222
223 // There is no vector conditional select instruction
224 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32, MVT::v2i64,
225 MVT::v2f64})
227
228 // Expand integer operations supported for scalars but not SIMD
229 for (auto Op :
231 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64})
233
234 // But we do have integer min and max operations
235 for (auto Op : {ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX})
236 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32})
238
239 // And we have popcnt for i8x16. It can be used to expand ctlz/cttz.
240 setOperationAction(ISD::CTPOP, MVT::v16i8, Legal);
241 setOperationAction(ISD::CTLZ, MVT::v16i8, Expand);
242 setOperationAction(ISD::CTTZ, MVT::v16i8, Expand);
243
244 // Custom lower bit counting operations for other types to scalarize them.
245 for (auto Op : {ISD::CTLZ, ISD::CTTZ, ISD::CTPOP})
246 for (auto T : {MVT::v8i16, MVT::v4i32, MVT::v2i64})
248
249 // Expand float operations supported for scalars but not SIMD
252 for (auto T : {MVT::v4f32, MVT::v2f64})
254
255 // Unsigned comparison operations are unavailable for i64x2 vectors.
257 setCondCodeAction(CC, MVT::v2i64, Custom);
258
259 // 64x2 conversions are not in the spec
260 for (auto Op :
262 for (auto T : {MVT::v2i64, MVT::v2f64})
264
265 // But saturating fp_to_int converstions are
267 setOperationAction(Op, MVT::v4i32, Custom);
268
269 // Support vector extending
273 }
274 }
275
276 // As a special case, these operators use the type to mean the type to
277 // sign-extend from.
279 if (!Subtarget->hasSignExt()) {
280 // Sign extends are legal only when extending a vector extract
281 auto Action = Subtarget->hasSIMD128() ? Custom : Expand;
282 for (auto T : {MVT::i8, MVT::i16, MVT::i32})
284 }
287
288 // Dynamic stack allocation: use the default expansion.
292
296
297 // Expand these forms; we pattern-match the forms that we can handle in isel.
298 for (auto T : {MVT::i32, MVT::i64, MVT::f32, MVT::f64})
299 for (auto Op : {ISD::BR_CC, ISD::SELECT_CC})
301
302 // We have custom switch handling.
304
305 // WebAssembly doesn't have:
306 // - Floating-point extending loads.
307 // - Floating-point truncating stores.
308 // - i1 extending loads.
309 // - truncating SIMD stores and most extending loads
310 setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f32, Expand);
311 setTruncStoreAction(MVT::f64, MVT::f32, Expand);
312 for (auto T : MVT::integer_valuetypes())
313 for (auto Ext : {ISD::EXTLOAD, ISD::ZEXTLOAD, ISD::SEXTLOAD})
314 setLoadExtAction(Ext, T, MVT::i1, Promote);
315 if (Subtarget->hasSIMD128()) {
316 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64, MVT::v4f32,
317 MVT::v2f64}) {
318 for (auto MemT : MVT::fixedlen_vector_valuetypes()) {
319 if (MVT(T) != MemT) {
321 for (auto Ext : {ISD::EXTLOAD, ISD::ZEXTLOAD, ISD::SEXTLOAD})
322 setLoadExtAction(Ext, T, MemT, Expand);
323 }
324 }
325 }
326 // But some vector extending loads are legal
327 for (auto Ext : {ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}) {
328 setLoadExtAction(Ext, MVT::v8i16, MVT::v8i8, Legal);
329 setLoadExtAction(Ext, MVT::v4i32, MVT::v4i16, Legal);
330 setLoadExtAction(Ext, MVT::v2i64, MVT::v2i32, Legal);
331 }
332 setLoadExtAction(ISD::EXTLOAD, MVT::v2f64, MVT::v2f32, Legal);
333 }
334
335 // Don't do anything clever with build_pairs
337
338 // Trap lowers to wasm unreachable
339 setOperationAction(ISD::TRAP, MVT::Other, Legal);
341
342 // Exception handling intrinsics
346
348
349 // Override the __gnu_f2h_ieee/__gnu_h2f_ieee names so that the f32 name is
350 // consistent with the f64 and f128 names.
351 setLibcallName(RTLIB::FPEXT_F16_F32, "__extendhfsf2");
352 setLibcallName(RTLIB::FPROUND_F32_F16, "__truncsfhf2");
353
354 // Define the emscripten name for return address helper.
355 // TODO: when implementing other Wasm backends, make this generic or only do
356 // this on emscripten depending on what they end up doing.
357 setLibcallName(RTLIB::RETURN_ADDRESS, "emscripten_return_address");
358
359 // Always convert switches to br_tables unless there is only one case, which
360 // is equivalent to a simple branch. This reduces code size for wasm, and we
361 // defer possible jump table optimizations to the VM.
363}
364
366 uint32_t AS) const {
368 return MVT::externref;
370 return MVT::funcref;
372}
373
375 uint32_t AS) const {
377 return MVT::externref;
379 return MVT::funcref;
381}
382
384WebAssemblyTargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const {
385 // We have wasm instructions for these
386 switch (AI->getOperation()) {
394 default:
395 break;
396 }
398}
399
400bool WebAssemblyTargetLowering::shouldScalarizeBinop(SDValue VecOp) const {
401 // Implementation copied from X86TargetLowering.
402 unsigned Opc = VecOp.getOpcode();
403
404 // Assume target opcodes can't be scalarized.
405 // TODO - do we have any exceptions?
406 if (Opc >= ISD::BUILTIN_OP_END)
407 return false;
408
409 // If the vector op is not supported, try to convert to scalar.
410 EVT VecVT = VecOp.getValueType();
411 if (!isOperationLegalOrCustomOrPromote(Opc, VecVT))
412 return true;
413
414 // If the vector op is supported, but the scalar op is not, the transform may
415 // not be worthwhile.
416 EVT ScalarVT = VecVT.getScalarType();
417 return isOperationLegalOrCustomOrPromote(Opc, ScalarVT);
418}
419
420FastISel *WebAssemblyTargetLowering::createFastISel(
421 FunctionLoweringInfo &FuncInfo, const TargetLibraryInfo *LibInfo) const {
422 return WebAssembly::createFastISel(FuncInfo, LibInfo);
423}
424
425MVT WebAssemblyTargetLowering::getScalarShiftAmountTy(const DataLayout & /*DL*/,
426 EVT VT) const {
427 unsigned BitWidth = NextPowerOf2(VT.getSizeInBits() - 1);
428 if (BitWidth > 1 && BitWidth < 8)
429 BitWidth = 8;
430
431 if (BitWidth > 64) {
432 // The shift will be lowered to a libcall, and compiler-rt libcalls expect
433 // the count to be an i32.
434 BitWidth = 32;
436 "32-bit shift counts ought to be enough for anyone");
437 }
438
441 "Unable to represent scalar shift amount type");
442 return Result;
443}
444
445// Lower an fp-to-int conversion operator from the LLVM opcode, which has an
446// undefined result on invalid/overflow, to the WebAssembly opcode, which
447// traps on invalid/overflow.
450 const TargetInstrInfo &TII,
451 bool IsUnsigned, bool Int64,
452 bool Float64, unsigned LoweredOpcode) {
454
455 Register OutReg = MI.getOperand(0).getReg();
456 Register InReg = MI.getOperand(1).getReg();
457
458 unsigned Abs = Float64 ? WebAssembly::ABS_F64 : WebAssembly::ABS_F32;
459 unsigned FConst = Float64 ? WebAssembly::CONST_F64 : WebAssembly::CONST_F32;
460 unsigned LT = Float64 ? WebAssembly::LT_F64 : WebAssembly::LT_F32;
461 unsigned GE = Float64 ? WebAssembly::GE_F64 : WebAssembly::GE_F32;
462 unsigned IConst = Int64 ? WebAssembly::CONST_I64 : WebAssembly::CONST_I32;
463 unsigned Eqz = WebAssembly::EQZ_I32;
464 unsigned And = WebAssembly::AND_I32;
465 int64_t Limit = Int64 ? INT64_MIN : INT32_MIN;
466 int64_t Substitute = IsUnsigned ? 0 : Limit;
467 double CmpVal = IsUnsigned ? -(double)Limit * 2.0 : -(double)Limit;
468 auto &Context = BB->getParent()->getFunction().getContext();
470
471 const BasicBlock *LLVMBB = BB->getBasicBlock();
472 MachineFunction *F = BB->getParent();
473 MachineBasicBlock *TrueMBB = F->CreateMachineBasicBlock(LLVMBB);
474 MachineBasicBlock *FalseMBB = F->CreateMachineBasicBlock(LLVMBB);
475 MachineBasicBlock *DoneMBB = F->CreateMachineBasicBlock(LLVMBB);
476
478 F->insert(It, FalseMBB);
479 F->insert(It, TrueMBB);
480 F->insert(It, DoneMBB);
481
482 // Transfer the remainder of BB and its successor edges to DoneMBB.
483 DoneMBB->splice(DoneMBB->begin(), BB, std::next(MI.getIterator()), BB->end());
485
486 BB->addSuccessor(TrueMBB);
487 BB->addSuccessor(FalseMBB);
488 TrueMBB->addSuccessor(DoneMBB);
489 FalseMBB->addSuccessor(DoneMBB);
490
491 unsigned Tmp0, Tmp1, CmpReg, EqzReg, FalseReg, TrueReg;
492 Tmp0 = MRI.createVirtualRegister(MRI.getRegClass(InReg));
493 Tmp1 = MRI.createVirtualRegister(MRI.getRegClass(InReg));
494 CmpReg = MRI.createVirtualRegister(&WebAssembly::I32RegClass);
495 EqzReg = MRI.createVirtualRegister(&WebAssembly::I32RegClass);
496 FalseReg = MRI.createVirtualRegister(MRI.getRegClass(OutReg));
497 TrueReg = MRI.createVirtualRegister(MRI.getRegClass(OutReg));
498
499 MI.eraseFromParent();
500 // For signed numbers, we can do a single comparison to determine whether
501 // fabs(x) is within range.
502 if (IsUnsigned) {
503 Tmp0 = InReg;
504 } else {
505 BuildMI(BB, DL, TII.get(Abs), Tmp0).addReg(InReg);
506 }
507 BuildMI(BB, DL, TII.get(FConst), Tmp1)
508 .addFPImm(cast<ConstantFP>(ConstantFP::get(Ty, CmpVal)));
509 BuildMI(BB, DL, TII.get(LT), CmpReg).addReg(Tmp0).addReg(Tmp1);
510
511 // For unsigned numbers, we have to do a separate comparison with zero.
512 if (IsUnsigned) {
513 Tmp1 = MRI.createVirtualRegister(MRI.getRegClass(InReg));
514 Register SecondCmpReg =
515 MRI.createVirtualRegister(&WebAssembly::I32RegClass);
516 Register AndReg = MRI.createVirtualRegister(&WebAssembly::I32RegClass);
517 BuildMI(BB, DL, TII.get(FConst), Tmp1)
518 .addFPImm(cast<ConstantFP>(ConstantFP::get(Ty, 0.0)));
519 BuildMI(BB, DL, TII.get(GE), SecondCmpReg).addReg(Tmp0).addReg(Tmp1);
520 BuildMI(BB, DL, TII.get(And), AndReg).addReg(CmpReg).addReg(SecondCmpReg);
521 CmpReg = AndReg;
522 }
523
524 BuildMI(BB, DL, TII.get(Eqz), EqzReg).addReg(CmpReg);
525
526 // Create the CFG diamond to select between doing the conversion or using
527 // the substitute value.
528 BuildMI(BB, DL, TII.get(WebAssembly::BR_IF)).addMBB(TrueMBB).addReg(EqzReg);
529 BuildMI(FalseMBB, DL, TII.get(LoweredOpcode), FalseReg).addReg(InReg);
530 BuildMI(FalseMBB, DL, TII.get(WebAssembly::BR)).addMBB(DoneMBB);
531 BuildMI(TrueMBB, DL, TII.get(IConst), TrueReg).addImm(Substitute);
532 BuildMI(*DoneMBB, DoneMBB->begin(), DL, TII.get(TargetOpcode::PHI), OutReg)
533 .addReg(FalseReg)
534 .addMBB(FalseMBB)
535 .addReg(TrueReg)
536 .addMBB(TrueMBB);
537
538 return DoneMBB;
539}
540
541static MachineBasicBlock *
543 const WebAssemblySubtarget *Subtarget,
544 const TargetInstrInfo &TII) {
545 MachineInstr &CallParams = *CallResults.getPrevNode();
546 assert(CallParams.getOpcode() == WebAssembly::CALL_PARAMS);
547 assert(CallResults.getOpcode() == WebAssembly::CALL_RESULTS ||
548 CallResults.getOpcode() == WebAssembly::RET_CALL_RESULTS);
549
550 bool IsIndirect =
551 CallParams.getOperand(0).isReg() || CallParams.getOperand(0).isFI();
552 bool IsRetCall = CallResults.getOpcode() == WebAssembly::RET_CALL_RESULTS;
553
554 bool IsFuncrefCall = false;
555 if (IsIndirect && CallParams.getOperand(0).isReg()) {
556 Register Reg = CallParams.getOperand(0).getReg();
557 const MachineFunction *MF = BB->getParent();
558 const MachineRegisterInfo &MRI = MF->getRegInfo();
559 const TargetRegisterClass *TRC = MRI.getRegClass(Reg);
560 IsFuncrefCall = (TRC == &WebAssembly::FUNCREFRegClass);
561 assert(!IsFuncrefCall || Subtarget->hasReferenceTypes());
562 }
563
564 unsigned CallOp;
565 if (IsIndirect && IsRetCall) {
566 CallOp = WebAssembly::RET_CALL_INDIRECT;
567 } else if (IsIndirect) {
568 CallOp = WebAssembly::CALL_INDIRECT;
569 } else if (IsRetCall) {
570 CallOp = WebAssembly::RET_CALL;
571 } else {
572 CallOp = WebAssembly::CALL;
573 }
574
575 MachineFunction &MF = *BB->getParent();
576 const MCInstrDesc &MCID = TII.get(CallOp);
577 MachineInstrBuilder MIB(MF, MF.CreateMachineInstr(MCID, DL));
578
579 // See if we must truncate the function pointer.
580 // CALL_INDIRECT takes an i32, but in wasm64 we represent function pointers
581 // as 64-bit for uniformity with other pointer types.
582 // See also: WebAssemblyFastISel::selectCall
583 if (IsIndirect && MF.getSubtarget<WebAssemblySubtarget>().hasAddr64()) {
584 Register Reg32 =
585 MF.getRegInfo().createVirtualRegister(&WebAssembly::I32RegClass);
586 auto &FnPtr = CallParams.getOperand(0);
587 BuildMI(*BB, CallResults.getIterator(), DL,
588 TII.get(WebAssembly::I32_WRAP_I64), Reg32)
589 .addReg(FnPtr.getReg());
590 FnPtr.setReg(Reg32);
591 }
592
593 // Move the function pointer to the end of the arguments for indirect calls
594 if (IsIndirect) {
595 auto FnPtr = CallParams.getOperand(0);
596 CallParams.removeOperand(0);
597
598 // For funcrefs, call_indirect is done through __funcref_call_table and the
599 // funcref is always installed in slot 0 of the table, therefore instead of
600 // having the function pointer added at the end of the params list, a zero
601 // (the index in
602 // __funcref_call_table is added).
603 if (IsFuncrefCall) {
604 Register RegZero =
605 MF.getRegInfo().createVirtualRegister(&WebAssembly::I32RegClass);
606 MachineInstrBuilder MIBC0 =
607 BuildMI(MF, DL, TII.get(WebAssembly::CONST_I32), RegZero).addImm(0);
608
609 BB->insert(CallResults.getIterator(), MIBC0);
610 MachineInstrBuilder(MF, CallParams).addReg(RegZero);
611 } else
612 CallParams.addOperand(FnPtr);
613 }
614
615 for (auto Def : CallResults.defs())
616 MIB.add(Def);
617
618 if (IsIndirect) {
619 // Placeholder for the type index.
620 MIB.addImm(0);
621 // The table into which this call_indirect indexes.
622 MCSymbolWasm *Table = IsFuncrefCall
624 MF.getContext(), Subtarget)
626 MF.getContext(), Subtarget);
627 if (Subtarget->hasReferenceTypes()) {
628 MIB.addSym(Table);
629 } else {
630 // For the MVP there is at most one table whose number is 0, but we can't
631 // write a table symbol or issue relocations. Instead we just ensure the
632 // table is live and write a zero.
633 Table->setNoStrip();
634 MIB.addImm(0);
635 }
636 }
637
638 for (auto Use : CallParams.uses())
639 MIB.add(Use);
640
641 BB->insert(CallResults.getIterator(), MIB);
642 CallParams.eraseFromParent();
643 CallResults.eraseFromParent();
644
645 // If this is a funcref call, to avoid hidden GC roots, we need to clear the
646 // table slot with ref.null upon call_indirect return.
647 //
648 // This generates the following code, which comes right after a call_indirect
649 // of a funcref:
650 //
651 // i32.const 0
652 // ref.null func
653 // table.set __funcref_call_table
654 if (IsIndirect && IsFuncrefCall) {
656 MF.getContext(), Subtarget);
657 Register RegZero =
658 MF.getRegInfo().createVirtualRegister(&WebAssembly::I32RegClass);
659 MachineInstr *Const0 =
660 BuildMI(MF, DL, TII.get(WebAssembly::CONST_I32), RegZero).addImm(0);
661 BB->insertAfter(MIB.getInstr()->getIterator(), Const0);
662
663 Register RegFuncref =
664 MF.getRegInfo().createVirtualRegister(&WebAssembly::FUNCREFRegClass);
665 MachineInstr *RefNull =
666 BuildMI(MF, DL, TII.get(WebAssembly::REF_NULL_FUNCREF), RegFuncref);
667 BB->insertAfter(Const0->getIterator(), RefNull);
668
669 MachineInstr *TableSet =
670 BuildMI(MF, DL, TII.get(WebAssembly::TABLE_SET_FUNCREF))
671 .addSym(Table)
672 .addReg(RegZero)
673 .addReg(RegFuncref);
674 BB->insertAfter(RefNull->getIterator(), TableSet);
675 }
676
677 return BB;
678}
679
680MachineBasicBlock *WebAssemblyTargetLowering::EmitInstrWithCustomInserter(
681 MachineInstr &MI, MachineBasicBlock *BB) const {
682 const TargetInstrInfo &TII = *Subtarget->getInstrInfo();
683 DebugLoc DL = MI.getDebugLoc();
684
685 switch (MI.getOpcode()) {
686 default:
687 llvm_unreachable("Unexpected instr type to insert");
688 case WebAssembly::FP_TO_SINT_I32_F32:
689 return LowerFPToInt(MI, DL, BB, TII, false, false, false,
690 WebAssembly::I32_TRUNC_S_F32);
691 case WebAssembly::FP_TO_UINT_I32_F32:
692 return LowerFPToInt(MI, DL, BB, TII, true, false, false,
693 WebAssembly::I32_TRUNC_U_F32);
694 case WebAssembly::FP_TO_SINT_I64_F32:
695 return LowerFPToInt(MI, DL, BB, TII, false, true, false,
696 WebAssembly::I64_TRUNC_S_F32);
697 case WebAssembly::FP_TO_UINT_I64_F32:
698 return LowerFPToInt(MI, DL, BB, TII, true, true, false,
699 WebAssembly::I64_TRUNC_U_F32);
700 case WebAssembly::FP_TO_SINT_I32_F64:
701 return LowerFPToInt(MI, DL, BB, TII, false, false, true,
702 WebAssembly::I32_TRUNC_S_F64);
703 case WebAssembly::FP_TO_UINT_I32_F64:
704 return LowerFPToInt(MI, DL, BB, TII, true, false, true,
705 WebAssembly::I32_TRUNC_U_F64);
706 case WebAssembly::FP_TO_SINT_I64_F64:
707 return LowerFPToInt(MI, DL, BB, TII, false, true, true,
708 WebAssembly::I64_TRUNC_S_F64);
709 case WebAssembly::FP_TO_UINT_I64_F64:
710 return LowerFPToInt(MI, DL, BB, TII, true, true, true,
711 WebAssembly::I64_TRUNC_U_F64);
712 case WebAssembly::CALL_RESULTS:
713 case WebAssembly::RET_CALL_RESULTS:
714 return LowerCallResults(MI, DL, BB, Subtarget, TII);
715 }
716}
717
718const char *
719WebAssemblyTargetLowering::getTargetNodeName(unsigned Opcode) const {
720 switch (static_cast<WebAssemblyISD::NodeType>(Opcode)) {
723 break;
724#define HANDLE_NODETYPE(NODE) \
725 case WebAssemblyISD::NODE: \
726 return "WebAssemblyISD::" #NODE;
727#define HANDLE_MEM_NODETYPE(NODE) HANDLE_NODETYPE(NODE)
728#include "WebAssemblyISD.def"
729#undef HANDLE_MEM_NODETYPE
730#undef HANDLE_NODETYPE
731 }
732 return nullptr;
733}
734
735std::pair<unsigned, const TargetRegisterClass *>
736WebAssemblyTargetLowering::getRegForInlineAsmConstraint(
737 const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const {
738 // First, see if this is a constraint that directly corresponds to a
739 // WebAssembly register class.
740 if (Constraint.size() == 1) {
741 switch (Constraint[0]) {
742 case 'r':
743 assert(VT != MVT::iPTR && "Pointer MVT not expected here");
744 if (Subtarget->hasSIMD128() && VT.isVector()) {
745 if (VT.getSizeInBits() == 128)
746 return std::make_pair(0U, &WebAssembly::V128RegClass);
747 }
748 if (VT.isInteger() && !VT.isVector()) {
749 if (VT.getSizeInBits() <= 32)
750 return std::make_pair(0U, &WebAssembly::I32RegClass);
751 if (VT.getSizeInBits() <= 64)
752 return std::make_pair(0U, &WebAssembly::I64RegClass);
753 }
754 if (VT.isFloatingPoint() && !VT.isVector()) {
755 switch (VT.getSizeInBits()) {
756 case 32:
757 return std::make_pair(0U, &WebAssembly::F32RegClass);
758 case 64:
759 return std::make_pair(0U, &WebAssembly::F64RegClass);
760 default:
761 break;
762 }
763 }
764 break;
765 default:
766 break;
767 }
768 }
769
771}
772
773bool WebAssemblyTargetLowering::isCheapToSpeculateCttz(Type *Ty) const {
774 // Assume ctz is a relatively cheap operation.
775 return true;
776}
777
778bool WebAssemblyTargetLowering::isCheapToSpeculateCtlz(Type *Ty) const {
779 // Assume clz is a relatively cheap operation.
780 return true;
781}
782
783bool WebAssemblyTargetLowering::isLegalAddressingMode(const DataLayout &DL,
784 const AddrMode &AM,
785 Type *Ty, unsigned AS,
786 Instruction *I) const {
787 // WebAssembly offsets are added as unsigned without wrapping. The
788 // isLegalAddressingMode gives us no way to determine if wrapping could be
789 // happening, so we approximate this by accepting only non-negative offsets.
790 if (AM.BaseOffs < 0)
791 return false;
792
793 // WebAssembly has no scale register operands.
794 if (AM.Scale != 0)
795 return false;
796
797 // Everything else is legal.
798 return true;
799}
800
801bool WebAssemblyTargetLowering::allowsMisalignedMemoryAccesses(
802 EVT /*VT*/, unsigned /*AddrSpace*/, Align /*Align*/,
803 MachineMemOperand::Flags /*Flags*/, unsigned *Fast) const {
804 // WebAssembly supports unaligned accesses, though it should be declared
805 // with the p2align attribute on loads and stores which do so, and there
806 // may be a performance impact. We tell LLVM they're "fast" because
807 // for the kinds of things that LLVM uses this for (merging adjacent stores
808 // of constants, etc.), WebAssembly implementations will either want the
809 // unaligned access or they'll split anyway.
810 if (Fast)
811 *Fast = 1;
812 return true;
813}
814
815bool WebAssemblyTargetLowering::isIntDivCheap(EVT VT,
816 AttributeList Attr) const {
817 // The current thinking is that wasm engines will perform this optimization,
818 // so we can save on code size.
819 return true;
820}
821
822bool WebAssemblyTargetLowering::isVectorLoadExtDesirable(SDValue ExtVal) const {
823 EVT ExtT = ExtVal.getValueType();
824 EVT MemT = cast<LoadSDNode>(ExtVal->getOperand(0))->getValueType(0);
825 return (ExtT == MVT::v8i16 && MemT == MVT::v8i8) ||
826 (ExtT == MVT::v4i32 && MemT == MVT::v4i16) ||
827 (ExtT == MVT::v2i64 && MemT == MVT::v2i32);
828}
829
830bool WebAssemblyTargetLowering::isOffsetFoldingLegal(
831 const GlobalAddressSDNode *GA) const {
832 // Wasm doesn't support function addresses with offsets
833 const GlobalValue *GV = GA->getGlobal();
834 return isa<Function>(GV) ? false : TargetLowering::isOffsetFoldingLegal(GA);
835}
836
837bool WebAssemblyTargetLowering::shouldSinkOperands(
838 Instruction *I, SmallVectorImpl<Use *> &Ops) const {
839 using namespace llvm::PatternMatch;
840
841 if (!I->getType()->isVectorTy() || !I->isShift())
842 return false;
843
844 Value *V = I->getOperand(1);
845 // We dont need to sink constant splat.
846 if (dyn_cast<Constant>(V))
847 return false;
848
850 m_Value(), m_ZeroMask()))) {
851 // Sink insert
852 Ops.push_back(&cast<Instruction>(V)->getOperandUse(0));
853 // Sink shuffle
854 Ops.push_back(&I->getOperandUse(1));
855 return true;
856 }
857
858 return false;
859}
860
861EVT WebAssemblyTargetLowering::getSetCCResultType(const DataLayout &DL,
862 LLVMContext &C,
863 EVT VT) const {
864 if (VT.isVector())
866
867 // So far, all branch instructions in Wasm take an I32 condition.
868 // The default TargetLowering::getSetCCResultType returns the pointer size,
869 // which would be useful to reduce instruction counts when testing
870 // against 64-bit pointers/values if at some point Wasm supports that.
871 return EVT::getIntegerVT(C, 32);
872}
873
874bool WebAssemblyTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
875 const CallInst &I,
876 MachineFunction &MF,
877 unsigned Intrinsic) const {
878 switch (Intrinsic) {
879 case Intrinsic::wasm_memory_atomic_notify:
881 Info.memVT = MVT::i32;
882 Info.ptrVal = I.getArgOperand(0);
883 Info.offset = 0;
884 Info.align = Align(4);
885 // atomic.notify instruction does not really load the memory specified with
886 // this argument, but MachineMemOperand should either be load or store, so
887 // we set this to a load.
888 // FIXME Volatile isn't really correct, but currently all LLVM atomic
889 // instructions are treated as volatiles in the backend, so we should be
890 // consistent. The same applies for wasm_atomic_wait intrinsics too.
892 return true;
893 case Intrinsic::wasm_memory_atomic_wait32:
895 Info.memVT = MVT::i32;
896 Info.ptrVal = I.getArgOperand(0);
897 Info.offset = 0;
898 Info.align = Align(4);
900 return true;
901 case Intrinsic::wasm_memory_atomic_wait64:
903 Info.memVT = MVT::i64;
904 Info.ptrVal = I.getArgOperand(0);
905 Info.offset = 0;
906 Info.align = Align(8);
908 return true;
909 default:
910 return false;
911 }
912}
913
914void WebAssemblyTargetLowering::computeKnownBitsForTargetNode(
915 const SDValue Op, KnownBits &Known, const APInt &DemandedElts,
916 const SelectionDAG &DAG, unsigned Depth) const {
917 switch (Op.getOpcode()) {
918 default:
919 break;
921 unsigned IntNo = Op.getConstantOperandVal(0);
922 switch (IntNo) {
923 default:
924 break;
925 case Intrinsic::wasm_bitmask: {
926 unsigned BitWidth = Known.getBitWidth();
927 EVT VT = Op.getOperand(1).getSimpleValueType();
928 unsigned PossibleBits = VT.getVectorNumElements();
929 APInt ZeroMask = APInt::getHighBitsSet(BitWidth, BitWidth - PossibleBits);
930 Known.Zero |= ZeroMask;
931 break;
932 }
933 }
934 }
935 }
936}
937
939WebAssemblyTargetLowering::getPreferredVectorAction(MVT VT) const {
940 if (VT.isFixedLengthVector()) {
941 MVT EltVT = VT.getVectorElementType();
942 // We have legal vector types with these lane types, so widening the
943 // vector would let us use some of the lanes directly without having to
944 // extend or truncate values.
945 if (EltVT == MVT::i8 || EltVT == MVT::i16 || EltVT == MVT::i32 ||
946 EltVT == MVT::i64 || EltVT == MVT::f32 || EltVT == MVT::f64)
947 return TypeWidenVector;
948 }
949
951}
952
953bool WebAssemblyTargetLowering::shouldSimplifyDemandedVectorElts(
954 SDValue Op, const TargetLoweringOpt &TLO) const {
955 // ISel process runs DAGCombiner after legalization; this step is called
956 // SelectionDAG optimization phase. This post-legalization combining process
957 // runs DAGCombiner on each node, and if there was a change to be made,
958 // re-runs legalization again on it and its user nodes to make sure
959 // everythiing is in a legalized state.
960 //
961 // The legalization calls lowering routines, and we do our custom lowering for
962 // build_vectors (LowerBUILD_VECTOR), which converts undef vector elements
963 // into zeros. But there is a set of routines in DAGCombiner that turns unused
964 // (= not demanded) nodes into undef, among which SimplifyDemandedVectorElts
965 // turns unused vector elements into undefs. But this routine does not work
966 // with our custom LowerBUILD_VECTOR, which turns undefs into zeros. This
967 // combination can result in a infinite loop, in which undefs are converted to
968 // zeros in legalization and back to undefs in combining.
969 //
970 // So after DAG is legalized, we prevent SimplifyDemandedVectorElts from
971 // running for build_vectors.
972 if (Op.getOpcode() == ISD::BUILD_VECTOR && TLO.LegalOps && TLO.LegalTys)
973 return false;
974 return true;
975}
976
977//===----------------------------------------------------------------------===//
978// WebAssembly Lowering private implementation.
979//===----------------------------------------------------------------------===//
980
981//===----------------------------------------------------------------------===//
982// Lowering Code
983//===----------------------------------------------------------------------===//
984
985static void fail(const SDLoc &DL, SelectionDAG &DAG, const char *Msg) {
987 DAG.getContext()->diagnose(
988 DiagnosticInfoUnsupported(MF.getFunction(), Msg, DL.getDebugLoc()));
989}
990
991// Test whether the given calling convention is supported.
993 // We currently support the language-independent target-independent
994 // conventions. We don't yet have a way to annotate calls with properties like
995 // "cold", and we don't have any call-clobbered registers, so these are mostly
996 // all handled the same.
997 return CallConv == CallingConv::C || CallConv == CallingConv::Fast ||
998 CallConv == CallingConv::Cold ||
999 CallConv == CallingConv::PreserveMost ||
1000 CallConv == CallingConv::PreserveAll ||
1001 CallConv == CallingConv::CXX_FAST_TLS ||
1003 CallConv == CallingConv::Swift;
1004}
1005
1006SDValue
1007WebAssemblyTargetLowering::LowerCall(CallLoweringInfo &CLI,
1008 SmallVectorImpl<SDValue> &InVals) const {
1009 SelectionDAG &DAG = CLI.DAG;
1010 SDLoc DL = CLI.DL;
1011 SDValue Chain = CLI.Chain;
1012 SDValue Callee = CLI.Callee;
1014 auto Layout = MF.getDataLayout();
1015
1016 CallingConv::ID CallConv = CLI.CallConv;
1017 if (!callingConvSupported(CallConv))
1018 fail(DL, DAG,
1019 "WebAssembly doesn't support language-specific or target-specific "
1020 "calling conventions yet");
1021 if (CLI.IsPatchPoint)
1022 fail(DL, DAG, "WebAssembly doesn't support patch point yet");
1023
1024 if (CLI.IsTailCall) {
1025 auto NoTail = [&](const char *Msg) {
1026 if (CLI.CB && CLI.CB->isMustTailCall())
1027 fail(DL, DAG, Msg);
1028 CLI.IsTailCall = false;
1029 };
1030
1031 if (!Subtarget->hasTailCall())
1032 NoTail("WebAssembly 'tail-call' feature not enabled");
1033
1034 // Varargs calls cannot be tail calls because the buffer is on the stack
1035 if (CLI.IsVarArg)
1036 NoTail("WebAssembly does not support varargs tail calls");
1037
1038 // Do not tail call unless caller and callee return types match
1039 const Function &F = MF.getFunction();
1041 Type *RetTy = F.getReturnType();
1042 SmallVector<MVT, 4> CallerRetTys;
1043 SmallVector<MVT, 4> CalleeRetTys;
1044 computeLegalValueVTs(F, TM, RetTy, CallerRetTys);
1045 computeLegalValueVTs(F, TM, CLI.RetTy, CalleeRetTys);
1046 bool TypesMatch = CallerRetTys.size() == CalleeRetTys.size() &&
1047 std::equal(CallerRetTys.begin(), CallerRetTys.end(),
1048 CalleeRetTys.begin());
1049 if (!TypesMatch)
1050 NoTail("WebAssembly tail call requires caller and callee return types to "
1051 "match");
1052
1053 // If pointers to local stack values are passed, we cannot tail call
1054 if (CLI.CB) {
1055 for (auto &Arg : CLI.CB->args()) {
1056 Value *Val = Arg.get();
1057 // Trace the value back through pointer operations
1058 while (true) {
1059 Value *Src = Val->stripPointerCastsAndAliases();
1060 if (auto *GEP = dyn_cast<GetElementPtrInst>(Src))
1061 Src = GEP->getPointerOperand();
1062 if (Val == Src)
1063 break;
1064 Val = Src;
1065 }
1066 if (isa<AllocaInst>(Val)) {
1067 NoTail(
1068 "WebAssembly does not support tail calling with stack arguments");
1069 break;
1070 }
1071 }
1072 }
1073 }
1074
1076 SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
1077 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
1078
1079 // The generic code may have added an sret argument. If we're lowering an
1080 // invoke function, the ABI requires that the function pointer be the first
1081 // argument, so we may have to swap the arguments.
1082 if (CallConv == CallingConv::WASM_EmscriptenInvoke && Outs.size() >= 2 &&
1083 Outs[0].Flags.isSRet()) {
1084 std::swap(Outs[0], Outs[1]);
1085 std::swap(OutVals[0], OutVals[1]);
1086 }
1087
1088 bool HasSwiftSelfArg = false;
1089 bool HasSwiftErrorArg = false;
1090 unsigned NumFixedArgs = 0;
1091 for (unsigned I = 0; I < Outs.size(); ++I) {
1092 const ISD::OutputArg &Out = Outs[I];
1093 SDValue &OutVal = OutVals[I];
1094 HasSwiftSelfArg |= Out.Flags.isSwiftSelf();
1095 HasSwiftErrorArg |= Out.Flags.isSwiftError();
1096 if (Out.Flags.isNest())
1097 fail(DL, DAG, "WebAssembly hasn't implemented nest arguments");
1098 if (Out.Flags.isInAlloca())
1099 fail(DL, DAG, "WebAssembly hasn't implemented inalloca arguments");
1100 if (Out.Flags.isInConsecutiveRegs())
1101 fail(DL, DAG, "WebAssembly hasn't implemented cons regs arguments");
1103 fail(DL, DAG, "WebAssembly hasn't implemented cons regs last arguments");
1104 if (Out.Flags.isByVal() && Out.Flags.getByValSize() != 0) {
1105 auto &MFI = MF.getFrameInfo();
1106 int FI = MFI.CreateStackObject(Out.Flags.getByValSize(),
1108 /*isSS=*/false);
1109 SDValue SizeNode =
1110 DAG.getConstant(Out.Flags.getByValSize(), DL, MVT::i32);
1111 SDValue FINode = DAG.getFrameIndex(FI, getPointerTy(Layout));
1112 Chain = DAG.getMemcpy(
1113 Chain, DL, FINode, OutVal, SizeNode, Out.Flags.getNonZeroByValAlign(),
1114 /*isVolatile*/ false, /*AlwaysInline=*/false,
1115 /*isTailCall*/ false, MachinePointerInfo(), MachinePointerInfo());
1116 OutVal = FINode;
1117 }
1118 // Count the number of fixed args *after* legalization.
1119 NumFixedArgs += Out.IsFixed;
1120 }
1121
1122 bool IsVarArg = CLI.IsVarArg;
1123 auto PtrVT = getPointerTy(Layout);
1124
1125 // For swiftcc, emit additional swiftself and swifterror arguments
1126 // if there aren't. These additional arguments are also added for callee
1127 // signature They are necessary to match callee and caller signature for
1128 // indirect call.
1129 if (CallConv == CallingConv::Swift) {
1130 if (!HasSwiftSelfArg) {
1131 NumFixedArgs++;
1132 ISD::OutputArg Arg;
1133 Arg.Flags.setSwiftSelf();
1134 CLI.Outs.push_back(Arg);
1135 SDValue ArgVal = DAG.getUNDEF(PtrVT);
1136 CLI.OutVals.push_back(ArgVal);
1137 }
1138 if (!HasSwiftErrorArg) {
1139 NumFixedArgs++;
1140 ISD::OutputArg Arg;
1141 Arg.Flags.setSwiftError();
1142 CLI.Outs.push_back(Arg);
1143 SDValue ArgVal = DAG.getUNDEF(PtrVT);
1144 CLI.OutVals.push_back(ArgVal);
1145 }
1146 }
1147
1148 // Analyze operands of the call, assigning locations to each operand.
1150 CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
1151
1152 if (IsVarArg) {
1153 // Outgoing non-fixed arguments are placed in a buffer. First
1154 // compute their offsets and the total amount of buffer space needed.
1155 for (unsigned I = NumFixedArgs; I < Outs.size(); ++I) {
1156 const ISD::OutputArg &Out = Outs[I];
1157 SDValue &Arg = OutVals[I];
1158 EVT VT = Arg.getValueType();
1159 assert(VT != MVT::iPTR && "Legalized args should be concrete");
1160 Type *Ty = VT.getTypeForEVT(*DAG.getContext());
1161 Align Alignment =
1162 std::max(Out.Flags.getNonZeroOrigAlign(), Layout.getABITypeAlign(Ty));
1163 unsigned Offset =
1164 CCInfo.AllocateStack(Layout.getTypeAllocSize(Ty), Alignment);
1165 CCInfo.addLoc(CCValAssign::getMem(ArgLocs.size(), VT.getSimpleVT(),
1166 Offset, VT.getSimpleVT(),
1168 }
1169 }
1170
1171 unsigned NumBytes = CCInfo.getAlignedCallFrameSize();
1172
1173 SDValue FINode;
1174 if (IsVarArg && NumBytes) {
1175 // For non-fixed arguments, next emit stores to store the argument values
1176 // to the stack buffer at the offsets computed above.
1177 int FI = MF.getFrameInfo().CreateStackObject(NumBytes,
1178 Layout.getStackAlignment(),
1179 /*isSS=*/false);
1180 unsigned ValNo = 0;
1182 for (SDValue Arg : drop_begin(OutVals, NumFixedArgs)) {
1183 assert(ArgLocs[ValNo].getValNo() == ValNo &&
1184 "ArgLocs should remain in order and only hold varargs args");
1185 unsigned Offset = ArgLocs[ValNo++].getLocMemOffset();
1186 FINode = DAG.getFrameIndex(FI, getPointerTy(Layout));
1187 SDValue Add = DAG.getNode(ISD::ADD, DL, PtrVT, FINode,
1188 DAG.getConstant(Offset, DL, PtrVT));
1189 Chains.push_back(
1190 DAG.getStore(Chain, DL, Arg, Add,
1192 }
1193 if (!Chains.empty())
1194 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains);
1195 } else if (IsVarArg) {
1196 FINode = DAG.getIntPtrConstant(0, DL);
1197 }
1198
1199 if (Callee->getOpcode() == ISD::GlobalAddress) {
1200 // If the callee is a GlobalAddress node (quite common, every direct call
1201 // is) turn it into a TargetGlobalAddress node so that LowerGlobalAddress
1202 // doesn't at MO_GOT which is not needed for direct calls.
1203 GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Callee);
1206 GA->getOffset());
1207 Callee = DAG.getNode(WebAssemblyISD::Wrapper, DL,
1208 getPointerTy(DAG.getDataLayout()), Callee);
1209 }
1210
1211 // Compute the operands for the CALLn node.
1213 Ops.push_back(Chain);
1214 Ops.push_back(Callee);
1215
1216 // Add all fixed arguments. Note that for non-varargs calls, NumFixedArgs
1217 // isn't reliable.
1218 Ops.append(OutVals.begin(),
1219 IsVarArg ? OutVals.begin() + NumFixedArgs : OutVals.end());
1220 // Add a pointer to the vararg buffer.
1221 if (IsVarArg)
1222 Ops.push_back(FINode);
1223
1224 SmallVector<EVT, 8> InTys;
1225 for (const auto &In : Ins) {
1226 assert(!In.Flags.isByVal() && "byval is not valid for return values");
1227 assert(!In.Flags.isNest() && "nest is not valid for return values");
1228 if (In.Flags.isInAlloca())
1229 fail(DL, DAG, "WebAssembly hasn't implemented inalloca return values");
1230 if (In.Flags.isInConsecutiveRegs())
1231 fail(DL, DAG, "WebAssembly hasn't implemented cons regs return values");
1232 if (In.Flags.isInConsecutiveRegsLast())
1233 fail(DL, DAG,
1234 "WebAssembly hasn't implemented cons regs last return values");
1235 // Ignore In.getNonZeroOrigAlign() because all our arguments are passed in
1236 // registers.
1237 InTys.push_back(In.VT);
1238 }
1239
1240 // Lastly, if this is a call to a funcref we need to add an instruction
1241 // table.set to the chain and transform the call.
1243 CLI.CB->getCalledOperand()->getType())) {
1244 // In the absence of function references proposal where a funcref call is
1245 // lowered to call_ref, using reference types we generate a table.set to set
1246 // the funcref to a special table used solely for this purpose, followed by
1247 // a call_indirect. Here we just generate the table set, and return the
1248 // SDValue of the table.set so that LowerCall can finalize the lowering by
1249 // generating the call_indirect.
1250 SDValue Chain = Ops[0];
1251
1253 MF.getContext(), Subtarget);
1254 SDValue Sym = DAG.getMCSymbol(Table, PtrVT);
1255 SDValue TableSlot = DAG.getConstant(0, DL, MVT::i32);
1256 SDValue TableSetOps[] = {Chain, Sym, TableSlot, Callee};
1257 SDValue TableSet = DAG.getMemIntrinsicNode(
1258 WebAssemblyISD::TABLE_SET, DL, DAG.getVTList(MVT::Other), TableSetOps,
1259 MVT::funcref,
1260 // Machine Mem Operand args
1263 CLI.CB->getCalledOperand()->getPointerAlignment(DAG.getDataLayout()),
1265
1266 Ops[0] = TableSet; // The new chain is the TableSet itself
1267 }
1268
1269 if (CLI.IsTailCall) {
1270 // ret_calls do not return values to the current frame
1271 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
1272 return DAG.getNode(WebAssemblyISD::RET_CALL, DL, NodeTys, Ops);
1273 }
1274
1275 InTys.push_back(MVT::Other);
1276 SDVTList InTyList = DAG.getVTList(InTys);
1277 SDValue Res = DAG.getNode(WebAssemblyISD::CALL, DL, InTyList, Ops);
1278
1279 for (size_t I = 0; I < Ins.size(); ++I)
1280 InVals.push_back(Res.getValue(I));
1281
1282 // Return the chain
1283 return Res.getValue(Ins.size());
1284}
1285
1286bool WebAssemblyTargetLowering::CanLowerReturn(
1287 CallingConv::ID /*CallConv*/, MachineFunction & /*MF*/, bool /*IsVarArg*/,
1289 LLVMContext & /*Context*/) const {
1290 // WebAssembly can only handle returning tuples with multivalue enabled
1291 return Subtarget->hasMultivalue() || Outs.size() <= 1;
1292}
1293
1294SDValue WebAssemblyTargetLowering::LowerReturn(
1295 SDValue Chain, CallingConv::ID CallConv, bool /*IsVarArg*/,
1297 const SmallVectorImpl<SDValue> &OutVals, const SDLoc &DL,
1298 SelectionDAG &DAG) const {
1299 assert((Subtarget->hasMultivalue() || Outs.size() <= 1) &&
1300 "MVP WebAssembly can only return up to one value");
1301 if (!callingConvSupported(CallConv))
1302 fail(DL, DAG, "WebAssembly doesn't support non-C calling conventions");
1303
1304 SmallVector<SDValue, 4> RetOps(1, Chain);
1305 RetOps.append(OutVals.begin(), OutVals.end());
1306 Chain = DAG.getNode(WebAssemblyISD::RETURN, DL, MVT::Other, RetOps);
1307
1308 // Record the number and types of the return values.
1309 for (const ISD::OutputArg &Out : Outs) {
1310 assert(!Out.Flags.isByVal() && "byval is not valid for return values");
1311 assert(!Out.Flags.isNest() && "nest is not valid for return values");
1312 assert(Out.IsFixed && "non-fixed return value is not valid");
1313 if (Out.Flags.isInAlloca())
1314 fail(DL, DAG, "WebAssembly hasn't implemented inalloca results");
1315 if (Out.Flags.isInConsecutiveRegs())
1316 fail(DL, DAG, "WebAssembly hasn't implemented cons regs results");
1318 fail(DL, DAG, "WebAssembly hasn't implemented cons regs last results");
1319 }
1320
1321 return Chain;
1322}
1323
1324SDValue WebAssemblyTargetLowering::LowerFormalArguments(
1325 SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
1326 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
1327 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
1328 if (!callingConvSupported(CallConv))
1329 fail(DL, DAG, "WebAssembly doesn't support non-C calling conventions");
1330
1332 auto *MFI = MF.getInfo<WebAssemblyFunctionInfo>();
1333
1334 // Set up the incoming ARGUMENTS value, which serves to represent the liveness
1335 // of the incoming values before they're represented by virtual registers.
1336 MF.getRegInfo().addLiveIn(WebAssembly::ARGUMENTS);
1337
1338 bool HasSwiftErrorArg = false;
1339 bool HasSwiftSelfArg = false;
1340 for (const ISD::InputArg &In : Ins) {
1341 HasSwiftSelfArg |= In.Flags.isSwiftSelf();
1342 HasSwiftErrorArg |= In.Flags.isSwiftError();
1343 if (In.Flags.isInAlloca())
1344 fail(DL, DAG, "WebAssembly hasn't implemented inalloca arguments");
1345 if (In.Flags.isNest())
1346 fail(DL, DAG, "WebAssembly hasn't implemented nest arguments");
1347 if (In.Flags.isInConsecutiveRegs())
1348 fail(DL, DAG, "WebAssembly hasn't implemented cons regs arguments");
1349 if (In.Flags.isInConsecutiveRegsLast())
1350 fail(DL, DAG, "WebAssembly hasn't implemented cons regs last arguments");
1351 // Ignore In.getNonZeroOrigAlign() because all our arguments are passed in
1352 // registers.
1353 InVals.push_back(In.Used ? DAG.getNode(WebAssemblyISD::ARGUMENT, DL, In.VT,
1354 DAG.getTargetConstant(InVals.size(),
1355 DL, MVT::i32))
1356 : DAG.getUNDEF(In.VT));
1357
1358 // Record the number and types of arguments.
1359 MFI->addParam(In.VT);
1360 }
1361
1362 // For swiftcc, emit additional swiftself and swifterror arguments
1363 // if there aren't. These additional arguments are also added for callee
1364 // signature They are necessary to match callee and caller signature for
1365 // indirect call.
1366 auto PtrVT = getPointerTy(MF.getDataLayout());
1367 if (CallConv == CallingConv::Swift) {
1368 if (!HasSwiftSelfArg) {
1369 MFI->addParam(PtrVT);
1370 }
1371 if (!HasSwiftErrorArg) {
1372 MFI->addParam(PtrVT);
1373 }
1374 }
1375 // Varargs are copied into a buffer allocated by the caller, and a pointer to
1376 // the buffer is passed as an argument.
1377 if (IsVarArg) {
1378 MVT PtrVT = getPointerTy(MF.getDataLayout());
1379 Register VarargVreg =
1381 MFI->setVarargBufferVreg(VarargVreg);
1382 Chain = DAG.getCopyToReg(
1383 Chain, DL, VarargVreg,
1384 DAG.getNode(WebAssemblyISD::ARGUMENT, DL, PtrVT,
1385 DAG.getTargetConstant(Ins.size(), DL, MVT::i32)));
1386 MFI->addParam(PtrVT);
1387 }
1388
1389 // Record the number and types of arguments and results.
1390 SmallVector<MVT, 4> Params;
1393 MF.getFunction(), DAG.getTarget(), Params, Results);
1394 for (MVT VT : Results)
1395 MFI->addResult(VT);
1396 // TODO: Use signatures in WebAssemblyMachineFunctionInfo too and unify
1397 // the param logic here with ComputeSignatureVTs
1398 assert(MFI->getParams().size() == Params.size() &&
1399 std::equal(MFI->getParams().begin(), MFI->getParams().end(),
1400 Params.begin()));
1401
1402 return Chain;
1403}
1404
1405void WebAssemblyTargetLowering::ReplaceNodeResults(
1407 switch (N->getOpcode()) {
1409 // Do not add any results, signifying that N should not be custom lowered
1410 // after all. This happens because simd128 turns on custom lowering for
1411 // SIGN_EXTEND_INREG, but for non-vector sign extends the result might be an
1412 // illegal type.
1413 break;
1416 // Do not add any results, signifying that N should not be custom lowered.
1417 // EXTEND_VECTOR_INREG is implemented for some vectors, but not all.
1418 break;
1419 default:
1421 "ReplaceNodeResults not implemented for this op for WebAssembly!");
1422 }
1423}
1424
1425//===----------------------------------------------------------------------===//
1426// Custom lowering hooks.
1427//===----------------------------------------------------------------------===//
1428
1429SDValue WebAssemblyTargetLowering::LowerOperation(SDValue Op,
1430 SelectionDAG &DAG) const {
1431 SDLoc DL(Op);
1432 switch (Op.getOpcode()) {
1433 default:
1434 llvm_unreachable("unimplemented operation lowering");
1435 return SDValue();
1436 case ISD::FrameIndex:
1437 return LowerFrameIndex(Op, DAG);
1438 case ISD::GlobalAddress:
1439 return LowerGlobalAddress(Op, DAG);
1441 return LowerGlobalTLSAddress(Op, DAG);
1443 return LowerExternalSymbol(Op, DAG);
1444 case ISD::JumpTable:
1445 return LowerJumpTable(Op, DAG);
1446 case ISD::BR_JT:
1447 return LowerBR_JT(Op, DAG);
1448 case ISD::VASTART:
1449 return LowerVASTART(Op, DAG);
1450 case ISD::BlockAddress:
1451 case ISD::BRIND:
1452 fail(DL, DAG, "WebAssembly hasn't implemented computed gotos");
1453 return SDValue();
1454 case ISD::RETURNADDR:
1455 return LowerRETURNADDR(Op, DAG);
1456 case ISD::FRAMEADDR:
1457 return LowerFRAMEADDR(Op, DAG);
1458 case ISD::CopyToReg:
1459 return LowerCopyToReg(Op, DAG);
1462 return LowerAccessVectorElement(Op, DAG);
1466 return LowerIntrinsic(Op, DAG);
1468 return LowerSIGN_EXTEND_INREG(Op, DAG);
1471 return LowerEXTEND_VECTOR_INREG(Op, DAG);
1472 case ISD::BUILD_VECTOR:
1473 return LowerBUILD_VECTOR(Op, DAG);
1475 return LowerVECTOR_SHUFFLE(Op, DAG);
1476 case ISD::SETCC:
1477 return LowerSETCC(Op, DAG);
1478 case ISD::SHL:
1479 case ISD::SRA:
1480 case ISD::SRL:
1481 return LowerShift(Op, DAG);
1484 return LowerFP_TO_INT_SAT(Op, DAG);
1485 case ISD::LOAD:
1486 return LowerLoad(Op, DAG);
1487 case ISD::STORE:
1488 return LowerStore(Op, DAG);
1489 case ISD::CTPOP:
1490 case ISD::CTLZ:
1491 case ISD::CTTZ:
1492 return DAG.UnrollVectorOp(Op.getNode());
1493 }
1494}
1495
1497 if (const GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Op))
1499
1500 return false;
1501}
1502
1503static std::optional<unsigned> IsWebAssemblyLocal(SDValue Op,
1504 SelectionDAG &DAG) {
1505 const FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Op);
1506 if (!FI)
1507 return std::nullopt;
1508
1509 auto &MF = DAG.getMachineFunction();
1511}
1512
1513SDValue WebAssemblyTargetLowering::LowerStore(SDValue Op,
1514 SelectionDAG &DAG) const {
1515 SDLoc DL(Op);
1516 StoreSDNode *SN = cast<StoreSDNode>(Op.getNode());
1517 const SDValue &Value = SN->getValue();
1518 const SDValue &Base = SN->getBasePtr();
1519 const SDValue &Offset = SN->getOffset();
1520
1522 if (!Offset->isUndef())
1523 report_fatal_error("unexpected offset when storing to webassembly global",
1524 false);
1525
1526 SDVTList Tys = DAG.getVTList(MVT::Other);
1527 SDValue Ops[] = {SN->getChain(), Value, Base};
1528 return DAG.getMemIntrinsicNode(WebAssemblyISD::GLOBAL_SET, DL, Tys, Ops,
1529 SN->getMemoryVT(), SN->getMemOperand());
1530 }
1531
1532 if (std::optional<unsigned> Local = IsWebAssemblyLocal(Base, DAG)) {
1533 if (!Offset->isUndef())
1534 report_fatal_error("unexpected offset when storing to webassembly local",
1535 false);
1536
1537 SDValue Idx = DAG.getTargetConstant(*Local, Base, MVT::i32);
1538 SDVTList Tys = DAG.getVTList(MVT::Other); // The chain.
1539 SDValue Ops[] = {SN->getChain(), Idx, Value};
1540 return DAG.getNode(WebAssemblyISD::LOCAL_SET, DL, Tys, Ops);
1541 }
1542
1545 "Encountered an unlowerable store to the wasm_var address space",
1546 false);
1547
1548 return Op;
1549}
1550
1551SDValue WebAssemblyTargetLowering::LowerLoad(SDValue Op,
1552 SelectionDAG &DAG) const {
1553 SDLoc DL(Op);
1554 LoadSDNode *LN = cast<LoadSDNode>(Op.getNode());
1555 const SDValue &Base = LN->getBasePtr();
1556 const SDValue &Offset = LN->getOffset();
1557
1559 if (!Offset->isUndef())
1561 "unexpected offset when loading from webassembly global", false);
1562
1563 SDVTList Tys = DAG.getVTList(LN->getValueType(0), MVT::Other);
1564 SDValue Ops[] = {LN->getChain(), Base};
1565 return DAG.getMemIntrinsicNode(WebAssemblyISD::GLOBAL_GET, DL, Tys, Ops,
1566 LN->getMemoryVT(), LN->getMemOperand());
1567 }
1568
1569 if (std::optional<unsigned> Local = IsWebAssemblyLocal(Base, DAG)) {
1570 if (!Offset->isUndef())
1572 "unexpected offset when loading from webassembly local", false);
1573
1574 SDValue Idx = DAG.getTargetConstant(*Local, Base, MVT::i32);
1575 EVT LocalVT = LN->getValueType(0);
1576 SDValue LocalGet = DAG.getNode(WebAssemblyISD::LOCAL_GET, DL, LocalVT,
1577 {LN->getChain(), Idx});
1578 SDValue Result = DAG.getMergeValues({LocalGet, LN->getChain()}, DL);
1579 assert(Result->getNumValues() == 2 && "Loads must carry a chain!");
1580 return Result;
1581 }
1582
1585 "Encountered an unlowerable load from the wasm_var address space",
1586 false);
1587
1588 return Op;
1589}
1590
1591SDValue WebAssemblyTargetLowering::LowerCopyToReg(SDValue Op,
1592 SelectionDAG &DAG) const {
1593 SDValue Src = Op.getOperand(2);
1594 if (isa<FrameIndexSDNode>(Src.getNode())) {
1595 // CopyToReg nodes don't support FrameIndex operands. Other targets select
1596 // the FI to some LEA-like instruction, but since we don't have that, we
1597 // need to insert some kind of instruction that can take an FI operand and
1598 // produces a value usable by CopyToReg (i.e. in a vreg). So insert a dummy
1599 // local.copy between Op and its FI operand.
1600 SDValue Chain = Op.getOperand(0);
1601 SDLoc DL(Op);
1602 Register Reg = cast<RegisterSDNode>(Op.getOperand(1))->getReg();
1603 EVT VT = Src.getValueType();
1604 SDValue Copy(DAG.getMachineNode(VT == MVT::i32 ? WebAssembly::COPY_I32
1605 : WebAssembly::COPY_I64,
1606 DL, VT, Src),
1607 0);
1608 return Op.getNode()->getNumValues() == 1
1609 ? DAG.getCopyToReg(Chain, DL, Reg, Copy)
1610 : DAG.getCopyToReg(Chain, DL, Reg, Copy,
1611 Op.getNumOperands() == 4 ? Op.getOperand(3)
1612 : SDValue());
1613 }
1614 return SDValue();
1615}
1616
1617SDValue WebAssemblyTargetLowering::LowerFrameIndex(SDValue Op,
1618 SelectionDAG &DAG) const {
1619 int FI = cast<FrameIndexSDNode>(Op)->getIndex();
1620 return DAG.getTargetFrameIndex(FI, Op.getValueType());
1621}
1622
1623SDValue WebAssemblyTargetLowering::LowerRETURNADDR(SDValue Op,
1624 SelectionDAG &DAG) const {
1625 SDLoc DL(Op);
1626
1627 if (!Subtarget->getTargetTriple().isOSEmscripten()) {
1628 fail(DL, DAG,
1629 "Non-Emscripten WebAssembly hasn't implemented "
1630 "__builtin_return_address");
1631 return SDValue();
1632 }
1633
1635 return SDValue();
1636
1637 unsigned Depth = Op.getConstantOperandVal(0);
1638 MakeLibCallOptions CallOptions;
1639 return makeLibCall(DAG, RTLIB::RETURN_ADDRESS, Op.getValueType(),
1640 {DAG.getConstant(Depth, DL, MVT::i32)}, CallOptions, DL)
1641 .first;
1642}
1643
1644SDValue WebAssemblyTargetLowering::LowerFRAMEADDR(SDValue Op,
1645 SelectionDAG &DAG) const {
1646 // Non-zero depths are not supported by WebAssembly currently. Use the
1647 // legalizer's default expansion, which is to return 0 (what this function is
1648 // documented to do).
1649 if (Op.getConstantOperandVal(0) > 0)
1650 return SDValue();
1651
1653 EVT VT = Op.getValueType();
1654 Register FP =
1656 return DAG.getCopyFromReg(DAG.getEntryNode(), SDLoc(Op), FP, VT);
1657}
1658
1659SDValue
1660WebAssemblyTargetLowering::LowerGlobalTLSAddress(SDValue Op,
1661 SelectionDAG &DAG) const {
1662 SDLoc DL(Op);
1663 const auto *GA = cast<GlobalAddressSDNode>(Op);
1664
1667 report_fatal_error("cannot use thread-local storage without bulk memory",
1668 false);
1669
1670 const GlobalValue *GV = GA->getGlobal();
1671
1672 // Currently only Emscripten supports dynamic linking with threads. Therefore,
1673 // on other targets, if we have thread-local storage, only the local-exec
1674 // model is possible.
1675 auto model = Subtarget->getTargetTriple().isOSEmscripten()
1676 ? GV->getThreadLocalMode()
1678
1679 // Unsupported TLS modes
1682
1683 if (model == GlobalValue::LocalExecTLSModel ||
1686 getTargetMachine().shouldAssumeDSOLocal(*GV->getParent(), GV))) {
1687 // For DSO-local TLS variables we use offset from __tls_base
1688
1689 MVT PtrVT = getPointerTy(DAG.getDataLayout());
1690 auto GlobalGet = PtrVT == MVT::i64 ? WebAssembly::GLOBAL_GET_I64
1691 : WebAssembly::GLOBAL_GET_I32;
1692 const char *BaseName = MF.createExternalSymbolName("__tls_base");
1693
1695 DAG.getMachineNode(GlobalGet, DL, PtrVT,
1696 DAG.getTargetExternalSymbol(BaseName, PtrVT)),
1697 0);
1698
1699 SDValue TLSOffset = DAG.getTargetGlobalAddress(
1700 GV, DL, PtrVT, GA->getOffset(), WebAssemblyII::MO_TLS_BASE_REL);
1701 SDValue SymOffset =
1702 DAG.getNode(WebAssemblyISD::WrapperREL, DL, PtrVT, TLSOffset);
1703
1704 return DAG.getNode(ISD::ADD, DL, PtrVT, BaseAddr, SymOffset);
1705 }
1706
1708
1709 EVT VT = Op.getValueType();
1710 return DAG.getNode(WebAssemblyISD::Wrapper, DL, VT,
1711 DAG.getTargetGlobalAddress(GA->getGlobal(), DL, VT,
1712 GA->getOffset(),
1714}
1715
1716SDValue WebAssemblyTargetLowering::LowerGlobalAddress(SDValue Op,
1717 SelectionDAG &DAG) const {
1718 SDLoc DL(Op);
1719 const auto *GA = cast<GlobalAddressSDNode>(Op);
1720 EVT VT = Op.getValueType();
1721 assert(GA->getTargetFlags() == 0 &&
1722 "Unexpected target flags on generic GlobalAddressSDNode");
1724 fail(DL, DAG, "Invalid address space for WebAssembly target");
1725
1726 unsigned OperandFlags = 0;
1727 const GlobalValue *GV = GA->getGlobal();
1728 // Since WebAssembly tables cannot yet be shared accross modules, we don't
1729 // need special treatment for tables in PIC mode.
1730 if (isPositionIndependent() &&
1732 if (getTargetMachine().shouldAssumeDSOLocal(*GV->getParent(), GV)) {
1734 MVT PtrVT = getPointerTy(MF.getDataLayout());
1735 const char *BaseName;
1736 if (GV->getValueType()->isFunctionTy()) {
1737 BaseName = MF.createExternalSymbolName("__table_base");
1739 } else {
1740 BaseName = MF.createExternalSymbolName("__memory_base");
1742 }
1744 DAG.getNode(WebAssemblyISD::Wrapper, DL, PtrVT,
1745 DAG.getTargetExternalSymbol(BaseName, PtrVT));
1746
1747 SDValue SymAddr = DAG.getNode(
1748 WebAssemblyISD::WrapperREL, DL, VT,
1749 DAG.getTargetGlobalAddress(GA->getGlobal(), DL, VT, GA->getOffset(),
1750 OperandFlags));
1751
1752 return DAG.getNode(ISD::ADD, DL, VT, BaseAddr, SymAddr);
1753 }
1755 }
1756
1757 return DAG.getNode(WebAssemblyISD::Wrapper, DL, VT,
1758 DAG.getTargetGlobalAddress(GA->getGlobal(), DL, VT,
1759 GA->getOffset(), OperandFlags));
1760}
1761
1762SDValue
1763WebAssemblyTargetLowering::LowerExternalSymbol(SDValue Op,
1764 SelectionDAG &DAG) const {
1765 SDLoc DL(Op);
1766 const auto *ES = cast<ExternalSymbolSDNode>(Op);
1767 EVT VT = Op.getValueType();
1768 assert(ES->getTargetFlags() == 0 &&
1769 "Unexpected target flags on generic ExternalSymbolSDNode");
1770 return DAG.getNode(WebAssemblyISD::Wrapper, DL, VT,
1771 DAG.getTargetExternalSymbol(ES->getSymbol(), VT));
1772}
1773
1774SDValue WebAssemblyTargetLowering::LowerJumpTable(SDValue Op,
1775 SelectionDAG &DAG) const {
1776 // There's no need for a Wrapper node because we always incorporate a jump
1777 // table operand into a BR_TABLE instruction, rather than ever
1778 // materializing it in a register.
1779 const JumpTableSDNode *JT = cast<JumpTableSDNode>(Op);
1780 return DAG.getTargetJumpTable(JT->getIndex(), Op.getValueType(),
1781 JT->getTargetFlags());
1782}
1783
1784SDValue WebAssemblyTargetLowering::LowerBR_JT(SDValue Op,
1785 SelectionDAG &DAG) const {
1786 SDLoc DL(Op);
1787 SDValue Chain = Op.getOperand(0);
1788 const auto *JT = cast<JumpTableSDNode>(Op.getOperand(1));
1789 SDValue Index = Op.getOperand(2);
1790 assert(JT->getTargetFlags() == 0 && "WebAssembly doesn't set target flags");
1791
1793 Ops.push_back(Chain);
1794 Ops.push_back(Index);
1795
1797 const auto &MBBs = MJTI->getJumpTables()[JT->getIndex()].MBBs;
1798
1799 // Add an operand for each case.
1800 for (auto *MBB : MBBs)
1801 Ops.push_back(DAG.getBasicBlock(MBB));
1802
1803 // Add the first MBB as a dummy default target for now. This will be replaced
1804 // with the proper default target (and the preceding range check eliminated)
1805 // if possible by WebAssemblyFixBrTableDefaults.
1806 Ops.push_back(DAG.getBasicBlock(*MBBs.begin()));
1807 return DAG.getNode(WebAssemblyISD::BR_TABLE, DL, MVT::Other, Ops);
1808}
1809
1810SDValue WebAssemblyTargetLowering::LowerVASTART(SDValue Op,
1811 SelectionDAG &DAG) const {
1812 SDLoc DL(Op);
1814
1816 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
1817
1818 SDValue ArgN = DAG.getCopyFromReg(DAG.getEntryNode(), DL,
1819 MFI->getVarargBufferVreg(), PtrVT);
1820 return DAG.getStore(Op.getOperand(0), DL, ArgN, Op.getOperand(1),
1821 MachinePointerInfo(SV));
1822}
1823
1824SDValue WebAssemblyTargetLowering::LowerIntrinsic(SDValue Op,
1825 SelectionDAG &DAG) const {
1827 unsigned IntNo;
1828 switch (Op.getOpcode()) {
1831 IntNo = Op.getConstantOperandVal(1);
1832 break;
1834 IntNo = Op.getConstantOperandVal(0);
1835 break;
1836 default:
1837 llvm_unreachable("Invalid intrinsic");
1838 }
1839 SDLoc DL(Op);
1840
1841 switch (IntNo) {
1842 default:
1843 return SDValue(); // Don't custom lower most intrinsics.
1844
1845 case Intrinsic::wasm_lsda: {
1846 auto PtrVT = getPointerTy(MF.getDataLayout());
1847 const char *SymName = MF.createExternalSymbolName(
1848 "GCC_except_table" + std::to_string(MF.getFunctionNumber()));
1849 if (isPositionIndependent()) {
1851 SymName, PtrVT, WebAssemblyII::MO_MEMORY_BASE_REL);
1852 const char *BaseName = MF.createExternalSymbolName("__memory_base");
1854 DAG.getNode(WebAssemblyISD::Wrapper, DL, PtrVT,
1855 DAG.getTargetExternalSymbol(BaseName, PtrVT));
1856 SDValue SymAddr =
1857 DAG.getNode(WebAssemblyISD::WrapperREL, DL, PtrVT, Node);
1858 return DAG.getNode(ISD::ADD, DL, PtrVT, BaseAddr, SymAddr);
1859 }
1860 SDValue Node = DAG.getTargetExternalSymbol(SymName, PtrVT);
1861 return DAG.getNode(WebAssemblyISD::Wrapper, DL, PtrVT, Node);
1862 }
1863
1864 case Intrinsic::wasm_shuffle: {
1865 // Drop in-chain and replace undefs, but otherwise pass through unchanged
1866 SDValue Ops[18];
1867 size_t OpIdx = 0;
1868 Ops[OpIdx++] = Op.getOperand(1);
1869 Ops[OpIdx++] = Op.getOperand(2);
1870 while (OpIdx < 18) {
1871 const SDValue &MaskIdx = Op.getOperand(OpIdx + 1);
1872 if (MaskIdx.isUndef() ||
1873 cast<ConstantSDNode>(MaskIdx.getNode())->getZExtValue() >= 32) {
1874 bool isTarget = MaskIdx.getNode()->getOpcode() == ISD::TargetConstant;
1875 Ops[OpIdx++] = DAG.getConstant(0, DL, MVT::i32, isTarget);
1876 } else {
1877 Ops[OpIdx++] = MaskIdx;
1878 }
1879 }
1880 return DAG.getNode(WebAssemblyISD::SHUFFLE, DL, Op.getValueType(), Ops);
1881 }
1882 }
1883}
1884
1885SDValue
1886WebAssemblyTargetLowering::LowerSIGN_EXTEND_INREG(SDValue Op,
1887 SelectionDAG &DAG) const {
1888 SDLoc DL(Op);
1889 // If sign extension operations are disabled, allow sext_inreg only if operand
1890 // is a vector extract of an i8 or i16 lane. SIMD does not depend on sign
1891 // extension operations, but allowing sext_inreg in this context lets us have
1892 // simple patterns to select extract_lane_s instructions. Expanding sext_inreg
1893 // everywhere would be simpler in this file, but would necessitate large and
1894 // brittle patterns to undo the expansion and select extract_lane_s
1895 // instructions.
1896 assert(!Subtarget->hasSignExt() && Subtarget->hasSIMD128());
1897 if (Op.getOperand(0).getOpcode() != ISD::EXTRACT_VECTOR_ELT)
1898 return SDValue();
1899
1900 const SDValue &Extract = Op.getOperand(0);
1901 MVT VecT = Extract.getOperand(0).getSimpleValueType();
1902 if (VecT.getVectorElementType().getSizeInBits() > 32)
1903 return SDValue();
1904 MVT ExtractedLaneT =
1905 cast<VTSDNode>(Op.getOperand(1).getNode())->getVT().getSimpleVT();
1906 MVT ExtractedVecT =
1907 MVT::getVectorVT(ExtractedLaneT, 128 / ExtractedLaneT.getSizeInBits());
1908 if (ExtractedVecT == VecT)
1909 return Op;
1910
1911 // Bitcast vector to appropriate type to ensure ISel pattern coverage
1912 const SDNode *Index = Extract.getOperand(1).getNode();
1913 if (!isa<ConstantSDNode>(Index))
1914 return SDValue();
1915 unsigned IndexVal = cast<ConstantSDNode>(Index)->getZExtValue();
1916 unsigned Scale =
1917 ExtractedVecT.getVectorNumElements() / VecT.getVectorNumElements();
1918 assert(Scale > 1);
1919 SDValue NewIndex =
1920 DAG.getConstant(IndexVal * Scale, DL, Index->getValueType(0));
1921 SDValue NewExtract = DAG.getNode(
1923 DAG.getBitcast(ExtractedVecT, Extract.getOperand(0)), NewIndex);
1924 return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, Op.getValueType(), NewExtract,
1925 Op.getOperand(1));
1926}
1927
1928SDValue
1929WebAssemblyTargetLowering::LowerEXTEND_VECTOR_INREG(SDValue Op,
1930 SelectionDAG &DAG) const {
1931 SDLoc DL(Op);
1932 EVT VT = Op.getValueType();
1933 SDValue Src = Op.getOperand(0);
1934 EVT SrcVT = Src.getValueType();
1935
1936 if (SrcVT.getVectorElementType() == MVT::i1 ||
1937 SrcVT.getVectorElementType() == MVT::i64)
1938 return SDValue();
1939
1940 assert(VT.getScalarSizeInBits() % SrcVT.getScalarSizeInBits() == 0 &&
1941 "Unexpected extension factor.");
1942 unsigned Scale = VT.getScalarSizeInBits() / SrcVT.getScalarSizeInBits();
1943
1944 if (Scale != 2 && Scale != 4 && Scale != 8)
1945 return SDValue();
1946
1947 unsigned Ext;
1948 switch (Op.getOpcode()) {
1950 Ext = WebAssemblyISD::EXTEND_LOW_U;
1951 break;
1953 Ext = WebAssemblyISD::EXTEND_LOW_S;
1954 break;
1955 }
1956
1957 SDValue Ret = Src;
1958 while (Scale != 1) {
1959 Ret = DAG.getNode(Ext, DL,
1960 Ret.getValueType()
1961 .widenIntegerVectorElementType(*DAG.getContext())
1962 .getHalfNumVectorElementsVT(*DAG.getContext()),
1963 Ret);
1964 Scale /= 2;
1965 }
1966 assert(Ret.getValueType() == VT);
1967 return Ret;
1968}
1969
1971 SDLoc DL(Op);
1972 if (Op.getValueType() != MVT::v2f64)
1973 return SDValue();
1974
1975 auto GetConvertedLane = [](SDValue Op, unsigned &Opcode, SDValue &SrcVec,
1976 unsigned &Index) -> bool {
1977 switch (Op.getOpcode()) {
1978 case ISD::SINT_TO_FP:
1979 Opcode = WebAssemblyISD::CONVERT_LOW_S;
1980 break;
1981 case ISD::UINT_TO_FP:
1982 Opcode = WebAssemblyISD::CONVERT_LOW_U;
1983 break;
1984 case ISD::FP_EXTEND:
1985 Opcode = WebAssemblyISD::PROMOTE_LOW;
1986 break;
1987 default:
1988 return false;
1989 }
1990
1991 auto ExtractVector = Op.getOperand(0);
1992 if (ExtractVector.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
1993 return false;
1994
1995 if (!isa<ConstantSDNode>(ExtractVector.getOperand(1).getNode()))
1996 return false;
1997
1998 SrcVec = ExtractVector.getOperand(0);
1999 Index = ExtractVector.getConstantOperandVal(1);
2000 return true;
2001 };
2002
2003 unsigned LHSOpcode, RHSOpcode, LHSIndex, RHSIndex;
2004 SDValue LHSSrcVec, RHSSrcVec;
2005 if (!GetConvertedLane(Op.getOperand(0), LHSOpcode, LHSSrcVec, LHSIndex) ||
2006 !GetConvertedLane(Op.getOperand(1), RHSOpcode, RHSSrcVec, RHSIndex))
2007 return SDValue();
2008
2009 if (LHSOpcode != RHSOpcode)
2010 return SDValue();
2011
2012 MVT ExpectedSrcVT;
2013 switch (LHSOpcode) {
2014 case WebAssemblyISD::CONVERT_LOW_S:
2015 case WebAssemblyISD::CONVERT_LOW_U:
2016 ExpectedSrcVT = MVT::v4i32;
2017 break;
2018 case WebAssemblyISD::PROMOTE_LOW:
2019 ExpectedSrcVT = MVT::v4f32;
2020 break;
2021 }
2022 if (LHSSrcVec.getValueType() != ExpectedSrcVT)
2023 return SDValue();
2024
2025 auto Src = LHSSrcVec;
2026 if (LHSIndex != 0 || RHSIndex != 1 || LHSSrcVec != RHSSrcVec) {
2027 // Shuffle the source vector so that the converted lanes are the low lanes.
2028 Src = DAG.getVectorShuffle(
2029 ExpectedSrcVT, DL, LHSSrcVec, RHSSrcVec,
2030 {static_cast<int>(LHSIndex), static_cast<int>(RHSIndex) + 4, -1, -1});
2031 }
2032 return DAG.getNode(LHSOpcode, DL, MVT::v2f64, Src);
2033}
2034
2035SDValue WebAssemblyTargetLowering::LowerBUILD_VECTOR(SDValue Op,
2036 SelectionDAG &DAG) const {
2037 if (auto ConvertLow = LowerConvertLow(Op, DAG))
2038 return ConvertLow;
2039
2040 SDLoc DL(Op);
2041 const EVT VecT = Op.getValueType();
2042 const EVT LaneT = Op.getOperand(0).getValueType();
2043 const size_t Lanes = Op.getNumOperands();
2044 bool CanSwizzle = VecT == MVT::v16i8;
2045
2046 // BUILD_VECTORs are lowered to the instruction that initializes the highest
2047 // possible number of lanes at once followed by a sequence of replace_lane
2048 // instructions to individually initialize any remaining lanes.
2049
2050 // TODO: Tune this. For example, lanewise swizzling is very expensive, so
2051 // swizzled lanes should be given greater weight.
2052
2053 // TODO: Investigate looping rather than always extracting/replacing specific
2054 // lanes to fill gaps.
2055
2056 auto IsConstant = [](const SDValue &V) {
2057 return V.getOpcode() == ISD::Constant || V.getOpcode() == ISD::ConstantFP;
2058 };
2059
2060 // Returns the source vector and index vector pair if they exist. Checks for:
2061 // (extract_vector_elt
2062 // $src,
2063 // (sign_extend_inreg (extract_vector_elt $indices, $i))
2064 // )
2065 auto GetSwizzleSrcs = [](size_t I, const SDValue &Lane) {
2066 auto Bail = std::make_pair(SDValue(), SDValue());
2067 if (Lane->getOpcode() != ISD::EXTRACT_VECTOR_ELT)
2068 return Bail;
2069 const SDValue &SwizzleSrc = Lane->getOperand(0);
2070 const SDValue &IndexExt = Lane->getOperand(1);
2071 if (IndexExt->getOpcode() != ISD::SIGN_EXTEND_INREG)
2072 return Bail;
2073 const SDValue &Index = IndexExt->getOperand(0);
2074 if (Index->getOpcode() != ISD::EXTRACT_VECTOR_ELT)
2075 return Bail;
2076 const SDValue &SwizzleIndices = Index->getOperand(0);
2077 if (SwizzleSrc.getValueType() != MVT::v16i8 ||
2078 SwizzleIndices.getValueType() != MVT::v16i8 ||
2079 Index->getOperand(1)->getOpcode() != ISD::Constant ||
2080 Index->getConstantOperandVal(1) != I)
2081 return Bail;
2082 return std::make_pair(SwizzleSrc, SwizzleIndices);
2083 };
2084
2085 // If the lane is extracted from another vector at a constant index, return
2086 // that vector. The source vector must not have more lanes than the dest
2087 // because the shufflevector indices are in terms of the destination lanes and
2088 // would not be able to address the smaller individual source lanes.
2089 auto GetShuffleSrc = [&](const SDValue &Lane) {
2090 if (Lane->getOpcode() != ISD::EXTRACT_VECTOR_ELT)
2091 return SDValue();
2092 if (!isa<ConstantSDNode>(Lane->getOperand(1).getNode()))
2093 return SDValue();
2094 if (Lane->getOperand(0).getValueType().getVectorNumElements() >
2095 VecT.getVectorNumElements())
2096 return SDValue();
2097 return Lane->getOperand(0);
2098 };
2099
2100 using ValueEntry = std::pair<SDValue, size_t>;
2101 SmallVector<ValueEntry, 16> SplatValueCounts;
2102
2103 using SwizzleEntry = std::pair<std::pair<SDValue, SDValue>, size_t>;
2104 SmallVector<SwizzleEntry, 16> SwizzleCounts;
2105
2106 using ShuffleEntry = std::pair<SDValue, size_t>;
2107 SmallVector<ShuffleEntry, 16> ShuffleCounts;
2108
2109 auto AddCount = [](auto &Counts, const auto &Val) {
2110 auto CountIt =
2111 llvm::find_if(Counts, [&Val](auto E) { return E.first == Val; });
2112 if (CountIt == Counts.end()) {
2113 Counts.emplace_back(Val, 1);
2114 } else {
2115 CountIt->second++;
2116 }
2117 };
2118
2119 auto GetMostCommon = [](auto &Counts) {
2120 auto CommonIt =
2121 std::max_element(Counts.begin(), Counts.end(), llvm::less_second());
2122 assert(CommonIt != Counts.end() && "Unexpected all-undef build_vector");
2123 return *CommonIt;
2124 };
2125
2126 size_t NumConstantLanes = 0;
2127
2128 // Count eligible lanes for each type of vector creation op
2129 for (size_t I = 0; I < Lanes; ++I) {
2130 const SDValue &Lane = Op->getOperand(I);
2131 if (Lane.isUndef())
2132 continue;
2133
2134 AddCount(SplatValueCounts, Lane);
2135
2136 if (IsConstant(Lane))
2137 NumConstantLanes++;
2138 if (auto ShuffleSrc = GetShuffleSrc(Lane))
2139 AddCount(ShuffleCounts, ShuffleSrc);
2140 if (CanSwizzle) {
2141 auto SwizzleSrcs = GetSwizzleSrcs(I, Lane);
2142 if (SwizzleSrcs.first)
2143 AddCount(SwizzleCounts, SwizzleSrcs);
2144 }
2145 }
2146
2147 SDValue SplatValue;
2148 size_t NumSplatLanes;
2149 std::tie(SplatValue, NumSplatLanes) = GetMostCommon(SplatValueCounts);
2150
2151 SDValue SwizzleSrc;
2152 SDValue SwizzleIndices;
2153 size_t NumSwizzleLanes = 0;
2154 if (SwizzleCounts.size())
2155 std::forward_as_tuple(std::tie(SwizzleSrc, SwizzleIndices),
2156 NumSwizzleLanes) = GetMostCommon(SwizzleCounts);
2157
2158 // Shuffles can draw from up to two vectors, so find the two most common
2159 // sources.
2160 SDValue ShuffleSrc1, ShuffleSrc2;
2161 size_t NumShuffleLanes = 0;
2162 if (ShuffleCounts.size()) {
2163 std::tie(ShuffleSrc1, NumShuffleLanes) = GetMostCommon(ShuffleCounts);
2164 llvm::erase_if(ShuffleCounts,
2165 [&](const auto &Pair) { return Pair.first == ShuffleSrc1; });
2166 }
2167 if (ShuffleCounts.size()) {
2168 size_t AdditionalShuffleLanes;
2169 std::tie(ShuffleSrc2, AdditionalShuffleLanes) =
2170 GetMostCommon(ShuffleCounts);
2171 NumShuffleLanes += AdditionalShuffleLanes;
2172 }
2173
2174 // Predicate returning true if the lane is properly initialized by the
2175 // original instruction
2176 std::function<bool(size_t, const SDValue &)> IsLaneConstructed;
2178 // Prefer swizzles over shuffles over vector consts over splats
2179 if (NumSwizzleLanes >= NumShuffleLanes &&
2180 NumSwizzleLanes >= NumConstantLanes && NumSwizzleLanes >= NumSplatLanes) {
2181 Result = DAG.getNode(WebAssemblyISD::SWIZZLE, DL, VecT, SwizzleSrc,
2182 SwizzleIndices);
2183 auto Swizzled = std::make_pair(SwizzleSrc, SwizzleIndices);
2184 IsLaneConstructed = [&, Swizzled](size_t I, const SDValue &Lane) {
2185 return Swizzled == GetSwizzleSrcs(I, Lane);
2186 };
2187 } else if (NumShuffleLanes >= NumConstantLanes &&
2188 NumShuffleLanes >= NumSplatLanes) {
2189 size_t DestLaneSize = VecT.getVectorElementType().getFixedSizeInBits() / 8;
2190 size_t DestLaneCount = VecT.getVectorNumElements();
2191 size_t Scale1 = 1;
2192 size_t Scale2 = 1;
2193 SDValue Src1 = ShuffleSrc1;
2194 SDValue Src2 = ShuffleSrc2 ? ShuffleSrc2 : DAG.getUNDEF(VecT);
2195 if (Src1.getValueType() != VecT) {
2196 size_t LaneSize =
2198 assert(LaneSize > DestLaneSize);
2199 Scale1 = LaneSize / DestLaneSize;
2200 Src1 = DAG.getBitcast(VecT, Src1);
2201 }
2202 if (Src2.getValueType() != VecT) {
2203 size_t LaneSize =
2205 assert(LaneSize > DestLaneSize);
2206 Scale2 = LaneSize / DestLaneSize;
2207 Src2 = DAG.getBitcast(VecT, Src2);
2208 }
2209
2210 int Mask[16];
2211 assert(DestLaneCount <= 16);
2212 for (size_t I = 0; I < DestLaneCount; ++I) {
2213 const SDValue &Lane = Op->getOperand(I);
2214 SDValue Src = GetShuffleSrc(Lane);
2215 if (Src == ShuffleSrc1) {
2216 Mask[I] = Lane->getConstantOperandVal(1) * Scale1;
2217 } else if (Src && Src == ShuffleSrc2) {
2218 Mask[I] = DestLaneCount + Lane->getConstantOperandVal(1) * Scale2;
2219 } else {
2220 Mask[I] = -1;
2221 }
2222 }
2223 ArrayRef<int> MaskRef(Mask, DestLaneCount);
2224 Result = DAG.getVectorShuffle(VecT, DL, Src1, Src2, MaskRef);
2225 IsLaneConstructed = [&](size_t, const SDValue &Lane) {
2226 auto Src = GetShuffleSrc(Lane);
2227 return Src == ShuffleSrc1 || (Src && Src == ShuffleSrc2);
2228 };
2229 } else if (NumConstantLanes >= NumSplatLanes) {
2230 SmallVector<SDValue, 16> ConstLanes;
2231 for (const SDValue &Lane : Op->op_values()) {
2232 if (IsConstant(Lane)) {
2233 // Values may need to be fixed so that they will sign extend to be
2234 // within the expected range during ISel. Check whether the value is in
2235 // bounds based on the lane bit width and if it is out of bounds, lop
2236 // off the extra bits and subtract 2^n to reflect giving the high bit
2237 // value -2^(n-1) rather than +2^(n-1). Skip the i64 case because it
2238 // cannot possibly be out of range.
2239 auto *Const = dyn_cast<ConstantSDNode>(Lane.getNode());
2240 int64_t Val = Const ? Const->getSExtValue() : 0;
2241 uint64_t LaneBits = 128 / Lanes;
2242 assert((LaneBits == 64 || Val >= -(1ll << (LaneBits - 1))) &&
2243 "Unexpected out of bounds negative value");
2244 if (Const && LaneBits != 64 && Val > (1ll << (LaneBits - 1)) - 1) {
2245 uint64_t Mask = (1ll << LaneBits) - 1;
2246 auto NewVal = (((uint64_t)Val & Mask) - (1ll << LaneBits)) & Mask;
2247 ConstLanes.push_back(DAG.getConstant(NewVal, SDLoc(Lane), LaneT));
2248 } else {
2249 ConstLanes.push_back(Lane);
2250 }
2251 } else if (LaneT.isFloatingPoint()) {
2252 ConstLanes.push_back(DAG.getConstantFP(0, DL, LaneT));
2253 } else {
2254 ConstLanes.push_back(DAG.getConstant(0, DL, LaneT));
2255 }
2256 }
2257 Result = DAG.getBuildVector(VecT, DL, ConstLanes);
2258 IsLaneConstructed = [&IsConstant](size_t _, const SDValue &Lane) {
2259 return IsConstant(Lane);
2260 };
2261 } else {
2262 // Use a splat (which might be selected as a load splat)
2263 Result = DAG.getSplatBuildVector(VecT, DL, SplatValue);
2264 IsLaneConstructed = [&SplatValue](size_t _, const SDValue &Lane) {
2265 return Lane == SplatValue;
2266 };
2267 }
2268
2269 assert(Result);
2270 assert(IsLaneConstructed);
2271
2272 // Add replace_lane instructions for any unhandled values
2273 for (size_t I = 0; I < Lanes; ++I) {
2274 const SDValue &Lane = Op->getOperand(I);
2275 if (!Lane.isUndef() && !IsLaneConstructed(I, Lane))
2276 Result = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VecT, Result, Lane,
2277 DAG.getConstant(I, DL, MVT::i32));
2278 }
2279
2280 return Result;
2281}
2282
2283SDValue
2284WebAssemblyTargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,
2285 SelectionDAG &DAG) const {
2286 SDLoc DL(Op);
2287 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Op.getNode())->getMask();
2288 MVT VecType = Op.getOperand(0).getSimpleValueType();
2289 assert(VecType.is128BitVector() && "Unexpected shuffle vector type");
2290 size_t LaneBytes = VecType.getVectorElementType().getSizeInBits() / 8;
2291
2292 // Space for two vector args and sixteen mask indices
2293 SDValue Ops[18];
2294 size_t OpIdx = 0;
2295 Ops[OpIdx++] = Op.getOperand(0);
2296 Ops[OpIdx++] = Op.getOperand(1);
2297
2298 // Expand mask indices to byte indices and materialize them as operands
2299 for (int M : Mask) {
2300 for (size_t J = 0; J < LaneBytes; ++J) {
2301 // Lower undefs (represented by -1 in mask) to {0..J}, which use a
2302 // whole lane of vector input, to allow further reduction at VM. E.g.
2303 // match an 8x16 byte shuffle to an equivalent cheaper 32x4 shuffle.
2304 uint64_t ByteIndex = M == -1 ? J : (uint64_t)M * LaneBytes + J;
2305 Ops[OpIdx++] = DAG.getConstant(ByteIndex, DL, MVT::i32);
2306 }
2307 }
2308
2309 return DAG.getNode(WebAssemblyISD::SHUFFLE, DL, Op.getValueType(), Ops);
2310}
2311
2312SDValue WebAssemblyTargetLowering::LowerSETCC(SDValue Op,
2313 SelectionDAG &DAG) const {
2314 SDLoc DL(Op);
2315 // The legalizer does not know how to expand the unsupported comparison modes
2316 // of i64x2 vectors, so we manually unroll them here.
2317 assert(Op->getOperand(0)->getSimpleValueType(0) == MVT::v2i64);
2319 DAG.ExtractVectorElements(Op->getOperand(0), LHS);
2320 DAG.ExtractVectorElements(Op->getOperand(1), RHS);
2321 const SDValue &CC = Op->getOperand(2);
2322 auto MakeLane = [&](unsigned I) {
2323 return DAG.getNode(ISD::SELECT_CC, DL, MVT::i64, LHS[I], RHS[I],
2324 DAG.getConstant(uint64_t(-1), DL, MVT::i64),
2325 DAG.getConstant(uint64_t(0), DL, MVT::i64), CC);
2326 };
2327 return DAG.getBuildVector(Op->getValueType(0), DL,
2328 {MakeLane(0), MakeLane(1)});
2329}
2330
2331SDValue
2332WebAssemblyTargetLowering::LowerAccessVectorElement(SDValue Op,
2333 SelectionDAG &DAG) const {
2334 // Allow constant lane indices, expand variable lane indices
2335 SDNode *IdxNode = Op.getOperand(Op.getNumOperands() - 1).getNode();
2336 if (isa<ConstantSDNode>(IdxNode)) {
2337 // Ensure the index type is i32 to match the tablegen patterns
2338 uint64_t Idx = cast<ConstantSDNode>(IdxNode)->getZExtValue();
2339 SmallVector<SDValue, 3> Ops(Op.getNode()->ops());
2340 Ops[Op.getNumOperands() - 1] =
2341 DAG.getConstant(Idx, SDLoc(IdxNode), MVT::i32);
2342 return DAG.getNode(Op.getOpcode(), SDLoc(Op), Op.getValueType(), Ops);
2343 }
2344 // Perform default expansion
2345 return SDValue();
2346}
2347
2349 EVT LaneT = Op.getSimpleValueType().getVectorElementType();
2350 // 32-bit and 64-bit unrolled shifts will have proper semantics
2351 if (LaneT.bitsGE(MVT::i32))
2352 return DAG.UnrollVectorOp(Op.getNode());
2353 // Otherwise mask the shift value to get proper semantics from 32-bit shift
2354 SDLoc DL(Op);
2355 size_t NumLanes = Op.getSimpleValueType().getVectorNumElements();
2356 SDValue Mask = DAG.getConstant(LaneT.getSizeInBits() - 1, DL, MVT::i32);
2357 unsigned ShiftOpcode = Op.getOpcode();
2358 SmallVector<SDValue, 16> ShiftedElements;
2359 DAG.ExtractVectorElements(Op.getOperand(0), ShiftedElements, 0, 0, MVT::i32);
2360 SmallVector<SDValue, 16> ShiftElements;
2361 DAG.ExtractVectorElements(Op.getOperand(1), ShiftElements, 0, 0, MVT::i32);
2362 SmallVector<SDValue, 16> UnrolledOps;
2363 for (size_t i = 0; i < NumLanes; ++i) {
2364 SDValue MaskedShiftValue =
2365 DAG.getNode(ISD::AND, DL, MVT::i32, ShiftElements[i], Mask);
2366 SDValue ShiftedValue = ShiftedElements[i];
2367 if (ShiftOpcode == ISD::SRA)
2368 ShiftedValue = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i32,
2369 ShiftedValue, DAG.getValueType(LaneT));
2370 UnrolledOps.push_back(
2371 DAG.getNode(ShiftOpcode, DL, MVT::i32, ShiftedValue, MaskedShiftValue));
2372 }
2373 return DAG.getBuildVector(Op.getValueType(), DL, UnrolledOps);
2374}
2375
2376SDValue WebAssemblyTargetLowering::LowerShift(SDValue Op,
2377 SelectionDAG &DAG) const {
2378 SDLoc DL(Op);
2379
2380 // Only manually lower vector shifts
2381 assert(Op.getSimpleValueType().isVector());
2382
2383 uint64_t LaneBits = Op.getValueType().getScalarSizeInBits();
2384 auto ShiftVal = Op.getOperand(1);
2385
2386 // Try to skip bitmask operation since it is implied inside shift instruction
2387 auto SkipImpliedMask = [](SDValue MaskOp, uint64_t MaskBits) {
2388 if (MaskOp.getOpcode() != ISD::AND)
2389 return MaskOp;
2390 SDValue LHS = MaskOp.getOperand(0);
2391 SDValue RHS = MaskOp.getOperand(1);
2392 if (MaskOp.getValueType().isVector()) {
2393 APInt MaskVal;
2394 if (!ISD::isConstantSplatVector(RHS.getNode(), MaskVal))
2395 std::swap(LHS, RHS);
2396
2397 if (ISD::isConstantSplatVector(RHS.getNode(), MaskVal) &&
2398 MaskVal == MaskBits)
2399 MaskOp = LHS;
2400 } else {
2401 if (!isa<ConstantSDNode>(RHS.getNode()))
2402 std::swap(LHS, RHS);
2403
2404 auto ConstantRHS = dyn_cast<ConstantSDNode>(RHS.getNode());
2405 if (ConstantRHS && ConstantRHS->getAPIntValue() == MaskBits)
2406 MaskOp = LHS;
2407 }
2408
2409 return MaskOp;
2410 };
2411
2412 // Skip vector and operation
2413 ShiftVal = SkipImpliedMask(ShiftVal, LaneBits - 1);
2414 ShiftVal = DAG.getSplatValue(ShiftVal);
2415 if (!ShiftVal)
2416 return unrollVectorShift(Op, DAG);
2417
2418 // Skip scalar and operation
2419 ShiftVal = SkipImpliedMask(ShiftVal, LaneBits - 1);
2420 // Use anyext because none of the high bits can affect the shift
2421 ShiftVal = DAG.getAnyExtOrTrunc(ShiftVal, DL, MVT::i32);
2422
2423 unsigned Opcode;
2424 switch (Op.getOpcode()) {
2425 case ISD::SHL:
2426 Opcode = WebAssemblyISD::VEC_SHL;
2427 break;
2428 case ISD::SRA:
2429 Opcode = WebAssemblyISD::VEC_SHR_S;
2430 break;
2431 case ISD::SRL:
2432 Opcode = WebAssemblyISD::VEC_SHR_U;
2433 break;
2434 default:
2435 llvm_unreachable("unexpected opcode");
2436 }
2437
2438 return DAG.getNode(Opcode, DL, Op.getValueType(), Op.getOperand(0), ShiftVal);
2439}
2440
2441SDValue WebAssemblyTargetLowering::LowerFP_TO_INT_SAT(SDValue Op,
2442 SelectionDAG &DAG) const {
2443 SDLoc DL(Op);
2444 EVT ResT = Op.getValueType();
2445 EVT SatVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
2446
2447 if ((ResT == MVT::i32 || ResT == MVT::i64) &&
2448 (SatVT == MVT::i32 || SatVT == MVT::i64))
2449 return Op;
2450
2451 if (ResT == MVT::v4i32 && SatVT == MVT::i32)
2452 return Op;
2453
2454 return SDValue();
2455}
2456
2457//===----------------------------------------------------------------------===//
2458// Custom DAG combine hooks
2459//===----------------------------------------------------------------------===//
2460static SDValue
2462 auto &DAG = DCI.DAG;
2463 auto Shuffle = cast<ShuffleVectorSDNode>(N);
2464
2465 // Hoist vector bitcasts that don't change the number of lanes out of unary
2466 // shuffles, where they are less likely to get in the way of other combines.
2467 // (shuffle (vNxT1 (bitcast (vNxT0 x))), undef, mask) ->
2468 // (vNxT1 (bitcast (vNxT0 (shuffle x, undef, mask))))
2469 SDValue Bitcast = N->getOperand(0);
2470 if (Bitcast.getOpcode() != ISD::BITCAST)
2471 return SDValue();
2472 if (!N->getOperand(1).isUndef())
2473 return SDValue();
2474 SDValue CastOp = Bitcast.getOperand(0);
2475 EVT SrcType = CastOp.getValueType();
2476 EVT DstType = Bitcast.getValueType();
2477 if (!SrcType.is128BitVector() ||
2478 SrcType.getVectorNumElements() != DstType.getVectorNumElements())
2479 return SDValue();
2480 SDValue NewShuffle = DAG.getVectorShuffle(
2481 SrcType, SDLoc(N), CastOp, DAG.getUNDEF(SrcType), Shuffle->getMask());
2482 return DAG.getBitcast(DstType, NewShuffle);
2483}
2484
2485/// Convert ({u,s}itofp vec) --> ({u,s}itofp ({s,z}ext vec)) so it doesn't get
2486/// split up into scalar instructions during legalization, and the vector
2487/// extending instructions are selected in performVectorExtendCombine below.
2488static SDValue
2491 auto &DAG = DCI.DAG;
2492 assert(N->getOpcode() == ISD::UINT_TO_FP ||
2493 N->getOpcode() == ISD::SINT_TO_FP);
2494
2495 EVT InVT = N->getOperand(0)->getValueType(0);
2496 EVT ResVT = N->getValueType(0);
2497 MVT ExtVT;
2498 if (ResVT == MVT::v4f32 && (InVT == MVT::v4i16 || InVT == MVT::v4i8))
2499 ExtVT = MVT::v4i32;
2500 else if (ResVT == MVT::v2f64 && (InVT == MVT::v2i16 || InVT == MVT::v2i8))
2501 ExtVT = MVT::v2i32;
2502 else
2503 return SDValue();
2504
2505 unsigned Op =
2507 SDValue Conv = DAG.getNode(Op, SDLoc(N), ExtVT, N->getOperand(0));
2508 return DAG.getNode(N->getOpcode(), SDLoc(N), ResVT, Conv);
2509}
2510
2511static SDValue
2513 auto &DAG = DCI.DAG;
2514 assert(N->getOpcode() == ISD::SIGN_EXTEND ||
2515 N->getOpcode() == ISD::ZERO_EXTEND);
2516
2517 // Combine ({s,z}ext (extract_subvector src, i)) into a widening operation if
2518 // possible before the extract_subvector can be expanded.
2519 auto Extract = N->getOperand(0);
2520 if (Extract.getOpcode() != ISD::EXTRACT_SUBVECTOR)
2521 return SDValue();
2522 auto Source = Extract.getOperand(0);
2523 auto *IndexNode = dyn_cast<ConstantSDNode>(Extract.getOperand(1));
2524 if (IndexNode == nullptr)
2525 return SDValue();
2526 auto Index = IndexNode->getZExtValue();
2527
2528 // Only v8i8, v4i16, and v2i32 extracts can be widened, and only if the
2529 // extracted subvector is the low or high half of its source.
2530 EVT ResVT = N->getValueType(0);
2531 if (ResVT == MVT::v8i16) {
2532 if (Extract.getValueType() != MVT::v8i8 ||
2533 Source.getValueType() != MVT::v16i8 || (Index != 0 && Index != 8))
2534 return SDValue();
2535 } else if (ResVT == MVT::v4i32) {
2536 if (Extract.getValueType() != MVT::v4i16 ||
2537 Source.getValueType() != MVT::v8i16 || (Index != 0 && Index != 4))
2538 return SDValue();
2539 } else if (ResVT == MVT::v2i64) {
2540 if (Extract.getValueType() != MVT::v2i32 ||
2541 Source.getValueType() != MVT::v4i32 || (Index != 0 && Index != 2))
2542 return SDValue();
2543 } else {
2544 return SDValue();
2545 }
2546
2547 bool IsSext = N->getOpcode() == ISD::SIGN_EXTEND;
2548 bool IsLow = Index == 0;
2549
2550 unsigned Op = IsSext ? (IsLow ? WebAssemblyISD::EXTEND_LOW_S
2551 : WebAssemblyISD::EXTEND_HIGH_S)
2552 : (IsLow ? WebAssemblyISD::EXTEND_LOW_U
2553 : WebAssemblyISD::EXTEND_HIGH_U);
2554
2555 return DAG.getNode(Op, SDLoc(N), ResVT, Source);
2556}
2557
2558static SDValue
2560 auto &DAG = DCI.DAG;
2561
2562 auto GetWasmConversionOp = [](unsigned Op) {
2563 switch (Op) {
2565 return WebAssemblyISD::TRUNC_SAT_ZERO_S;
2567 return WebAssemblyISD::TRUNC_SAT_ZERO_U;
2568 case ISD::FP_ROUND:
2569 return WebAssemblyISD::DEMOTE_ZERO;
2570 }
2571 llvm_unreachable("unexpected op");
2572 };
2573
2574 auto IsZeroSplat = [](SDValue SplatVal) {
2575 auto *Splat = dyn_cast<BuildVectorSDNode>(SplatVal.getNode());
2576 APInt SplatValue, SplatUndef;
2577 unsigned SplatBitSize;
2578 bool HasAnyUndefs;
2579 // Endianness doesn't matter in this context because we are looking for
2580 // an all-zero value.
2581 return Splat &&
2582 Splat->isConstantSplat(SplatValue, SplatUndef, SplatBitSize,
2583 HasAnyUndefs) &&
2584 SplatValue == 0;
2585 };
2586
2587 if (N->getOpcode() == ISD::CONCAT_VECTORS) {
2588 // Combine this:
2589 //
2590 // (concat_vectors (v2i32 (fp_to_{s,u}int_sat $x, 32)), (v2i32 (splat 0)))
2591 //
2592 // into (i32x4.trunc_sat_f64x2_zero_{s,u} $x).
2593 //
2594 // Or this:
2595 //
2596 // (concat_vectors (v2f32 (fp_round (v2f64 $x))), (v2f32 (splat 0)))
2597 //
2598 // into (f32x4.demote_zero_f64x2 $x).
2599 EVT ResVT;
2600 EVT ExpectedConversionType;
2601 auto Conversion = N->getOperand(0);
2602 auto ConversionOp = Conversion.getOpcode();
2603 switch (ConversionOp) {
2606 ResVT = MVT::v4i32;
2607 ExpectedConversionType = MVT::v2i32;
2608 break;
2609 case ISD::FP_ROUND:
2610 ResVT = MVT::v4f32;
2611 ExpectedConversionType = MVT::v2f32;
2612 break;
2613 default:
2614 return SDValue();
2615 }
2616
2617 if (N->getValueType(0) != ResVT)
2618 return SDValue();
2619
2620 if (Conversion.getValueType() != ExpectedConversionType)
2621 return SDValue();
2622
2623 auto Source = Conversion.getOperand(0);
2624 if (Source.getValueType() != MVT::v2f64)
2625 return SDValue();
2626
2627 if (!IsZeroSplat(N->getOperand(1)) ||
2628 N->getOperand(1).getValueType() != ExpectedConversionType)
2629 return SDValue();
2630
2631 unsigned Op = GetWasmConversionOp(ConversionOp);
2632 return DAG.getNode(Op, SDLoc(N), ResVT, Source);
2633 }
2634
2635 // Combine this:
2636 //
2637 // (fp_to_{s,u}int_sat (concat_vectors $x, (v2f64 (splat 0))), 32)
2638 //
2639 // into (i32x4.trunc_sat_f64x2_zero_{s,u} $x).
2640 //
2641 // Or this:
2642 //
2643 // (v4f32 (fp_round (concat_vectors $x, (v2f64 (splat 0)))))
2644 //
2645 // into (f32x4.demote_zero_f64x2 $x).
2646 EVT ResVT;
2647 auto ConversionOp = N->getOpcode();
2648 switch (ConversionOp) {
2651 ResVT = MVT::v4i32;
2652 break;
2653 case ISD::FP_ROUND:
2654 ResVT = MVT::v4f32;
2655 break;
2656 default:
2657 llvm_unreachable("unexpected op");
2658 }
2659
2660 if (N->getValueType(0) != ResVT)
2661 return SDValue();
2662
2663 auto Concat = N->getOperand(0);
2664 if (Concat.getValueType() != MVT::v4f64)
2665 return SDValue();
2666
2667 auto Source = Concat.getOperand(0);
2668 if (Source.getValueType() != MVT::v2f64)
2669 return SDValue();
2670
2671 if (!IsZeroSplat(Concat.getOperand(1)) ||
2672 Concat.getOperand(1).getValueType() != MVT::v2f64)
2673 return SDValue();
2674
2675 unsigned Op = GetWasmConversionOp(ConversionOp);
2676 return DAG.getNode(Op, SDLoc(N), ResVT, Source);
2677}
2678
2679// Helper to extract VectorWidth bits from Vec, starting from IdxVal.
2680static SDValue extractSubVector(SDValue Vec, unsigned IdxVal, SelectionDAG &DAG,
2681 const SDLoc &DL, unsigned VectorWidth) {
2682 EVT VT = Vec.getValueType();
2683 EVT ElVT = VT.getVectorElementType();
2684 unsigned Factor = VT.getSizeInBits() / VectorWidth;
2685 EVT ResultVT = EVT::getVectorVT(*DAG.getContext(), ElVT,
2686 VT.getVectorNumElements() / Factor);
2687
2688 // Extract the relevant VectorWidth bits. Generate an EXTRACT_SUBVECTOR
2689 unsigned ElemsPerChunk = VectorWidth / ElVT.getSizeInBits();
2690 assert(isPowerOf2_32(ElemsPerChunk) && "Elements per chunk not power of 2");
2691
2692 // This is the index of the first element of the VectorWidth-bit chunk
2693 // we want. Since ElemsPerChunk is a power of 2 just need to clear bits.
2694 IdxVal &= ~(ElemsPerChunk - 1);
2695
2696 // If the input is a buildvector just emit a smaller one.
2697 if (Vec.getOpcode() == ISD::BUILD_VECTOR)
2698 return DAG.getBuildVector(ResultVT, DL,
2699 Vec->ops().slice(IdxVal, ElemsPerChunk));
2700
2701 SDValue VecIdx = DAG.getIntPtrConstant(IdxVal, DL);
2702 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ResultVT, Vec, VecIdx);
2703}
2704
2705// Helper to recursively truncate vector elements in half with NARROW_U. DstVT
2706// is the expected destination value type after recursion. In is the initial
2707// input. Note that the input should have enough leading zero bits to prevent
2708// NARROW_U from saturating results.
2710 SelectionDAG &DAG) {
2711 EVT SrcVT = In.getValueType();
2712
2713 // No truncation required, we might get here due to recursive calls.
2714 if (SrcVT == DstVT)
2715 return In;
2716
2717 unsigned SrcSizeInBits = SrcVT.getSizeInBits();
2718 unsigned NumElems = SrcVT.getVectorNumElements();
2719 if (!isPowerOf2_32(NumElems))
2720 return SDValue();
2721 assert(DstVT.getVectorNumElements() == NumElems && "Illegal truncation");
2722 assert(SrcSizeInBits > DstVT.getSizeInBits() && "Illegal truncation");
2723
2724 LLVMContext &Ctx = *DAG.getContext();
2725 EVT PackedSVT = EVT::getIntegerVT(Ctx, SrcVT.getScalarSizeInBits() / 2);
2726
2727 // Narrow to the largest type possible:
2728 // vXi64/vXi32 -> i16x8.narrow_i32x4_u and vXi16 -> i8x16.narrow_i16x8_u.
2729 EVT InVT = MVT::i16, OutVT = MVT::i8;
2730 if (SrcVT.getScalarSizeInBits() > 16) {
2731 InVT = MVT::i32;
2732 OutVT = MVT::i16;
2733 }
2734 unsigned SubSizeInBits = SrcSizeInBits / 2;
2735 InVT = EVT::getVectorVT(Ctx, InVT, SubSizeInBits / InVT.getSizeInBits());
2736 OutVT = EVT::getVectorVT(Ctx, OutVT, SubSizeInBits / OutVT.getSizeInBits());
2737
2738 // Split lower/upper subvectors.
2739 SDValue Lo = extractSubVector(In, 0, DAG, DL, SubSizeInBits);
2740 SDValue Hi = extractSubVector(In, NumElems / 2, DAG, DL, SubSizeInBits);
2741
2742 // 256bit -> 128bit truncate - Narrow lower/upper 128-bit subvectors.
2743 if (SrcVT.is256BitVector() && DstVT.is128BitVector()) {
2744 Lo = DAG.getBitcast(InVT, Lo);
2745 Hi = DAG.getBitcast(InVT, Hi);
2746 SDValue Res = DAG.getNode(WebAssemblyISD::NARROW_U, DL, OutVT, Lo, Hi);
2747 return DAG.getBitcast(DstVT, Res);
2748 }
2749
2750 // Recursively narrow lower/upper subvectors, concat result and narrow again.
2751 EVT PackedVT = EVT::getVectorVT(Ctx, PackedSVT, NumElems / 2);
2752 Lo = truncateVectorWithNARROW(PackedVT, Lo, DL, DAG);
2753 Hi = truncateVectorWithNARROW(PackedVT, Hi, DL, DAG);
2754
2755 PackedVT = EVT::getVectorVT(Ctx, PackedSVT, NumElems);
2756 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, DL, PackedVT, Lo, Hi);
2757 return truncateVectorWithNARROW(DstVT, Res, DL, DAG);
2758}
2759
2762 auto &DAG = DCI.DAG;
2763
2764 SDValue In = N->getOperand(0);
2765 EVT InVT = In.getValueType();
2766 if (!InVT.isSimple())
2767 return SDValue();
2768
2769 EVT OutVT = N->getValueType(0);
2770 if (!OutVT.isVector())
2771 return SDValue();
2772
2773 EVT OutSVT = OutVT.getVectorElementType();
2774 EVT InSVT = InVT.getVectorElementType();
2775 // Currently only cover truncate to v16i8 or v8i16.
2776 if (!((InSVT == MVT::i16 || InSVT == MVT::i32 || InSVT == MVT::i64) &&
2777 (OutSVT == MVT::i8 || OutSVT == MVT::i16) && OutVT.is128BitVector()))
2778 return SDValue();
2779
2780 SDLoc DL(N);
2782 OutVT.getScalarSizeInBits());
2783 In = DAG.getNode(ISD::AND, DL, InVT, In, DAG.getConstant(Mask, DL, InVT));
2784 return truncateVectorWithNARROW(OutVT, In, DL, DAG);
2785}
2786
2789 auto &DAG = DCI.DAG;
2790 SDLoc DL(N);
2791 SDValue Src = N->getOperand(0);
2792 EVT VT = N->getValueType(0);
2793 EVT SrcVT = Src.getValueType();
2794
2795 // bitcast <N x i1> to iN
2796 // ==> bitmask
2797 if (DCI.isBeforeLegalize() && VT.isScalarInteger() &&
2798 SrcVT.isFixedLengthVector() && SrcVT.getScalarType() == MVT::i1) {
2799 unsigned NumElts = SrcVT.getVectorNumElements();
2800 if (NumElts != 2 && NumElts != 4 && NumElts != 8 && NumElts != 16)
2801 return SDValue();
2802 EVT Width = MVT::getIntegerVT(128 / NumElts);
2803 return DAG.getZExtOrTrunc(
2804 DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, MVT::i32,
2805 {DAG.getConstant(Intrinsic::wasm_bitmask, DL, MVT::i32),
2806 DAG.getSExtOrTrunc(N->getOperand(0), DL,
2807 SrcVT.changeVectorElementType(Width))}),
2808 DL, VT);
2809 }
2810
2811 return SDValue();
2812}
2813
2816 auto &DAG = DCI.DAG;
2817
2818 SDValue LHS = N->getOperand(0);
2819 SDValue RHS = N->getOperand(1);
2820 ISD::CondCode Cond = cast<CondCodeSDNode>(N->getOperand(2))->get();
2821 SDLoc DL(N);
2822 EVT VT = N->getValueType(0);
2823
2824 // setcc (iN (bitcast (vNi1 X))), 0, ne
2825 // ==> any_true (vNi1 X)
2826 // setcc (iN (bitcast (vNi1 X))), 0, eq
2827 // ==> xor (any_true (vNi1 X)), -1
2828 // setcc (iN (bitcast (vNi1 X))), -1, eq
2829 // ==> all_true (vNi1 X)
2830 // setcc (iN (bitcast (vNi1 X))), -1, ne
2831 // ==> xor (all_true (vNi1 X)), -1
2832 if (DCI.isBeforeLegalize() && VT.isScalarInteger() &&
2833 (Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
2835 LHS->getOpcode() == ISD::BITCAST) {
2836 EVT FromVT = LHS->getOperand(0).getValueType();
2837 if (FromVT.isFixedLengthVector() &&
2838 FromVT.getVectorElementType() == MVT::i1) {
2839 int Intrin = isNullConstant(RHS) ? Intrinsic::wasm_anytrue
2840 : Intrinsic::wasm_alltrue;
2841 unsigned NumElts = FromVT.getVectorNumElements();
2842 if (NumElts != 2 && NumElts != 4 && NumElts != 8 && NumElts != 16)
2843 return SDValue();
2844 EVT Width = MVT::getIntegerVT(128 / NumElts);
2845 SDValue Ret = DAG.getZExtOrTrunc(
2846 DAG.getNode(
2847 ISD::INTRINSIC_WO_CHAIN, DL, MVT::i32,
2848 {DAG.getConstant(Intrin, DL, MVT::i32),
2849 DAG.getSExtOrTrunc(LHS->getOperand(0), DL,
2850 FromVT.changeVectorElementType(Width))}),
2851 DL, MVT::i1);
2852 if ((isNullConstant(RHS) && (Cond == ISD::SETEQ)) ||
2853 (isAllOnesConstant(RHS) && (Cond == ISD::SETNE))) {
2854 Ret = DAG.getNOT(DL, Ret, MVT::i1);
2855 }
2856 return DAG.getZExtOrTrunc(Ret, DL, VT);
2857 }
2858 }
2859
2860 return SDValue();
2861}
2862
2863SDValue
2864WebAssemblyTargetLowering::PerformDAGCombine(SDNode *N,
2865 DAGCombinerInfo &DCI) const {
2866 switch (N->getOpcode()) {
2867 default:
2868 return SDValue();
2869 case ISD::BITCAST:
2870 return performBitcastCombine(N, DCI);
2871 case ISD::SETCC:
2872 return performSETCCCombine(N, DCI);
2874 return performVECTOR_SHUFFLECombine(N, DCI);
2875 case ISD::SIGN_EXTEND:
2876 case ISD::ZERO_EXTEND:
2877 return performVectorExtendCombine(N, DCI);
2878 case ISD::UINT_TO_FP:
2879 case ISD::SINT_TO_FP:
2880 return performVectorExtendToFPCombine(N, DCI);
2883 case ISD::FP_ROUND:
2885 return performVectorTruncZeroCombine(N, DCI);
2886 case ISD::TRUNCATE:
2887 return performTruncateCombine(N, DCI);
2888 }
2889}
unsigned const MachineRegisterInfo * MRI
static SDValue performTruncateCombine(SDNode *N, SelectionDAG &DAG)
static SDValue performSETCCCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
MachineBasicBlock & MBB
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Function Alias Analysis Results
static void fail(const SDLoc &DL, SelectionDAG &DAG, const Twine &Msg, SDValue Val={})
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
Analysis containing CSE Info
Definition: CSEInfo.cpp:27
return RetTy
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
Symbol * Sym
Definition: ELF_riscv.cpp:477
Hexagon Common GEP
const HexagonInstrInfo * TII
#define _
IRTranslator LLVM IR MI
static unsigned NumFixedArgs
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
unsigned const TargetRegisterInfo * TRI
LLVMContext & Context
const char LLVMTargetMachineRef TM
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static MachineBasicBlock * LowerFPToInt(MachineInstr &MI, DebugLoc DL, MachineBasicBlock *BB, const TargetInstrInfo &TII, bool IsUnsigned, bool Int64, bool Float64, unsigned LoweredOpcode)
static bool callingConvSupported(CallingConv::ID CallConv)
static std::optional< unsigned > IsWebAssemblyLocal(SDValue Op, SelectionDAG &DAG)
static SDValue performVectorExtendCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue unrollVectorShift(SDValue Op, SelectionDAG &DAG)
static MachineBasicBlock * LowerCallResults(MachineInstr &CallResults, DebugLoc DL, MachineBasicBlock *BB, const WebAssemblySubtarget *Subtarget, const TargetInstrInfo &TII)
static SDValue performVECTOR_SHUFFLECombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue performVectorTruncZeroCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static bool IsWebAssemblyGlobal(SDValue Op)
static SDValue performVectorExtendToFPCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
Convert ({u,s}itofp vec) --> ({u,s}itofp ({s,z}ext vec)) so it doesn't get split up into scalar instr...
static SDValue LowerConvertLow(SDValue Op, SelectionDAG &DAG)
static SDValue extractSubVector(SDValue Vec, unsigned IdxVal, SelectionDAG &DAG, const SDLoc &DL, unsigned VectorWidth)
static SDValue performBitcastCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue truncateVectorWithNARROW(EVT DstVT, SDValue In, const SDLoc &DL, SelectionDAG &DAG)
This file defines the interfaces that WebAssembly uses to lower LLVM code into a selection DAG.
This file provides WebAssembly-specific target descriptions.
This file declares WebAssembly-specific per-machine-function information.
This file declares the WebAssembly-specific subclass of TargetSubtarget.
This file declares the WebAssembly-specific subclass of TargetMachine.
This file contains the declaration of the WebAssembly-specific type parsing utility functions.
This file contains the declaration of the WebAssembly-specific utility functions.
X86 cmov Conversion
static constexpr int Concat[]
Value * RHS
Value * LHS
static constexpr uint32_t Opcode
Definition: aarch32.h:200
Class for arbitrary precision integers.
Definition: APInt.h:76
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
Definition: APInt.h:284
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Constructs an APInt value that has the top hiBitsSet bits set.
Definition: APInt.h:274
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
an instruction that atomically reads a memory location, combines it with another value,...
Definition: Instructions.h:726
@ Add
*p = old + v
Definition: Instructions.h:742
@ Or
*p = old | v
Definition: Instructions.h:750
@ Sub
*p = old - v
Definition: Instructions.h:744
@ And
*p = old & v
Definition: Instructions.h:746
@ Xor
*p = old ^ v
Definition: Instructions.h:752
BinOp getOperation() const
Definition: Instructions.h:820
LLVM Basic Block Representation.
Definition: BasicBlock.h:60
CCState - This class holds information needed while lowering arguments and return values.
static CCValAssign getMem(unsigned ValNo, MVT ValVT, int64_t Offset, MVT LocVT, LocInfo HTP, bool IsCustom=false)
This class represents a function call, abstracting a target machine's calling convention.
static Constant * get(Type *Ty, double V)
This returns a ConstantFP, or a vector containing a splat of a ConstantFP, for the specified value in...
Definition: Constants.cpp:927
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:110
A debug info location.
Definition: DebugLoc.h:33
Diagnostic information for unsupported feature in backend.
This is a fast-path instruction selection class that generates poor code and doesn't support illegal ...
Definition: FastISel.h:66
FunctionLoweringInfo - This contains information that is global to a function that is used when lower...
FunctionType * getFunctionType() const
Returns the FunctionType for me.
Definition: Function.h:200
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Definition: Function.cpp:341
unsigned getAddressSpace() const
const GlobalValue * getGlobal() const
ThreadLocalMode getThreadLocalMode() const
Definition: GlobalValue.h:267
Module * getParent()
Get the module that this global value is contained inside of...
Definition: GlobalValue.h:652
Type * getValueType() const
Definition: GlobalValue.h:292
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:67
void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
This class is used to represent ISD::LOAD nodes.
const SDValue & getBasePtr() const
const SDValue & getOffset() const
Describe properties that are true of each instruction in the target description file.
Definition: MCInstrDesc.h:198
void setNoStrip() const
Definition: MCSymbolWasm.h:65
Machine Value Type.
static auto integer_fixedlen_vector_valuetypes()
@ INVALID_SIMPLE_VALUE_TYPE
unsigned getVectorNumElements() const
bool isVector() const
Return true if this is a vector value type.
bool isInteger() const
Return true if this is an integer or a vector integer type.
static MVT getVT(Type *Ty, bool HandleUnknown=false)
Return the value type corresponding to the specified type.
Definition: ValueTypes.cpp:581
static auto integer_valuetypes()
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
static auto fixedlen_vector_valuetypes()
bool isFixedLengthVector() const
static MVT getVectorVT(MVT VT, unsigned NumElements)
MVT getVectorElementType() const
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
static MVT getIntegerVT(unsigned BitWidth)
void transferSuccessorsAndUpdatePHIs(MachineBasicBlock *FromMBB)
Transfers all the successors, as in transferSuccessors, and update PHI operands in the successor bloc...
instr_iterator insert(instr_iterator I, MachineInstr *M)
Insert MI into the instruction list before I, possibly inside a bundle.
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
iterator insertAfter(iterator I, MachineInstr *MI)
Insert MI into the instruction list after I.
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it.
void setFrameAddressIsTaken(bool T)
unsigned getFunctionNumber() const
getFunctionNumber - Return a unique ID for the current function.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineInstr * CreateMachineInstr(const MCInstrDesc &MCID, DebugLoc DL, bool NoImplicit=false)
CreateMachineInstr - Allocate a new MachineInstr.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
const char * createExternalSymbolName(StringRef Name)
Allocate a string and populate it with the given external symbol name.
MCContext & getContext() const
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
Function & getFunction()
Return the LLVM function that this machine code represents.
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
const MachineJumpTableInfo * getJumpTableInfo() const
getJumpTableInfo - Return the jump table info object for the current function.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & addSym(MCSymbol *Sym, unsigned char TargetFlags=0) const
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addFPImm(const ConstantFP *Val) const
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
MachineInstr * getInstr() const
If conversion operators fail, use this method to get the MachineInstr explicitly.
Representation of each machine instruction.
Definition: MachineInstr.h:68
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
Definition: MachineInstr.h:543
iterator_range< mop_iterator > uses()
Returns a range that includes all operands that are register uses.
Definition: MachineInstr.h:707
void addOperand(MachineFunction &MF, const MachineOperand &Op)
Add the specified operand to the instruction.
iterator_range< mop_iterator > defs()
Returns a range over all explicit operands that are register definitions.
Definition: MachineInstr.h:696
void eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
void removeOperand(unsigned OpNo)
Erase an operand from an instruction, leaving it with one fewer operand than it started with.
const MachineOperand & getOperand(unsigned i) const
Definition: MachineInstr.h:553
const std::vector< MachineJumpTableEntry > & getJumpTables() const
Flags
Flags values. These may be or'd together.
@ MOVolatile
The memory access is volatile.
@ MOLoad
The memory access reads data.
@ MOStore
The memory access writes data.
bool isReg() const
isReg - Tests if this is a MO_Register operand.
Register getReg() const
getReg - Returns the register number.
bool isFI() const
isFI - Tests if this is a MO_FrameIndex operand.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
Register createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name="")
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
void addLiveIn(MCRegister Reg, Register vreg=Register())
addLiveIn - Add the specified register as a live-in.
unsigned getAddressSpace() const
Return the address space for the associated pointer.
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
const SDValue & getChain() const
EVT getMemoryVT() const
Return the type of the in-memory value.
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
ArrayRef< SDUse > ops() const
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
const SDValue & getOperand(unsigned Num) const
uint64_t getConstantOperandVal(unsigned Num) const
Helper method returns the integer value of a ConstantSDNode operand.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
bool isUndef() const
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
const SDValue & getOperand(unsigned i) const
MVT getSimpleValueType() const
Return the simple ValueType of the referenced return value.
unsigned getOpcode() const
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:225
SDValue getTargetGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, unsigned TargetFlags=0)
Definition: SelectionDAG.h:720
SDValue getMergeValues(ArrayRef< SDValue > Ops, const SDLoc &dl)
Create a MERGE_VALUES node from the given operands.
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
SDValue getSplatValue(SDValue V, bool LegalTypes=false)
If V is a splat vector, return its scalar source operand by extracting that element from the source v...
MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
void ExtractVectorElements(SDValue Op, SmallVectorImpl< SDValue > &Args, unsigned Start=0, unsigned Count=0, EVT EltVT=EVT())
Append the extracted elements from Start to Count out of the vector Op in Args.
SDValue UnrollVectorOp(SDNode *N, unsigned ResNE=0)
Utility function used by legalize and lowering to "unroll" a vector operation by splitting out the sc...
SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, bool isTarget=false)
Create a ConstantFPSDNode wrapping a constant value.
SDValue getNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a bitwise NOT operation as (XOR Val, -1).
SDValue getTargetJumpTable(int JTI, EVT VT, unsigned TargetFlags=0)
Definition: SelectionDAG.h:730
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
Definition: SelectionDAG.h:826
SDValue getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVol, bool AlwaysInline, bool isTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo=AAMDNodes(), AAResults *AA=nullptr)
SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
const DataLayout & getDataLayout() const
Definition: SelectionDAG.h:472
SDValue getTargetFrameIndex(int FI, EVT VT)
Definition: SelectionDAG.h:725
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
SDValue getBasicBlock(MachineBasicBlock *MBB)
const TargetMachine & getTarget() const
Definition: SelectionDAG.h:473
SDValue getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either any-extending or truncat...
SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, unsigned Reg, SDValue N)
Definition: SelectionDAG.h:771
SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef< SDValue > Ops, EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags Flags=MachineMemOperand::MOLoad|MachineMemOperand::MOStore, uint64_t Size=0, const AAMDNodes &AAInfo=AAMDNodes())
Creates a MemIntrinsicNode that may produce a result and takes a list of operands.
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
SDValue getValueType(EVT)
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
Definition: SelectionDAG.h:674
MachineFunction & getMachineFunction() const
Definition: SelectionDAG.h:469
SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, unsigned Reg, EVT VT)
Definition: SelectionDAG.h:797
SDValue getSplatBuildVector(EVT VT, const SDLoc &DL, SDValue Op)
Return a splat ISD::BUILD_VECTOR node, consisting of Op splatted to all elements.
Definition: SelectionDAG.h:843
SDValue getFrameIndex(int FI, EVT VT, bool isTarget=false)
SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
LLVMContext * getContext() const
Definition: SelectionDAG.h:485
SDValue getTargetExternalSymbol(const char *Sym, EVT VT, unsigned TargetFlags=0)
SDValue getMCSymbol(MCSymbol *Sym, EVT VT)
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
Definition: SelectionDAG.h:554
SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
bool empty() const
Definition: SmallVector.h:94
size_t size() const
Definition: SmallVector.h:91
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: SmallVector.h:577
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
Definition: SmallVector.h:687
void push_back(const T &Elt)
Definition: SmallVector.h:416
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1200
This class is used to represent ISD::STORE nodes.
const SDValue & getBasePtr() const
const SDValue & getOffset() const
const SDValue & getValue() const
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
constexpr size_t size() const
size - Get the string size.
Definition: StringRef.h:137
TargetInstrInfo - Interface to description of machine instruction set.
Provides information about what library functions are available for the current target.
void setBooleanVectorContents(BooleanContent Ty)
Specify how the target extends the result of a vector boolean value from a vector of i1 to a wider ty...
void setOperationAction(unsigned Op, MVT VT, LegalizeAction Action)
Indicate that the specified operation does not work with the specified type and indicate what to do a...
virtual const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent=false) const
Return the register class that should be used for the specified value type.
const TargetMachine & getTargetMachine() const
LegalizeTypeAction
This enum indicates whether a types are legal for a target, and if not, what action should be used to...
void setMaxAtomicSizeInBitsSupported(unsigned SizeInBits)
Set the maximum atomic operation size supported by the backend.
virtual TargetLoweringBase::LegalizeTypeAction getPreferredVectorAction(MVT VT) const
Return the preferred vector type legalization action.
void setBooleanContents(BooleanContent Ty)
Specify how the target extends the result of integer and floating point boolean values from i1 to a w...
void computeRegisterProperties(const TargetRegisterInfo *TRI)
Once all of the register classes are added, this allows us to compute derived properties we expose.
void addRegisterClass(MVT VT, const TargetRegisterClass *RC)
Add the specified register class as an available regclass for the specified value type.
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
void setLibcallName(RTLIB::Libcall Call, const char *Name)
Rename the default libcall routine name for the specified libcall.
void setMinimumJumpTableEntries(unsigned Val)
Indicate the minimum number of blocks to generate jump tables.
void setTruncStoreAction(MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified truncating store does not work with the specified type and indicate what ...
void setStackPointerRegisterToSaveRestore(Register R)
If set to a physical register, this specifies the register that llvm.savestack/llvm....
AtomicExpansionKind
Enum that specifies what an atomic load/AtomicRMWInst is expanded to, if at all.
void setCondCodeAction(ArrayRef< ISD::CondCode > CCs, MVT VT, LegalizeAction Action)
Indicate that the specified condition code is or isn't supported on the target and indicate what to d...
void setTargetDAGCombine(ArrayRef< ISD::NodeType > NTs)
Targets should invoke this method for each target independent node that they want to provide a custom...
void setLoadExtAction(unsigned ExtType, MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified load with extension does not work with the specified type and indicate wh...
virtual MVT getPointerMemTy(const DataLayout &DL, uint32_t AS=0) const
Return the in-memory pointer type for the given address space, defaults to the pointer type from the ...
void setSchedulingPreference(Sched::Preference Pref)
Specify the target scheduling preference.
bool isOperationLegalOrCustomOrPromote(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
std::pair< SDValue, SDValue > makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT, ArrayRef< SDValue > Ops, MakeLibCallOptions CallOptions, const SDLoc &dl, SDValue Chain=SDValue()) const
Returns a pair of (return value, chain).
bool isPositionIndependent() const
virtual std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const
Given a physical register constraint (e.g.
bool verifyReturnAddressArgumentIsConstant(SDValue Op, SelectionDAG &DAG) const
virtual bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const
Return true if folding a constant offset with the given GlobalAddress is legal.
Primary interface to the complete machine description for the target machine.
Definition: TargetMachine.h:78
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
bool isOSEmscripten() const
Tests whether the OS is Emscripten.
Definition: Triple.h:657
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
static Type * getDoubleTy(LLVMContext &C)
bool isFunctionTy() const
True if this is an instance of FunctionType.
Definition: Type.h:246
static Type * getFloatTy(LLVMContext &C)
A Use represents the edge between a Value definition and its users.
Definition: Use.h:43
LLVM Value Representation.
Definition: Value.h:74
const Value * stripPointerCastsAndAliases() const
Strip off pointer casts, all-zero GEPs, address space casts, and aliases.
Definition: Value.cpp:697
static std::optional< unsigned > getLocalForStackObject(MachineFunction &MF, int FrameIndex)
This class is derived from MachineFunctionInfo and contains private WebAssembly-specific information ...
Register getFrameRegister(const MachineFunction &MF) const override
const Triple & getTargetTriple() const
const WebAssemblyInstrInfo * getInstrInfo() const override
const WebAssemblyRegisterInfo * getRegisterInfo() const override
WebAssemblyTargetLowering(const TargetMachine &TM, const WebAssemblySubtarget &STI)
MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const override
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
MVT getPointerMemTy(const DataLayout &DL, uint32_t AS=0) const override
Return the in-memory pointer type for the given address space, defaults to the pointer type from the ...
self_iterator getIterator()
Definition: ilist_node.h:109
#define INT64_MIN
Definition: DataTypes.h:74
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
const CustomOperand< const MCSubtargetInfo & > Msg[]
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition: BitmaskEnum.h:121
@ Swift
Calling convention for Swift.
Definition: CallingConv.h:69
@ PreserveMost
Used for runtime calls that preserves most registers.
Definition: CallingConv.h:63
@ CXX_FAST_TLS
Used for access functions.
Definition: CallingConv.h:72
@ WASM_EmscriptenInvoke
For emscripten __invoke_* functions.
Definition: CallingConv.h:226
@ Cold
Attempts to make code in the caller as efficient as possible under the assumption that the call is no...
Definition: CallingConv.h:47
@ PreserveAll
Used for runtime calls that preserves (almost) all registers.
Definition: CallingConv.h:66
@ Fast
Attempts to make calls as fast as possible (e.g.
Definition: CallingConv.h:41
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
Definition: ISDOpcodes.h:750
@ STACKRESTORE
STACKRESTORE has two operands, an input chain and a pointer to restore to it returns an output chain.
Definition: ISDOpcodes.h:1124
@ STACKSAVE
STACKSAVE - STACKSAVE has one operand, an input chain.
Definition: ISDOpcodes.h:1120
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
Definition: ISDOpcodes.h:250
@ BSWAP
Byte Swap and Counting operators.
Definition: ISDOpcodes.h:714
@ VAEND
VAEND, VASTART - VAEND and VASTART have three operands: an input chain, pointer, and a SRCVALUE.
Definition: ISDOpcodes.h:1153
@ ConstantFP
Definition: ISDOpcodes.h:77
@ ADDC
Carry-setting nodes for multiple precision addition and subtraction.
Definition: ISDOpcodes.h:269
@ ADD
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:239
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
Definition: ISDOpcodes.h:1029
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
Definition: ISDOpcodes.h:483
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
Definition: ISDOpcodes.h:199
@ RETURNADDR
Definition: ISDOpcodes.h:95
@ GlobalAddress
Definition: ISDOpcodes.h:78
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition: ISDOpcodes.h:790
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
Definition: ISDOpcodes.h:543
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
Definition: ISDOpcodes.h:688
@ SIGN_EXTEND_VECTOR_INREG
SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register sign-extension of the low ...
Definition: ISDOpcodes.h:820
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
Definition: ISDOpcodes.h:255
@ FP16_TO_FP
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
Definition: ISDOpcodes.h:913
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:903
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
Definition: ISDOpcodes.h:229
@ BUILTIN_OP_END
BUILTIN_OP_END - This must be the last enum value in this list.
Definition: ISDOpcodes.h:1383
@ GlobalTLSAddress
Definition: ISDOpcodes.h:79
@ FrameIndex
Definition: ISDOpcodes.h:80
@ SIGN_EXTEND
Conversion operators.
Definition: ISDOpcodes.h:774
@ FSINCOS
FSINCOS - Compute both fsin and fcos as a single operation.
Definition: ISDOpcodes.h:986
@ BR_CC
BR_CC - Conditional branch.
Definition: ISDOpcodes.h:1075
@ BRIND
BRIND - Indirect branch.
Definition: ISDOpcodes.h:1050
@ BR_JT
BR_JT - Jumptable branch.
Definition: ISDOpcodes.h:1054
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
Definition: ISDOpcodes.h:627
@ VACOPY
VACOPY - VACOPY has 5 operands: an input chain, a destination pointer, a source pointer,...
Definition: ISDOpcodes.h:1149
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
Definition: ISDOpcodes.h:651
@ SHL
Shift and rotation operations.
Definition: ISDOpcodes.h:705
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
Definition: ISDOpcodes.h:600
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
Definition: ISDOpcodes.h:573
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition: ISDOpcodes.h:535
@ CopyToReg
CopyToReg - This node has three operands: a chain, a register number to set to this value,...
Definition: ISDOpcodes.h:203
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:780
@ DEBUGTRAP
DEBUGTRAP - Trap intended to get the attention of a debugger.
Definition: ISDOpcodes.h:1203
@ FP_TO_UINT_SAT
Definition: ISDOpcodes.h:856
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
Definition: ISDOpcodes.h:742
@ DYNAMIC_STACKALLOC
DYNAMIC_STACKALLOC - Allocate some number of bytes on the stack aligned to a specified boundary.
Definition: ISDOpcodes.h:1039
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
Definition: ISDOpcodes.h:798
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
Definition: ISDOpcodes.h:674
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
Definition: ISDOpcodes.h:888
@ FRAMEADDR
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG.
Definition: ISDOpcodes.h:94
@ FMINIMUM
FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 as less than 0....
Definition: ISDOpcodes.h:982
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:836
@ TargetConstant
TargetConstant* - Like Constant*, but the DAG does not do any folding, simplification,...
Definition: ISDOpcodes.h:158
@ AND
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:680
@ TRAP
TRAP - Trapping instruction.
Definition: ISDOpcodes.h:1200
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
Definition: ISDOpcodes.h:184
@ ADDE
Carry-using nodes for multiple precision addition and subtraction.
Definition: ISDOpcodes.h:279
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
Definition: ISDOpcodes.h:524
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
Definition: ISDOpcodes.h:52
@ ExternalSymbol
Definition: ISDOpcodes.h:83
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
Definition: ISDOpcodes.h:869
@ ZERO_EXTEND_VECTOR_INREG
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
Definition: ISDOpcodes.h:831
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
Definition: ISDOpcodes.h:855
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
Definition: ISDOpcodes.h:786
@ VAARG
VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE, and the alignment.
Definition: ISDOpcodes.h:1144
@ BlockAddress
Definition: ISDOpcodes.h:84
@ SHL_PARTS
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations.
Definition: ISDOpcodes.h:763
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
Definition: ISDOpcodes.h:493
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
Definition: ISDOpcodes.h:340
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
Definition: ISDOpcodes.h:192
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
Definition: ISDOpcodes.h:515
bool isConstantSplatVector(const SDNode *N, APInt &SplatValue)
Node predicates.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
Definition: ISDOpcodes.h:1506
OperandFlags
These are flags set on operands, but should be considered private, all access should go through the M...
Definition: MCInstrDesc.h:50
bool match(Val *V, const Pattern &P)
Definition: PatternMatch.h:49
cst_pred_ty< is_zero_int > m_ZeroInt()
Match an integer 0 or a vector with all elements equal to 0.
Definition: PatternMatch.h:532
TwoOps_match< V1_t, V2_t, Instruction::ShuffleVector > m_Shuffle(const V1_t &v1, const V2_t &v2)
Matches ShuffleVectorInst independently of mask value.
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
Definition: PatternMatch.h:76
ThreeOps_match< Val_t, Elt_t, Idx_t, Instruction::InsertElement > m_InsertElt(const Val_t &Val, const Elt_t &Elt, const Idx_t &Idx)
Matches InsertElementInst.
MCSymbolWasm * getOrCreateFunctionTableSymbol(MCContext &Ctx, const WebAssemblySubtarget *Subtarget)
Returns the __indirect_function_table, for use in call_indirect and in function bitcasts.
bool isWebAssemblyFuncrefType(const Type *Ty)
Return true if this is a WebAssembly Funcref Type.
bool isWebAssemblyTableType(const Type *Ty)
Return true if the table represents a WebAssembly table type.
MCSymbolWasm * getOrCreateFuncrefCallTableSymbol(MCContext &Ctx, const WebAssemblySubtarget *Subtarget)
Returns the __funcref_call_table, for use in funcref calls when lowered to table.set + call_indirect.
FastISel * createFastISel(FunctionLoweringInfo &funcInfo, const TargetLibraryInfo *libInfo)
bool isValidAddressSpace(unsigned AS)
bool isWasmVarAddressSpace(unsigned AS)
Reg
All possible values of the reg field in the ModR/M byte.
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
Definition: STLExtras.h:329
unsigned Log2_32_Ceil(uint32_t Value)
Return the ceil log base 2 of the specified value, 32 if the value is zero.
Definition: MathExtras.h:326
@ Offset
Definition: DWP.cpp:456
void computeSignatureVTs(const FunctionType *Ty, const Function *TargetFunc, const Function &ContextFunc, const TargetMachine &TM, SmallVectorImpl< MVT > &Params, SmallVectorImpl< MVT > &Results)
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition: MathExtras.h:264
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:156
@ And
Bitwise or logical AND of integers.
@ Add
Sum of integers.
DWARFExpression::Operation Op
constexpr unsigned BitWidth
Definition: BitmaskEnum.h:191
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1753
void erase_if(Container &C, UnaryPredicate P)
Provide a container algorithm similar to C++ Library Fundamentals v2's erase_if which is equivalent t...
Definition: STLExtras.h:2020
void computeLegalValueVTs(const WebAssemblyTargetLowering &TLI, LLVMContext &Ctx, const DataLayout &DL, Type *Ty, SmallVectorImpl< MVT > &ValueVTs)
bool isAllOnesConstant(SDValue V)
Returns true if V is an integer constant with all bits set.
constexpr uint64_t NextPowerOf2(uint64_t A)
Returns the next power of two (in 64-bits) that is strictly greater than A.
Definition: MathExtras.h:349
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:860
#define N
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
Extended Value Type.
Definition: ValueTypes.h:34
EVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
Definition: ValueTypes.h:93
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
Definition: ValueTypes.h:129
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
Definition: ValueTypes.h:73
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
Definition: ValueTypes.h:139
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:351
uint64_t getScalarSizeInBits() const
Definition: ValueTypes.h:363
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition: ValueTypes.h:299
bool is128BitVector() const
Return true if this is a 128-bit vector type.
Definition: ValueTypes.h:196
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
Definition: ValueTypes.h:64
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
Definition: ValueTypes.h:359
bool isFixedLengthVector() const
Definition: ValueTypes.h:170
bool isVector() const
Return true if this is a vector value type.
Definition: ValueTypes.h:160
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Definition: ValueTypes.h:306
bool bitsGE(EVT VT) const
Return true if this has no less bits than VT.
Definition: ValueTypes.h:275
bool is256BitVector() const
Return true if this is a 256-bit vector type.
Definition: ValueTypes.h:201
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
Definition: ValueTypes.cpp:202
EVT getVectorElementType() const
Given a vector type, return the type of each element.
Definition: ValueTypes.h:311
bool isScalarInteger() const
Return true if this is an integer, but not a vector.
Definition: ValueTypes.h:149
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
Definition: ValueTypes.h:319
bool isInConsecutiveRegs() const
Align getNonZeroOrigAlign() const
unsigned getByValSize() const
bool isInConsecutiveRegsLast() const
Align getNonZeroByValAlign() const
InputArg - This struct carries flags and type information about a single incoming (formal) argument o...
OutputArg - This struct carries flags and a value for a single outgoing (actual) argument or outgoing...
bool IsFixed
IsFixed - Is this a "fixed" value, ie not passed through a vararg "...".
unsigned getBitWidth() const
Get the bit width of this value.
Definition: KnownBits.h:40
This class contains a discriminated union of information about pointers in memory operands,...
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
Function object to check whether the second component of a container supported by std::get (like std:...
Definition: STLExtras.h:1463