LLVM 17.0.0git
WebAssemblyISelLowering.cpp
Go to the documentation of this file.
1//=- WebAssemblyISelLowering.cpp - WebAssembly DAG Lowering Implementation -==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// This file implements the WebAssemblyTargetLowering class.
11///
12//===----------------------------------------------------------------------===//
13
32#include "llvm/IR/Function.h"
33#include "llvm/IR/Intrinsics.h"
34#include "llvm/IR/IntrinsicsWebAssembly.h"
35#include "llvm/Support/Debug.h"
41using namespace llvm;
42
43#define DEBUG_TYPE "wasm-lower"
44
46 const TargetMachine &TM, const WebAssemblySubtarget &STI)
47 : TargetLowering(TM), Subtarget(&STI) {
48 auto MVTPtr = Subtarget->hasAddr64() ? MVT::i64 : MVT::i32;
49
50 // Booleans always contain 0 or 1.
52 // Except in SIMD vectors
54 // We don't know the microarchitecture here, so just reduce register pressure.
56 // Tell ISel that we have a stack pointer.
58 Subtarget->hasAddr64() ? WebAssembly::SP64 : WebAssembly::SP32);
59 // Set up the register classes.
60 addRegisterClass(MVT::i32, &WebAssembly::I32RegClass);
61 addRegisterClass(MVT::i64, &WebAssembly::I64RegClass);
62 addRegisterClass(MVT::f32, &WebAssembly::F32RegClass);
63 addRegisterClass(MVT::f64, &WebAssembly::F64RegClass);
64 if (Subtarget->hasSIMD128()) {
65 addRegisterClass(MVT::v16i8, &WebAssembly::V128RegClass);
66 addRegisterClass(MVT::v8i16, &WebAssembly::V128RegClass);
67 addRegisterClass(MVT::v4i32, &WebAssembly::V128RegClass);
68 addRegisterClass(MVT::v4f32, &WebAssembly::V128RegClass);
69 addRegisterClass(MVT::v2i64, &WebAssembly::V128RegClass);
70 addRegisterClass(MVT::v2f64, &WebAssembly::V128RegClass);
71 }
72 if (Subtarget->hasReferenceTypes()) {
73 addRegisterClass(MVT::externref, &WebAssembly::EXTERNREFRegClass);
74 addRegisterClass(MVT::funcref, &WebAssembly::FUNCREFRegClass);
75 }
76 // Compute derived properties from the register classes.
78
79 // Transform loads and stores to pointers in address space 1 to loads and
80 // stores to WebAssembly global variables, outside linear memory.
81 for (auto T : {MVT::i32, MVT::i64, MVT::f32, MVT::f64}) {
84 }
85 if (Subtarget->hasSIMD128()) {
87 MVT::v2f64}) {
90 }
91 }
92 if (Subtarget->hasReferenceTypes()) {
93 // We need custom load and store lowering for both externref, funcref and
94 // Other. The MVT::Other here represents tables of reference types.
95 for (auto T : {MVT::externref, MVT::funcref, MVT::Other}) {
98 }
99 }
100
107
108 // Take the default expansion for va_arg, va_copy, and va_end. There is no
109 // default action for va_start, so we do that custom.
114
115 for (auto T : {MVT::f32, MVT::f64, MVT::v4f32, MVT::v2f64}) {
116 // Don't expand the floating-point types to constant pools.
118 // Expand floating-point comparisons.
122 // Expand floating-point library function operators.
123 for (auto Op :
126 // Note supported floating-point library function operators that otherwise
127 // default to expand.
128 for (auto Op :
131 // Support minimum and maximum, which otherwise default to expand.
134 // WebAssembly currently has no builtin f16 support.
139 }
140
141 // Expand unavailable integer operations.
142 for (auto Op :
146 for (auto T : {MVT::i32, MVT::i64})
148 if (Subtarget->hasSIMD128())
151 }
152
153 if (Subtarget->hasNontrappingFPToInt())
155 for (auto T : {MVT::i32, MVT::i64})
157
158 // SIMD-specific configuration
159 if (Subtarget->hasSIMD128()) {
160 // Hoist bitcasts out of shuffles
162
163 // Combine extends of extract_subvectors into widening ops
165
166 // Combine int_to_fp or fp_extend of extract_vectors and vice versa into
167 // conversions ops
170
171 // Combine fp_to_{s,u}int_sat or fp_round of concat_vectors or vice versa
172 // into conversion ops
175
177
178 // Support saturating add for i8x16 and i16x8
179 for (auto Op : {ISD::SADDSAT, ISD::UADDSAT})
180 for (auto T : {MVT::v16i8, MVT::v8i16})
182
183 // Support integer abs
186
187 // Custom lower BUILD_VECTORs to minimize number of replace_lanes
189 MVT::v2f64})
191
192 // We have custom shuffle lowering to expose the shuffle mask
194 MVT::v2f64})
196
197 // Support splatting
199 MVT::v2f64})
201
202 // Custom lowering since wasm shifts must have a scalar shift amount
203 for (auto Op : {ISD::SHL, ISD::SRA, ISD::SRL})
206
207 // Custom lower lane accesses to expand out variable indices
210 MVT::v2f64})
212
213 // There is no i8x16.mul instruction
215
216 // There is no vector conditional select instruction
218 MVT::v2f64})
220
221 // Expand integer operations supported for scalars but not SIMD
222 for (auto Op :
226
227 // But we do have integer min and max operations
228 for (auto Op : {ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX})
229 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32})
231
232 // And we have popcnt for i8x16. It can be used to expand ctlz/cttz.
236
237 // Custom lower bit counting operations for other types to scalarize them.
238 for (auto Op : {ISD::CTLZ, ISD::CTTZ, ISD::CTPOP})
239 for (auto T : {MVT::v8i16, MVT::v4i32, MVT::v2i64})
241
242 // Expand float operations supported for scalars but not SIMD
245 for (auto T : {MVT::v4f32, MVT::v2f64})
247
248 // Unsigned comparison operations are unavailable for i64x2 vectors.
251
252 // 64x2 conversions are not in the spec
253 for (auto Op :
255 for (auto T : {MVT::v2i64, MVT::v2f64})
257
258 // But saturating fp_to_int converstions are
261 }
262
263 // As a special case, these operators use the type to mean the type to
264 // sign-extend from.
266 if (!Subtarget->hasSignExt()) {
267 // Sign extends are legal only when extending a vector extract
268 auto Action = Subtarget->hasSIMD128() ? Custom : Expand;
269 for (auto T : {MVT::i8, MVT::i16, MVT::i32})
271 }
274
275 // Dynamic stack allocation: use the default expansion.
279
283
284 // Expand these forms; we pattern-match the forms that we can handle in isel.
285 for (auto T : {MVT::i32, MVT::i64, MVT::f32, MVT::f64})
286 for (auto Op : {ISD::BR_CC, ISD::SELECT_CC})
288
289 // We have custom switch handling.
291
292 // WebAssembly doesn't have:
293 // - Floating-point extending loads.
294 // - Floating-point truncating stores.
295 // - i1 extending loads.
296 // - truncating SIMD stores and most extending loads
299 for (auto T : MVT::integer_valuetypes())
300 for (auto Ext : {ISD::EXTLOAD, ISD::ZEXTLOAD, ISD::SEXTLOAD})
302 if (Subtarget->hasSIMD128()) {
304 MVT::v2f64}) {
305 for (auto MemT : MVT::fixedlen_vector_valuetypes()) {
306 if (MVT(T) != MemT) {
308 for (auto Ext : {ISD::EXTLOAD, ISD::ZEXTLOAD, ISD::SEXTLOAD})
309 setLoadExtAction(Ext, T, MemT, Expand);
310 }
311 }
312 }
313 // But some vector extending loads are legal
314 for (auto Ext : {ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}) {
318 }
320 }
321
322 // Don't do anything clever with build_pairs
324
325 // Trap lowers to wasm unreachable
328
329 // Exception handling intrinsics
333
335
336 // Override the __gnu_f2h_ieee/__gnu_h2f_ieee names so that the f32 name is
337 // consistent with the f64 and f128 names.
338 setLibcallName(RTLIB::FPEXT_F16_F32, "__extendhfsf2");
339 setLibcallName(RTLIB::FPROUND_F32_F16, "__truncsfhf2");
340
341 // Define the emscripten name for return address helper.
342 // TODO: when implementing other Wasm backends, make this generic or only do
343 // this on emscripten depending on what they end up doing.
344 setLibcallName(RTLIB::RETURN_ADDRESS, "emscripten_return_address");
345
346 // Always convert switches to br_tables unless there is only one case, which
347 // is equivalent to a simple branch. This reduces code size for wasm, and we
348 // defer possible jump table optimizations to the VM.
350}
351
353 uint32_t AS) const {
355 return MVT::externref;
357 return MVT::funcref;
359}
360
362 uint32_t AS) const {
364 return MVT::externref;
366 return MVT::funcref;
368}
369
371WebAssemblyTargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const {
372 // We have wasm instructions for these
373 switch (AI->getOperation()) {
381 default:
382 break;
383 }
385}
386
387bool WebAssemblyTargetLowering::shouldScalarizeBinop(SDValue VecOp) const {
388 // Implementation copied from X86TargetLowering.
389 unsigned Opc = VecOp.getOpcode();
390
391 // Assume target opcodes can't be scalarized.
392 // TODO - do we have any exceptions?
393 if (Opc >= ISD::BUILTIN_OP_END)
394 return false;
395
396 // If the vector op is not supported, try to convert to scalar.
397 EVT VecVT = VecOp.getValueType();
398 if (!isOperationLegalOrCustomOrPromote(Opc, VecVT))
399 return true;
400
401 // If the vector op is supported, but the scalar op is not, the transform may
402 // not be worthwhile.
403 EVT ScalarVT = VecVT.getScalarType();
404 return isOperationLegalOrCustomOrPromote(Opc, ScalarVT);
405}
406
407FastISel *WebAssemblyTargetLowering::createFastISel(
408 FunctionLoweringInfo &FuncInfo, const TargetLibraryInfo *LibInfo) const {
409 return WebAssembly::createFastISel(FuncInfo, LibInfo);
410}
411
412MVT WebAssemblyTargetLowering::getScalarShiftAmountTy(const DataLayout & /*DL*/,
413 EVT VT) const {
414 unsigned BitWidth = NextPowerOf2(VT.getSizeInBits() - 1);
415 if (BitWidth > 1 && BitWidth < 8)
416 BitWidth = 8;
417
418 if (BitWidth > 64) {
419 // The shift will be lowered to a libcall, and compiler-rt libcalls expect
420 // the count to be an i32.
421 BitWidth = 32;
423 "32-bit shift counts ought to be enough for anyone");
424 }
425
428 "Unable to represent scalar shift amount type");
429 return Result;
430}
431
432// Lower an fp-to-int conversion operator from the LLVM opcode, which has an
433// undefined result on invalid/overflow, to the WebAssembly opcode, which
434// traps on invalid/overflow.
437 const TargetInstrInfo &TII,
438 bool IsUnsigned, bool Int64,
439 bool Float64, unsigned LoweredOpcode) {
441
442 Register OutReg = MI.getOperand(0).getReg();
443 Register InReg = MI.getOperand(1).getReg();
444
445 unsigned Abs = Float64 ? WebAssembly::ABS_F64 : WebAssembly::ABS_F32;
446 unsigned FConst = Float64 ? WebAssembly::CONST_F64 : WebAssembly::CONST_F32;
447 unsigned LT = Float64 ? WebAssembly::LT_F64 : WebAssembly::LT_F32;
448 unsigned GE = Float64 ? WebAssembly::GE_F64 : WebAssembly::GE_F32;
449 unsigned IConst = Int64 ? WebAssembly::CONST_I64 : WebAssembly::CONST_I32;
450 unsigned Eqz = WebAssembly::EQZ_I32;
451 unsigned And = WebAssembly::AND_I32;
452 int64_t Limit = Int64 ? INT64_MIN : INT32_MIN;
453 int64_t Substitute = IsUnsigned ? 0 : Limit;
454 double CmpVal = IsUnsigned ? -(double)Limit * 2.0 : -(double)Limit;
455 auto &Context = BB->getParent()->getFunction().getContext();
457
458 const BasicBlock *LLVMBB = BB->getBasicBlock();
459 MachineFunction *F = BB->getParent();
460 MachineBasicBlock *TrueMBB = F->CreateMachineBasicBlock(LLVMBB);
461 MachineBasicBlock *FalseMBB = F->CreateMachineBasicBlock(LLVMBB);
462 MachineBasicBlock *DoneMBB = F->CreateMachineBasicBlock(LLVMBB);
463
465 F->insert(It, FalseMBB);
466 F->insert(It, TrueMBB);
467 F->insert(It, DoneMBB);
468
469 // Transfer the remainder of BB and its successor edges to DoneMBB.
470 DoneMBB->splice(DoneMBB->begin(), BB, std::next(MI.getIterator()), BB->end());
472
473 BB->addSuccessor(TrueMBB);
474 BB->addSuccessor(FalseMBB);
475 TrueMBB->addSuccessor(DoneMBB);
476 FalseMBB->addSuccessor(DoneMBB);
477
478 unsigned Tmp0, Tmp1, CmpReg, EqzReg, FalseReg, TrueReg;
479 Tmp0 = MRI.createVirtualRegister(MRI.getRegClass(InReg));
480 Tmp1 = MRI.createVirtualRegister(MRI.getRegClass(InReg));
481 CmpReg = MRI.createVirtualRegister(&WebAssembly::I32RegClass);
482 EqzReg = MRI.createVirtualRegister(&WebAssembly::I32RegClass);
483 FalseReg = MRI.createVirtualRegister(MRI.getRegClass(OutReg));
484 TrueReg = MRI.createVirtualRegister(MRI.getRegClass(OutReg));
485
486 MI.eraseFromParent();
487 // For signed numbers, we can do a single comparison to determine whether
488 // fabs(x) is within range.
489 if (IsUnsigned) {
490 Tmp0 = InReg;
491 } else {
492 BuildMI(BB, DL, TII.get(Abs), Tmp0).addReg(InReg);
493 }
494 BuildMI(BB, DL, TII.get(FConst), Tmp1)
495 .addFPImm(cast<ConstantFP>(ConstantFP::get(Ty, CmpVal)));
496 BuildMI(BB, DL, TII.get(LT), CmpReg).addReg(Tmp0).addReg(Tmp1);
497
498 // For unsigned numbers, we have to do a separate comparison with zero.
499 if (IsUnsigned) {
500 Tmp1 = MRI.createVirtualRegister(MRI.getRegClass(InReg));
501 Register SecondCmpReg =
502 MRI.createVirtualRegister(&WebAssembly::I32RegClass);
503 Register AndReg = MRI.createVirtualRegister(&WebAssembly::I32RegClass);
504 BuildMI(BB, DL, TII.get(FConst), Tmp1)
505 .addFPImm(cast<ConstantFP>(ConstantFP::get(Ty, 0.0)));
506 BuildMI(BB, DL, TII.get(GE), SecondCmpReg).addReg(Tmp0).addReg(Tmp1);
507 BuildMI(BB, DL, TII.get(And), AndReg).addReg(CmpReg).addReg(SecondCmpReg);
508 CmpReg = AndReg;
509 }
510
511 BuildMI(BB, DL, TII.get(Eqz), EqzReg).addReg(CmpReg);
512
513 // Create the CFG diamond to select between doing the conversion or using
514 // the substitute value.
515 BuildMI(BB, DL, TII.get(WebAssembly::BR_IF)).addMBB(TrueMBB).addReg(EqzReg);
516 BuildMI(FalseMBB, DL, TII.get(LoweredOpcode), FalseReg).addReg(InReg);
517 BuildMI(FalseMBB, DL, TII.get(WebAssembly::BR)).addMBB(DoneMBB);
518 BuildMI(TrueMBB, DL, TII.get(IConst), TrueReg).addImm(Substitute);
519 BuildMI(*DoneMBB, DoneMBB->begin(), DL, TII.get(TargetOpcode::PHI), OutReg)
520 .addReg(FalseReg)
521 .addMBB(FalseMBB)
522 .addReg(TrueReg)
523 .addMBB(TrueMBB);
524
525 return DoneMBB;
526}
527
528static MachineBasicBlock *
530 const WebAssemblySubtarget *Subtarget,
531 const TargetInstrInfo &TII) {
532 MachineInstr &CallParams = *CallResults.getPrevNode();
533 assert(CallParams.getOpcode() == WebAssembly::CALL_PARAMS);
534 assert(CallResults.getOpcode() == WebAssembly::CALL_RESULTS ||
535 CallResults.getOpcode() == WebAssembly::RET_CALL_RESULTS);
536
537 bool IsIndirect = CallParams.getOperand(0).isReg();
538 bool IsRetCall = CallResults.getOpcode() == WebAssembly::RET_CALL_RESULTS;
539
540 bool IsFuncrefCall = false;
541 if (IsIndirect) {
542 Register Reg = CallParams.getOperand(0).getReg();
543 const MachineFunction *MF = BB->getParent();
544 const MachineRegisterInfo &MRI = MF->getRegInfo();
545 const TargetRegisterClass *TRC = MRI.getRegClass(Reg);
546 IsFuncrefCall = (TRC == &WebAssembly::FUNCREFRegClass);
547 assert(!IsFuncrefCall || Subtarget->hasReferenceTypes());
548 }
549
550 unsigned CallOp;
551 if (IsIndirect && IsRetCall) {
552 CallOp = WebAssembly::RET_CALL_INDIRECT;
553 } else if (IsIndirect) {
554 CallOp = WebAssembly::CALL_INDIRECT;
555 } else if (IsRetCall) {
556 CallOp = WebAssembly::RET_CALL;
557 } else {
558 CallOp = WebAssembly::CALL;
559 }
560
561 MachineFunction &MF = *BB->getParent();
562 const MCInstrDesc &MCID = TII.get(CallOp);
563 MachineInstrBuilder MIB(MF, MF.CreateMachineInstr(MCID, DL));
564
565 // See if we must truncate the function pointer.
566 // CALL_INDIRECT takes an i32, but in wasm64 we represent function pointers
567 // as 64-bit for uniformity with other pointer types.
568 // See also: WebAssemblyFastISel::selectCall
569 if (IsIndirect && MF.getSubtarget<WebAssemblySubtarget>().hasAddr64()) {
570 Register Reg32 =
571 MF.getRegInfo().createVirtualRegister(&WebAssembly::I32RegClass);
572 auto &FnPtr = CallParams.getOperand(0);
573 BuildMI(*BB, CallResults.getIterator(), DL,
574 TII.get(WebAssembly::I32_WRAP_I64), Reg32)
575 .addReg(FnPtr.getReg());
576 FnPtr.setReg(Reg32);
577 }
578
579 // Move the function pointer to the end of the arguments for indirect calls
580 if (IsIndirect) {
581 auto FnPtr = CallParams.getOperand(0);
582 CallParams.removeOperand(0);
583
584 // For funcrefs, call_indirect is done through __funcref_call_table and the
585 // funcref is always installed in slot 0 of the table, therefore instead of
586 // having the function pointer added at the end of the params list, a zero
587 // (the index in
588 // __funcref_call_table is added).
589 if (IsFuncrefCall) {
590 Register RegZero =
591 MF.getRegInfo().createVirtualRegister(&WebAssembly::I32RegClass);
592 MachineInstrBuilder MIBC0 =
593 BuildMI(MF, DL, TII.get(WebAssembly::CONST_I32), RegZero).addImm(0);
594
595 BB->insert(CallResults.getIterator(), MIBC0);
596 MachineInstrBuilder(MF, CallParams).addReg(RegZero);
597 } else
598 CallParams.addOperand(FnPtr);
599 }
600
601 for (auto Def : CallResults.defs())
602 MIB.add(Def);
603
604 if (IsIndirect) {
605 // Placeholder for the type index.
606 MIB.addImm(0);
607 // The table into which this call_indirect indexes.
608 MCSymbolWasm *Table = IsFuncrefCall
610 MF.getContext(), Subtarget)
612 MF.getContext(), Subtarget);
613 if (Subtarget->hasReferenceTypes()) {
614 MIB.addSym(Table);
615 } else {
616 // For the MVP there is at most one table whose number is 0, but we can't
617 // write a table symbol or issue relocations. Instead we just ensure the
618 // table is live and write a zero.
619 Table->setNoStrip();
620 MIB.addImm(0);
621 }
622 }
623
624 for (auto Use : CallParams.uses())
625 MIB.add(Use);
626
627 BB->insert(CallResults.getIterator(), MIB);
628 CallParams.eraseFromParent();
629 CallResults.eraseFromParent();
630
631 // If this is a funcref call, to avoid hidden GC roots, we need to clear the
632 // table slot with ref.null upon call_indirect return.
633 //
634 // This generates the following code, which comes right after a call_indirect
635 // of a funcref:
636 //
637 // i32.const 0
638 // ref.null func
639 // table.set __funcref_call_table
640 if (IsIndirect && IsFuncrefCall) {
642 MF.getContext(), Subtarget);
643 Register RegZero =
644 MF.getRegInfo().createVirtualRegister(&WebAssembly::I32RegClass);
645 MachineInstr *Const0 =
646 BuildMI(MF, DL, TII.get(WebAssembly::CONST_I32), RegZero).addImm(0);
647 BB->insertAfter(MIB.getInstr()->getIterator(), Const0);
648
649 Register RegFuncref =
650 MF.getRegInfo().createVirtualRegister(&WebAssembly::FUNCREFRegClass);
651 MachineInstr *RefNull =
652 BuildMI(MF, DL, TII.get(WebAssembly::REF_NULL_FUNCREF), RegFuncref);
653 BB->insertAfter(Const0->getIterator(), RefNull);
654
655 MachineInstr *TableSet =
656 BuildMI(MF, DL, TII.get(WebAssembly::TABLE_SET_FUNCREF))
657 .addSym(Table)
658 .addReg(RegZero)
659 .addReg(RegFuncref);
660 BB->insertAfter(RefNull->getIterator(), TableSet);
661 }
662
663 return BB;
664}
665
666MachineBasicBlock *WebAssemblyTargetLowering::EmitInstrWithCustomInserter(
667 MachineInstr &MI, MachineBasicBlock *BB) const {
668 const TargetInstrInfo &TII = *Subtarget->getInstrInfo();
669 DebugLoc DL = MI.getDebugLoc();
670
671 switch (MI.getOpcode()) {
672 default:
673 llvm_unreachable("Unexpected instr type to insert");
674 case WebAssembly::FP_TO_SINT_I32_F32:
675 return LowerFPToInt(MI, DL, BB, TII, false, false, false,
676 WebAssembly::I32_TRUNC_S_F32);
677 case WebAssembly::FP_TO_UINT_I32_F32:
678 return LowerFPToInt(MI, DL, BB, TII, true, false, false,
679 WebAssembly::I32_TRUNC_U_F32);
680 case WebAssembly::FP_TO_SINT_I64_F32:
681 return LowerFPToInt(MI, DL, BB, TII, false, true, false,
682 WebAssembly::I64_TRUNC_S_F32);
683 case WebAssembly::FP_TO_UINT_I64_F32:
684 return LowerFPToInt(MI, DL, BB, TII, true, true, false,
685 WebAssembly::I64_TRUNC_U_F32);
686 case WebAssembly::FP_TO_SINT_I32_F64:
687 return LowerFPToInt(MI, DL, BB, TII, false, false, true,
688 WebAssembly::I32_TRUNC_S_F64);
689 case WebAssembly::FP_TO_UINT_I32_F64:
690 return LowerFPToInt(MI, DL, BB, TII, true, false, true,
691 WebAssembly::I32_TRUNC_U_F64);
692 case WebAssembly::FP_TO_SINT_I64_F64:
693 return LowerFPToInt(MI, DL, BB, TII, false, true, true,
694 WebAssembly::I64_TRUNC_S_F64);
695 case WebAssembly::FP_TO_UINT_I64_F64:
696 return LowerFPToInt(MI, DL, BB, TII, true, true, true,
697 WebAssembly::I64_TRUNC_U_F64);
698 case WebAssembly::CALL_RESULTS:
699 case WebAssembly::RET_CALL_RESULTS:
700 return LowerCallResults(MI, DL, BB, Subtarget, TII);
701 }
702}
703
704const char *
705WebAssemblyTargetLowering::getTargetNodeName(unsigned Opcode) const {
706 switch (static_cast<WebAssemblyISD::NodeType>(Opcode)) {
709 break;
710#define HANDLE_NODETYPE(NODE) \
711 case WebAssemblyISD::NODE: \
712 return "WebAssemblyISD::" #NODE;
713#define HANDLE_MEM_NODETYPE(NODE) HANDLE_NODETYPE(NODE)
714#include "WebAssemblyISD.def"
715#undef HANDLE_MEM_NODETYPE
716#undef HANDLE_NODETYPE
717 }
718 return nullptr;
719}
720
721std::pair<unsigned, const TargetRegisterClass *>
722WebAssemblyTargetLowering::getRegForInlineAsmConstraint(
723 const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const {
724 // First, see if this is a constraint that directly corresponds to a
725 // WebAssembly register class.
726 if (Constraint.size() == 1) {
727 switch (Constraint[0]) {
728 case 'r':
729 assert(VT != MVT::iPTR && "Pointer MVT not expected here");
730 if (Subtarget->hasSIMD128() && VT.isVector()) {
731 if (VT.getSizeInBits() == 128)
732 return std::make_pair(0U, &WebAssembly::V128RegClass);
733 }
734 if (VT.isInteger() && !VT.isVector()) {
735 if (VT.getSizeInBits() <= 32)
736 return std::make_pair(0U, &WebAssembly::I32RegClass);
737 if (VT.getSizeInBits() <= 64)
738 return std::make_pair(0U, &WebAssembly::I64RegClass);
739 }
740 if (VT.isFloatingPoint() && !VT.isVector()) {
741 switch (VT.getSizeInBits()) {
742 case 32:
743 return std::make_pair(0U, &WebAssembly::F32RegClass);
744 case 64:
745 return std::make_pair(0U, &WebAssembly::F64RegClass);
746 default:
747 break;
748 }
749 }
750 break;
751 default:
752 break;
753 }
754 }
755
757}
758
759bool WebAssemblyTargetLowering::isCheapToSpeculateCttz(Type *Ty) const {
760 // Assume ctz is a relatively cheap operation.
761 return true;
762}
763
764bool WebAssemblyTargetLowering::isCheapToSpeculateCtlz(Type *Ty) const {
765 // Assume clz is a relatively cheap operation.
766 return true;
767}
768
769bool WebAssemblyTargetLowering::isLegalAddressingMode(const DataLayout &DL,
770 const AddrMode &AM,
771 Type *Ty, unsigned AS,
772 Instruction *I) const {
773 // WebAssembly offsets are added as unsigned without wrapping. The
774 // isLegalAddressingMode gives us no way to determine if wrapping could be
775 // happening, so we approximate this by accepting only non-negative offsets.
776 if (AM.BaseOffs < 0)
777 return false;
778
779 // WebAssembly has no scale register operands.
780 if (AM.Scale != 0)
781 return false;
782
783 // Everything else is legal.
784 return true;
785}
786
787bool WebAssemblyTargetLowering::allowsMisalignedMemoryAccesses(
788 EVT /*VT*/, unsigned /*AddrSpace*/, Align /*Align*/,
789 MachineMemOperand::Flags /*Flags*/, unsigned *Fast) const {
790 // WebAssembly supports unaligned accesses, though it should be declared
791 // with the p2align attribute on loads and stores which do so, and there
792 // may be a performance impact. We tell LLVM they're "fast" because
793 // for the kinds of things that LLVM uses this for (merging adjacent stores
794 // of constants, etc.), WebAssembly implementations will either want the
795 // unaligned access or they'll split anyway.
796 if (Fast)
797 *Fast = 1;
798 return true;
799}
800
801bool WebAssemblyTargetLowering::isIntDivCheap(EVT VT,
802 AttributeList Attr) const {
803 // The current thinking is that wasm engines will perform this optimization,
804 // so we can save on code size.
805 return true;
806}
807
808bool WebAssemblyTargetLowering::isVectorLoadExtDesirable(SDValue ExtVal) const {
809 EVT ExtT = ExtVal.getValueType();
810 EVT MemT = cast<LoadSDNode>(ExtVal->getOperand(0))->getValueType(0);
811 return (ExtT == MVT::v8i16 && MemT == MVT::v8i8) ||
812 (ExtT == MVT::v4i32 && MemT == MVT::v4i16) ||
813 (ExtT == MVT::v2i64 && MemT == MVT::v2i32);
814}
815
816bool WebAssemblyTargetLowering::isOffsetFoldingLegal(
817 const GlobalAddressSDNode *GA) const {
818 // Wasm doesn't support function addresses with offsets
819 const GlobalValue *GV = GA->getGlobal();
820 return isa<Function>(GV) ? false : TargetLowering::isOffsetFoldingLegal(GA);
821}
822
823EVT WebAssemblyTargetLowering::getSetCCResultType(const DataLayout &DL,
824 LLVMContext &C,
825 EVT VT) const {
826 if (VT.isVector())
828
829 // So far, all branch instructions in Wasm take an I32 condition.
830 // The default TargetLowering::getSetCCResultType returns the pointer size,
831 // which would be useful to reduce instruction counts when testing
832 // against 64-bit pointers/values if at some point Wasm supports that.
833 return EVT::getIntegerVT(C, 32);
834}
835
836bool WebAssemblyTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
837 const CallInst &I,
838 MachineFunction &MF,
839 unsigned Intrinsic) const {
840 switch (Intrinsic) {
841 case Intrinsic::wasm_memory_atomic_notify:
843 Info.memVT = MVT::i32;
844 Info.ptrVal = I.getArgOperand(0);
845 Info.offset = 0;
846 Info.align = Align(4);
847 // atomic.notify instruction does not really load the memory specified with
848 // this argument, but MachineMemOperand should either be load or store, so
849 // we set this to a load.
850 // FIXME Volatile isn't really correct, but currently all LLVM atomic
851 // instructions are treated as volatiles in the backend, so we should be
852 // consistent. The same applies for wasm_atomic_wait intrinsics too.
854 return true;
855 case Intrinsic::wasm_memory_atomic_wait32:
857 Info.memVT = MVT::i32;
858 Info.ptrVal = I.getArgOperand(0);
859 Info.offset = 0;
860 Info.align = Align(4);
862 return true;
863 case Intrinsic::wasm_memory_atomic_wait64:
865 Info.memVT = MVT::i64;
866 Info.ptrVal = I.getArgOperand(0);
867 Info.offset = 0;
868 Info.align = Align(8);
870 return true;
871 default:
872 return false;
873 }
874}
875
876void WebAssemblyTargetLowering::computeKnownBitsForTargetNode(
877 const SDValue Op, KnownBits &Known, const APInt &DemandedElts,
878 const SelectionDAG &DAG, unsigned Depth) const {
879 switch (Op.getOpcode()) {
880 default:
881 break;
883 unsigned IntNo = Op.getConstantOperandVal(0);
884 switch (IntNo) {
885 default:
886 break;
887 case Intrinsic::wasm_bitmask: {
888 unsigned BitWidth = Known.getBitWidth();
889 EVT VT = Op.getOperand(1).getSimpleValueType();
890 unsigned PossibleBits = VT.getVectorNumElements();
891 APInt ZeroMask = APInt::getHighBitsSet(BitWidth, BitWidth - PossibleBits);
892 Known.Zero |= ZeroMask;
893 break;
894 }
895 }
896 }
897 }
898}
899
901WebAssemblyTargetLowering::getPreferredVectorAction(MVT VT) const {
902 if (VT.isFixedLengthVector()) {
903 MVT EltVT = VT.getVectorElementType();
904 // We have legal vector types with these lane types, so widening the
905 // vector would let us use some of the lanes directly without having to
906 // extend or truncate values.
907 if (EltVT == MVT::i8 || EltVT == MVT::i16 || EltVT == MVT::i32 ||
908 EltVT == MVT::i64 || EltVT == MVT::f32 || EltVT == MVT::f64)
909 return TypeWidenVector;
910 }
911
913}
914
915bool WebAssemblyTargetLowering::shouldSimplifyDemandedVectorElts(
916 SDValue Op, const TargetLoweringOpt &TLO) const {
917 // ISel process runs DAGCombiner after legalization; this step is called
918 // SelectionDAG optimization phase. This post-legalization combining process
919 // runs DAGCombiner on each node, and if there was a change to be made,
920 // re-runs legalization again on it and its user nodes to make sure
921 // everythiing is in a legalized state.
922 //
923 // The legalization calls lowering routines, and we do our custom lowering for
924 // build_vectors (LowerBUILD_VECTOR), which converts undef vector elements
925 // into zeros. But there is a set of routines in DAGCombiner that turns unused
926 // (= not demanded) nodes into undef, among which SimplifyDemandedVectorElts
927 // turns unused vector elements into undefs. But this routine does not work
928 // with our custom LowerBUILD_VECTOR, which turns undefs into zeros. This
929 // combination can result in a infinite loop, in which undefs are converted to
930 // zeros in legalization and back to undefs in combining.
931 //
932 // So after DAG is legalized, we prevent SimplifyDemandedVectorElts from
933 // running for build_vectors.
934 if (Op.getOpcode() == ISD::BUILD_VECTOR && TLO.LegalOps && TLO.LegalTys)
935 return false;
936 return true;
937}
938
939//===----------------------------------------------------------------------===//
940// WebAssembly Lowering private implementation.
941//===----------------------------------------------------------------------===//
942
943//===----------------------------------------------------------------------===//
944// Lowering Code
945//===----------------------------------------------------------------------===//
946
947static void fail(const SDLoc &DL, SelectionDAG &DAG, const char *Msg) {
949 DAG.getContext()->diagnose(
950 DiagnosticInfoUnsupported(MF.getFunction(), Msg, DL.getDebugLoc()));
951}
952
953// Test whether the given calling convention is supported.
955 // We currently support the language-independent target-independent
956 // conventions. We don't yet have a way to annotate calls with properties like
957 // "cold", and we don't have any call-clobbered registers, so these are mostly
958 // all handled the same.
959 return CallConv == CallingConv::C || CallConv == CallingConv::Fast ||
960 CallConv == CallingConv::Cold ||
961 CallConv == CallingConv::PreserveMost ||
962 CallConv == CallingConv::PreserveAll ||
963 CallConv == CallingConv::CXX_FAST_TLS ||
965 CallConv == CallingConv::Swift;
966}
967
969WebAssemblyTargetLowering::LowerCall(CallLoweringInfo &CLI,
970 SmallVectorImpl<SDValue> &InVals) const {
971 SelectionDAG &DAG = CLI.DAG;
972 SDLoc DL = CLI.DL;
973 SDValue Chain = CLI.Chain;
974 SDValue Callee = CLI.Callee;
976 auto Layout = MF.getDataLayout();
977
978 CallingConv::ID CallConv = CLI.CallConv;
979 if (!callingConvSupported(CallConv))
980 fail(DL, DAG,
981 "WebAssembly doesn't support language-specific or target-specific "
982 "calling conventions yet");
983 if (CLI.IsPatchPoint)
984 fail(DL, DAG, "WebAssembly doesn't support patch point yet");
985
986 if (CLI.IsTailCall) {
987 auto NoTail = [&](const char *Msg) {
988 if (CLI.CB && CLI.CB->isMustTailCall())
989 fail(DL, DAG, Msg);
990 CLI.IsTailCall = false;
991 };
992
993 if (!Subtarget->hasTailCall())
994 NoTail("WebAssembly 'tail-call' feature not enabled");
995
996 // Varargs calls cannot be tail calls because the buffer is on the stack
997 if (CLI.IsVarArg)
998 NoTail("WebAssembly does not support varargs tail calls");
999
1000 // Do not tail call unless caller and callee return types match
1001 const Function &F = MF.getFunction();
1003 Type *RetTy = F.getReturnType();
1004 SmallVector<MVT, 4> CallerRetTys;
1005 SmallVector<MVT, 4> CalleeRetTys;
1006 computeLegalValueVTs(F, TM, RetTy, CallerRetTys);
1007 computeLegalValueVTs(F, TM, CLI.RetTy, CalleeRetTys);
1008 bool TypesMatch = CallerRetTys.size() == CalleeRetTys.size() &&
1009 std::equal(CallerRetTys.begin(), CallerRetTys.end(),
1010 CalleeRetTys.begin());
1011 if (!TypesMatch)
1012 NoTail("WebAssembly tail call requires caller and callee return types to "
1013 "match");
1014
1015 // If pointers to local stack values are passed, we cannot tail call
1016 if (CLI.CB) {
1017 for (auto &Arg : CLI.CB->args()) {
1018 Value *Val = Arg.get();
1019 // Trace the value back through pointer operations
1020 while (true) {
1021 Value *Src = Val->stripPointerCastsAndAliases();
1022 if (auto *GEP = dyn_cast<GetElementPtrInst>(Src))
1023 Src = GEP->getPointerOperand();
1024 if (Val == Src)
1025 break;
1026 Val = Src;
1027 }
1028 if (isa<AllocaInst>(Val)) {
1029 NoTail(
1030 "WebAssembly does not support tail calling with stack arguments");
1031 break;
1032 }
1033 }
1034 }
1035 }
1036
1038 SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
1039 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
1040
1041 // The generic code may have added an sret argument. If we're lowering an
1042 // invoke function, the ABI requires that the function pointer be the first
1043 // argument, so we may have to swap the arguments.
1044 if (CallConv == CallingConv::WASM_EmscriptenInvoke && Outs.size() >= 2 &&
1045 Outs[0].Flags.isSRet()) {
1046 std::swap(Outs[0], Outs[1]);
1047 std::swap(OutVals[0], OutVals[1]);
1048 }
1049
1050 bool HasSwiftSelfArg = false;
1051 bool HasSwiftErrorArg = false;
1052 unsigned NumFixedArgs = 0;
1053 for (unsigned I = 0; I < Outs.size(); ++I) {
1054 const ISD::OutputArg &Out = Outs[I];
1055 SDValue &OutVal = OutVals[I];
1056 HasSwiftSelfArg |= Out.Flags.isSwiftSelf();
1057 HasSwiftErrorArg |= Out.Flags.isSwiftError();
1058 if (Out.Flags.isNest())
1059 fail(DL, DAG, "WebAssembly hasn't implemented nest arguments");
1060 if (Out.Flags.isInAlloca())
1061 fail(DL, DAG, "WebAssembly hasn't implemented inalloca arguments");
1062 if (Out.Flags.isInConsecutiveRegs())
1063 fail(DL, DAG, "WebAssembly hasn't implemented cons regs arguments");
1065 fail(DL, DAG, "WebAssembly hasn't implemented cons regs last arguments");
1066 if (Out.Flags.isByVal() && Out.Flags.getByValSize() != 0) {
1067 auto &MFI = MF.getFrameInfo();
1068 int FI = MFI.CreateStackObject(Out.Flags.getByValSize(),
1070 /*isSS=*/false);
1071 SDValue SizeNode =
1073 SDValue FINode = DAG.getFrameIndex(FI, getPointerTy(Layout));
1074 Chain = DAG.getMemcpy(
1075 Chain, DL, FINode, OutVal, SizeNode, Out.Flags.getNonZeroByValAlign(),
1076 /*isVolatile*/ false, /*AlwaysInline=*/false,
1077 /*isTailCall*/ false, MachinePointerInfo(), MachinePointerInfo());
1078 OutVal = FINode;
1079 }
1080 // Count the number of fixed args *after* legalization.
1081 NumFixedArgs += Out.IsFixed;
1082 }
1083
1084 bool IsVarArg = CLI.IsVarArg;
1085 auto PtrVT = getPointerTy(Layout);
1086
1087 // For swiftcc, emit additional swiftself and swifterror arguments
1088 // if there aren't. These additional arguments are also added for callee
1089 // signature They are necessary to match callee and caller signature for
1090 // indirect call.
1091 if (CallConv == CallingConv::Swift) {
1092 if (!HasSwiftSelfArg) {
1093 NumFixedArgs++;
1095 Arg.Flags.setSwiftSelf();
1096 CLI.Outs.push_back(Arg);
1097 SDValue ArgVal = DAG.getUNDEF(PtrVT);
1098 CLI.OutVals.push_back(ArgVal);
1099 }
1100 if (!HasSwiftErrorArg) {
1101 NumFixedArgs++;
1103 Arg.Flags.setSwiftError();
1104 CLI.Outs.push_back(Arg);
1105 SDValue ArgVal = DAG.getUNDEF(PtrVT);
1106 CLI.OutVals.push_back(ArgVal);
1107 }
1108 }
1109
1110 // Analyze operands of the call, assigning locations to each operand.
1112 CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
1113
1114 if (IsVarArg) {
1115 // Outgoing non-fixed arguments are placed in a buffer. First
1116 // compute their offsets and the total amount of buffer space needed.
1117 for (unsigned I = NumFixedArgs; I < Outs.size(); ++I) {
1118 const ISD::OutputArg &Out = Outs[I];
1119 SDValue &Arg = OutVals[I];
1120 EVT VT = Arg.getValueType();
1121 assert(VT != MVT::iPTR && "Legalized args should be concrete");
1122 Type *Ty = VT.getTypeForEVT(*DAG.getContext());
1123 Align Alignment =
1124 std::max(Out.Flags.getNonZeroOrigAlign(), Layout.getABITypeAlign(Ty));
1125 unsigned Offset =
1126 CCInfo.AllocateStack(Layout.getTypeAllocSize(Ty), Alignment);
1127 CCInfo.addLoc(CCValAssign::getMem(ArgLocs.size(), VT.getSimpleVT(),
1128 Offset, VT.getSimpleVT(),
1130 }
1131 }
1132
1133 unsigned NumBytes = CCInfo.getAlignedCallFrameSize();
1134
1135 SDValue FINode;
1136 if (IsVarArg && NumBytes) {
1137 // For non-fixed arguments, next emit stores to store the argument values
1138 // to the stack buffer at the offsets computed above.
1139 int FI = MF.getFrameInfo().CreateStackObject(NumBytes,
1140 Layout.getStackAlignment(),
1141 /*isSS=*/false);
1142 unsigned ValNo = 0;
1144 for (SDValue Arg : drop_begin(OutVals, NumFixedArgs)) {
1145 assert(ArgLocs[ValNo].getValNo() == ValNo &&
1146 "ArgLocs should remain in order and only hold varargs args");
1147 unsigned Offset = ArgLocs[ValNo++].getLocMemOffset();
1148 FINode = DAG.getFrameIndex(FI, getPointerTy(Layout));
1149 SDValue Add = DAG.getNode(ISD::ADD, DL, PtrVT, FINode,
1150 DAG.getConstant(Offset, DL, PtrVT));
1151 Chains.push_back(
1152 DAG.getStore(Chain, DL, Arg, Add,
1154 }
1155 if (!Chains.empty())
1156 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains);
1157 } else if (IsVarArg) {
1158 FINode = DAG.getIntPtrConstant(0, DL);
1159 }
1160
1161 if (Callee->getOpcode() == ISD::GlobalAddress) {
1162 // If the callee is a GlobalAddress node (quite common, every direct call
1163 // is) turn it into a TargetGlobalAddress node so that LowerGlobalAddress
1164 // doesn't at MO_GOT which is not needed for direct calls.
1165 GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Callee);
1168 GA->getOffset());
1169 Callee = DAG.getNode(WebAssemblyISD::Wrapper, DL,
1171 }
1172
1173 // Compute the operands for the CALLn node.
1175 Ops.push_back(Chain);
1176 Ops.push_back(Callee);
1177
1178 // Add all fixed arguments. Note that for non-varargs calls, NumFixedArgs
1179 // isn't reliable.
1180 Ops.append(OutVals.begin(),
1181 IsVarArg ? OutVals.begin() + NumFixedArgs : OutVals.end());
1182 // Add a pointer to the vararg buffer.
1183 if (IsVarArg)
1184 Ops.push_back(FINode);
1185
1186 SmallVector<EVT, 8> InTys;
1187 for (const auto &In : Ins) {
1188 assert(!In.Flags.isByVal() && "byval is not valid for return values");
1189 assert(!In.Flags.isNest() && "nest is not valid for return values");
1190 if (In.Flags.isInAlloca())
1191 fail(DL, DAG, "WebAssembly hasn't implemented inalloca return values");
1192 if (In.Flags.isInConsecutiveRegs())
1193 fail(DL, DAG, "WebAssembly hasn't implemented cons regs return values");
1194 if (In.Flags.isInConsecutiveRegsLast())
1195 fail(DL, DAG,
1196 "WebAssembly hasn't implemented cons regs last return values");
1197 // Ignore In.getNonZeroOrigAlign() because all our arguments are passed in
1198 // registers.
1199 InTys.push_back(In.VT);
1200 }
1201
1202 // Lastly, if this is a call to a funcref we need to add an instruction
1203 // table.set to the chain and transform the call.
1204 if (CLI.CB &&
1205 WebAssembly::isFuncrefType(CLI.CB->getCalledOperand()->getType())) {
1206 // In the absence of function references proposal where a funcref call is
1207 // lowered to call_ref, using reference types we generate a table.set to set
1208 // the funcref to a special table used solely for this purpose, followed by
1209 // a call_indirect. Here we just generate the table set, and return the
1210 // SDValue of the table.set so that LowerCall can finalize the lowering by
1211 // generating the call_indirect.
1212 SDValue Chain = Ops[0];
1213
1215 MF.getContext(), Subtarget);
1216 SDValue Sym = DAG.getMCSymbol(Table, PtrVT);
1217 SDValue TableSlot = DAG.getConstant(0, DL, MVT::i32);
1218 SDValue TableSetOps[] = {Chain, Sym, TableSlot, Callee};
1219 SDValue TableSet = DAG.getMemIntrinsicNode(
1220 WebAssemblyISD::TABLE_SET, DL, DAG.getVTList(MVT::Other), TableSetOps,
1222 // Machine Mem Operand args
1225 CLI.CB->getCalledOperand()->getPointerAlignment(DAG.getDataLayout()),
1227
1228 Ops[0] = TableSet; // The new chain is the TableSet itself
1229 }
1230
1231 if (CLI.IsTailCall) {
1232 // ret_calls do not return values to the current frame
1233 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
1234 return DAG.getNode(WebAssemblyISD::RET_CALL, DL, NodeTys, Ops);
1235 }
1236
1237 InTys.push_back(MVT::Other);
1238 SDVTList InTyList = DAG.getVTList(InTys);
1239 SDValue Res = DAG.getNode(WebAssemblyISD::CALL, DL, InTyList, Ops);
1240
1241 for (size_t I = 0; I < Ins.size(); ++I)
1242 InVals.push_back(Res.getValue(I));
1243
1244 // Return the chain
1245 return Res.getValue(Ins.size());
1246}
1247
1248bool WebAssemblyTargetLowering::CanLowerReturn(
1249 CallingConv::ID /*CallConv*/, MachineFunction & /*MF*/, bool /*IsVarArg*/,
1251 LLVMContext & /*Context*/) const {
1252 // WebAssembly can only handle returning tuples with multivalue enabled
1253 return Subtarget->hasMultivalue() || Outs.size() <= 1;
1254}
1255
1256SDValue WebAssemblyTargetLowering::LowerReturn(
1257 SDValue Chain, CallingConv::ID CallConv, bool /*IsVarArg*/,
1259 const SmallVectorImpl<SDValue> &OutVals, const SDLoc &DL,
1260 SelectionDAG &DAG) const {
1261 assert((Subtarget->hasMultivalue() || Outs.size() <= 1) &&
1262 "MVP WebAssembly can only return up to one value");
1263 if (!callingConvSupported(CallConv))
1264 fail(DL, DAG, "WebAssembly doesn't support non-C calling conventions");
1265
1266 SmallVector<SDValue, 4> RetOps(1, Chain);
1267 RetOps.append(OutVals.begin(), OutVals.end());
1268 Chain = DAG.getNode(WebAssemblyISD::RETURN, DL, MVT::Other, RetOps);
1269
1270 // Record the number and types of the return values.
1271 for (const ISD::OutputArg &Out : Outs) {
1272 assert(!Out.Flags.isByVal() && "byval is not valid for return values");
1273 assert(!Out.Flags.isNest() && "nest is not valid for return values");
1274 assert(Out.IsFixed && "non-fixed return value is not valid");
1275 if (Out.Flags.isInAlloca())
1276 fail(DL, DAG, "WebAssembly hasn't implemented inalloca results");
1277 if (Out.Flags.isInConsecutiveRegs())
1278 fail(DL, DAG, "WebAssembly hasn't implemented cons regs results");
1280 fail(DL, DAG, "WebAssembly hasn't implemented cons regs last results");
1281 }
1282
1283 return Chain;
1284}
1285
1286SDValue WebAssemblyTargetLowering::LowerFormalArguments(
1287 SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
1288 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
1289 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
1290 if (!callingConvSupported(CallConv))
1291 fail(DL, DAG, "WebAssembly doesn't support non-C calling conventions");
1292
1294 auto *MFI = MF.getInfo<WebAssemblyFunctionInfo>();
1295
1296 // Set up the incoming ARGUMENTS value, which serves to represent the liveness
1297 // of the incoming values before they're represented by virtual registers.
1298 MF.getRegInfo().addLiveIn(WebAssembly::ARGUMENTS);
1299
1300 bool HasSwiftErrorArg = false;
1301 bool HasSwiftSelfArg = false;
1302 for (const ISD::InputArg &In : Ins) {
1303 HasSwiftSelfArg |= In.Flags.isSwiftSelf();
1304 HasSwiftErrorArg |= In.Flags.isSwiftError();
1305 if (In.Flags.isInAlloca())
1306 fail(DL, DAG, "WebAssembly hasn't implemented inalloca arguments");
1307 if (In.Flags.isNest())
1308 fail(DL, DAG, "WebAssembly hasn't implemented nest arguments");
1309 if (In.Flags.isInConsecutiveRegs())
1310 fail(DL, DAG, "WebAssembly hasn't implemented cons regs arguments");
1311 if (In.Flags.isInConsecutiveRegsLast())
1312 fail(DL, DAG, "WebAssembly hasn't implemented cons regs last arguments");
1313 // Ignore In.getNonZeroOrigAlign() because all our arguments are passed in
1314 // registers.
1315 InVals.push_back(In.Used ? DAG.getNode(WebAssemblyISD::ARGUMENT, DL, In.VT,
1316 DAG.getTargetConstant(InVals.size(),
1317 DL, MVT::i32))
1318 : DAG.getUNDEF(In.VT));
1319
1320 // Record the number and types of arguments.
1321 MFI->addParam(In.VT);
1322 }
1323
1324 // For swiftcc, emit additional swiftself and swifterror arguments
1325 // if there aren't. These additional arguments are also added for callee
1326 // signature They are necessary to match callee and caller signature for
1327 // indirect call.
1328 auto PtrVT = getPointerTy(MF.getDataLayout());
1329 if (CallConv == CallingConv::Swift) {
1330 if (!HasSwiftSelfArg) {
1331 MFI->addParam(PtrVT);
1332 }
1333 if (!HasSwiftErrorArg) {
1334 MFI->addParam(PtrVT);
1335 }
1336 }
1337 // Varargs are copied into a buffer allocated by the caller, and a pointer to
1338 // the buffer is passed as an argument.
1339 if (IsVarArg) {
1340 MVT PtrVT = getPointerTy(MF.getDataLayout());
1341 Register VarargVreg =
1343 MFI->setVarargBufferVreg(VarargVreg);
1344 Chain = DAG.getCopyToReg(
1345 Chain, DL, VarargVreg,
1346 DAG.getNode(WebAssemblyISD::ARGUMENT, DL, PtrVT,
1347 DAG.getTargetConstant(Ins.size(), DL, MVT::i32)));
1348 MFI->addParam(PtrVT);
1349 }
1350
1351 // Record the number and types of arguments and results.
1352 SmallVector<MVT, 4> Params;
1355 MF.getFunction(), DAG.getTarget(), Params, Results);
1356 for (MVT VT : Results)
1357 MFI->addResult(VT);
1358 // TODO: Use signatures in WebAssemblyMachineFunctionInfo too and unify
1359 // the param logic here with ComputeSignatureVTs
1360 assert(MFI->getParams().size() == Params.size() &&
1361 std::equal(MFI->getParams().begin(), MFI->getParams().end(),
1362 Params.begin()));
1363
1364 return Chain;
1365}
1366
1367void WebAssemblyTargetLowering::ReplaceNodeResults(
1369 switch (N->getOpcode()) {
1371 // Do not add any results, signifying that N should not be custom lowered
1372 // after all. This happens because simd128 turns on custom lowering for
1373 // SIGN_EXTEND_INREG, but for non-vector sign extends the result might be an
1374 // illegal type.
1375 break;
1376 default:
1378 "ReplaceNodeResults not implemented for this op for WebAssembly!");
1379 }
1380}
1381
1382//===----------------------------------------------------------------------===//
1383// Custom lowering hooks.
1384//===----------------------------------------------------------------------===//
1385
1386SDValue WebAssemblyTargetLowering::LowerOperation(SDValue Op,
1387 SelectionDAG &DAG) const {
1388 SDLoc DL(Op);
1389 switch (Op.getOpcode()) {
1390 default:
1391 llvm_unreachable("unimplemented operation lowering");
1392 return SDValue();
1393 case ISD::FrameIndex:
1394 return LowerFrameIndex(Op, DAG);
1395 case ISD::GlobalAddress:
1396 return LowerGlobalAddress(Op, DAG);
1398 return LowerGlobalTLSAddress(Op, DAG);
1400 return LowerExternalSymbol(Op, DAG);
1401 case ISD::JumpTable:
1402 return LowerJumpTable(Op, DAG);
1403 case ISD::BR_JT:
1404 return LowerBR_JT(Op, DAG);
1405 case ISD::VASTART:
1406 return LowerVASTART(Op, DAG);
1407 case ISD::BlockAddress:
1408 case ISD::BRIND:
1409 fail(DL, DAG, "WebAssembly hasn't implemented computed gotos");
1410 return SDValue();
1411 case ISD::RETURNADDR:
1412 return LowerRETURNADDR(Op, DAG);
1413 case ISD::FRAMEADDR:
1414 return LowerFRAMEADDR(Op, DAG);
1415 case ISD::CopyToReg:
1416 return LowerCopyToReg(Op, DAG);
1419 return LowerAccessVectorElement(Op, DAG);
1423 return LowerIntrinsic(Op, DAG);
1425 return LowerSIGN_EXTEND_INREG(Op, DAG);
1426 case ISD::BUILD_VECTOR:
1427 return LowerBUILD_VECTOR(Op, DAG);
1429 return LowerVECTOR_SHUFFLE(Op, DAG);
1430 case ISD::SETCC:
1431 return LowerSETCC(Op, DAG);
1432 case ISD::SHL:
1433 case ISD::SRA:
1434 case ISD::SRL:
1435 return LowerShift(Op, DAG);
1438 return LowerFP_TO_INT_SAT(Op, DAG);
1439 case ISD::LOAD:
1440 return LowerLoad(Op, DAG);
1441 case ISD::STORE:
1442 return LowerStore(Op, DAG);
1443 case ISD::CTPOP:
1444 case ISD::CTLZ:
1445 case ISD::CTTZ:
1446 return DAG.UnrollVectorOp(Op.getNode());
1447 }
1448}
1449
1451 if (const GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Op))
1453
1454 return false;
1455}
1456
1457static std::optional<unsigned> IsWebAssemblyLocal(SDValue Op,
1458 SelectionDAG &DAG) {
1459 const FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Op);
1460 if (!FI)
1461 return std::nullopt;
1462
1463 auto &MF = DAG.getMachineFunction();
1465}
1466
1467SDValue WebAssemblyTargetLowering::LowerStore(SDValue Op,
1468 SelectionDAG &DAG) const {
1469 SDLoc DL(Op);
1470 StoreSDNode *SN = cast<StoreSDNode>(Op.getNode());
1471 const SDValue &Value = SN->getValue();
1472 const SDValue &Base = SN->getBasePtr();
1473 const SDValue &Offset = SN->getOffset();
1474
1476 if (!Offset->isUndef())
1477 report_fatal_error("unexpected offset when storing to webassembly global",
1478 false);
1479
1480 SDVTList Tys = DAG.getVTList(MVT::Other);
1481 SDValue Ops[] = {SN->getChain(), Value, Base};
1482 return DAG.getMemIntrinsicNode(WebAssemblyISD::GLOBAL_SET, DL, Tys, Ops,
1483 SN->getMemoryVT(), SN->getMemOperand());
1484 }
1485
1486 if (std::optional<unsigned> Local = IsWebAssemblyLocal(Base, DAG)) {
1487 if (!Offset->isUndef())
1488 report_fatal_error("unexpected offset when storing to webassembly local",
1489 false);
1490
1491 SDValue Idx = DAG.getTargetConstant(*Local, Base, MVT::i32);
1492 SDVTList Tys = DAG.getVTList(MVT::Other); // The chain.
1493 SDValue Ops[] = {SN->getChain(), Idx, Value};
1494 return DAG.getNode(WebAssemblyISD::LOCAL_SET, DL, Tys, Ops);
1495 }
1496
1499 "Encountered an unlowerable store to the wasm_var address space",
1500 false);
1501
1502 return Op;
1503}
1504
1505SDValue WebAssemblyTargetLowering::LowerLoad(SDValue Op,
1506 SelectionDAG &DAG) const {
1507 SDLoc DL(Op);
1508 LoadSDNode *LN = cast<LoadSDNode>(Op.getNode());
1509 const SDValue &Base = LN->getBasePtr();
1510 const SDValue &Offset = LN->getOffset();
1511
1513 if (!Offset->isUndef())
1515 "unexpected offset when loading from webassembly global", false);
1516
1517 SDVTList Tys = DAG.getVTList(LN->getValueType(0), MVT::Other);
1518 SDValue Ops[] = {LN->getChain(), Base};
1519 return DAG.getMemIntrinsicNode(WebAssemblyISD::GLOBAL_GET, DL, Tys, Ops,
1520 LN->getMemoryVT(), LN->getMemOperand());
1521 }
1522
1523 if (std::optional<unsigned> Local = IsWebAssemblyLocal(Base, DAG)) {
1524 if (!Offset->isUndef())
1526 "unexpected offset when loading from webassembly local", false);
1527
1528 SDValue Idx = DAG.getTargetConstant(*Local, Base, MVT::i32);
1529 EVT LocalVT = LN->getValueType(0);
1530 SDValue LocalGet = DAG.getNode(WebAssemblyISD::LOCAL_GET, DL, LocalVT,
1531 {LN->getChain(), Idx});
1532 SDValue Result = DAG.getMergeValues({LocalGet, LN->getChain()}, DL);
1533 assert(Result->getNumValues() == 2 && "Loads must carry a chain!");
1534 return Result;
1535 }
1536
1539 "Encountered an unlowerable load from the wasm_var address space",
1540 false);
1541
1542 return Op;
1543}
1544
1545SDValue WebAssemblyTargetLowering::LowerCopyToReg(SDValue Op,
1546 SelectionDAG &DAG) const {
1547 SDValue Src = Op.getOperand(2);
1548 if (isa<FrameIndexSDNode>(Src.getNode())) {
1549 // CopyToReg nodes don't support FrameIndex operands. Other targets select
1550 // the FI to some LEA-like instruction, but since we don't have that, we
1551 // need to insert some kind of instruction that can take an FI operand and
1552 // produces a value usable by CopyToReg (i.e. in a vreg). So insert a dummy
1553 // local.copy between Op and its FI operand.
1554 SDValue Chain = Op.getOperand(0);
1555 SDLoc DL(Op);
1556 Register Reg = cast<RegisterSDNode>(Op.getOperand(1))->getReg();
1557 EVT VT = Src.getValueType();
1558 SDValue Copy(DAG.getMachineNode(VT == MVT::i32 ? WebAssembly::COPY_I32
1559 : WebAssembly::COPY_I64,
1560 DL, VT, Src),
1561 0);
1562 return Op.getNode()->getNumValues() == 1
1563 ? DAG.getCopyToReg(Chain, DL, Reg, Copy)
1564 : DAG.getCopyToReg(Chain, DL, Reg, Copy,
1565 Op.getNumOperands() == 4 ? Op.getOperand(3)
1566 : SDValue());
1567 }
1568 return SDValue();
1569}
1570
1571SDValue WebAssemblyTargetLowering::LowerFrameIndex(SDValue Op,
1572 SelectionDAG &DAG) const {
1573 int FI = cast<FrameIndexSDNode>(Op)->getIndex();
1574 return DAG.getTargetFrameIndex(FI, Op.getValueType());
1575}
1576
1577SDValue WebAssemblyTargetLowering::LowerRETURNADDR(SDValue Op,
1578 SelectionDAG &DAG) const {
1579 SDLoc DL(Op);
1580
1581 if (!Subtarget->getTargetTriple().isOSEmscripten()) {
1582 fail(DL, DAG,
1583 "Non-Emscripten WebAssembly hasn't implemented "
1584 "__builtin_return_address");
1585 return SDValue();
1586 }
1587
1589 return SDValue();
1590
1591 unsigned Depth = Op.getConstantOperandVal(0);
1592 MakeLibCallOptions CallOptions;
1593 return makeLibCall(DAG, RTLIB::RETURN_ADDRESS, Op.getValueType(),
1594 {DAG.getConstant(Depth, DL, MVT::i32)}, CallOptions, DL)
1595 .first;
1596}
1597
1598SDValue WebAssemblyTargetLowering::LowerFRAMEADDR(SDValue Op,
1599 SelectionDAG &DAG) const {
1600 // Non-zero depths are not supported by WebAssembly currently. Use the
1601 // legalizer's default expansion, which is to return 0 (what this function is
1602 // documented to do).
1603 if (Op.getConstantOperandVal(0) > 0)
1604 return SDValue();
1605
1607 EVT VT = Op.getValueType();
1608 Register FP =
1610 return DAG.getCopyFromReg(DAG.getEntryNode(), SDLoc(Op), FP, VT);
1611}
1612
1613SDValue
1614WebAssemblyTargetLowering::LowerGlobalTLSAddress(SDValue Op,
1615 SelectionDAG &DAG) const {
1616 SDLoc DL(Op);
1617 const auto *GA = cast<GlobalAddressSDNode>(Op);
1618
1621 report_fatal_error("cannot use thread-local storage without bulk memory",
1622 false);
1623
1624 const GlobalValue *GV = GA->getGlobal();
1625
1626 // Currently only Emscripten supports dynamic linking with threads. Therefore,
1627 // on other targets, if we have thread-local storage, only the local-exec
1628 // model is possible.
1629 auto model = Subtarget->getTargetTriple().isOSEmscripten()
1630 ? GV->getThreadLocalMode()
1632
1633 // Unsupported TLS modes
1636
1637 if (model == GlobalValue::LocalExecTLSModel ||
1640 getTargetMachine().shouldAssumeDSOLocal(*GV->getParent(), GV))) {
1641 // For DSO-local TLS variables we use offset from __tls_base
1642
1643 MVT PtrVT = getPointerTy(DAG.getDataLayout());
1644 auto GlobalGet = PtrVT == MVT::i64 ? WebAssembly::GLOBAL_GET_I64
1645 : WebAssembly::GLOBAL_GET_I32;
1646 const char *BaseName = MF.createExternalSymbolName("__tls_base");
1647
1649 DAG.getMachineNode(GlobalGet, DL, PtrVT,
1650 DAG.getTargetExternalSymbol(BaseName, PtrVT)),
1651 0);
1652
1653 SDValue TLSOffset = DAG.getTargetGlobalAddress(
1654 GV, DL, PtrVT, GA->getOffset(), WebAssemblyII::MO_TLS_BASE_REL);
1655 SDValue SymOffset =
1656 DAG.getNode(WebAssemblyISD::WrapperREL, DL, PtrVT, TLSOffset);
1657
1658 return DAG.getNode(ISD::ADD, DL, PtrVT, BaseAddr, SymOffset);
1659 }
1660
1662
1663 EVT VT = Op.getValueType();
1664 return DAG.getNode(WebAssemblyISD::Wrapper, DL, VT,
1665 DAG.getTargetGlobalAddress(GA->getGlobal(), DL, VT,
1666 GA->getOffset(),
1668}
1669
1670SDValue WebAssemblyTargetLowering::LowerGlobalAddress(SDValue Op,
1671 SelectionDAG &DAG) const {
1672 SDLoc DL(Op);
1673 const auto *GA = cast<GlobalAddressSDNode>(Op);
1674 EVT VT = Op.getValueType();
1675 assert(GA->getTargetFlags() == 0 &&
1676 "Unexpected target flags on generic GlobalAddressSDNode");
1678 fail(DL, DAG, "Invalid address space for WebAssembly target");
1679
1680 unsigned OperandFlags = 0;
1681 if (isPositionIndependent()) {
1682 const GlobalValue *GV = GA->getGlobal();
1683 if (getTargetMachine().shouldAssumeDSOLocal(*GV->getParent(), GV)) {
1685 MVT PtrVT = getPointerTy(MF.getDataLayout());
1686 const char *BaseName;
1687 if (GV->getValueType()->isFunctionTy()) {
1688 BaseName = MF.createExternalSymbolName("__table_base");
1690 } else {
1691 BaseName = MF.createExternalSymbolName("__memory_base");
1693 }
1695 DAG.getNode(WebAssemblyISD::Wrapper, DL, PtrVT,
1696 DAG.getTargetExternalSymbol(BaseName, PtrVT));
1697
1698 SDValue SymAddr = DAG.getNode(
1699 WebAssemblyISD::WrapperREL, DL, VT,
1700 DAG.getTargetGlobalAddress(GA->getGlobal(), DL, VT, GA->getOffset(),
1701 OperandFlags));
1702
1703 return DAG.getNode(ISD::ADD, DL, VT, BaseAddr, SymAddr);
1704 }
1706 }
1707
1708 return DAG.getNode(WebAssemblyISD::Wrapper, DL, VT,
1709 DAG.getTargetGlobalAddress(GA->getGlobal(), DL, VT,
1710 GA->getOffset(), OperandFlags));
1711}
1712
1713SDValue
1714WebAssemblyTargetLowering::LowerExternalSymbol(SDValue Op,
1715 SelectionDAG &DAG) const {
1716 SDLoc DL(Op);
1717 const auto *ES = cast<ExternalSymbolSDNode>(Op);
1718 EVT VT = Op.getValueType();
1719 assert(ES->getTargetFlags() == 0 &&
1720 "Unexpected target flags on generic ExternalSymbolSDNode");
1721 return DAG.getNode(WebAssemblyISD::Wrapper, DL, VT,
1722 DAG.getTargetExternalSymbol(ES->getSymbol(), VT));
1723}
1724
1725SDValue WebAssemblyTargetLowering::LowerJumpTable(SDValue Op,
1726 SelectionDAG &DAG) const {
1727 // There's no need for a Wrapper node because we always incorporate a jump
1728 // table operand into a BR_TABLE instruction, rather than ever
1729 // materializing it in a register.
1730 const JumpTableSDNode *JT = cast<JumpTableSDNode>(Op);
1731 return DAG.getTargetJumpTable(JT->getIndex(), Op.getValueType(),
1732 JT->getTargetFlags());
1733}
1734
1735SDValue WebAssemblyTargetLowering::LowerBR_JT(SDValue Op,
1736 SelectionDAG &DAG) const {
1737 SDLoc DL(Op);
1738 SDValue Chain = Op.getOperand(0);
1739 const auto *JT = cast<JumpTableSDNode>(Op.getOperand(1));
1740 SDValue Index = Op.getOperand(2);
1741 assert(JT->getTargetFlags() == 0 && "WebAssembly doesn't set target flags");
1742
1744 Ops.push_back(Chain);
1745 Ops.push_back(Index);
1746
1748 const auto &MBBs = MJTI->getJumpTables()[JT->getIndex()].MBBs;
1749
1750 // Add an operand for each case.
1751 for (auto *MBB : MBBs)
1752 Ops.push_back(DAG.getBasicBlock(MBB));
1753
1754 // Add the first MBB as a dummy default target for now. This will be replaced
1755 // with the proper default target (and the preceding range check eliminated)
1756 // if possible by WebAssemblyFixBrTableDefaults.
1757 Ops.push_back(DAG.getBasicBlock(*MBBs.begin()));
1758 return DAG.getNode(WebAssemblyISD::BR_TABLE, DL, MVT::Other, Ops);
1759}
1760
1761SDValue WebAssemblyTargetLowering::LowerVASTART(SDValue Op,
1762 SelectionDAG &DAG) const {
1763 SDLoc DL(Op);
1765
1767 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
1768
1769 SDValue ArgN = DAG.getCopyFromReg(DAG.getEntryNode(), DL,
1770 MFI->getVarargBufferVreg(), PtrVT);
1771 return DAG.getStore(Op.getOperand(0), DL, ArgN, Op.getOperand(1),
1772 MachinePointerInfo(SV));
1773}
1774
1775SDValue WebAssemblyTargetLowering::LowerIntrinsic(SDValue Op,
1776 SelectionDAG &DAG) const {
1778 unsigned IntNo;
1779 switch (Op.getOpcode()) {
1782 IntNo = Op.getConstantOperandVal(1);
1783 break;
1785 IntNo = Op.getConstantOperandVal(0);
1786 break;
1787 default:
1788 llvm_unreachable("Invalid intrinsic");
1789 }
1790 SDLoc DL(Op);
1791
1792 switch (IntNo) {
1793 default:
1794 return SDValue(); // Don't custom lower most intrinsics.
1795
1796 case Intrinsic::wasm_lsda: {
1797 auto PtrVT = getPointerTy(MF.getDataLayout());
1798 const char *SymName = MF.createExternalSymbolName(
1799 "GCC_except_table" + std::to_string(MF.getFunctionNumber()));
1800 if (isPositionIndependent()) {
1802 SymName, PtrVT, WebAssemblyII::MO_MEMORY_BASE_REL);
1803 const char *BaseName = MF.createExternalSymbolName("__memory_base");
1805 DAG.getNode(WebAssemblyISD::Wrapper, DL, PtrVT,
1806 DAG.getTargetExternalSymbol(BaseName, PtrVT));
1807 SDValue SymAddr =
1808 DAG.getNode(WebAssemblyISD::WrapperREL, DL, PtrVT, Node);
1809 return DAG.getNode(ISD::ADD, DL, PtrVT, BaseAddr, SymAddr);
1810 }
1811 SDValue Node = DAG.getTargetExternalSymbol(SymName, PtrVT);
1812 return DAG.getNode(WebAssemblyISD::Wrapper, DL, PtrVT, Node);
1813 }
1814
1815 case Intrinsic::wasm_shuffle: {
1816 // Drop in-chain and replace undefs, but otherwise pass through unchanged
1817 SDValue Ops[18];
1818 size_t OpIdx = 0;
1819 Ops[OpIdx++] = Op.getOperand(1);
1820 Ops[OpIdx++] = Op.getOperand(2);
1821 while (OpIdx < 18) {
1822 const SDValue &MaskIdx = Op.getOperand(OpIdx + 1);
1823 if (MaskIdx.isUndef() ||
1824 cast<ConstantSDNode>(MaskIdx.getNode())->getZExtValue() >= 32) {
1825 Ops[OpIdx++] = DAG.getConstant(0, DL, MVT::i32);
1826 } else {
1827 Ops[OpIdx++] = MaskIdx;
1828 }
1829 }
1830 return DAG.getNode(WebAssemblyISD::SHUFFLE, DL, Op.getValueType(), Ops);
1831 }
1832 }
1833}
1834
1835SDValue
1836WebAssemblyTargetLowering::LowerSIGN_EXTEND_INREG(SDValue Op,
1837 SelectionDAG &DAG) const {
1838 SDLoc DL(Op);
1839 // If sign extension operations are disabled, allow sext_inreg only if operand
1840 // is a vector extract of an i8 or i16 lane. SIMD does not depend on sign
1841 // extension operations, but allowing sext_inreg in this context lets us have
1842 // simple patterns to select extract_lane_s instructions. Expanding sext_inreg
1843 // everywhere would be simpler in this file, but would necessitate large and
1844 // brittle patterns to undo the expansion and select extract_lane_s
1845 // instructions.
1846 assert(!Subtarget->hasSignExt() && Subtarget->hasSIMD128());
1847 if (Op.getOperand(0).getOpcode() != ISD::EXTRACT_VECTOR_ELT)
1848 return SDValue();
1849
1850 const SDValue &Extract = Op.getOperand(0);
1851 MVT VecT = Extract.getOperand(0).getSimpleValueType();
1852 if (VecT.getVectorElementType().getSizeInBits() > 32)
1853 return SDValue();
1854 MVT ExtractedLaneT =
1855 cast<VTSDNode>(Op.getOperand(1).getNode())->getVT().getSimpleVT();
1856 MVT ExtractedVecT =
1857 MVT::getVectorVT(ExtractedLaneT, 128 / ExtractedLaneT.getSizeInBits());
1858 if (ExtractedVecT == VecT)
1859 return Op;
1860
1861 // Bitcast vector to appropriate type to ensure ISel pattern coverage
1862 const SDNode *Index = Extract.getOperand(1).getNode();
1863 if (!isa<ConstantSDNode>(Index))
1864 return SDValue();
1865 unsigned IndexVal = cast<ConstantSDNode>(Index)->getZExtValue();
1866 unsigned Scale =
1867 ExtractedVecT.getVectorNumElements() / VecT.getVectorNumElements();
1868 assert(Scale > 1);
1869 SDValue NewIndex =
1870 DAG.getConstant(IndexVal * Scale, DL, Index->getValueType(0));
1871 SDValue NewExtract = DAG.getNode(
1873 DAG.getBitcast(ExtractedVecT, Extract.getOperand(0)), NewIndex);
1874 return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, Op.getValueType(), NewExtract,
1875 Op.getOperand(1));
1876}
1877
1879 SDLoc DL(Op);
1880 if (Op.getValueType() != MVT::v2f64)
1881 return SDValue();
1882
1883 auto GetConvertedLane = [](SDValue Op, unsigned &Opcode, SDValue &SrcVec,
1884 unsigned &Index) -> bool {
1885 switch (Op.getOpcode()) {
1886 case ISD::SINT_TO_FP:
1887 Opcode = WebAssemblyISD::CONVERT_LOW_S;
1888 break;
1889 case ISD::UINT_TO_FP:
1890 Opcode = WebAssemblyISD::CONVERT_LOW_U;
1891 break;
1892 case ISD::FP_EXTEND:
1893 Opcode = WebAssemblyISD::PROMOTE_LOW;
1894 break;
1895 default:
1896 return false;
1897 }
1898
1899 auto ExtractVector = Op.getOperand(0);
1900 if (ExtractVector.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
1901 return false;
1902
1903 if (!isa<ConstantSDNode>(ExtractVector.getOperand(1).getNode()))
1904 return false;
1905
1906 SrcVec = ExtractVector.getOperand(0);
1907 Index = ExtractVector.getConstantOperandVal(1);
1908 return true;
1909 };
1910
1911 unsigned LHSOpcode, RHSOpcode, LHSIndex, RHSIndex;
1912 SDValue LHSSrcVec, RHSSrcVec;
1913 if (!GetConvertedLane(Op.getOperand(0), LHSOpcode, LHSSrcVec, LHSIndex) ||
1914 !GetConvertedLane(Op.getOperand(1), RHSOpcode, RHSSrcVec, RHSIndex))
1915 return SDValue();
1916
1917 if (LHSOpcode != RHSOpcode)
1918 return SDValue();
1919
1920 MVT ExpectedSrcVT;
1921 switch (LHSOpcode) {
1922 case WebAssemblyISD::CONVERT_LOW_S:
1923 case WebAssemblyISD::CONVERT_LOW_U:
1924 ExpectedSrcVT = MVT::v4i32;
1925 break;
1926 case WebAssemblyISD::PROMOTE_LOW:
1927 ExpectedSrcVT = MVT::v4f32;
1928 break;
1929 }
1930 if (LHSSrcVec.getValueType() != ExpectedSrcVT)
1931 return SDValue();
1932
1933 auto Src = LHSSrcVec;
1934 if (LHSIndex != 0 || RHSIndex != 1 || LHSSrcVec != RHSSrcVec) {
1935 // Shuffle the source vector so that the converted lanes are the low lanes.
1936 Src = DAG.getVectorShuffle(
1937 ExpectedSrcVT, DL, LHSSrcVec, RHSSrcVec,
1938 {static_cast<int>(LHSIndex), static_cast<int>(RHSIndex) + 4, -1, -1});
1939 }
1940 return DAG.getNode(LHSOpcode, DL, MVT::v2f64, Src);
1941}
1942
1943SDValue WebAssemblyTargetLowering::LowerBUILD_VECTOR(SDValue Op,
1944 SelectionDAG &DAG) const {
1945 if (auto ConvertLow = LowerConvertLow(Op, DAG))
1946 return ConvertLow;
1947
1948 SDLoc DL(Op);
1949 const EVT VecT = Op.getValueType();
1950 const EVT LaneT = Op.getOperand(0).getValueType();
1951 const size_t Lanes = Op.getNumOperands();
1952 bool CanSwizzle = VecT == MVT::v16i8;
1953
1954 // BUILD_VECTORs are lowered to the instruction that initializes the highest
1955 // possible number of lanes at once followed by a sequence of replace_lane
1956 // instructions to individually initialize any remaining lanes.
1957
1958 // TODO: Tune this. For example, lanewise swizzling is very expensive, so
1959 // swizzled lanes should be given greater weight.
1960
1961 // TODO: Investigate looping rather than always extracting/replacing specific
1962 // lanes to fill gaps.
1963
1964 auto IsConstant = [](const SDValue &V) {
1965 return V.getOpcode() == ISD::Constant || V.getOpcode() == ISD::ConstantFP;
1966 };
1967
1968 // Returns the source vector and index vector pair if they exist. Checks for:
1969 // (extract_vector_elt
1970 // $src,
1971 // (sign_extend_inreg (extract_vector_elt $indices, $i))
1972 // )
1973 auto GetSwizzleSrcs = [](size_t I, const SDValue &Lane) {
1974 auto Bail = std::make_pair(SDValue(), SDValue());
1975 if (Lane->getOpcode() != ISD::EXTRACT_VECTOR_ELT)
1976 return Bail;
1977 const SDValue &SwizzleSrc = Lane->getOperand(0);
1978 const SDValue &IndexExt = Lane->getOperand(1);
1979 if (IndexExt->getOpcode() != ISD::SIGN_EXTEND_INREG)
1980 return Bail;
1981 const SDValue &Index = IndexExt->getOperand(0);
1982 if (Index->getOpcode() != ISD::EXTRACT_VECTOR_ELT)
1983 return Bail;
1984 const SDValue &SwizzleIndices = Index->getOperand(0);
1985 if (SwizzleSrc.getValueType() != MVT::v16i8 ||
1986 SwizzleIndices.getValueType() != MVT::v16i8 ||
1987 Index->getOperand(1)->getOpcode() != ISD::Constant ||
1988 Index->getConstantOperandVal(1) != I)
1989 return Bail;
1990 return std::make_pair(SwizzleSrc, SwizzleIndices);
1991 };
1992
1993 // If the lane is extracted from another vector at a constant index, return
1994 // that vector. The source vector must not have more lanes than the dest
1995 // because the shufflevector indices are in terms of the destination lanes and
1996 // would not be able to address the smaller individual source lanes.
1997 auto GetShuffleSrc = [&](const SDValue &Lane) {
1998 if (Lane->getOpcode() != ISD::EXTRACT_VECTOR_ELT)
1999 return SDValue();
2000 if (!isa<ConstantSDNode>(Lane->getOperand(1).getNode()))
2001 return SDValue();
2002 if (Lane->getOperand(0).getValueType().getVectorNumElements() >
2003 VecT.getVectorNumElements())
2004 return SDValue();
2005 return Lane->getOperand(0);
2006 };
2007
2008 using ValueEntry = std::pair<SDValue, size_t>;
2009 SmallVector<ValueEntry, 16> SplatValueCounts;
2010
2011 using SwizzleEntry = std::pair<std::pair<SDValue, SDValue>, size_t>;
2012 SmallVector<SwizzleEntry, 16> SwizzleCounts;
2013
2014 using ShuffleEntry = std::pair<SDValue, size_t>;
2015 SmallVector<ShuffleEntry, 16> ShuffleCounts;
2016
2017 auto AddCount = [](auto &Counts, const auto &Val) {
2018 auto CountIt =
2019 llvm::find_if(Counts, [&Val](auto E) { return E.first == Val; });
2020 if (CountIt == Counts.end()) {
2021 Counts.emplace_back(Val, 1);
2022 } else {
2023 CountIt->second++;
2024 }
2025 };
2026
2027 auto GetMostCommon = [](auto &Counts) {
2028 auto CommonIt =
2029 std::max_element(Counts.begin(), Counts.end(), llvm::less_second());
2030 assert(CommonIt != Counts.end() && "Unexpected all-undef build_vector");
2031 return *CommonIt;
2032 };
2033
2034 size_t NumConstantLanes = 0;
2035
2036 // Count eligible lanes for each type of vector creation op
2037 for (size_t I = 0; I < Lanes; ++I) {
2038 const SDValue &Lane = Op->getOperand(I);
2039 if (Lane.isUndef())
2040 continue;
2041
2042 AddCount(SplatValueCounts, Lane);
2043
2044 if (IsConstant(Lane))
2045 NumConstantLanes++;
2046 if (auto ShuffleSrc = GetShuffleSrc(Lane))
2047 AddCount(ShuffleCounts, ShuffleSrc);
2048 if (CanSwizzle) {
2049 auto SwizzleSrcs = GetSwizzleSrcs(I, Lane);
2050 if (SwizzleSrcs.first)
2051 AddCount(SwizzleCounts, SwizzleSrcs);
2052 }
2053 }
2054
2055 SDValue SplatValue;
2056 size_t NumSplatLanes;
2057 std::tie(SplatValue, NumSplatLanes) = GetMostCommon(SplatValueCounts);
2058
2059 SDValue SwizzleSrc;
2060 SDValue SwizzleIndices;
2061 size_t NumSwizzleLanes = 0;
2062 if (SwizzleCounts.size())
2063 std::forward_as_tuple(std::tie(SwizzleSrc, SwizzleIndices),
2064 NumSwizzleLanes) = GetMostCommon(SwizzleCounts);
2065
2066 // Shuffles can draw from up to two vectors, so find the two most common
2067 // sources.
2068 SDValue ShuffleSrc1, ShuffleSrc2;
2069 size_t NumShuffleLanes = 0;
2070 if (ShuffleCounts.size()) {
2071 std::tie(ShuffleSrc1, NumShuffleLanes) = GetMostCommon(ShuffleCounts);
2072 llvm::erase_if(ShuffleCounts,
2073 [&](const auto &Pair) { return Pair.first == ShuffleSrc1; });
2074 }
2075 if (ShuffleCounts.size()) {
2076 size_t AdditionalShuffleLanes;
2077 std::tie(ShuffleSrc2, AdditionalShuffleLanes) =
2078 GetMostCommon(ShuffleCounts);
2079 NumShuffleLanes += AdditionalShuffleLanes;
2080 }
2081
2082 // Predicate returning true if the lane is properly initialized by the
2083 // original instruction
2084 std::function<bool(size_t, const SDValue &)> IsLaneConstructed;
2086 // Prefer swizzles over shuffles over vector consts over splats
2087 if (NumSwizzleLanes >= NumShuffleLanes &&
2088 NumSwizzleLanes >= NumConstantLanes && NumSwizzleLanes >= NumSplatLanes) {
2089 Result = DAG.getNode(WebAssemblyISD::SWIZZLE, DL, VecT, SwizzleSrc,
2090 SwizzleIndices);
2091 auto Swizzled = std::make_pair(SwizzleSrc, SwizzleIndices);
2092 IsLaneConstructed = [&, Swizzled](size_t I, const SDValue &Lane) {
2093 return Swizzled == GetSwizzleSrcs(I, Lane);
2094 };
2095 } else if (NumShuffleLanes >= NumConstantLanes &&
2096 NumShuffleLanes >= NumSplatLanes) {
2097 size_t DestLaneSize = VecT.getVectorElementType().getFixedSizeInBits() / 8;
2098 size_t DestLaneCount = VecT.getVectorNumElements();
2099 size_t Scale1 = 1;
2100 size_t Scale2 = 1;
2101 SDValue Src1 = ShuffleSrc1;
2102 SDValue Src2 = ShuffleSrc2 ? ShuffleSrc2 : DAG.getUNDEF(VecT);
2103 if (Src1.getValueType() != VecT) {
2104 size_t LaneSize =
2106 assert(LaneSize > DestLaneSize);
2107 Scale1 = LaneSize / DestLaneSize;
2108 Src1 = DAG.getBitcast(VecT, Src1);
2109 }
2110 if (Src2.getValueType() != VecT) {
2111 size_t LaneSize =
2113 assert(LaneSize > DestLaneSize);
2114 Scale2 = LaneSize / DestLaneSize;
2115 Src2 = DAG.getBitcast(VecT, Src2);
2116 }
2117
2118 int Mask[16];
2119 assert(DestLaneCount <= 16);
2120 for (size_t I = 0; I < DestLaneCount; ++I) {
2121 const SDValue &Lane = Op->getOperand(I);
2122 SDValue Src = GetShuffleSrc(Lane);
2123 if (Src == ShuffleSrc1) {
2124 Mask[I] = Lane->getConstantOperandVal(1) * Scale1;
2125 } else if (Src && Src == ShuffleSrc2) {
2126 Mask[I] = DestLaneCount + Lane->getConstantOperandVal(1) * Scale2;
2127 } else {
2128 Mask[I] = -1;
2129 }
2130 }
2131 ArrayRef<int> MaskRef(Mask, DestLaneCount);
2132 Result = DAG.getVectorShuffle(VecT, DL, Src1, Src2, MaskRef);
2133 IsLaneConstructed = [&](size_t, const SDValue &Lane) {
2134 auto Src = GetShuffleSrc(Lane);
2135 return Src == ShuffleSrc1 || (Src && Src == ShuffleSrc2);
2136 };
2137 } else if (NumConstantLanes >= NumSplatLanes) {
2138 SmallVector<SDValue, 16> ConstLanes;
2139 for (const SDValue &Lane : Op->op_values()) {
2140 if (IsConstant(Lane)) {
2141 // Values may need to be fixed so that they will sign extend to be
2142 // within the expected range during ISel. Check whether the value is in
2143 // bounds based on the lane bit width and if it is out of bounds, lop
2144 // off the extra bits and subtract 2^n to reflect giving the high bit
2145 // value -2^(n-1) rather than +2^(n-1). Skip the i64 case because it
2146 // cannot possibly be out of range.
2147 auto *Const = dyn_cast<ConstantSDNode>(Lane.getNode());
2148 int64_t Val = Const ? Const->getSExtValue() : 0;
2149 uint64_t LaneBits = 128 / Lanes;
2150 assert((LaneBits == 64 || Val >= -(1ll << (LaneBits - 1))) &&
2151 "Unexpected out of bounds negative value");
2152 if (Const && LaneBits != 64 && Val > (1ll << (LaneBits - 1)) - 1) {
2153 auto NewVal = ((uint64_t)Val % (1ll << LaneBits)) - (1ll << LaneBits);
2154 ConstLanes.push_back(DAG.getConstant(NewVal, SDLoc(Lane), LaneT));
2155 } else {
2156 ConstLanes.push_back(Lane);
2157 }
2158 } else if (LaneT.isFloatingPoint()) {
2159 ConstLanes.push_back(DAG.getConstantFP(0, DL, LaneT));
2160 } else {
2161 ConstLanes.push_back(DAG.getConstant(0, DL, LaneT));
2162 }
2163 }
2164 Result = DAG.getBuildVector(VecT, DL, ConstLanes);
2165 IsLaneConstructed = [&IsConstant](size_t _, const SDValue &Lane) {
2166 return IsConstant(Lane);
2167 };
2168 } else {
2169 // Use a splat (which might be selected as a load splat)
2170 Result = DAG.getSplatBuildVector(VecT, DL, SplatValue);
2171 IsLaneConstructed = [&SplatValue](size_t _, const SDValue &Lane) {
2172 return Lane == SplatValue;
2173 };
2174 }
2175
2176 assert(Result);
2177 assert(IsLaneConstructed);
2178
2179 // Add replace_lane instructions for any unhandled values
2180 for (size_t I = 0; I < Lanes; ++I) {
2181 const SDValue &Lane = Op->getOperand(I);
2182 if (!Lane.isUndef() && !IsLaneConstructed(I, Lane))
2183 Result = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VecT, Result, Lane,
2184 DAG.getConstant(I, DL, MVT::i32));
2185 }
2186
2187 return Result;
2188}
2189
2190SDValue
2191WebAssemblyTargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,
2192 SelectionDAG &DAG) const {
2193 SDLoc DL(Op);
2194 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Op.getNode())->getMask();
2195 MVT VecType = Op.getOperand(0).getSimpleValueType();
2196 assert(VecType.is128BitVector() && "Unexpected shuffle vector type");
2197 size_t LaneBytes = VecType.getVectorElementType().getSizeInBits() / 8;
2198
2199 // Space for two vector args and sixteen mask indices
2200 SDValue Ops[18];
2201 size_t OpIdx = 0;
2202 Ops[OpIdx++] = Op.getOperand(0);
2203 Ops[OpIdx++] = Op.getOperand(1);
2204
2205 // Expand mask indices to byte indices and materialize them as operands
2206 for (int M : Mask) {
2207 for (size_t J = 0; J < LaneBytes; ++J) {
2208 // Lower undefs (represented by -1 in mask) to {0..J}, which use a
2209 // whole lane of vector input, to allow further reduction at VM. E.g.
2210 // match an 8x16 byte shuffle to an equivalent cheaper 32x4 shuffle.
2211 uint64_t ByteIndex = M == -1 ? J : (uint64_t)M * LaneBytes + J;
2212 Ops[OpIdx++] = DAG.getConstant(ByteIndex, DL, MVT::i32);
2213 }
2214 }
2215
2216 return DAG.getNode(WebAssemblyISD::SHUFFLE, DL, Op.getValueType(), Ops);
2217}
2218
2219SDValue WebAssemblyTargetLowering::LowerSETCC(SDValue Op,
2220 SelectionDAG &DAG) const {
2221 SDLoc DL(Op);
2222 // The legalizer does not know how to expand the unsupported comparison modes
2223 // of i64x2 vectors, so we manually unroll them here.
2224 assert(Op->getOperand(0)->getSimpleValueType(0) == MVT::v2i64);
2226 DAG.ExtractVectorElements(Op->getOperand(0), LHS);
2227 DAG.ExtractVectorElements(Op->getOperand(1), RHS);
2228 const SDValue &CC = Op->getOperand(2);
2229 auto MakeLane = [&](unsigned I) {
2230 return DAG.getNode(ISD::SELECT_CC, DL, MVT::i64, LHS[I], RHS[I],
2231 DAG.getConstant(uint64_t(-1), DL, MVT::i64),
2232 DAG.getConstant(uint64_t(0), DL, MVT::i64), CC);
2233 };
2234 return DAG.getBuildVector(Op->getValueType(0), DL,
2235 {MakeLane(0), MakeLane(1)});
2236}
2237
2238SDValue
2239WebAssemblyTargetLowering::LowerAccessVectorElement(SDValue Op,
2240 SelectionDAG &DAG) const {
2241 // Allow constant lane indices, expand variable lane indices
2242 SDNode *IdxNode = Op.getOperand(Op.getNumOperands() - 1).getNode();
2243 if (isa<ConstantSDNode>(IdxNode) || IdxNode->isUndef()) {
2244 // Ensure the index type is i32 to match the tablegen patterns
2245 uint64_t Idx = cast<ConstantSDNode>(IdxNode)->getZExtValue();
2246 SmallVector<SDValue, 3> Ops(Op.getNode()->ops());
2247 Ops[Op.getNumOperands() - 1] =
2248 DAG.getConstant(Idx, SDLoc(IdxNode), MVT::i32);
2249 return DAG.getNode(Op.getOpcode(), SDLoc(Op), Op.getValueType(), Ops);
2250 }
2251 // Perform default expansion
2252 return SDValue();
2253}
2254
2256 EVT LaneT = Op.getSimpleValueType().getVectorElementType();
2257 // 32-bit and 64-bit unrolled shifts will have proper semantics
2258 if (LaneT.bitsGE(MVT::i32))
2259 return DAG.UnrollVectorOp(Op.getNode());
2260 // Otherwise mask the shift value to get proper semantics from 32-bit shift
2261 SDLoc DL(Op);
2262 size_t NumLanes = Op.getSimpleValueType().getVectorNumElements();
2263 SDValue Mask = DAG.getConstant(LaneT.getSizeInBits() - 1, DL, MVT::i32);
2264 unsigned ShiftOpcode = Op.getOpcode();
2265 SmallVector<SDValue, 16> ShiftedElements;
2266 DAG.ExtractVectorElements(Op.getOperand(0), ShiftedElements, 0, 0, MVT::i32);
2267 SmallVector<SDValue, 16> ShiftElements;
2268 DAG.ExtractVectorElements(Op.getOperand(1), ShiftElements, 0, 0, MVT::i32);
2269 SmallVector<SDValue, 16> UnrolledOps;
2270 for (size_t i = 0; i < NumLanes; ++i) {
2271 SDValue MaskedShiftValue =
2272 DAG.getNode(ISD::AND, DL, MVT::i32, ShiftElements[i], Mask);
2273 SDValue ShiftedValue = ShiftedElements[i];
2274 if (ShiftOpcode == ISD::SRA)
2275 ShiftedValue = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i32,
2276 ShiftedValue, DAG.getValueType(LaneT));
2277 UnrolledOps.push_back(
2278 DAG.getNode(ShiftOpcode, DL, MVT::i32, ShiftedValue, MaskedShiftValue));
2279 }
2280 return DAG.getBuildVector(Op.getValueType(), DL, UnrolledOps);
2281}
2282
2283SDValue WebAssemblyTargetLowering::LowerShift(SDValue Op,
2284 SelectionDAG &DAG) const {
2285 SDLoc DL(Op);
2286
2287 // Only manually lower vector shifts
2288 assert(Op.getSimpleValueType().isVector());
2289
2290 uint64_t LaneBits = Op.getValueType().getScalarSizeInBits();
2291 auto ShiftVal = Op.getOperand(1);
2292
2293 // Try to skip bitmask operation since it is implied inside shift instruction
2294 auto SkipImpliedMask = [](SDValue MaskOp, uint64_t MaskBits) {
2295 if (MaskOp.getOpcode() != ISD::AND)
2296 return MaskOp;
2297 SDValue LHS = MaskOp.getOperand(0);
2298 SDValue RHS = MaskOp.getOperand(1);
2299 if (MaskOp.getValueType().isVector()) {
2300 APInt MaskVal;
2301 if (!ISD::isConstantSplatVector(RHS.getNode(), MaskVal))
2302 std::swap(LHS, RHS);
2303
2304 if (ISD::isConstantSplatVector(RHS.getNode(), MaskVal) &&
2305 MaskVal == MaskBits)
2306 MaskOp = LHS;
2307 } else {
2308 if (!isa<ConstantSDNode>(RHS.getNode()))
2309 std::swap(LHS, RHS);
2310
2311 auto ConstantRHS = dyn_cast<ConstantSDNode>(RHS.getNode());
2312 if (ConstantRHS && ConstantRHS->getAPIntValue() == MaskBits)
2313 MaskOp = LHS;
2314 }
2315
2316 return MaskOp;
2317 };
2318
2319 // Skip vector and operation
2320 ShiftVal = SkipImpliedMask(ShiftVal, LaneBits - 1);
2321 ShiftVal = DAG.getSplatValue(ShiftVal);
2322 if (!ShiftVal)
2323 return unrollVectorShift(Op, DAG);
2324
2325 // Skip scalar and operation
2326 ShiftVal = SkipImpliedMask(ShiftVal, LaneBits - 1);
2327 // Use anyext because none of the high bits can affect the shift
2328 ShiftVal = DAG.getAnyExtOrTrunc(ShiftVal, DL, MVT::i32);
2329
2330 unsigned Opcode;
2331 switch (Op.getOpcode()) {
2332 case ISD::SHL:
2333 Opcode = WebAssemblyISD::VEC_SHL;
2334 break;
2335 case ISD::SRA:
2336 Opcode = WebAssemblyISD::VEC_SHR_S;
2337 break;
2338 case ISD::SRL:
2339 Opcode = WebAssemblyISD::VEC_SHR_U;
2340 break;
2341 default:
2342 llvm_unreachable("unexpected opcode");
2343 }
2344
2345 return DAG.getNode(Opcode, DL, Op.getValueType(), Op.getOperand(0), ShiftVal);
2346}
2347
2348SDValue WebAssemblyTargetLowering::LowerFP_TO_INT_SAT(SDValue Op,
2349 SelectionDAG &DAG) const {
2350 SDLoc DL(Op);
2351 EVT ResT = Op.getValueType();
2352 EVT SatVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
2353
2354 if ((ResT == MVT::i32 || ResT == MVT::i64) &&
2355 (SatVT == MVT::i32 || SatVT == MVT::i64))
2356 return Op;
2357
2358 if (ResT == MVT::v4i32 && SatVT == MVT::i32)
2359 return Op;
2360
2361 return SDValue();
2362}
2363
2364//===----------------------------------------------------------------------===//
2365// Custom DAG combine hooks
2366//===----------------------------------------------------------------------===//
2367static SDValue
2369 auto &DAG = DCI.DAG;
2370 auto Shuffle = cast<ShuffleVectorSDNode>(N);
2371
2372 // Hoist vector bitcasts that don't change the number of lanes out of unary
2373 // shuffles, where they are less likely to get in the way of other combines.
2374 // (shuffle (vNxT1 (bitcast (vNxT0 x))), undef, mask) ->
2375 // (vNxT1 (bitcast (vNxT0 (shuffle x, undef, mask))))
2376 SDValue Bitcast = N->getOperand(0);
2377 if (Bitcast.getOpcode() != ISD::BITCAST)
2378 return SDValue();
2379 if (!N->getOperand(1).isUndef())
2380 return SDValue();
2381 SDValue CastOp = Bitcast.getOperand(0);
2382 MVT SrcType = CastOp.getSimpleValueType();
2383 MVT DstType = Bitcast.getSimpleValueType();
2384 if (!SrcType.is128BitVector() ||
2385 SrcType.getVectorNumElements() != DstType.getVectorNumElements())
2386 return SDValue();
2387 SDValue NewShuffle = DAG.getVectorShuffle(
2388 SrcType, SDLoc(N), CastOp, DAG.getUNDEF(SrcType), Shuffle->getMask());
2389 return DAG.getBitcast(DstType, NewShuffle);
2390}
2391
2392/// Convert ({u,s}itofp vec) --> ({u,s}itofp ({s,z}ext vec)) so it doesn't get
2393/// split up into scalar instructions during legalization, and the vector
2394/// extending instructions are selected in performVectorExtendCombine below.
2395static SDValue
2398 auto &DAG = DCI.DAG;
2399 assert(N->getOpcode() == ISD::UINT_TO_FP ||
2400 N->getOpcode() == ISD::SINT_TO_FP);
2401
2402 EVT InVT = N->getOperand(0)->getValueType(0);
2403 EVT ResVT = N->getValueType(0);
2404 MVT ExtVT;
2405 if (ResVT == MVT::v4f32 && (InVT == MVT::v4i16 || InVT == MVT::v4i8))
2406 ExtVT = MVT::v4i32;
2407 else if (ResVT == MVT::v2f64 && (InVT == MVT::v2i16 || InVT == MVT::v2i8))
2408 ExtVT = MVT::v2i32;
2409 else
2410 return SDValue();
2411
2412 unsigned Op =
2414 SDValue Conv = DAG.getNode(Op, SDLoc(N), ExtVT, N->getOperand(0));
2415 return DAG.getNode(N->getOpcode(), SDLoc(N), ResVT, Conv);
2416}
2417
2418static SDValue
2420 auto &DAG = DCI.DAG;
2421 assert(N->getOpcode() == ISD::SIGN_EXTEND ||
2422 N->getOpcode() == ISD::ZERO_EXTEND);
2423
2424 // Combine ({s,z}ext (extract_subvector src, i)) into a widening operation if
2425 // possible before the extract_subvector can be expanded.
2426 auto Extract = N->getOperand(0);
2427 if (Extract.getOpcode() != ISD::EXTRACT_SUBVECTOR)
2428 return SDValue();
2429 auto Source = Extract.getOperand(0);
2430 auto *IndexNode = dyn_cast<ConstantSDNode>(Extract.getOperand(1));
2431 if (IndexNode == nullptr)
2432 return SDValue();
2433 auto Index = IndexNode->getZExtValue();
2434
2435 // Only v8i8, v4i16, and v2i32 extracts can be widened, and only if the
2436 // extracted subvector is the low or high half of its source.
2437 EVT ResVT = N->getValueType(0);
2438 if (ResVT == MVT::v8i16) {
2439 if (Extract.getValueType() != MVT::v8i8 ||
2440 Source.getValueType() != MVT::v16i8 || (Index != 0 && Index != 8))
2441 return SDValue();
2442 } else if (ResVT == MVT::v4i32) {
2443 if (Extract.getValueType() != MVT::v4i16 ||
2444 Source.getValueType() != MVT::v8i16 || (Index != 0 && Index != 4))
2445 return SDValue();
2446 } else if (ResVT == MVT::v2i64) {
2447 if (Extract.getValueType() != MVT::v2i32 ||
2448 Source.getValueType() != MVT::v4i32 || (Index != 0 && Index != 2))
2449 return SDValue();
2450 } else {
2451 return SDValue();
2452 }
2453
2454 bool IsSext = N->getOpcode() == ISD::SIGN_EXTEND;
2455 bool IsLow = Index == 0;
2456
2457 unsigned Op = IsSext ? (IsLow ? WebAssemblyISD::EXTEND_LOW_S
2458 : WebAssemblyISD::EXTEND_HIGH_S)
2459 : (IsLow ? WebAssemblyISD::EXTEND_LOW_U
2460 : WebAssemblyISD::EXTEND_HIGH_U);
2461
2462 return DAG.getNode(Op, SDLoc(N), ResVT, Source);
2463}
2464
2465static SDValue
2467 auto &DAG = DCI.DAG;
2468
2469 auto GetWasmConversionOp = [](unsigned Op) {
2470 switch (Op) {
2472 return WebAssemblyISD::TRUNC_SAT_ZERO_S;
2474 return WebAssemblyISD::TRUNC_SAT_ZERO_U;
2475 case ISD::FP_ROUND:
2476 return WebAssemblyISD::DEMOTE_ZERO;
2477 }
2478 llvm_unreachable("unexpected op");
2479 };
2480
2481 auto IsZeroSplat = [](SDValue SplatVal) {
2482 auto *Splat = dyn_cast<BuildVectorSDNode>(SplatVal.getNode());
2483 APInt SplatValue, SplatUndef;
2484 unsigned SplatBitSize;
2485 bool HasAnyUndefs;
2486 return Splat &&
2487 Splat->isConstantSplat(SplatValue, SplatUndef, SplatBitSize,
2488 HasAnyUndefs) &&
2489 SplatValue == 0;
2490 };
2491
2492 if (N->getOpcode() == ISD::CONCAT_VECTORS) {
2493 // Combine this:
2494 //
2495 // (concat_vectors (v2i32 (fp_to_{s,u}int_sat $x, 32)), (v2i32 (splat 0)))
2496 //
2497 // into (i32x4.trunc_sat_f64x2_zero_{s,u} $x).
2498 //
2499 // Or this:
2500 //
2501 // (concat_vectors (v2f32 (fp_round (v2f64 $x))), (v2f32 (splat 0)))
2502 //
2503 // into (f32x4.demote_zero_f64x2 $x).
2504 EVT ResVT;
2505 EVT ExpectedConversionType;
2506 auto Conversion = N->getOperand(0);
2507 auto ConversionOp = Conversion.getOpcode();
2508 switch (ConversionOp) {
2511 ResVT = MVT::v4i32;
2512 ExpectedConversionType = MVT::v2i32;
2513 break;
2514 case ISD::FP_ROUND:
2515 ResVT = MVT::v4f32;
2516 ExpectedConversionType = MVT::v2f32;
2517 break;
2518 default:
2519 return SDValue();
2520 }
2521
2522 if (N->getValueType(0) != ResVT)
2523 return SDValue();
2524
2525 if (Conversion.getValueType() != ExpectedConversionType)
2526 return SDValue();
2527
2528 auto Source = Conversion.getOperand(0);
2529 if (Source.getValueType() != MVT::v2f64)
2530 return SDValue();
2531
2532 if (!IsZeroSplat(N->getOperand(1)) ||
2533 N->getOperand(1).getValueType() != ExpectedConversionType)
2534 return SDValue();
2535
2536 unsigned Op = GetWasmConversionOp(ConversionOp);
2537 return DAG.getNode(Op, SDLoc(N), ResVT, Source);
2538 }
2539
2540 // Combine this:
2541 //
2542 // (fp_to_{s,u}int_sat (concat_vectors $x, (v2f64 (splat 0))), 32)
2543 //
2544 // into (i32x4.trunc_sat_f64x2_zero_{s,u} $x).
2545 //
2546 // Or this:
2547 //
2548 // (v4f32 (fp_round (concat_vectors $x, (v2f64 (splat 0)))))
2549 //
2550 // into (f32x4.demote_zero_f64x2 $x).
2551 EVT ResVT;
2552 auto ConversionOp = N->getOpcode();
2553 switch (ConversionOp) {
2556 ResVT = MVT::v4i32;
2557 break;
2558 case ISD::FP_ROUND:
2559 ResVT = MVT::v4f32;
2560 break;
2561 default:
2562 llvm_unreachable("unexpected op");
2563 }
2564
2565 if (N->getValueType(0) != ResVT)
2566 return SDValue();
2567
2568 auto Concat = N->getOperand(0);
2569 if (Concat.getValueType() != MVT::v4f64)
2570 return SDValue();
2571
2572 auto Source = Concat.getOperand(0);
2573 if (Source.getValueType() != MVT::v2f64)
2574 return SDValue();
2575
2576 if (!IsZeroSplat(Concat.getOperand(1)) ||
2577 Concat.getOperand(1).getValueType() != MVT::v2f64)
2578 return SDValue();
2579
2580 unsigned Op = GetWasmConversionOp(ConversionOp);
2581 return DAG.getNode(Op, SDLoc(N), ResVT, Source);
2582}
2583
2584// Helper to extract VectorWidth bits from Vec, starting from IdxVal.
2585static SDValue extractSubVector(SDValue Vec, unsigned IdxVal, SelectionDAG &DAG,
2586 const SDLoc &DL, unsigned VectorWidth) {
2587 EVT VT = Vec.getValueType();
2588 EVT ElVT = VT.getVectorElementType();
2589 unsigned Factor = VT.getSizeInBits() / VectorWidth;
2590 EVT ResultVT = EVT::getVectorVT(*DAG.getContext(), ElVT,
2591 VT.getVectorNumElements() / Factor);
2592
2593 // Extract the relevant VectorWidth bits. Generate an EXTRACT_SUBVECTOR
2594 unsigned ElemsPerChunk = VectorWidth / ElVT.getSizeInBits();
2595 assert(isPowerOf2_32(ElemsPerChunk) && "Elements per chunk not power of 2");
2596
2597 // This is the index of the first element of the VectorWidth-bit chunk
2598 // we want. Since ElemsPerChunk is a power of 2 just need to clear bits.
2599 IdxVal &= ~(ElemsPerChunk - 1);
2600
2601 // If the input is a buildvector just emit a smaller one.
2602 if (Vec.getOpcode() == ISD::BUILD_VECTOR)
2603 return DAG.getBuildVector(ResultVT, DL,
2604 Vec->ops().slice(IdxVal, ElemsPerChunk));
2605
2606 SDValue VecIdx = DAG.getIntPtrConstant(IdxVal, DL);
2607 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ResultVT, Vec, VecIdx);
2608}
2609
2610// Helper to recursively truncate vector elements in half with NARROW_U. DstVT
2611// is the expected destination value type after recursion. In is the initial
2612// input. Note that the input should have enough leading zero bits to prevent
2613// NARROW_U from saturating results.
2615 SelectionDAG &DAG) {
2616 EVT SrcVT = In.getValueType();
2617
2618 // No truncation required, we might get here due to recursive calls.
2619 if (SrcVT == DstVT)
2620 return In;
2621
2622 unsigned SrcSizeInBits = SrcVT.getSizeInBits();
2623 unsigned NumElems = SrcVT.getVectorNumElements();
2624 if (!isPowerOf2_32(NumElems))
2625 return SDValue();
2626 assert(DstVT.getVectorNumElements() == NumElems && "Illegal truncation");
2627 assert(SrcSizeInBits > DstVT.getSizeInBits() && "Illegal truncation");
2628
2629 LLVMContext &Ctx = *DAG.getContext();
2630 EVT PackedSVT = EVT::getIntegerVT(Ctx, SrcVT.getScalarSizeInBits() / 2);
2631
2632 // Narrow to the largest type possible:
2633 // vXi64/vXi32 -> i16x8.narrow_i32x4_u and vXi16 -> i8x16.narrow_i16x8_u.
2634 EVT InVT = MVT::i16, OutVT = MVT::i8;
2635 if (SrcVT.getScalarSizeInBits() > 16) {
2636 InVT = MVT::i32;
2637 OutVT = MVT::i16;
2638 }
2639 unsigned SubSizeInBits = SrcSizeInBits / 2;
2640 InVT = EVT::getVectorVT(Ctx, InVT, SubSizeInBits / InVT.getSizeInBits());
2641 OutVT = EVT::getVectorVT(Ctx, OutVT, SubSizeInBits / OutVT.getSizeInBits());
2642
2643 // Split lower/upper subvectors.
2644 SDValue Lo = extractSubVector(In, 0, DAG, DL, SubSizeInBits);
2645 SDValue Hi = extractSubVector(In, NumElems / 2, DAG, DL, SubSizeInBits);
2646
2647 // 256bit -> 128bit truncate - Narrow lower/upper 128-bit subvectors.
2648 if (SrcVT.is256BitVector() && DstVT.is128BitVector()) {
2649 Lo = DAG.getBitcast(InVT, Lo);
2650 Hi = DAG.getBitcast(InVT, Hi);
2651 SDValue Res = DAG.getNode(WebAssemblyISD::NARROW_U, DL, OutVT, Lo, Hi);
2652 return DAG.getBitcast(DstVT, Res);
2653 }
2654
2655 // Recursively narrow lower/upper subvectors, concat result and narrow again.
2656 EVT PackedVT = EVT::getVectorVT(Ctx, PackedSVT, NumElems / 2);
2657 Lo = truncateVectorWithNARROW(PackedVT, Lo, DL, DAG);
2658 Hi = truncateVectorWithNARROW(PackedVT, Hi, DL, DAG);
2659
2660 PackedVT = EVT::getVectorVT(Ctx, PackedSVT, NumElems);
2661 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, DL, PackedVT, Lo, Hi);
2662 return truncateVectorWithNARROW(DstVT, Res, DL, DAG);
2663}
2664
2667 auto &DAG = DCI.DAG;
2668
2669 SDValue In = N->getOperand(0);
2670 EVT InVT = In.getValueType();
2671 if (!InVT.isSimple())
2672 return SDValue();
2673
2674 EVT OutVT = N->getValueType(0);
2675 if (!OutVT.isVector())
2676 return SDValue();
2677
2678 EVT OutSVT = OutVT.getVectorElementType();
2679 EVT InSVT = InVT.getVectorElementType();
2680 // Currently only cover truncate to v16i8 or v8i16.
2681 if (!((InSVT == MVT::i16 || InSVT == MVT::i32 || InSVT == MVT::i64) &&
2682 (OutSVT == MVT::i8 || OutSVT == MVT::i16) && OutVT.is128BitVector()))
2683 return SDValue();
2684
2685 SDLoc DL(N);
2687 OutVT.getScalarSizeInBits());
2688 In = DAG.getNode(ISD::AND, DL, InVT, In, DAG.getConstant(Mask, DL, InVT));
2689 return truncateVectorWithNARROW(OutVT, In, DL, DAG);
2690}
2691
2692SDValue
2693WebAssemblyTargetLowering::PerformDAGCombine(SDNode *N,
2694 DAGCombinerInfo &DCI) const {
2695 switch (N->getOpcode()) {
2696 default:
2697 return SDValue();
2699 return performVECTOR_SHUFFLECombine(N, DCI);
2700 case ISD::SIGN_EXTEND:
2701 case ISD::ZERO_EXTEND:
2702 return performVectorExtendCombine(N, DCI);
2703 case ISD::UINT_TO_FP:
2704 case ISD::SINT_TO_FP:
2705 return performVectorExtendToFPCombine(N, DCI);
2708 case ISD::FP_ROUND:
2710 return performVectorTruncZeroCombine(N, DCI);
2711 case ISD::TRUNCATE:
2712 return performTruncateCombine(N, DCI);
2713 }
2714}
unsigned const MachineRegisterInfo * MRI
static SDValue performTruncateCombine(SDNode *N, SelectionDAG &DAG)
MachineBasicBlock & MBB
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
amdgpu Simplify well known AMD library false FunctionCallee Callee
amdgpu Simplify well known AMD library false FunctionCallee Value * Arg
Function Alias Analysis Results
static void fail(const SDLoc &DL, SelectionDAG &DAG, const Twine &Msg)
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
Analysis containing CSE Info
Definition: CSEInfo.cpp:27
return RetTy
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
Hexagon Common GEP
const HexagonInstrInfo * TII
#define _
IRTranslator LLVM IR MI
static unsigned NumFixedArgs
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
unsigned const TargetRegisterInfo * TRI
typename CallsiteContextGraph< DerivedCCG, FuncTy, CallTy >::FuncInfo FuncInfo
LLVMContext & Context
const char LLVMTargetMachineRef TM
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static MachineBasicBlock * LowerFPToInt(MachineInstr &MI, DebugLoc DL, MachineBasicBlock *BB, const TargetInstrInfo &TII, bool IsUnsigned, bool Int64, bool Float64, unsigned LoweredOpcode)
static bool callingConvSupported(CallingConv::ID CallConv)
static std::optional< unsigned > IsWebAssemblyLocal(SDValue Op, SelectionDAG &DAG)
static SDValue performVectorExtendCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue unrollVectorShift(SDValue Op, SelectionDAG &DAG)
static MachineBasicBlock * LowerCallResults(MachineInstr &CallResults, DebugLoc DL, MachineBasicBlock *BB, const WebAssemblySubtarget *Subtarget, const TargetInstrInfo &TII)
static SDValue performVECTOR_SHUFFLECombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue performVectorTruncZeroCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static bool IsWebAssemblyGlobal(SDValue Op)
static SDValue performVectorExtendToFPCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
Convert ({u,s}itofp vec) --> ({u,s}itofp ({s,z}ext vec)) so it doesn't get split up into scalar instr...
static SDValue LowerConvertLow(SDValue Op, SelectionDAG &DAG)
static SDValue extractSubVector(SDValue Vec, unsigned IdxVal, SelectionDAG &DAG, const SDLoc &DL, unsigned VectorWidth)
static SDValue truncateVectorWithNARROW(EVT DstVT, SDValue In, const SDLoc &DL, SelectionDAG &DAG)
This file defines the interfaces that WebAssembly uses to lower LLVM code into a selection DAG.
This file provides WebAssembly-specific target descriptions.
This file declares WebAssembly-specific per-machine-function information.
This file declares the WebAssembly-specific subclass of TargetSubtarget.
This file declares the WebAssembly-specific subclass of TargetMachine.
This file contains the declaration of the WebAssembly-specific type parsing utility functions.
This file contains the declaration of the WebAssembly-specific utility functions.
X86 cmov Conversion
static constexpr int Concat[]
Value * RHS
Value * LHS
Class for arbitrary precision integers.
Definition: APInt.h:75
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
Definition: APInt.h:289
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Constructs an APInt value that has the top hiBitsSet bits set.
Definition: APInt.h:279
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
an instruction that atomically reads a memory location, combines it with another value,...
Definition: Instructions.h:718
@ Add
*p = old + v
Definition: Instructions.h:734
@ Or
*p = old | v
Definition: Instructions.h:742
@ Sub
*p = old - v
Definition: Instructions.h:736
@ And
*p = old & v
Definition: Instructions.h:738
@ Xor
*p = old ^ v
Definition: Instructions.h:744
BinOp getOperation() const
Definition: Instructions.h:812
LLVM Basic Block Representation.
Definition: BasicBlock.h:56
CCState - This class holds information needed while lowering arguments and return values.
static CCValAssign getMem(unsigned ValNo, MVT ValVT, unsigned Offset, MVT LocVT, LocInfo HTP, bool IsCustom=false)
This class represents a function call, abstracting a target machine's calling convention.
static Constant * get(Type *Ty, double V)
This returns a ConstantFP, or a vector containing a splat of a ConstantFP, for the specified value in...
Definition: Constants.cpp:927
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:110
A debug info location.
Definition: DebugLoc.h:33
Diagnostic information for unsupported feature in backend.
This is a fast-path instruction selection class that generates poor code and doesn't support illegal ...
Definition: FastISel.h:66
FunctionLoweringInfo - This contains information that is global to a function that is used when lower...
FunctionType * getFunctionType() const
Returns the FunctionType for me.
Definition: Function.h:174
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Definition: Function.cpp:319
unsigned getAddressSpace() const
const GlobalValue * getGlobal() const
ThreadLocalMode getThreadLocalMode() const
Definition: GlobalValue.h:267
Module * getParent()
Get the module that this global value is contained inside of...
Definition: GlobalValue.h:652
Type * getValueType() const
Definition: GlobalValue.h:292
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:67
void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
This class is used to represent ISD::LOAD nodes.
const SDValue & getBasePtr() const
const SDValue & getOffset() const
Describe properties that are true of each instruction in the target description file.
Definition: MCInstrDesc.h:198
void setNoStrip() const
Definition: MCSymbolWasm.h:65
Machine Value Type.
bool is128BitVector() const
Return true if this is a 128-bit vector type.
static auto integer_fixedlen_vector_valuetypes()
@ INVALID_SIMPLE_VALUE_TYPE
unsigned getVectorNumElements() const
bool isVector() const
Return true if this is a vector value type.
bool isInteger() const
Return true if this is an integer or a vector integer type.
static MVT getVT(Type *Ty, bool HandleUnknown=false)
Return the value type corresponding to the specified type.
Definition: ValueTypes.cpp:573
static auto integer_valuetypes()
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
static auto fixedlen_vector_valuetypes()
bool isFixedLengthVector() const
static MVT getVectorVT(MVT VT, unsigned NumElements)
MVT getVectorElementType() const
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
static MVT getIntegerVT(unsigned BitWidth)
void transferSuccessorsAndUpdatePHIs(MachineBasicBlock *FromMBB)
Transfers all the successors, as in transferSuccessors, and update PHI operands in the successor bloc...
instr_iterator insert(instr_iterator I, MachineInstr *M)
Insert MI into the instruction list before I, possibly inside a bundle.
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
iterator insertAfter(iterator I, MachineInstr *MI)
Insert MI into the instruction list after I.
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it.
void setFrameAddressIsTaken(bool T)
unsigned getFunctionNumber() const
getFunctionNumber - Return a unique ID for the current function.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineInstr * CreateMachineInstr(const MCInstrDesc &MCID, DebugLoc DL, bool NoImplicit=false)
CreateMachineInstr - Allocate a new MachineInstr.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
const char * createExternalSymbolName(StringRef Name)
Allocate a string and populate it with the given external symbol name.
MCContext & getContext() const
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
Function & getFunction()
Return the LLVM function that this machine code represents.
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
const MachineJumpTableInfo * getJumpTableInfo() const
getJumpTableInfo - Return the jump table info object for the current function.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & addSym(MCSymbol *Sym, unsigned char TargetFlags=0) const
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addFPImm(const ConstantFP *Val) const
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
MachineInstr * getInstr() const
If conversion operators fail, use this method to get the MachineInstr explicitly.
Representation of each machine instruction.
Definition: MachineInstr.h:68
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
Definition: MachineInstr.h:516
iterator_range< mop_iterator > uses()
Returns a range that includes all operands that are register uses.
Definition: MachineInstr.h:689
void addOperand(MachineFunction &MF, const MachineOperand &Op)
Add the specified operand to the instruction.
iterator_range< mop_iterator > defs()
Returns a range over all explicit operands that are register definitions.
Definition: MachineInstr.h:678
void eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
void removeOperand(unsigned OpNo)
Erase an operand from an instruction, leaving it with one fewer operand than it started with.
const MachineOperand & getOperand(unsigned i) const
Definition: MachineInstr.h:526
const std::vector< MachineJumpTableEntry > & getJumpTables() const
Flags
Flags values. These may be or'd together.
@ MOVolatile
The memory access is volatile.
@ MOLoad
The memory access reads data.
@ MOStore
The memory access writes data.
bool isReg() const
isReg - Tests if this is a MO_Register operand.
Register getReg() const
getReg - Returns the register number.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
Register createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name="")
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
void addLiveIn(MCRegister Reg, Register vreg=Register())
addLiveIn - Add the specified register as a live-in.
unsigned getAddressSpace() const
Return the address space for the associated pointer.
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
const SDValue & getChain() const
EVT getMemoryVT() const
Return the type of the in-memory value.
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
ArrayRef< SDUse > ops() const
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
const SDValue & getOperand(unsigned Num) const
uint64_t getConstantOperandVal(unsigned Num) const
Helper method returns the integer value of a ConstantSDNode operand.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
bool isUndef() const
Return true if the type of the node type undefined.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
bool isUndef() const
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
const SDValue & getOperand(unsigned i) const
MVT getSimpleValueType() const
Return the simple ValueType of the referenced return value.
unsigned getOpcode() const
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:225
SDValue getTargetGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, unsigned TargetFlags=0)
Definition: SelectionDAG.h:721
SDValue getMergeValues(ArrayRef< SDValue > Ops, const SDLoc &dl)
Create a MERGE_VALUES node from the given operands.
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
SDValue getSplatValue(SDValue V, bool LegalTypes=false)
If V is a splat vector, return its scalar source operand by extracting that element from the source v...
MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
void ExtractVectorElements(SDValue Op, SmallVectorImpl< SDValue > &Args, unsigned Start=0, unsigned Count=0, EVT EltVT=EVT())
Append the extracted elements from Start to Count out of the vector Op in Args.
SDValue UnrollVectorOp(SDNode *N, unsigned ResNE=0)
Utility function used by legalize and lowering to "unroll" a vector operation by splitting out the sc...
SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, bool isTarget=false)
Create a ConstantFPSDNode wrapping a constant value.
SDValue getTargetJumpTable(int JTI, EVT VT, unsigned TargetFlags=0)
Definition: SelectionDAG.h:731
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
Definition: SelectionDAG.h:828
SDValue getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVol, bool AlwaysInline, bool isTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo=AAMDNodes(), AAResults *AA=nullptr)
SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
const DataLayout & getDataLayout() const
Definition: SelectionDAG.h:472
SDValue getTargetFrameIndex(int FI, EVT VT)
Definition: SelectionDAG.h:726
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
SDValue getBasicBlock(MachineBasicBlock *MBB)
const TargetMachine & getTarget() const
Definition: SelectionDAG.h:473
SDValue getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either any-extending or truncat...
SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, unsigned Reg, SDValue N)
Definition: SelectionDAG.h:773
SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef< SDValue > Ops, EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags Flags=MachineMemOperand::MOLoad|MachineMemOperand::MOStore, uint64_t Size=0, const AAMDNodes &AAInfo=AAMDNodes())
Creates a MemIntrinsicNode that may produce a result and takes a list of operands.
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
SDValue getValueType(EVT)
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
Definition: SelectionDAG.h:675
MachineFunction & getMachineFunction() const
Definition: SelectionDAG.h:469
SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, unsigned Reg, EVT VT)
Definition: SelectionDAG.h:799
SDValue getSplatBuildVector(EVT VT, const SDLoc &DL, SDValue Op)
Return a splat ISD::BUILD_VECTOR node, consisting of Op splatted to all elements.
Definition: SelectionDAG.h:845
SDValue getFrameIndex(int FI, EVT VT, bool isTarget=false)
LLVMContext * getContext() const
Definition: SelectionDAG.h:485
SDValue getTargetExternalSymbol(const char *Sym, EVT VT, unsigned TargetFlags=0)
SDValue getMCSymbol(MCSymbol *Sym, EVT VT)
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
Definition: SelectionDAG.h:554
SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
bool empty() const
Definition: SmallVector.h:94
size_t size() const
Definition: SmallVector.h:91
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: SmallVector.h:577
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
Definition: SmallVector.h:687
void push_back(const T &Elt)
Definition: SmallVector.h:416
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1200
This class is used to represent ISD::STORE nodes.
const SDValue & getBasePtr() const
const SDValue & getOffset() const
const SDValue & getValue() const
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
constexpr size_t size() const
size - Get the string size.
Definition: StringRef.h:137
TargetInstrInfo - Interface to description of machine instruction set.
Provides information about what library functions are available for the current target.
void setBooleanVectorContents(BooleanContent Ty)
Specify how the target extends the result of a vector boolean value from a vector of i1 to a wider ty...
void setOperationAction(unsigned Op, MVT VT, LegalizeAction Action)
Indicate that the specified operation does not work with the specified type and indicate what to do a...
virtual const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent=false) const
Return the register class that should be used for the specified value type.
const TargetMachine & getTargetMachine() const
LegalizeTypeAction
This enum indicates whether a types are legal for a target, and if not, what action should be used to...
void setMaxAtomicSizeInBitsSupported(unsigned SizeInBits)
Set the maximum atomic operation size supported by the backend.
virtual TargetLoweringBase::LegalizeTypeAction getPreferredVectorAction(MVT VT) const
Return the preferred vector type legalization action.
void setBooleanContents(BooleanContent Ty)
Specify how the target extends the result of integer and floating point boolean values from i1 to a w...
void computeRegisterProperties(const TargetRegisterInfo *TRI)
Once all of the register classes are added, this allows us to compute derived properties we expose.
void addRegisterClass(MVT VT, const TargetRegisterClass *RC)
Add the specified register class as an available regclass for the specified value type.
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
void setLibcallName(RTLIB::Libcall Call, const char *Name)
Rename the default libcall routine name for the specified libcall.
void setMinimumJumpTableEntries(unsigned Val)
Indicate the minimum number of blocks to generate jump tables.
void setTruncStoreAction(MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified truncating store does not work with the specified type and indicate what ...
void setStackPointerRegisterToSaveRestore(Register R)
If set to a physical register, this specifies the register that llvm.savestack/llvm....
AtomicExpansionKind
Enum that specifies what an atomic load/AtomicRMWInst is expanded to, if at all.
void setCondCodeAction(ArrayRef< ISD::CondCode > CCs, MVT VT, LegalizeAction Action)
Indicate that the specified condition code is or isn't supported on the target and indicate what to d...
void setTargetDAGCombine(ArrayRef< ISD::NodeType > NTs)
Targets should invoke this method for each target independent node that they want to provide a custom...
void setLoadExtAction(unsigned ExtType, MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified load with extension does not work with the specified type and indicate wh...
virtual MVT getPointerMemTy(const DataLayout &DL, uint32_t AS=0) const
Return the in-memory pointer type for the given address space, defaults to the pointer type from the ...
void setSchedulingPreference(Sched::Preference Pref)
Specify the target scheduling preference.
bool isOperationLegalOrCustomOrPromote(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
std::pair< SDValue, SDValue > makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT, ArrayRef< SDValue > Ops, MakeLibCallOptions CallOptions, const SDLoc &dl, SDValue Chain=SDValue()) const
Returns a pair of (return value, chain).
bool isPositionIndependent() const
virtual std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const
Given a physical register constraint (e.g.
bool verifyReturnAddressArgumentIsConstant(SDValue Op, SelectionDAG &DAG) const
virtual bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const
Return true if folding a constant offset with the given GlobalAddress is legal.
Primary interface to the complete machine description for the target machine.
Definition: TargetMachine.h:78
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
bool isOSEmscripten() const
Tests whether the OS is Emscripten.
Definition: Triple.h:658
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
static Type * getDoubleTy(LLVMContext &C)
bool isFunctionTy() const
True if this is an instance of FunctionType.
Definition: Type.h:249
static Type * getFloatTy(LLVMContext &C)
A Use represents the edge between a Value definition and its users.
Definition: Use.h:43
LLVM Value Representation.
Definition: Value.h:74
const Value * stripPointerCastsAndAliases() const
Strip off pointer casts, all-zero GEPs, address space casts, and aliases.
Definition: Value.cpp:689
static std::optional< unsigned > getLocalForStackObject(MachineFunction &MF, int FrameIndex)
This class is derived from MachineFunctionInfo and contains private WebAssembly-specific information ...
Register getFrameRegister(const MachineFunction &MF) const override
const Triple & getTargetTriple() const
const WebAssemblyInstrInfo * getInstrInfo() const override
const WebAssemblyRegisterInfo * getRegisterInfo() const override
WebAssemblyTargetLowering(const TargetMachine &TM, const WebAssemblySubtarget &STI)
MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const override
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
MVT getPointerMemTy(const DataLayout &DL, uint32_t AS=0) const override
Return the in-memory pointer type for the given address space, defaults to the pointer type from the ...
self_iterator getIterator()
Definition: ilist_node.h:82
#define INT64_MIN
Definition: DataTypes.h:74
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
const CustomOperand< const MCSubtargetInfo & > Msg[]
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition: BitmaskEnum.h:119
@ Swift
Calling convention for Swift.
Definition: CallingConv.h:69
@ PreserveMost
Used for runtime calls that preserves most registers.
Definition: CallingConv.h:63
@ CXX_FAST_TLS
Used for access functions.
Definition: CallingConv.h:72
@ WASM_EmscriptenInvoke
For emscripten __invoke_* functions.
Definition: CallingConv.h:226
@ Cold
Attempts to make code in the caller as efficient as possible under the assumption that the call is no...
Definition: CallingConv.h:47
@ PreserveAll
Used for runtime calls that preserves (almost) all registers.
Definition: CallingConv.h:66
@ Fast
Attempts to make calls as fast as possible (e.g.
Definition: CallingConv.h:41
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
Definition: ISDOpcodes.h:749
@ STACKRESTORE
STACKRESTORE has two operands, an input chain and a pointer to restore to it returns an output chain.
Definition: ISDOpcodes.h:1069
@ STACKSAVE
STACKSAVE - STACKSAVE has one operand, an input chain.
Definition: ISDOpcodes.h:1065
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
Definition: ISDOpcodes.h:250
@ BSWAP
Byte Swap and Counting operators.
Definition: ISDOpcodes.h:713
@ VAEND
VAEND, VASTART - VAEND and VASTART have three operands: an input chain, pointer, and a SRCVALUE.
Definition: ISDOpcodes.h:1098
@ ConstantFP
Definition: ISDOpcodes.h:77
@ ADDC
Carry-setting nodes for multiple precision addition and subtraction.
Definition: ISDOpcodes.h:269
@ ADD
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:239
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
Definition: ISDOpcodes.h:978
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
Definition: ISDOpcodes.h:482
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
Definition: ISDOpcodes.h:199
@ RETURNADDR
Definition: ISDOpcodes.h:95
@ GlobalAddress
Definition: ISDOpcodes.h:78
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition: ISDOpcodes.h:786
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
Definition: ISDOpcodes.h:542
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
Definition: ISDOpcodes.h:687
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
Definition: ISDOpcodes.h:255
@ FP16_TO_FP
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
Definition: ISDOpcodes.h:908
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:898
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
Definition: ISDOpcodes.h:229
@ BUILTIN_OP_END
BUILTIN_OP_END - This must be the last enum value in this list.
Definition: ISDOpcodes.h:1324
@ GlobalTLSAddress
Definition: ISDOpcodes.h:79
@ FrameIndex
Definition: ISDOpcodes.h:80
@ SIGN_EXTEND
Conversion operators.
Definition: ISDOpcodes.h:773
@ FSINCOS
FSINCOS - Compute both fsin and fcos as a single operation.
Definition: ISDOpcodes.h:972
@ BR_CC
BR_CC - Conditional branch.
Definition: ISDOpcodes.h:1020
@ BRIND
BRIND - Indirect branch.
Definition: ISDOpcodes.h:999
@ BR_JT
BR_JT - Jumptable branch.
Definition: ISDOpcodes.h:1003
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
Definition: ISDOpcodes.h:626
@ VACOPY
VACOPY - VACOPY has 5 operands: an input chain, a destination pointer, a source pointer,...
Definition: ISDOpcodes.h:1094
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
Definition: ISDOpcodes.h:650
@ SHL
Shift and rotation operations.
Definition: ISDOpcodes.h:704
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
Definition: ISDOpcodes.h:599
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
Definition: ISDOpcodes.h:572
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition: ISDOpcodes.h:534
@ CopyToReg
CopyToReg - This node has three operands: a chain, a register number to set to this value,...
Definition: ISDOpcodes.h:203
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:776
@ DEBUGTRAP
DEBUGTRAP - Trap intended to get the attention of a debugger.
Definition: ISDOpcodes.h:1148
@ FP_TO_UINT_SAT
Definition: ISDOpcodes.h:852
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
Definition: ISDOpcodes.h:741
@ DYNAMIC_STACKALLOC
DYNAMIC_STACKALLOC - Allocate some number of bytes on the stack aligned to a specified boundary.
Definition: ISDOpcodes.h:988
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
Definition: ISDOpcodes.h:794
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
Definition: ISDOpcodes.h:673
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
Definition: ISDOpcodes.h:883
@ FRAMEADDR
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG.
Definition: ISDOpcodes.h:94
@ FMINIMUM
FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 as less than 0....
Definition: ISDOpcodes.h:968
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:832
@ AND
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:679
@ TRAP
TRAP - Trapping instruction.
Definition: ISDOpcodes.h:1145
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
Definition: ISDOpcodes.h:184
@ ADDE
Carry-using nodes for multiple precision addition and subtraction.
Definition: ISDOpcodes.h:279
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
Definition: ISDOpcodes.h:523
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
Definition: ISDOpcodes.h:52
@ ExternalSymbol
Definition: ISDOpcodes.h:83
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
Definition: ISDOpcodes.h:865
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
Definition: ISDOpcodes.h:851
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
Definition: ISDOpcodes.h:782
@ VAARG
VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE, and the alignment.
Definition: ISDOpcodes.h:1089
@ BlockAddress
Definition: ISDOpcodes.h:84
@ SHL_PARTS
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations.
Definition: ISDOpcodes.h:762
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
Definition: ISDOpcodes.h:492
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
Definition: ISDOpcodes.h:340
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
Definition: ISDOpcodes.h:192
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
Definition: ISDOpcodes.h:514
bool isConstantSplatVector(const SDNode *N, APInt &SplatValue)
Node predicates.
OperandFlags
These are flags set on operands, but should be considered private, all access should go through the M...
Definition: MCInstrDesc.h:50
MCSymbolWasm * getOrCreateFunctionTableSymbol(MCContext &Ctx, const WebAssemblySubtarget *Subtarget)
Returns the __indirect_function_table, for use in call_indirect and in function bitcasts.
MCSymbolWasm * getOrCreateFuncrefCallTableSymbol(MCContext &Ctx, const WebAssemblySubtarget *Subtarget)
Returns the __funcref_call_table, for use in funcref calls when lowered to table.set + call_indirect.
bool isFuncrefType(const Type *Ty)
FastISel * createFastISel(FunctionLoweringInfo &funcInfo, const TargetLibraryInfo *libInfo)
bool isValidAddressSpace(unsigned AS)
bool isWasmVarAddressSpace(unsigned AS)
Reg
All possible values of the reg field in the ModR/M byte.
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
Definition: STLExtras.h:413
unsigned Log2_32_Ceil(uint32_t Value)
Return the ceil log base 2 of the specified value, 32 if the value is zero.
Definition: MathExtras.h:395
@ Offset
Definition: DWP.cpp:406
void computeSignatureVTs(const FunctionType *Ty, const Function *TargetFunc, const Function &ContextFunc, const TargetMachine &TM, SmallVectorImpl< MVT > &Params, SmallVectorImpl< MVT > &Results)
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition: MathExtras.h:292
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:145
@ And
Bitwise or logical AND of integers.
@ Add
Sum of integers.
constexpr unsigned BitWidth
Definition: BitmaskEnum.h:184
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1846
void erase_if(Container &C, UnaryPredicate P)
Provide a container algorithm similar to C++ Library Fundamentals v2's erase_if which is equivalent t...
Definition: STLExtras.h:2113
void computeLegalValueVTs(const WebAssemblyTargetLowering &TLI, LLVMContext &Ctx, const DataLayout &DL, Type *Ty, SmallVectorImpl< MVT > &ValueVTs)
constexpr uint64_t NextPowerOf2(uint64_t A)
Returns the next power of two (in 64-bits) that is strictly greater than A.
Definition: MathExtras.h:450
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:860
#define N
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
Extended Value Type.
Definition: ValueTypes.h:34
EVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
Definition: ValueTypes.h:93
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
Definition: ValueTypes.h:129
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
Definition: ValueTypes.h:73
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
Definition: ValueTypes.h:139
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:351
uint64_t getScalarSizeInBits() const
Definition: ValueTypes.h:363
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition: ValueTypes.h:299
bool is128BitVector() const
Return true if this is a 128-bit vector type.
Definition: ValueTypes.h:196
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
Definition: ValueTypes.h:64
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
Definition: ValueTypes.h:359
bool isVector() const
Return true if this is a vector value type.
Definition: ValueTypes.h:160
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Definition: ValueTypes.h:306
bool bitsGE(EVT VT) const
Return true if this has no less bits than VT.
Definition: ValueTypes.h:275
bool is256BitVector() const
Return true if this is a 256-bit vector type.
Definition: ValueTypes.h:201
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
Definition: ValueTypes.cpp:194
EVT getVectorElementType() const
Given a vector type, return the type of each element.
Definition: ValueTypes.h:311
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
Definition: ValueTypes.h:319
bool isInConsecutiveRegs() const
Align getNonZeroOrigAlign() const
unsigned getByValSize() const
bool isInConsecutiveRegsLast() const
Align getNonZeroByValAlign() const
InputArg - This struct carries flags and type information about a single incoming (formal) argument o...
OutputArg - This struct carries flags and a value for a single outgoing (actual) argument or outgoing...
bool IsFixed
IsFixed - Is this a "fixed" value, ie not passed through a vararg "...".
unsigned getBitWidth() const
Get the bit width of this value.
Definition: KnownBits.h:40
This class contains a discriminated union of information about pointers in memory operands,...
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
Function object to check whether the second component of a container supported by std::get (like std:...
Definition: STLExtras.h:1546