summaryrefslogtreecommitdiff
path: root/lib/Target/ARM64/ARM64InstrInfo.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'lib/Target/ARM64/ARM64InstrInfo.cpp')
-rw-r--r--lib/Target/ARM64/ARM64InstrInfo.cpp1864
1 files changed, 1864 insertions, 0 deletions
diff --git a/lib/Target/ARM64/ARM64InstrInfo.cpp b/lib/Target/ARM64/ARM64InstrInfo.cpp
new file mode 100644
index 0000000000..8f117573fd
--- /dev/null
+++ b/lib/Target/ARM64/ARM64InstrInfo.cpp
@@ -0,0 +1,1864 @@
+//===- ARM64InstrInfo.cpp - ARM64 Instruction Information -----------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains the ARM64 implementation of the TargetInstrInfo class.
+//
+//===----------------------------------------------------------------------===//
+
+#include "ARM64InstrInfo.h"
+#include "ARM64Subtarget.h"
+#include "MCTargetDesc/ARM64AddressingModes.h"
+#include "llvm/CodeGen/MachineFrameInfo.h"
+#include "llvm/CodeGen/MachineInstrBuilder.h"
+#include "llvm/CodeGen/MachineMemOperand.h"
+#include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/CodeGen/PseudoSourceValue.h"
+#include "llvm/MC/MCInst.h"
+#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/TargetRegistry.h"
+
+#define GET_INSTRINFO_CTOR_DTOR
+#include "ARM64GenInstrInfo.inc"
+
+using namespace llvm;
+
+ARM64InstrInfo::ARM64InstrInfo(const ARM64Subtarget &STI)
+ : ARM64GenInstrInfo(ARM64::ADJCALLSTACKDOWN, ARM64::ADJCALLSTACKUP),
+ RI(this, &STI), Subtarget(STI) {}
+
+/// GetInstSize - Return the number of bytes of code the specified
+/// instruction may be. This returns the maximum number of bytes.
+unsigned ARM64InstrInfo::GetInstSizeInBytes(const MachineInstr *MI) const {
+ const MCInstrDesc &Desc = MI->getDesc();
+
+ switch (Desc.getOpcode()) {
+ default:
+ // Anything not explicitly designated otherwise is a nomal 4-byte insn.
+ return 4;
+ case TargetOpcode::DBG_VALUE:
+ case TargetOpcode::EH_LABEL:
+ case TargetOpcode::IMPLICIT_DEF:
+ case TargetOpcode::KILL:
+ return 0;
+ }
+
+ llvm_unreachable("GetInstSizeInBytes()- Unable to determin insn size");
+}
+
+static void parseCondBranch(MachineInstr *LastInst, MachineBasicBlock *&Target,
+ SmallVectorImpl<MachineOperand> &Cond) {
+ // Block ends with fall-through condbranch.
+ switch (LastInst->getOpcode()) {
+ default:
+ llvm_unreachable("Unknown branch instruction?");
+ case ARM64::Bcc:
+ Target = LastInst->getOperand(1).getMBB();
+ Cond.push_back(LastInst->getOperand(0));
+ break;
+ case ARM64::CBZW:
+ case ARM64::CBZX:
+ case ARM64::CBNZW:
+ case ARM64::CBNZX:
+ Target = LastInst->getOperand(1).getMBB();
+ Cond.push_back(MachineOperand::CreateImm(-1));
+ Cond.push_back(MachineOperand::CreateImm(LastInst->getOpcode()));
+ Cond.push_back(LastInst->getOperand(0));
+ break;
+ case ARM64::TBZ:
+ case ARM64::TBNZ:
+ Target = LastInst->getOperand(2).getMBB();
+ Cond.push_back(MachineOperand::CreateImm(-1));
+ Cond.push_back(MachineOperand::CreateImm(LastInst->getOpcode()));
+ Cond.push_back(LastInst->getOperand(0));
+ Cond.push_back(LastInst->getOperand(1));
+ }
+}
+
+// Branch analysis.
+bool ARM64InstrInfo::AnalyzeBranch(MachineBasicBlock &MBB,
+ MachineBasicBlock *&TBB,
+ MachineBasicBlock *&FBB,
+ SmallVectorImpl<MachineOperand> &Cond,
+ bool AllowModify) const {
+ // If the block has no terminators, it just falls into the block after it.
+ MachineBasicBlock::iterator I = MBB.end();
+ if (I == MBB.begin())
+ return false;
+ --I;
+ while (I->isDebugValue()) {
+ if (I == MBB.begin())
+ return false;
+ --I;
+ }
+ if (!isUnpredicatedTerminator(I))
+ return false;
+
+ // Get the last instruction in the block.
+ MachineInstr *LastInst = I;
+
+ // If there is only one terminator instruction, process it.
+ unsigned LastOpc = LastInst->getOpcode();
+ if (I == MBB.begin() || !isUnpredicatedTerminator(--I)) {
+ if (isUncondBranchOpcode(LastOpc)) {
+ TBB = LastInst->getOperand(0).getMBB();
+ return false;
+ }
+ if (isCondBranchOpcode(LastOpc)) {
+ // Block ends with fall-through condbranch.
+ parseCondBranch(LastInst, TBB, Cond);
+ return false;
+ }
+ return true; // Can't handle indirect branch.
+ }
+
+ // Get the instruction before it if it is a terminator.
+ MachineInstr *SecondLastInst = I;
+ unsigned SecondLastOpc = SecondLastInst->getOpcode();
+
+ // If AllowModify is true and the block ends with two or more unconditional
+ // branches, delete all but the first unconditional branch.
+ if (AllowModify && isUncondBranchOpcode(LastOpc)) {
+ while (isUncondBranchOpcode(SecondLastOpc)) {
+ LastInst->eraseFromParent();
+ LastInst = SecondLastInst;
+ LastOpc = LastInst->getOpcode();
+ if (I == MBB.begin() || !isUnpredicatedTerminator(--I)) {
+ // Return now the only terminator is an unconditional branch.
+ TBB = LastInst->getOperand(0).getMBB();
+ return false;
+ } else {
+ SecondLastInst = I;
+ SecondLastOpc = SecondLastInst->getOpcode();
+ }
+ }
+ }
+
+ // If there are three terminators, we don't know what sort of block this is.
+ if (SecondLastInst && I != MBB.begin() && isUnpredicatedTerminator(--I))
+ return true;
+
+ // If the block ends with a B and a Bcc, handle it.
+ if (isCondBranchOpcode(SecondLastOpc) && isUncondBranchOpcode(LastOpc)) {
+ parseCondBranch(SecondLastInst, TBB, Cond);
+ FBB = LastInst->getOperand(0).getMBB();
+ return false;
+ }
+
+ // If the block ends with two unconditional branches, handle it. The second
+ // one is not executed, so remove it.
+ if (isUncondBranchOpcode(SecondLastOpc) && isUncondBranchOpcode(LastOpc)) {
+ TBB = SecondLastInst->getOperand(0).getMBB();
+ I = LastInst;
+ if (AllowModify)
+ I->eraseFromParent();
+ return false;
+ }
+
+ // ...likewise if it ends with an indirect branch followed by an unconditional
+ // branch.
+ if (isIndirectBranchOpcode(SecondLastOpc) && isUncondBranchOpcode(LastOpc)) {
+ I = LastInst;
+ if (AllowModify)
+ I->eraseFromParent();
+ return true;
+ }
+
+ // Otherwise, can't handle this.
+ return true;
+}
+
+bool ARM64InstrInfo::ReverseBranchCondition(
+ SmallVectorImpl<MachineOperand> &Cond) const {
+ if (Cond[0].getImm() != -1) {
+ // Regular Bcc
+ ARM64CC::CondCode CC = (ARM64CC::CondCode)(int)Cond[0].getImm();
+ Cond[0].setImm(ARM64CC::getInvertedCondCode(CC));
+ } else {
+ // Folded compare-and-branch
+ switch (Cond[1].getImm()) {
+ default:
+ llvm_unreachable("Unknown conditional branch!");
+ case ARM64::CBZW:
+ Cond[1].setImm(ARM64::CBNZW);
+ break;
+ case ARM64::CBNZW:
+ Cond[1].setImm(ARM64::CBZW);
+ break;
+ case ARM64::CBZX:
+ Cond[1].setImm(ARM64::CBNZX);
+ break;
+ case ARM64::CBNZX:
+ Cond[1].setImm(ARM64::CBZX);
+ break;
+ case ARM64::TBZ:
+ Cond[1].setImm(ARM64::TBNZ);
+ break;
+ case ARM64::TBNZ:
+ Cond[1].setImm(ARM64::TBZ);
+ break;
+ }
+ }
+
+ return false;
+}
+
+unsigned ARM64InstrInfo::RemoveBranch(MachineBasicBlock &MBB) const {
+ MachineBasicBlock::iterator I = MBB.end();
+ if (I == MBB.begin())
+ return 0;
+ --I;
+ while (I->isDebugValue()) {
+ if (I == MBB.begin())
+ return 0;
+ --I;
+ }
+ if (!isUncondBranchOpcode(I->getOpcode()) &&
+ !isCondBranchOpcode(I->getOpcode()))
+ return 0;
+
+ // Remove the branch.
+ I->eraseFromParent();
+
+ I = MBB.end();
+
+ if (I == MBB.begin())
+ return 1;
+ --I;
+ if (!isCondBranchOpcode(I->getOpcode()))
+ return 1;
+
+ // Remove the branch.
+ I->eraseFromParent();
+ return 2;
+}
+
+void ARM64InstrInfo::instantiateCondBranch(
+ MachineBasicBlock &MBB, DebugLoc DL, MachineBasicBlock *TBB,
+ const SmallVectorImpl<MachineOperand> &Cond) const {
+ if (Cond[0].getImm() != -1) {
+ // Regular Bcc
+ BuildMI(&MBB, DL, get(ARM64::Bcc)).addImm(Cond[0].getImm()).addMBB(TBB);
+ } else {
+ // Folded compare-and-branch
+ const MachineInstrBuilder MIB =
+ BuildMI(&MBB, DL, get(Cond[1].getImm())).addReg(Cond[2].getReg());
+ if (Cond.size() > 3)
+ MIB.addImm(Cond[3].getImm());
+ MIB.addMBB(TBB);
+ }
+}
+
+unsigned ARM64InstrInfo::InsertBranch(
+ MachineBasicBlock &MBB, MachineBasicBlock *TBB, MachineBasicBlock *FBB,
+ const SmallVectorImpl<MachineOperand> &Cond, DebugLoc DL) const {
+ // Shouldn't be a fall through.
+ assert(TBB && "InsertBranch must not be told to insert a fallthrough");
+
+ if (FBB == 0) {
+ if (Cond.empty()) // Unconditional branch?
+ BuildMI(&MBB, DL, get(ARM64::B)).addMBB(TBB);
+ else
+ instantiateCondBranch(MBB, DL, TBB, Cond);
+ return 1;
+ }
+
+ // Two-way conditional branch.
+ instantiateCondBranch(MBB, DL, TBB, Cond);
+ BuildMI(&MBB, DL, get(ARM64::B)).addMBB(FBB);
+ return 2;
+}
+
+// Find the original register that VReg is copied from.
+static unsigned removeCopies(const MachineRegisterInfo &MRI, unsigned VReg) {
+ while (TargetRegisterInfo::isVirtualRegister(VReg)) {
+ const MachineInstr *DefMI = MRI.getVRegDef(VReg);
+ if (!DefMI->isFullCopy())
+ return VReg;
+ VReg = DefMI->getOperand(1).getReg();
+ }
+ return VReg;
+}
+
+// Determine if VReg is defined by an instruction that can be folded into a
+// csel instruction. If so, return the folded opcode, and the replacement
+// register.
+static unsigned canFoldIntoCSel(const MachineRegisterInfo &MRI, unsigned VReg,
+ unsigned *NewVReg = 0) {
+ VReg = removeCopies(MRI, VReg);
+ if (!TargetRegisterInfo::isVirtualRegister(VReg))
+ return 0;
+
+ bool Is64Bit = ARM64::GPR64allRegClass.hasSubClassEq(MRI.getRegClass(VReg));
+ const MachineInstr *DefMI = MRI.getVRegDef(VReg);
+ unsigned Opc = 0;
+ unsigned SrcOpNum = 0;
+ switch (DefMI->getOpcode()) {
+ case ARM64::ADDSXri:
+ case ARM64::ADDSWri:
+ // if CPSR is used, do not fold.
+ if (DefMI->findRegisterDefOperandIdx(ARM64::CPSR, true) == -1)
+ return 0;
+ // fall-through to ADDXri and ADDWri.
+ case ARM64::ADDXri:
+ case ARM64::ADDWri:
+ // add x, 1 -> csinc.
+ if (!DefMI->getOperand(2).isImm() || DefMI->getOperand(2).getImm() != 1 ||
+ DefMI->getOperand(3).getImm() != 0)
+ return 0;
+ SrcOpNum = 1;
+ Opc = Is64Bit ? ARM64::CSINCXr : ARM64::CSINCWr;
+ break;
+
+ case ARM64::ORNXrr:
+ case ARM64::ORNWrr: {
+ // not x -> csinv, represented as orn dst, xzr, src.
+ unsigned ZReg = removeCopies(MRI, DefMI->getOperand(1).getReg());
+ if (ZReg != ARM64::XZR && ZReg != ARM64::WZR)
+ return 0;
+ SrcOpNum = 2;
+ Opc = Is64Bit ? ARM64::CSINVXr : ARM64::CSINVWr;
+ break;
+ }
+
+ case ARM64::SUBSXrr:
+ case ARM64::SUBSWrr:
+ // if CPSR is used, do not fold.
+ if (DefMI->findRegisterDefOperandIdx(ARM64::CPSR, true) == -1)
+ return 0;
+ // fall-through to SUBXrr and SUBWrr.
+ case ARM64::SUBXrr:
+ case ARM64::SUBWrr: {
+ // neg x -> csneg, represented as sub dst, xzr, src.
+ unsigned ZReg = removeCopies(MRI, DefMI->getOperand(1).getReg());
+ if (ZReg != ARM64::XZR && ZReg != ARM64::WZR)
+ return 0;
+ SrcOpNum = 2;
+ Opc = Is64Bit ? ARM64::CSNEGXr : ARM64::CSNEGWr;
+ break;
+ }
+ default:
+ return 0;
+ }
+ assert(Opc && SrcOpNum && "Missing parameters");
+
+ if (NewVReg)
+ *NewVReg = DefMI->getOperand(SrcOpNum).getReg();
+ return Opc;
+}
+
+bool ARM64InstrInfo::canInsertSelect(
+ const MachineBasicBlock &MBB, const SmallVectorImpl<MachineOperand> &Cond,
+ unsigned TrueReg, unsigned FalseReg, int &CondCycles, int &TrueCycles,
+ int &FalseCycles) const {
+ // Check register classes.
+ const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
+ const TargetRegisterClass *RC =
+ RI.getCommonSubClass(MRI.getRegClass(TrueReg), MRI.getRegClass(FalseReg));
+ if (!RC)
+ return false;
+
+ // Expanding cbz/tbz requires an extra cycle of latency on the condition.
+ unsigned ExtraCondLat = Cond.size() != 1;
+
+ // GPRs are handled by csel.
+ // FIXME: Fold in x+1, -x, and ~x when applicable.
+ if (ARM64::GPR64allRegClass.hasSubClassEq(RC) ||
+ ARM64::GPR32allRegClass.hasSubClassEq(RC)) {
+ // Single-cycle csel, csinc, csinv, and csneg.
+ CondCycles = 1 + ExtraCondLat;
+ TrueCycles = FalseCycles = 1;
+ if (canFoldIntoCSel(MRI, TrueReg))
+ TrueCycles = 0;
+ else if (canFoldIntoCSel(MRI, FalseReg))
+ FalseCycles = 0;
+ return true;
+ }
+
+ // Scalar floating point is handled by fcsel.
+ // FIXME: Form fabs, fmin, and fmax when applicable.
+ if (ARM64::FPR64RegClass.hasSubClassEq(RC) ||
+ ARM64::FPR32RegClass.hasSubClassEq(RC)) {
+ CondCycles = 5 + ExtraCondLat;
+ TrueCycles = FalseCycles = 2;
+ return true;
+ }
+
+ // Can't do vectors.
+ return false;
+}
+
+void ARM64InstrInfo::insertSelect(MachineBasicBlock &MBB,
+ MachineBasicBlock::iterator I, DebugLoc DL,
+ unsigned DstReg,
+ const SmallVectorImpl<MachineOperand> &Cond,
+ unsigned TrueReg, unsigned FalseReg) const {
+ MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
+
+ // Parse the condition code, see parseCondBranch() above.
+ ARM64CC::CondCode CC;
+ switch (Cond.size()) {
+ default:
+ llvm_unreachable("Unknown condition opcode in Cond");
+ case 1: // b.cc
+ CC = ARM64CC::CondCode(Cond[0].getImm());
+ break;
+ case 3: { // cbz/cbnz
+ // We must insert a compare against 0.
+ bool Is64Bit;
+ switch (Cond[1].getImm()) {
+ default:
+ llvm_unreachable("Unknown branch opcode in Cond");
+ case ARM64::CBZW:
+ Is64Bit = 0;
+ CC = ARM64CC::EQ;
+ break;
+ case ARM64::CBZX:
+ Is64Bit = 1;
+ CC = ARM64CC::EQ;
+ break;
+ case ARM64::CBNZW:
+ Is64Bit = 0;
+ CC = ARM64CC::NE;
+ break;
+ case ARM64::CBNZX:
+ Is64Bit = 1;
+ CC = ARM64CC::NE;
+ break;
+ }
+ unsigned SrcReg = Cond[2].getReg();
+ if (Is64Bit) {
+ // cmp reg, #0 is actually subs xzr, reg, #0.
+ MRI.constrainRegClass(SrcReg, &ARM64::GPR64spRegClass);
+ BuildMI(MBB, I, DL, get(ARM64::SUBSXri), ARM64::XZR)
+ .addReg(SrcReg)
+ .addImm(0)
+ .addImm(0);
+ } else {
+ MRI.constrainRegClass(SrcReg, &ARM64::GPR32spRegClass);
+ BuildMI(MBB, I, DL, get(ARM64::SUBSWri), ARM64::WZR)
+ .addReg(SrcReg)
+ .addImm(0)
+ .addImm(0);
+ }
+ break;
+ }
+ case 4: { // tbz/tbnz
+ // We must insert a tst instruction.
+ switch (Cond[1].getImm()) {
+ default:
+ llvm_unreachable("Unknown branch opcode in Cond");
+ case ARM64::TBZ:
+ CC = ARM64CC::EQ;
+ break;
+ case ARM64::TBNZ:
+ CC = ARM64CC::NE;
+ break;
+ }
+ // cmp reg, #foo is actually ands xzr, reg, #1<<foo.
+ BuildMI(MBB, I, DL, get(ARM64::ANDSXri), ARM64::XZR)
+ .addReg(Cond[2].getReg())
+ .addImm(ARM64_AM::encodeLogicalImmediate(1ull << Cond[3].getImm(), 64));
+ break;
+ }
+ }
+
+ unsigned Opc = 0;
+ const TargetRegisterClass *RC = 0;
+ bool TryFold = false;
+ if (MRI.constrainRegClass(DstReg, &ARM64::GPR64RegClass)) {
+ RC = &ARM64::GPR64RegClass;
+ Opc = ARM64::CSELXr;
+ TryFold = true;
+ } else if (MRI.constrainRegClass(DstReg, &ARM64::GPR32RegClass)) {
+ RC = &ARM64::GPR32RegClass;
+ Opc = ARM64::CSELWr;
+ TryFold = true;
+ } else if (MRI.constrainRegClass(DstReg, &ARM64::FPR64RegClass)) {
+ RC = &ARM64::FPR64RegClass;
+ Opc = ARM64::FCSELDrrr;
+ } else if (MRI.constrainRegClass(DstReg, &ARM64::FPR32RegClass)) {
+ RC = &ARM64::FPR32RegClass;
+ Opc = ARM64::FCSELSrrr;
+ }
+ assert(RC && "Unsupported regclass");
+
+ // Try folding simple instructions into the csel.
+ if (TryFold) {
+ unsigned NewVReg = 0;
+ unsigned FoldedOpc = canFoldIntoCSel(MRI, TrueReg, &NewVReg);
+ if (FoldedOpc) {
+ // The folded opcodes csinc, csinc and csneg apply the operation to
+ // FalseReg, so we need to invert the condition.
+ CC = ARM64CC::getInvertedCondCode(CC);
+ TrueReg = FalseReg;
+ } else
+ FoldedOpc = canFoldIntoCSel(MRI, FalseReg, &NewVReg);
+
+ // Fold the operation. Leave any dead instructions for DCE to clean up.
+ if (FoldedOpc) {
+ FalseReg = NewVReg;
+ Opc = FoldedOpc;
+ // The extends the live range of NewVReg.
+ MRI.clearKillFlags(NewVReg);
+ }
+ }
+
+ // Pull all virtual register into the appropriate class.
+ MRI.constrainRegClass(TrueReg, RC);
+ MRI.constrainRegClass(FalseReg, RC);
+
+ // Insert the csel.
+ BuildMI(MBB, I, DL, get(Opc), DstReg).addReg(TrueReg).addReg(FalseReg).addImm(
+ CC);
+}
+
+bool ARM64InstrInfo::isCoalescableExtInstr(const MachineInstr &MI,
+ unsigned &SrcReg, unsigned &DstReg,
+ unsigned &SubIdx) const {
+ switch (MI.getOpcode()) {
+ default:
+ return false;
+ case ARM64::SBFMXri: // aka sxtw
+ case ARM64::UBFMXri: // aka uxtw
+ // Check for the 32 -> 64 bit extension case, these instructions can do
+ // much more.
+ if (MI.getOperand(2).getImm() != 0 || MI.getOperand(3).getImm() != 31)
+ return false;
+ // This is a signed or unsigned 32 -> 64 bit extension.
+ SrcReg = MI.getOperand(1).getReg();
+ DstReg = MI.getOperand(0).getReg();
+ SubIdx = ARM64::sub_32;
+ return true;
+ }
+}
+
+/// analyzeCompare - For a comparison instruction, return the source registers
+/// in SrcReg and SrcReg2, and the value it compares against in CmpValue.
+/// Return true if the comparison instruction can be analyzed.
+bool ARM64InstrInfo::analyzeCompare(const MachineInstr *MI, unsigned &SrcReg,
+ unsigned &SrcReg2, int &CmpMask,
+ int &CmpValue) const {
+ switch (MI->getOpcode()) {
+ default:
+ break;
+ case ARM64::SUBSWrr:
+ case ARM64::SUBSWrs:
+ case ARM64::SUBSWrx:
+ case ARM64::SUBSXrr:
+ case ARM64::SUBSXrs:
+ case ARM64::SUBSXrx:
+ case ARM64::ADDSWrr:
+ case ARM64::ADDSWrs:
+ case ARM64::ADDSWrx:
+ case ARM64::ADDSXrr:
+ case ARM64::ADDSXrs:
+ case ARM64::ADDSXrx:
+ // Replace SUBSWrr with SUBWrr if CPSR is not used.
+ SrcReg = MI->getOperand(1).getReg();
+ SrcReg2 = MI->getOperand(2).getReg();
+ CmpMask = ~0;
+ CmpValue = 0;
+ return true;
+ case ARM64::SUBSWri:
+ case ARM64::ADDSWri:
+ case ARM64::ANDSWri:
+ case ARM64::SUBSXri:
+ case ARM64::ADDSXri:
+ case ARM64::ANDSXri:
+ SrcReg = MI->getOperand(1).getReg();
+ SrcReg2 = 0;
+ CmpMask = ~0;
+ CmpValue = MI->getOperand(2).getImm();
+ return true;
+ }
+
+ return false;
+}
+
+static bool UpdateOperandRegClass(MachineInstr *Instr) {
+ MachineBasicBlock *MBB = Instr->getParent();
+ assert(MBB && "Can't get MachineBasicBlock here");
+ MachineFunction *MF = MBB->getParent();
+ assert(MF && "Can't get MachineFunction here");
+ const TargetMachine *TM = &MF->getTarget();
+ const TargetInstrInfo *TII = TM->getInstrInfo();
+ const TargetRegisterInfo *TRI = TM->getRegisterInfo();
+ MachineRegisterInfo *MRI = &MF->getRegInfo();
+
+ for (unsigned OpIdx = 0, EndIdx = Instr->getNumOperands(); OpIdx < EndIdx;
+ ++OpIdx) {
+ MachineOperand &MO = Instr->getOperand(OpIdx);
+ const TargetRegisterClass *OpRegCstraints =
+ Instr->getRegClassConstraint(OpIdx, TII, TRI);
+
+ // If there's no constraint, there's nothing to do.
+ if (!OpRegCstraints)
+ continue;
+ // If the operand is a frame index, there's nothing to do here.
+ // A frame index operand will resolve correctly during PEI.
+ if (MO.isFI())
+ continue;
+
+ assert(MO.isReg() &&
+ "Operand has register constraints without being a register!");
+
+ unsigned Reg = MO.getReg();
+ if (TargetRegisterInfo::isPhysicalRegister(Reg)) {
+ if (!OpRegCstraints->contains(Reg))
+ return false;
+ } else if (!OpRegCstraints->hasSubClassEq(MRI->getRegClass(Reg)) &&
+ !MRI->constrainRegClass(Reg, OpRegCstraints))
+ return false;
+ }
+
+ return true;
+}
+
+/// optimizeCompareInstr - Convert the instruction supplying the argument to the
+/// comparison into one that sets the zero bit in the flags register.
+bool ARM64InstrInfo::optimizeCompareInstr(
+ MachineInstr *CmpInstr, unsigned SrcReg, unsigned SrcReg2, int CmpMask,
+ int CmpValue, const MachineRegisterInfo *MRI) const {
+
+ // Replace SUBSWrr with SUBWrr if CPSR is not used.
+ int Cmp_CPSR = CmpInstr->findRegisterDefOperandIdx(ARM64::CPSR, true);
+ if (Cmp_CPSR != -1) {
+ unsigned NewOpc;
+ switch (CmpInstr->getOpcode()) {
+ default:
+ return false;
+ case ARM64::ADDSWrr: NewOpc = ARM64::ADDWrr; break;
+ case ARM64::ADDSWri: NewOpc = ARM64::ADDWri; break;
+ case ARM64::ADDSWrs: NewOpc = ARM64::ADDWrs; break;
+ case ARM64::ADDSWrx: NewOpc = ARM64::ADDWrx; break;
+ case ARM64::ADDSXrr: NewOpc = ARM64::ADDXrr; break;
+ case ARM64::ADDSXri: NewOpc = ARM64::ADDXri; break;
+ case ARM64::ADDSXrs: NewOpc = ARM64::ADDXrs; break;
+ case ARM64::ADDSXrx: NewOpc = ARM64::ADDXrx; break;
+ case ARM64::SUBSWrr: NewOpc = ARM64::SUBWrr; break;
+ case ARM64::SUBSWri: NewOpc = ARM64::SUBWri; break;
+ case ARM64::SUBSWrs: NewOpc = ARM64::SUBWrs; break;
+ case ARM64::SUBSWrx: NewOpc = ARM64::SUBWrx; break;
+ case ARM64::SUBSXrr: NewOpc = ARM64::SUBXrr; break;
+ case ARM64::SUBSXri: NewOpc = ARM64::SUBXri; break;
+ case ARM64::SUBSXrs: NewOpc = ARM64::SUBXrs; break;
+ case ARM64::SUBSXrx: NewOpc = ARM64::SUBXrx; break;
+ }
+
+ const MCInstrDesc &MCID = get(NewOpc);
+ CmpInstr->setDesc(MCID);
+ CmpInstr->RemoveOperand(Cmp_CPSR);
+ bool succeeded = UpdateOperandRegClass(CmpInstr);
+ (void)succeeded;
+ assert(succeeded && "Some operands reg class are incompatible!");
+ return true;
+ }
+
+ // Continue only if we have a "ri" where immediate is zero.
+ if (CmpValue != 0 || SrcReg2 != 0)
+ return false;
+
+ // CmpInstr is a Compare instruction if destination register is not used.
+ if (!MRI->use_nodbg_empty(CmpInstr->getOperand(0).getReg()))
+ return false;
+
+ // Get the unique definition of SrcReg.
+ MachineInstr *MI = MRI->getUniqueVRegDef(SrcReg);
+ if (!MI)
+ return false;
+
+ // We iterate backward, starting from the instruction before CmpInstr and
+ // stop when reaching the definition of the source register or done with the
+ // basic block, to check whether CPSR is used or modified in between.
+ MachineBasicBlock::iterator I = CmpInstr, E = MI,
+ B = CmpInstr->getParent()->begin();
+
+ // Early exit if CmpInstr is at the beginning of the BB.
+ if (I == B)
+ return false;
+
+ // Check whether the definition of SrcReg is in the same basic block as
+ // Compare. If not, we can't optimize away the Compare.
+ if (MI->getParent() != CmpInstr->getParent())
+ return false;
+
+ // Check that CPSR isn't set between the comparison instruction and the one we
+ // want to change.
+ const TargetRegisterInfo *TRI = &getRegisterInfo();
+ for (--I; I != E; --I) {
+ const MachineInstr &Instr = *I;
+
+ if (Instr.modifiesRegister(ARM64::CPSR, TRI) ||
+ Instr.readsRegister(ARM64::CPSR, TRI))
+ // This instruction modifies or uses CPSR after the one we want to
+ // change. We can't do this transformation.
+ return false;
+ if (I == B)
+ // The 'and' is below the comparison instruction.
+ return false;
+ }
+
+ unsigned NewOpc = MI->getOpcode();
+ switch (MI->getOpcode()) {
+ default:
+ return false;
+ case ARM64::ADDSWrr:
+ case ARM64::ADDSWri:
+ case ARM64::ADDSXrr:
+ case ARM64::ADDSXri:
+ case ARM64::SUBSWrr:
+ case ARM64::SUBSWri:
+ case ARM64::SUBSXrr:
+ case ARM64::SUBSXri:
+ break;
+ case ARM64::ADDWrr: NewOpc = ARM64::ADDSWrr; break;
+ case ARM64::ADDWri: NewOpc = ARM64::ADDSWri; break;
+ case ARM64::ADDXrr: NewOpc = ARM64::ADDSXrr; break;
+ case ARM64::ADDXri: NewOpc = ARM64::ADDSXri; break;
+ case ARM64::ADCWr: NewOpc = ARM64::ADCSWr; break;
+ case ARM64::ADCXr: NewOpc = ARM64::ADCSXr; break;
+ case ARM64::SUBWrr: NewOpc = ARM64::SUBSWrr; break;
+ case ARM64::SUBWri: NewOpc = ARM64::SUBSWri; break;
+ case ARM64::SUBXrr: NewOpc = ARM64::SUBSXrr; break;
+ case ARM64::SUBXri: NewOpc = ARM64::SUBSXri; break;
+ case ARM64::SBCWr: NewOpc = ARM64::SBCSWr; break;
+ case ARM64::SBCXr: NewOpc = ARM64::SBCSXr; break;
+ case ARM64::ANDWri: NewOpc = ARM64::ANDSWri; break;
+ case ARM64::ANDXri: NewOpc = ARM64::ANDSXri; break;
+ }
+
+ // Scan forward for the use of CPSR.
+ // When checking against MI: if it's a conditional code requires
+ // checking of V bit, then this is not safe to do.
+ // It is safe to remove CmpInstr if CPSR is redefined or killed.
+ // If we are done with the basic block, we need to check whether CPSR is
+ // live-out.
+ bool IsSafe = false;
+ for (MachineBasicBlock::iterator I = CmpInstr,
+ E = CmpInstr->getParent()->end();
+ !IsSafe && ++I != E;) {
+ const MachineInstr &Instr = *I;
+ for (unsigned IO = 0, EO = Instr.getNumOperands(); !IsSafe && IO != EO;
+ ++IO) {
+ const MachineOperand &MO = Instr.getOperand(IO);
+ if (MO.isRegMask() && MO.clobbersPhysReg(ARM64::CPSR)) {
+ IsSafe = true;
+ break;
+ }
+ if (!MO.isReg() || MO.getReg() != ARM64::CPSR)
+ continue;
+ if (MO.isDef()) {
+ IsSafe = true;
+ break;
+ }
+
+ // Decode the condition code.
+ unsigned Opc = Instr.getOpcode();
+ ARM64CC::CondCode CC;
+ switch (Opc) {
+ default:
+ return false;
+ case ARM64::Bcc:
+ CC = (ARM64CC::CondCode)Instr.getOperand(IO - 2).getImm();
+ break;
+ case ARM64::CSINVWr:
+ case ARM64::CSINVXr:
+ case ARM64::CSINCWr:
+ case ARM64::CSINCXr:
+ case ARM64::CSELWr:
+ case ARM64::CSELXr:
+ case ARM64::CSNEGWr:
+ case ARM64::CSNEGXr:
+ CC = (ARM64CC::CondCode)Instr.getOperand(IO - 1).getImm();
+ break;
+ }
+
+ // It is not safe to remove Compare instruction if Overflow(V) is used.
+ switch (CC) {
+ default:
+ // CPSR can be used multiple times, we should continue.
+ break;
+ case ARM64CC::VS:
+ case ARM64CC::VC:
+ case ARM64CC::GE:
+ case ARM64CC::LT:
+ case ARM64CC::GT:
+ case ARM64CC::LE:
+ return false;
+ }
+ }
+ }
+
+ // If CPSR is not killed nor re-defined, we should check whether it is
+ // live-out. If it is live-out, do not optimize.
+ if (!IsSafe) {
+ MachineBasicBlock *MBB = CmpInstr->getParent();
+ for (MachineBasicBlock::succ_iterator SI = MBB->succ_begin(),
+ SE = MBB->succ_end();
+ SI != SE; ++SI)
+ if ((*SI)->isLiveIn(ARM64::CPSR))
+ return false;
+ }
+
+ // Update the instruction to set CPSR.
+ MI->setDesc(get(NewOpc));
+ CmpInstr->eraseFromParent();
+ bool succeeded = UpdateOperandRegClass(MI);
+ (void)succeeded;
+ assert(succeeded && "Some operands reg class are incompatible!");
+ MI->addRegisterDefined(ARM64::CPSR, TRI);
+ return true;
+}
+
+// Return true if this instruction simply sets its single destination register
+// to zero. This is equivalent to a register rename of the zero-register.
+bool ARM64InstrInfo::isGPRZero(const MachineInstr *MI) const {
+ switch (MI->getOpcode()) {
+ default:
+ break;
+ case ARM64::MOVZWi:
+ case ARM64::MOVZXi: // movz Rd, #0 (LSL #0)
+ if (MI->getOperand(1).isImm() && MI->getOperand(1).getImm() == 0) {
+ assert(MI->getDesc().getNumOperands() == 3 &&
+ MI->getOperand(2).getImm() == 0 && "invalid MOVZi operands");
+ return true;
+ }
+ break;
+ case ARM64::ANDWri: // and Rd, Rzr, #imm
+ return MI->getOperand(1).getReg() == ARM64::WZR;
+ case ARM64::ANDXri:
+ return MI->getOperand(1).getReg() == ARM64::XZR;
+ case TargetOpcode::COPY:
+ return MI->getOperand(1).getReg() == ARM64::WZR;
+ }
+ return false;
+}
+
+// Return true if this instruction simply renames a general register without
+// modifying bits.
+bool ARM64InstrInfo::isGPRCopy(const MachineInstr *MI) const {
+ switch (MI->getOpcode()) {
+ default:
+ break;
+ case TargetOpcode::COPY: {
+ // GPR32 copies will by lowered to ORRXrs
+ unsigned DstReg = MI->getOperand(0).getReg();
+ return (ARM64::GPR32RegClass.contains(DstReg) ||
+ ARM64::GPR64RegClass.contains(DstReg));
+ }
+ case ARM64::ORRXrs: // orr Xd, Xzr, Xm (LSL #0)
+ if (MI->getOperand(1).getReg() == ARM64::XZR) {
+ assert(MI->getDesc().getNumOperands() == 4 &&
+ MI->getOperand(3).getImm() == 0 && "invalid ORRrs operands");
+ return true;
+ }
+ case ARM64::ADDXri: // add Xd, Xn, #0 (LSL #0)
+ if (MI->getOperand(2).getImm() == 0) {
+ assert(MI->getDesc().getNumOperands() == 4 &&
+ MI->getOperand(3).getImm() == 0 && "invalid ADDXri operands");
+ return true;
+ }
+ }
+ return false;
+}
+
+// Return true if this instruction simply renames a general register without
+// modifying bits.
+bool ARM64InstrInfo::isFPRCopy(const MachineInstr *MI) const {
+ switch (MI->getOpcode()) {
+ default:
+ break;
+ case TargetOpcode::COPY: {
+ // FPR64 copies will by lowered to ORR.16b
+ unsigned DstReg = MI->getOperand(0).getReg();
+ return (ARM64::FPR64RegClass.contains(DstReg) ||
+ ARM64::FPR128RegClass.contains(DstReg));
+ }
+ case ARM64::ORRv16i8:
+ if (MI->getOperand(1).getReg() == MI->getOperand(2).getReg()) {
+ assert(MI->getDesc().getNumOperands() == 3 && MI->getOperand(0).isReg() &&
+ "invalid ORRv16i8 operands");
+ return true;
+ }
+ }
+ return false;
+}
+
+unsigned ARM64InstrInfo::isLoadFromStackSlot(const MachineInstr *MI,
+ int &FrameIndex) const {
+ switch (MI->getOpcode()) {
+ default:
+ break;
+ case ARM64::LDRWui:
+ case ARM64::LDRXui:
+ case ARM64::LDRBui:
+ case ARM64::LDRHui:
+ case ARM64::LDRSui:
+ case ARM64::LDRDui:
+ case ARM64::LDRQui:
+ if (MI->getOperand(0).getSubReg() == 0 && MI->getOperand(1).isFI() &&
+ MI->getOperand(2).isImm() && MI->getOperand(2).getImm() == 0) {
+ FrameIndex = MI->getOperand(1).getIndex();
+ return MI->getOperand(0).getReg();
+ }
+ break;
+ }
+
+ return 0;
+}
+
+unsigned ARM64InstrInfo::isStoreToStackSlot(const MachineInstr *MI,
+ int &FrameIndex) const {
+ switch (MI->getOpcode()) {
+ default:
+ break;
+ case ARM64::STRWui:
+ case ARM64::STRXui:
+ case ARM64::STRBui:
+ case ARM64::STRHui:
+ case ARM64::STRSui:
+ case ARM64::STRDui:
+ case ARM64::STRQui:
+ if (MI->getOperand(0).getSubReg() == 0 && MI->getOperand(1).isFI() &&
+ MI->getOperand(2).isImm() && MI->getOperand(2).getImm() == 0) {
+ FrameIndex = MI->getOperand(1).getIndex();
+ return MI->getOperand(0).getReg();
+ }
+ break;
+ }
+ return 0;
+}
+
+/// Return true if this is load/store scales or extends its register offset.
+/// This refers to scaling a dynamic index as opposed to scaled immediates.
+/// MI should be a memory op that allows scaled addressing.
+bool ARM64InstrInfo::isScaledAddr(const MachineInstr *MI) const {
+ switch (MI->getOpcode()) {
+ default:
+ break;
+ case ARM64::LDRBBro:
+ case ARM64::LDRBro:
+ case ARM64::LDRDro:
+ case ARM64::LDRHHro:
+ case ARM64::LDRHro:
+ case ARM64::LDRQro:
+ case ARM64::LDRSBWro:
+ case ARM64::LDRSBXro:
+ case ARM64::LDRSHWro:
+ case ARM64::LDRSHXro:
+ case ARM64::LDRSWro:
+ case ARM64::LDRSro:
+ case ARM64::LDRWro:
+ case ARM64::LDRXro:
+ case ARM64::STRBBro:
+ case ARM64::STRBro:
+ case ARM64::STRDro:
+ case ARM64::STRHHro:
+ case ARM64::STRHro:
+ case ARM64::STRQro:
+ case ARM64::STRSro:
+ case ARM64::STRWro:
+ case ARM64::STRXro:
+ unsigned Val = MI->getOperand(3).getImm();
+ ARM64_AM::ExtendType ExtType = ARM64_AM::getMemExtendType(Val);
+ return (ExtType != ARM64_AM::UXTX) || ARM64_AM::getMemDoShift(Val);
+ }
+ return false;
+}
+
+/// Check all MachineMemOperands for a hint to suppress pairing.
+bool ARM64InstrInfo::isLdStPairSuppressed(const MachineInstr *MI) const {
+ assert(MOSuppressPair < (1 << MachineMemOperand::MOTargetNumBits) &&
+ "Too many target MO flags");
+ for (MachineInstr::mmo_iterator MM = MI->memoperands_begin(),
+ E = MI->memoperands_end();
+ MM != E; ++MM) {
+
+ if ((*MM)->getFlags() &
+ (MOSuppressPair << MachineMemOperand::MOTargetStartBit)) {
+ return true;
+ }
+ }
+ return false;
+}
+
+/// Set a flag on the first MachineMemOperand to suppress pairing.
+void ARM64InstrInfo::suppressLdStPair(MachineInstr *MI) const {
+ if (MI->memoperands_empty())
+ return;
+
+ assert(MOSuppressPair < (1 << MachineMemOperand::MOTargetNumBits) &&
+ "Too many target MO flags");
+ (*MI->memoperands_begin())
+ ->setFlags(MOSuppressPair << MachineMemOperand::MOTargetStartBit);
+}
+
+bool ARM64InstrInfo::getLdStBaseRegImmOfs(MachineInstr *LdSt, unsigned &BaseReg,
+ unsigned &Offset,
+ const TargetRegisterInfo *TRI) const {
+ switch (LdSt->getOpcode()) {
+ default:
+ return false;
+ case ARM64::STRSui:
+ case ARM64::STRDui:
+ case ARM64::STRQui:
+ case ARM64::STRXui:
+ case ARM64::STRWui:
+ case ARM64::LDRSui:
+ case ARM64::LDRDui:
+ case ARM64::LDRQui:
+ case ARM64::LDRXui:
+ case ARM64::LDRWui:
+ if (!LdSt->getOperand(1).isReg() || !LdSt->getOperand(2).isImm())
+ return false;
+ BaseReg = LdSt->getOperand(1).getReg();
+ MachineFunction &MF = *LdSt->getParent()->getParent();
+ unsigned Width = getRegClass(LdSt->getDesc(), 0, TRI, MF)->getSize();
+ Offset = LdSt->getOperand(2).getImm() * Width;
+ return true;
+ };
+}
+
+/// Detect opportunities for ldp/stp formation.
+///
+/// Only called for LdSt for which getLdStBaseRegImmOfs returns true.
+bool ARM64InstrInfo::shouldClusterLoads(MachineInstr *FirstLdSt,
+ MachineInstr *SecondLdSt,
+ unsigned NumLoads) const {
+ // Only cluster up to a single pair.
+ if (NumLoads > 1)
+ return false;
+ if (FirstLdSt->getOpcode() != SecondLdSt->getOpcode())
+ return false;
+ // getLdStBaseRegImmOfs guarantees that oper 2 isImm.
+ unsigned Ofs1 = FirstLdSt->getOperand(2).getImm();
+ // Allow 6 bits of positive range.
+ if (Ofs1 > 64)
+ return false;
+ // The caller should already have ordered First/SecondLdSt by offset.
+ unsigned Ofs2 = SecondLdSt->getOperand(2).getImm();
+ return Ofs1 + 1 == Ofs2;
+}
+
+bool ARM64InstrInfo::shouldScheduleAdjacent(MachineInstr *First,
+ MachineInstr *Second) const {
+ // Cyclone can fuse CMN, CMP followed by Bcc.
+
+ // FIXME: B0 can also fuse:
+ // AND, BIC, ORN, ORR, or EOR (optional S) followed by Bcc or CBZ or CBNZ.
+ if (Second->getOpcode() != ARM64::Bcc)
+ return false;
+ switch (First->getOpcode()) {
+ default:
+ return false;
+ case ARM64::SUBSWri:
+ case ARM64::ADDSWri:
+ case ARM64::ANDSWri:
+ case ARM64::SUBSXri:
+ case ARM64::ADDSXri:
+ case ARM64::ANDSXri:
+ return true;
+ }
+}
+
+MachineInstr *ARM64InstrInfo::emitFrameIndexDebugValue(MachineFunction &MF,
+ int FrameIx,
+ uint64_t Offset,
+ const MDNode *MDPtr,
+ DebugLoc DL) const {
+ MachineInstrBuilder MIB = BuildMI(MF, DL, get(ARM64::DBG_VALUE))
+ .addFrameIndex(FrameIx)
+ .addImm(0)
+ .addImm(Offset)
+ .addMetadata(MDPtr);
+ return &*MIB;
+}
+
+static const MachineInstrBuilder &AddSubReg(const MachineInstrBuilder &MIB,
+ unsigned Reg, unsigned SubIdx,
+ unsigned State,
+ const TargetRegisterInfo *TRI) {
+ if (!SubIdx)
+ return MIB.addReg(Reg, State);
+
+ if (TargetRegisterInfo::isPhysicalRegister(Reg))
+ return MIB.addReg(TRI->getSubReg(Reg, SubIdx), State);
+ return MIB.addReg(Reg, State, SubIdx);
+}
+
+static bool forwardCopyWillClobberTuple(unsigned DestReg, unsigned SrcReg,
+ unsigned NumRegs) {
+ // We really want the positive remainder mod 32 here, that happens to be
+ // easily obtainable with a mask.
+ return ((DestReg - SrcReg) & 0x1f) < NumRegs;
+}
+
+void ARM64InstrInfo::copyPhysRegTuple(MachineBasicBlock &MBB,
+ MachineBasicBlock::iterator I,
+ DebugLoc DL, unsigned DestReg,
+ unsigned SrcReg, bool KillSrc,
+ unsigned Opcode,
+ llvm::ArrayRef<unsigned> Indices) const {
+ const TargetRegisterInfo *TRI = &getRegisterInfo();
+ uint16_t DestEncoding = TRI->getEncodingValue(DestReg);
+ uint16_t SrcEncoding = TRI->getEncodingValue(SrcReg);
+ unsigned NumRegs = Indices.size();
+
+ int SubReg = 0, End = NumRegs, Incr = 1;
+ if (forwardCopyWillClobberTuple(DestEncoding, SrcEncoding, NumRegs)) {
+ SubReg = NumRegs - 1;
+ End = -1;
+ Incr = -1;
+ }
+
+ for (; SubReg != End; SubReg += Incr) {
+ const MachineInstrBuilder &MIB = BuildMI(MBB, I, DL, get(Opcode));
+ AddSubReg(MIB, DestReg, Indices[SubReg], RegState::Define, TRI);
+ AddSubReg(MIB, SrcReg, Indices[SubReg], 0, TRI);
+ AddSubReg(MIB, SrcReg, Indices[SubReg], getKillRegState(KillSrc), TRI);
+ }
+}
+
+void ARM64InstrInfo::copyPhysReg(MachineBasicBlock &MBB,
+ MachineBasicBlock::iterator I, DebugLoc DL,
+ unsigned DestReg, unsigned SrcReg,
+ bool KillSrc) const {
+ if (ARM64::GPR32spRegClass.contains(DestReg) &&
+ (ARM64::GPR32spRegClass.contains(SrcReg) || SrcReg == ARM64::WZR)) {
+ const TargetRegisterInfo *TRI = &getRegisterInfo();
+
+ if (DestReg == ARM64::WSP || SrcReg == ARM64::WSP) {
+ // If either operand is WSP, expand to ADD #0.
+ if (Subtarget.hasZeroCycleRegMove()) {
+ // Cyclone recognizes "ADD Xd, Xn, #0" as a zero-cycle register move.
+ unsigned DestRegX = TRI->getMatchingSuperReg(DestReg, ARM64::sub_32,
+ &ARM64::GPR64spRegClass);
+ unsigned SrcRegX = TRI->getMatchingSuperReg(SrcReg, ARM64::sub_32,
+ &ARM64::GPR64spRegClass);
+ // This instruction is reading and writing X registers. This may upset
+ // the register scavenger and machine verifier, so we need to indicate
+ // that we are reading an undefined value from SrcRegX, but a proper
+ // value from SrcReg.
+ BuildMI(MBB, I, DL, get(ARM64::ADDXri), DestRegX)
+ .addReg(SrcRegX, RegState::Undef)
+ .addImm(0)
+ .addImm(ARM64_AM::getShifterImm(ARM64_AM::LSL, 0))
+ .addReg(SrcReg, RegState::Implicit | getKillRegState(KillSrc));
+ } else {
+ BuildMI(MBB, I, DL, get(ARM64::ADDWri), DestReg)
+ .addReg(SrcReg, getKillRegState(KillSrc))
+ .addImm(0)
+ .addImm(ARM64_AM::getShifterImm(ARM64_AM::LSL, 0));
+ }
+ } else if (SrcReg == ARM64::WZR && Subtarget.hasZeroCycleZeroing()) {
+ BuildMI(MBB, I, DL, get(ARM64::MOVZWi), DestReg).addImm(0).addImm(
+ ARM64_AM::getShifterImm(ARM64_AM::LSL, 0));
+ } else {
+ if (Subtarget.hasZeroCycleRegMove()) {
+ // Cyclone recognizes "ORR Xd, XZR, Xm" as a zero-cycle register move.
+ unsigned DestRegX = TRI->getMatchingSuperReg(DestReg, ARM64::sub_32,
+ &ARM64::GPR64spRegClass);
+ unsigned SrcRegX = TRI->getMatchingSuperReg(SrcReg, ARM64::sub_32,
+ &ARM64::GPR64spRegClass);
+ // This instruction is reading and writing X registers. This may upset
+ // the register scavenger and machine verifier, so we need to indicate
+ // that we are reading an undefined value from SrcRegX, but a proper
+ // value from SrcReg.
+ BuildMI(MBB, I, DL, get(ARM64::ORRXrr), DestRegX)
+ .addReg(ARM64::XZR)
+ .addReg(SrcRegX, RegState::Undef)
+ .addReg(SrcReg, RegState::Implicit | getKillRegState(KillSrc));
+ } else {
+ // Otherwise, expand to ORR WZR.
+ BuildMI(MBB, I, DL, get(ARM64::ORRWrr), DestReg)
+ .addReg(ARM64::WZR)
+ .addReg(SrcReg, getKillRegState(KillSrc));
+ }
+ }
+ return;
+ }
+
+ if (ARM64::GPR64spRegClass.contains(DestReg) &&
+ (ARM64::GPR64spRegClass.contains(SrcReg) || SrcReg == ARM64::XZR)) {
+ if (DestReg == ARM64::SP || SrcReg == ARM64::SP) {
+ // If either operand is SP, expand to ADD #0.
+ BuildMI(MBB, I, DL, get(ARM64::ADDXri), DestReg)
+ .addReg(SrcReg, getKillRegState(KillSrc))
+ .addImm(0)
+ .addImm(ARM64_AM::getShifterImm(ARM64_AM::LSL, 0));
+ } else if (SrcReg == ARM64::XZR && Subtarget.hasZeroCycleZeroing()) {
+ BuildMI(MBB, I, DL, get(ARM64::MOVZXi), DestReg).addImm(0).addImm(
+ ARM64_AM::getShifterImm(ARM64_AM::LSL, 0));
+ } else {
+ // Otherwise, expand to ORR XZR.
+ BuildMI(MBB, I, DL, get(ARM64::ORRXrr), DestReg)
+ .addReg(ARM64::XZR)
+ .addReg(SrcReg, getKillRegState(KillSrc));
+ }
+ return;
+ }
+
+ // Copy a DDDD register quad by copying the individual sub-registers.
+ if (ARM64::DDDDRegClass.contains(DestReg) &&
+ ARM64::DDDDRegClass.contains(SrcReg)) {
+ static const unsigned Indices[] = { ARM64::dsub0, ARM64::dsub1,
+ ARM64::dsub2, ARM64::dsub3 };
+ copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, ARM64::ORRv8i8,
+ Indices);
+ return;
+ }
+
+ // Copy a DDD register triple by copying the individual sub-registers.
+ if (ARM64::DDDRegClass.contains(DestReg) &&
+ ARM64::DDDRegClass.contains(SrcReg)) {
+ static const unsigned Indices[] = { ARM64::dsub0, ARM64::dsub1,
+ ARM64::dsub2 };
+ copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, ARM64::ORRv8i8,
+ Indices);
+ return;
+ }
+
+ // Copy a DD register pair by copying the individual sub-registers.
+ if (ARM64::DDRegClass.contains(DestReg) &&
+ ARM64::DDRegClass.contains(SrcReg)) {
+ static const unsigned Indices[] = { ARM64::dsub0, ARM64::dsub1 };
+ copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, ARM64::ORRv8i8,
+ Indices);
+ return;
+ }
+
+ // Copy a QQQQ register quad by copying the individual sub-registers.
+ if (ARM64::QQQQRegClass.contains(DestReg) &&
+ ARM64::QQQQRegClass.contains(SrcReg)) {
+ static const unsigned Indices[] = { ARM64::qsub0, ARM64::qsub1,
+ ARM64::qsub2, ARM64::qsub3 };
+ copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, ARM64::ORRv16i8,
+ Indices);
+ return;
+ }
+
+ // Copy a QQQ register triple by copying the individual sub-registers.
+ if (ARM64::QQQRegClass.contains(DestReg) &&
+ ARM64::QQQRegClass.contains(SrcReg)) {
+ static const unsigned Indices[] = { ARM64::qsub0, ARM64::qsub1,
+ ARM64::qsub2 };
+ copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, ARM64::ORRv16i8,
+ Indices);
+ return;
+ }
+
+ // Copy a QQ register pair by copying the individual sub-registers.
+ if (ARM64::QQRegClass.contains(DestReg) &&
+ ARM64::QQRegClass.contains(SrcReg)) {
+ static const unsigned Indices[] = { ARM64::qsub0, ARM64::qsub1 };
+ copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, ARM64::ORRv16i8,
+ Indices);
+ return;
+ }
+
+ if (ARM64::FPR128RegClass.contains(DestReg) &&
+ ARM64::FPR128RegClass.contains(SrcReg)) {
+ BuildMI(MBB, I, DL, get(ARM64::ORRv16i8), DestReg).addReg(SrcReg).addReg(
+ SrcReg, getKillRegState(KillSrc));
+ return;
+ }
+
+ if (ARM64::FPR64RegClass.contains(DestReg) &&
+ ARM64::FPR64RegClass.contains(SrcReg)) {
+ DestReg =
+ RI.getMatchingSuperReg(DestReg, ARM64::dsub, &ARM64::FPR128RegClass);
+ SrcReg =
+ RI.getMatchingSuperReg(SrcReg, ARM64::dsub, &ARM64::FPR128RegClass);
+ BuildMI(MBB, I, DL, get(ARM64::ORRv16i8), DestReg).addReg(SrcReg).addReg(
+ SrcReg, getKillRegState(KillSrc));
+ return;
+ }
+
+ if (ARM64::FPR32RegClass.contains(DestReg) &&
+ ARM64::FPR32RegClass.contains(SrcReg)) {
+ DestReg =
+ RI.getMatchingSuperReg(DestReg, ARM64::ssub, &ARM64::FPR128RegClass);
+ SrcReg =
+ RI.getMatchingSuperReg(SrcReg, ARM64::ssub, &ARM64::FPR128RegClass);
+ BuildMI(MBB, I, DL, get(ARM64::ORRv16i8), DestReg).addReg(SrcReg).addReg(
+ SrcReg, getKillRegState(KillSrc));
+ return;
+ }
+
+ if (ARM64::FPR16RegClass.contains(DestReg) &&
+ ARM64::FPR16RegClass.contains(SrcReg)) {
+ DestReg =
+ RI.getMatchingSuperReg(DestReg, ARM64::hsub, &ARM64::FPR128RegClass);
+ SrcReg =
+ RI.getMatchingSuperReg(SrcReg, ARM64::hsub, &ARM64::FPR128RegClass);
+ BuildMI(MBB, I, DL, get(ARM64::ORRv16i8), DestReg).addReg(SrcReg).addReg(
+ SrcReg, getKillRegState(KillSrc));
+ return;
+ }
+
+ if (ARM64::FPR8RegClass.contains(DestReg) &&
+ ARM64::FPR8RegClass.contains(SrcReg)) {
+ DestReg =
+ RI.getMatchingSuperReg(DestReg, ARM64::bsub, &ARM64::FPR128RegClass);
+ SrcReg =
+ RI.getMatchingSuperReg(SrcReg, ARM64::bsub, &ARM64::FPR128RegClass);
+ BuildMI(MBB, I, DL, get(ARM64::ORRv16i8), DestReg).addReg(SrcReg).addReg(
+ SrcReg, getKillRegState(KillSrc));
+ return;
+ }
+
+ // Copies between GPR64 and FPR64.
+ if (ARM64::FPR64RegClass.contains(DestReg) &&
+ ARM64::GPR64RegClass.contains(SrcReg)) {
+ BuildMI(MBB, I, DL, get(ARM64::FMOVXDr), DestReg)
+ .addReg(SrcReg, getKillRegState(KillSrc));
+ return;
+ }
+ if (ARM64::GPR64RegClass.contains(DestReg) &&
+ ARM64::FPR64RegClass.contains(SrcReg)) {
+ BuildMI(MBB, I, DL, get(ARM64::FMOVDXr), DestReg)
+ .addReg(SrcReg, getKillRegState(KillSrc));
+ return;
+ }
+ // Copies between GPR32 and FPR32.
+ if (ARM64::FPR32RegClass.contains(DestReg) &&
+ ARM64::GPR32RegClass.contains(SrcReg)) {
+ BuildMI(MBB, I, DL, get(ARM64::FMOVWSr), DestReg)
+ .addReg(SrcReg, getKillRegState(KillSrc));
+ return;
+ }
+ if (ARM64::GPR32RegClass.contains(DestReg) &&
+ ARM64::FPR32RegClass.contains(SrcReg)) {
+ BuildMI(MBB, I, DL, get(ARM64::FMOVSWr), DestReg)
+ .addReg(SrcReg, getKillRegState(KillSrc));
+ return;
+ }
+
+ assert(0 && "unimplemented reg-to-reg copy");
+}
+
+void ARM64InstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB,
+ MachineBasicBlock::iterator MBBI,
+ unsigned SrcReg, bool isKill, int FI,
+ const TargetRegisterClass *RC,
+ const TargetRegisterInfo *TRI) const {
+ DebugLoc DL;
+ if (MBBI != MBB.end())
+ DL = MBBI->getDebugLoc();
+ MachineFunction &MF = *MBB.getParent();
+ MachineFrameInfo &MFI = *MF.getFrameInfo();
+ unsigned Align = MFI.getObjectAlignment(FI);
+
+ MachinePointerInfo PtrInfo(PseudoSourceValue::getFixedStack(FI));
+ MachineMemOperand *MMO = MF.getMachineMemOperand(
+ PtrInfo, MachineMemOperand::MOStore, MFI.getObjectSize(FI), Align);
+ unsigned Opc = 0;
+ bool Offset = true;
+ switch (RC->getSize()) {
+ case 1:
+ if (ARM64::FPR8RegClass.hasSubClassEq(RC))
+ Opc = ARM64::STRBui;
+ break;
+ case 2:
+ if (ARM64::FPR16RegClass.hasSubClassEq(RC))
+ Opc = ARM64::STRHui;
+ break;
+ case 4:
+ if (ARM64::GPR32allRegClass.hasSubClassEq(RC)) {
+ Opc = ARM64::STRWui;
+ if (TargetRegisterInfo::isVirtualRegister(SrcReg))
+ MF.getRegInfo().constrainRegClass(SrcReg, &ARM64::GPR32RegClass);
+ else
+ assert(SrcReg != ARM64::WSP);
+ } else if (ARM64::FPR32RegClass.hasSubClassEq(RC))
+ Opc = ARM64::STRSui;
+ break;
+ case 8:
+ if (ARM64::GPR64allRegClass.hasSubClassEq(RC)) {
+ Opc = ARM64::STRXui;
+ if (TargetRegisterInfo::isVirtualRegister(SrcReg))
+ MF.getRegInfo().constrainRegClass(SrcReg, &ARM64::GPR64RegClass);
+ else
+ assert(SrcReg != ARM64::SP);
+ } else if (ARM64::FPR64RegClass.hasSubClassEq(RC))
+ Opc = ARM64::STRDui;
+ break;
+ case 16:
+ if (ARM64::FPR128RegClass.hasSubClassEq(RC))
+ Opc = ARM64::STRQui;
+ else if (ARM64::DDRegClass.hasSubClassEq(RC))
+ Opc = ARM64::ST1Twov1d, Offset = false;
+ break;
+ case 24:
+ if (ARM64::DDDRegClass.hasSubClassEq(RC))
+ Opc = ARM64::ST1Threev1d, Offset = false;
+ break;
+ case 32:
+ if (ARM64::DDDDRegClass.hasSubClassEq(RC))
+ Opc = ARM64::ST1Fourv1d, Offset = false;
+ else if (ARM64::QQRegClass.hasSubClassEq(RC))
+ Opc = ARM64::ST1Twov2d, Offset = false;
+ break;
+ case 48:
+ if (ARM64::QQQRegClass.hasSubClassEq(RC))
+ Opc = ARM64::ST1Threev2d, Offset = false;
+ break;
+ case 64:
+ if (ARM64::QQQQRegClass.hasSubClassEq(RC))
+ Opc = ARM64::ST1Fourv2d, Offset = false;
+ break;
+ }
+ assert(Opc && "Unknown register class");
+
+ const MachineInstrBuilder &MI = BuildMI(MBB, MBBI, DL, get(Opc))
+ .addReg(SrcReg, getKillRegState(isKill))
+ .addFrameIndex(FI);
+
+ if (Offset)
+ MI.addImm(0);
+ MI.addMemOperand(MMO);
+}
+
+void ARM64InstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
+ MachineBasicBlock::iterator MBBI,
+ unsigned DestReg, int FI,
+ const TargetRegisterClass *RC,
+ const TargetRegisterInfo *TRI) const {
+ DebugLoc DL;
+ if (MBBI != MBB.end())
+ DL = MBBI->getDebugLoc();
+ MachineFunction &MF = *MBB.getParent();
+ MachineFrameInfo &MFI = *MF.getFrameInfo();
+ unsigned Align = MFI.getObjectAlignment(FI);
+ MachinePointerInfo PtrInfo(PseudoSourceValue::getFixedStack(FI));
+ MachineMemOperand *MMO = MF.getMachineMemOperand(
+ PtrInfo, MachineMemOperand::MOLoad, MFI.getObjectSize(FI), Align);
+
+ unsigned Opc = 0;
+ bool Offset = true;
+ switch (RC->getSize()) {
+ case 1:
+ if (ARM64::FPR8RegClass.hasSubClassEq(RC))
+ Opc = ARM64::LDRBui;
+ break;
+ case 2:
+ if (ARM64::FPR16RegClass.hasSubClassEq(RC))
+ Opc = ARM64::LDRHui;
+ break;
+ case 4:
+ if (ARM64::GPR32allRegClass.hasSubClassEq(RC)) {
+ Opc = ARM64::LDRWui;
+ if (TargetRegisterInfo::isVirtualRegister(DestReg))
+ MF.getRegInfo().constrainRegClass(DestReg, &ARM64::GPR32RegClass);
+ else
+ assert(DestReg != ARM64::WSP);
+ } else if (ARM64::FPR32RegClass.hasSubClassEq(RC))
+ Opc = ARM64::LDRSui;
+ break;
+ case 8:
+ if (ARM64::GPR64allRegClass.hasSubClassEq(RC)) {
+ Opc = ARM64::LDRXui;
+ if (TargetRegisterInfo::isVirtualRegister(DestReg))
+ MF.getRegInfo().constrainRegClass(DestReg, &ARM64::GPR64RegClass);
+ else
+ assert(DestReg != ARM64::SP);
+ } else if (ARM64::FPR64RegClass.hasSubClassEq(RC))
+ Opc = ARM64::LDRDui;
+ break;
+ case 16:
+ if (ARM64::FPR128RegClass.hasSubClassEq(RC))
+ Opc = ARM64::LDRQui;
+ else if (ARM64::DDRegClass.hasSubClassEq(RC))
+ Opc = ARM64::LD1Twov1d, Offset = false;
+ break;
+ case 24:
+ if (ARM64::DDDRegClass.hasSubClassEq(RC))
+ Opc = ARM64::LD1Threev1d, Offset = false;
+ break;
+ case 32:
+ if (ARM64::DDDDRegClass.hasSubClassEq(RC))
+ Opc = ARM64::LD1Fourv1d, Offset = false;
+ else if (ARM64::QQRegClass.hasSubClassEq(RC))
+ Opc = ARM64::LD1Twov2d, Offset = false;
+ break;
+ case 48:
+ if (ARM64::QQQRegClass.hasSubClassEq(RC))
+ Opc = ARM64::LD1Threev2d, Offset = false;
+ break;
+ case 64:
+ if (ARM64::QQQQRegClass.hasSubClassEq(RC))
+ Opc = ARM64::LD1Fourv2d, Offset = false;
+ break;
+ }
+ assert(Opc && "Unknown register class");
+
+ const MachineInstrBuilder &MI = BuildMI(MBB, MBBI, DL, get(Opc))
+ .addReg(DestReg, getDefRegState(true))
+ .addFrameIndex(FI);
+ if (Offset)
+ MI.addImm(0);
+ MI.addMemOperand(MMO);
+}
+
+void llvm::emitFrameOffset(MachineBasicBlock &MBB,
+ MachineBasicBlock::iterator MBBI, DebugLoc DL,
+ unsigned DestReg, unsigned SrcReg, int Offset,
+ const ARM64InstrInfo *TII, MachineInstr::MIFlag Flag,
+ bool SetCPSR) {
+ if (DestReg == SrcReg && Offset == 0)
+ return;
+
+ bool isSub = Offset < 0;
+ if (isSub)
+ Offset = -Offset;
+
+ // FIXME: If the offset won't fit in 24-bits, compute the offset into a
+ // scratch register. If DestReg is a virtual register, use it as the
+ // scratch register; otherwise, create a new virtual register (to be
+ // replaced by the scavenger at the end of PEI). That case can be optimized
+ // slightly if DestReg is SP which is always 16-byte aligned, so the scratch
+ // register can be loaded with offset%8 and the add/sub can use an extending
+ // instruction with LSL#3.
+ // Currently the function handles any offsets but generates a poor sequence
+ // of code.
+ // assert(Offset < (1 << 24) && "unimplemented reg plus immediate");
+
+ unsigned Opc;
+ if (SetCPSR)
+ Opc = isSub ? ARM64::SUBSXri : ARM64::ADDSXri;
+ else
+ Opc = isSub ? ARM64::SUBXri : ARM64::ADDXri;
+ const unsigned MaxEncoding = 0xfff;
+ const unsigned ShiftSize = 12;
+ const unsigned MaxEncodableValue = MaxEncoding << ShiftSize;
+ while (((unsigned)Offset) >= (1 << ShiftSize)) {
+ unsigned ThisVal;
+ if (((unsigned)Offset) > MaxEncodableValue) {
+ ThisVal = MaxEncodableValue;
+ } else {
+ ThisVal = Offset & MaxEncodableValue;
+ }
+ assert((ThisVal >> ShiftSize) <= MaxEncoding &&
+ "Encoding cannot handle value that big");
+ BuildMI(MBB, MBBI, DL, TII->get(Opc), DestReg)
+ .addReg(SrcReg)
+ .addImm(ThisVal >> ShiftSize)
+ .addImm(ARM64_AM::getShifterImm(ARM64_AM::LSL, ShiftSize))
+ .setMIFlag(Flag);
+
+ SrcReg = DestReg;
+ Offset -= ThisVal;
+ if (Offset == 0)
+ return;
+ }
+ BuildMI(MBB, MBBI, DL, TII->get(Opc), DestReg)
+ .addReg(SrcReg)
+ .addImm(Offset)
+ .addImm(ARM64_AM::getShifterImm(ARM64_AM::LSL, 0))
+ .setMIFlag(Flag);
+}
+
+MachineInstr *
+ARM64InstrInfo::foldMemoryOperandImpl(MachineFunction &MF, MachineInstr *MI,
+ const SmallVectorImpl<unsigned> &Ops,
+ int FrameIndex) const {
+ // This is a bit of a hack. Consider this instruction:
+ //
+ // %vreg0<def> = COPY %SP; GPR64all:%vreg0
+ //
+ // We explicitly chose GPR64all for the virtual register so such a copy might
+ // be eliminated by RegisterCoalescer. However, that may not be possible, and
+ // %vreg0 may even spill. We can't spill %SP, and since it is in the GPR64all
+ // register class, TargetInstrInfo::foldMemoryOperand() is going to try.
+ //
+ // To prevent that, we are going to constrain the %vreg0 register class here.
+ //
+ // <rdar://problem/11522048>
+ //
+ if (MI->isCopy()) {
+ unsigned DstReg = MI->getOperand(0).getReg();
+ unsigned SrcReg = MI->getOperand(1).getReg();
+ if (SrcReg == ARM64::SP && TargetRegisterInfo::isVirtualRegister(DstReg)) {
+ MF.getRegInfo().constrainRegClass(DstReg, &ARM64::GPR64RegClass);
+ return 0;
+ }
+ if (DstReg == ARM64::SP && TargetRegisterInfo::isVirtualRegister(SrcReg)) {
+ MF.getRegInfo().constrainRegClass(SrcReg, &ARM64::GPR64RegClass);
+ return 0;
+ }
+ }
+
+ // Cannot fold.
+ return 0;
+}
+
+int llvm::isARM64FrameOffsetLegal(const MachineInstr &MI, int &Offset,
+ bool *OutUseUnscaledOp,
+ unsigned *OutUnscaledOp,
+ int *EmittableOffset) {
+ int Scale = 1;
+ bool IsSigned = false;
+ // The ImmIdx should be changed case by case if it is not 2.
+ unsigned ImmIdx = 2;
+ unsigned UnscaledOp = 0;
+ // Set output values in case of early exit.
+ if (EmittableOffset)
+ *EmittableOffset = 0;
+ if (OutUseUnscaledOp)
+ *OutUseUnscaledOp = false;
+ if (OutUnscaledOp)
+ *OutUnscaledOp = 0;
+ switch (MI.getOpcode()) {
+ default:
+ assert(0 && "unhandled opcode in rewriteARM64FrameIndex");
+ // Vector spills/fills can't take an immediate offset.
+ case ARM64::LD1Twov2d:
+ case ARM64::LD1Threev2d:
+ case ARM64::LD1Fourv2d:
+ case ARM64::LD1Twov1d:
+ case ARM64::LD1Threev1d:
+ case ARM64::LD1Fourv1d:
+ case ARM64::ST1Twov2d:
+ case ARM64::ST1Threev2d:
+ case ARM64::ST1Fourv2d:
+ case ARM64::ST1Twov1d:
+ case ARM64::ST1Threev1d:
+ case ARM64::ST1Fourv1d:
+ return ARM64FrameOffsetCannotUpdate;
+ case ARM64::PRFMui:
+ Scale = 8;
+ UnscaledOp = ARM64::PRFUMi;
+ break;
+ case ARM64::LDRXui:
+ Scale = 8;
+ UnscaledOp = ARM64::LDURXi;
+ break;
+ case ARM64::LDRWui:
+ Scale = 4;
+ UnscaledOp = ARM64::LDURWi;
+ break;
+ case ARM64::LDRBui:
+ Scale = 1;
+ UnscaledOp = ARM64::LDURBi;
+ break;
+ case ARM64::LDRHui:
+ Scale = 2;
+ UnscaledOp = ARM64::LDURHi;
+ break;
+ case ARM64::LDRSui:
+ Scale = 4;
+ UnscaledOp = ARM64::LDURSi;
+ break;
+ case ARM64::LDRDui:
+ Scale = 8;
+ UnscaledOp = ARM64::LDURDi;
+ break;
+ case ARM64::LDRQui:
+ Scale = 16;
+ UnscaledOp = ARM64::LDURQi;
+ break;
+ case ARM64::LDRBBui:
+ Scale = 1;
+ UnscaledOp = ARM64::LDURBBi;
+ break;
+ case ARM64::LDRHHui:
+ Scale = 2;
+ UnscaledOp = ARM64::LDURHHi;
+ break;
+ case ARM64::LDRSBXui:
+ Scale = 1;
+ UnscaledOp = ARM64::LDURSBXi;
+ break;
+ case ARM64::LDRSBWui:
+ Scale = 1;
+ UnscaledOp = ARM64::LDURSBWi;
+ break;
+ case ARM64::LDRSHXui:
+ Scale = 2;
+ UnscaledOp = ARM64::LDURSHXi;
+ break;
+ case ARM64::LDRSHWui:
+ Scale = 2;
+ UnscaledOp = ARM64::LDURSHWi;
+ break;
+ case ARM64::LDRSWui:
+ Scale = 4;
+ UnscaledOp = ARM64::LDURSWi;
+ break;
+
+ case ARM64::STRXui:
+ Scale = 8;
+ UnscaledOp = ARM64::STURXi;
+ break;
+ case ARM64::STRWui:
+ Scale = 4;
+ UnscaledOp = ARM64::STURWi;
+ break;
+ case ARM64::STRBui:
+ Scale = 1;
+ UnscaledOp = ARM64::STURBi;
+ break;
+ case ARM64::STRHui:
+ Scale = 2;
+ UnscaledOp = ARM64::STURHi;
+ break;
+ case ARM64::STRSui:
+ Scale = 4;
+ UnscaledOp = ARM64::STURSi;
+ break;
+ case ARM64::STRDui:
+ Scale = 8;
+ UnscaledOp = ARM64::STURDi;
+ break;
+ case ARM64::STRQui:
+ Scale = 16;
+ UnscaledOp = ARM64::STURQi;
+ break;
+ case ARM64::STRBBui:
+ Scale = 1;
+ UnscaledOp = ARM64::STURBBi;
+ break;
+ case ARM64::STRHHui:
+ Scale = 2;
+ UnscaledOp = ARM64::STURHHi;
+ break;
+
+ case ARM64::LDPXi:
+ case ARM64::LDPDi:
+ case ARM64::STPXi:
+ case ARM64::STPDi:
+ IsSigned = true;
+ Scale = 8;
+ break;
+ case ARM64::LDPQi:
+ case ARM64::STPQi:
+ IsSigned = true;
+ Scale = 16;
+ break;
+ case ARM64::LDPWi:
+ case ARM64::LDPSi:
+ case ARM64::STPWi:
+ case ARM64::STPSi:
+ IsSigned = true;
+ Scale = 4;
+ break;
+
+ case ARM64::LDURXi:
+ case ARM64::LDURWi:
+ case ARM64::LDURBi:
+ case ARM64::LDURHi:
+ case ARM64::LDURSi:
+ case ARM64::LDURDi:
+ case ARM64::LDURQi:
+ case ARM64::LDURHHi:
+ case ARM64::LDURBBi:
+ case ARM64::LDURSBXi:
+ case ARM64::LDURSBWi:
+ case ARM64::LDURSHXi:
+ case ARM64::LDURSHWi:
+ case ARM64::LDURSWi:
+ case ARM64::STURXi:
+ case ARM64::STURWi:
+ case ARM64::STURBi:
+ case ARM64::STURHi:
+ case ARM64::STURSi:
+ case ARM64::STURDi:
+ case ARM64::STURQi:
+ case ARM64::STURBBi:
+ case ARM64::STURHHi:
+ Scale = 1;
+ break;
+ }
+
+ Offset += MI.getOperand(ImmIdx).getImm() * Scale;
+
+ bool useUnscaledOp = false;
+ // If the offset doesn't match the scale, we rewrite the instruction to
+ // use the unscaled instruction instead. Likewise, if we have a negative
+ // offset (and have an unscaled op to use).
+ if ((Offset & (Scale - 1)) != 0 || (Offset < 0 && UnscaledOp != 0))
+ useUnscaledOp = true;
+
+ // Use an unscaled addressing mode if the instruction has a negative offset
+ // (or if the instruction is already using an unscaled addressing mode).
+ unsigned MaskBits;
+ if (IsSigned) {
+ // ldp/stp instructions.
+ MaskBits = 7;
+ Offset /= Scale;
+ } else if (UnscaledOp == 0 || useUnscaledOp) {
+ MaskBits = 9;
+ IsSigned = true;
+ Scale = 1;
+ } else {
+ MaskBits = 12;
+ IsSigned = false;
+ Offset /= Scale;
+ }
+
+ // Attempt to fold address computation.
+ int MaxOff = (1 << (MaskBits - IsSigned)) - 1;
+ int MinOff = (IsSigned ? (-MaxOff - 1) : 0);
+ if (Offset >= MinOff && Offset <= MaxOff) {
+ if (EmittableOffset)
+ *EmittableOffset = Offset;
+ Offset = 0;
+ } else {
+ int NewOff = Offset < 0 ? MinOff : MaxOff;
+ if (EmittableOffset)
+ *EmittableOffset = NewOff;
+ Offset = (Offset - NewOff) * Scale;
+ }
+ if (OutUseUnscaledOp)
+ *OutUseUnscaledOp = useUnscaledOp;
+ if (OutUnscaledOp)
+ *OutUnscaledOp = UnscaledOp;
+ return ARM64FrameOffsetCanUpdate |
+ (Offset == 0 ? ARM64FrameOffsetIsLegal : 0);
+}
+
+bool llvm::rewriteARM64FrameIndex(MachineInstr &MI, unsigned FrameRegIdx,
+ unsigned FrameReg, int &Offset,
+ const ARM64InstrInfo *TII) {
+ unsigned Opcode = MI.getOpcode();
+ unsigned ImmIdx = FrameRegIdx + 1;
+
+ if (Opcode == ARM64::ADDSXri || Opcode == ARM64::ADDXri) {
+ Offset += MI.getOperand(ImmIdx).getImm();
+ emitFrameOffset(*MI.getParent(), MI, MI.getDebugLoc(),
+ MI.getOperand(0).getReg(), FrameReg, Offset, TII,
+ MachineInstr::NoFlags, (Opcode == ARM64::ADDSXri));
+ MI.eraseFromParent();
+ Offset = 0;
+ return true;
+ }
+
+ int NewOffset;
+ unsigned UnscaledOp;
+ bool UseUnscaledOp;
+ int Status = isARM64FrameOffsetLegal(MI, Offset, &UseUnscaledOp, &UnscaledOp,
+ &NewOffset);
+ if (Status & ARM64FrameOffsetCanUpdate) {
+ if (Status & ARM64FrameOffsetIsLegal)
+ // Replace the FrameIndex with FrameReg.
+ MI.getOperand(FrameRegIdx).ChangeToRegister(FrameReg, false);
+ if (UseUnscaledOp)
+ MI.setDesc(TII->get(UnscaledOp));
+
+ MI.getOperand(ImmIdx).ChangeToImmediate(NewOffset);
+ return Offset == 0;
+ }
+
+ return false;
+}
+
+void ARM64InstrInfo::getNoopForMachoTarget(MCInst &NopInst) const {
+ NopInst.setOpcode(ARM64::HINT);
+ NopInst.addOperand(MCOperand::CreateImm(0));
+}
generated by cgit v1.2.3 (git 2.25.1) at 2024-06-04 15:50:11 +0000