summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--include/llvm/CodeGen/MachineBasicBlock.h3
-rw-r--r--lib/CodeGen/IfConversion.cpp47
-rw-r--r--lib/CodeGen/MachineBasicBlock.cpp7
-rw-r--r--test/CodeGen/ARM/ifcvt-branch-weight.ll42
-rw-r--r--test/CodeGen/Thumb2/v8_IT_5.ll3
5 files changed, 101 insertions, 1 deletions
diff --git a/include/llvm/CodeGen/MachineBasicBlock.h b/include/llvm/CodeGen/MachineBasicBlock.h
index e196419e90..b2dac195fd 100644
--- a/include/llvm/CodeGen/MachineBasicBlock.h
+++ b/include/llvm/CodeGen/MachineBasicBlock.h
@@ -363,6 +363,9 @@ public:
///
void addSuccessor(MachineBasicBlock *succ, uint32_t weight = 0);
+ /// Set successor weight of a given iterator.
+ void setSuccWeight(succ_iterator I, uint32_t weight);
+
/// removeSuccessor - Remove successor from the successors list of this
/// MachineBasicBlock. The Predecessors list of succ is automatically updated.
///
diff --git a/lib/CodeGen/IfConversion.cpp b/lib/CodeGen/IfConversion.cpp
index 75a7f361fd..f0e8c4775c 100644
--- a/lib/CodeGen/IfConversion.cpp
+++ b/lib/CodeGen/IfConversion.cpp
@@ -1102,6 +1102,28 @@ bool IfConverter::IfConvertSimple(BBInfo &BBI, IfcvtKind Kind) {
return true;
}
+/// Scale down weights to fit into uint32_t. NewTrue is the new weight
+/// for successor TrueBB, and NewFalse is the new weight for successor
+/// FalseBB.
+static void ScaleWeights(uint64_t NewTrue, uint64_t NewFalse,
+ MachineBasicBlock *MBB,
+ const MachineBasicBlock *TrueBB,
+ const MachineBasicBlock *FalseBB,
+ const MachineBranchProbabilityInfo *MBPI) {
+ uint64_t NewMax = (NewTrue > NewFalse) ? NewTrue : NewFalse;
+ uint32_t Scale = (NewMax / UINT32_MAX) + 1;
+ for (MachineBasicBlock::succ_iterator SI = MBB->succ_begin(),
+ SE = MBB->succ_end();
+ SI != SE; ++SI) {
+ if (*SI == TrueBB)
+ MBB->setSuccWeight(SI, (uint32_t)(NewTrue / Scale));
+ else if (*SI == FalseBB)
+ MBB->setSuccWeight(SI, (uint32_t)(NewFalse / Scale));
+ else
+ MBB->setSuccWeight(SI, MBPI->getEdgeWeight(MBB, SI) / Scale);
+ }
+}
+
/// IfConvertTriangle - If convert a triangle sub-CFG.
///
bool IfConverter::IfConvertTriangle(BBInfo &BBI, IfcvtKind Kind) {
@@ -1158,6 +1180,14 @@ bool IfConverter::IfConvertTriangle(BBInfo &BBI, IfcvtKind Kind) {
DontKill.clear();
bool HasEarlyExit = CvtBBI->FalseBB != NULL;
+ uint64_t CvtNext = 0, CvtFalse = 0, SumWeight = 0;
+ uint32_t WeightScale = 0;
+ if (HasEarlyExit) {
+ // Get weights before modifying CvtBBI->BB.
+ CvtNext = MBPI->getEdgeWeight(CvtBBI->BB, NextBBI->BB);
+ CvtFalse = MBPI->getEdgeWeight(CvtBBI->BB, CvtBBI->FalseBB);
+ SumWeight = MBPI->getSumForBlock(CvtBBI->BB, WeightScale);
+ }
if (CvtBBI->BB->pred_size() > 1) {
BBI.NonPredSize -= TII->RemoveBranch(*BBI.BB);
// Copy instructions in the true block, predicate them, and add them to
@@ -1185,6 +1215,23 @@ bool IfConverter::IfConvertTriangle(BBInfo &BBI, IfcvtKind Kind) {
llvm_unreachable("Unable to reverse branch condition!");
TII->InsertBranch(*BBI.BB, CvtBBI->FalseBB, NULL, RevCond, dl);
BBI.BB->addSuccessor(CvtBBI->FalseBB);
+ // Update the edge weight for both CvtBBI->FalseBB and NextBBI.
+ // New_Weight(BBI.BB, NextBBI->BB) =
+ // Weight(BBI.BB, NextBBI->BB) * getSumForBlock(CvtBBI->BB) +
+ // Weight(BBI.BB, CvtBBI->BB) * Weight(CvtBBI->BB, NextBBI->BB)
+ // New_Weight(BBI.BB, CvtBBI->FalseBB) =
+ // Weight(BBI.BB, CvtBBI->BB) * Weight(CvtBBI->BB, CvtBBI->FalseBB)
+
+ uint64_t BBNext = MBPI->getEdgeWeight(BBI.BB, NextBBI->BB);
+ uint64_t BBCvt = MBPI->getEdgeWeight(BBI.BB, CvtBBI->BB);
+
+ uint64_t NewNext = BBNext * SumWeight + (BBCvt * CvtNext) / WeightScale;
+ uint64_t NewFalse = (BBCvt * CvtFalse) / WeightScale;
+ // We need to scale down all weights of BBI.BB to fit uint32_t.
+ // Here BBI.BB is connected to CvtBBI->FalseBB and will fall through to
+ // the next block.
+ ScaleWeights(NewNext, NewFalse, BBI.BB, getNextBlock(BBI.BB),
+ CvtBBI->FalseBB, MBPI);
}
// Merge in the 'false' block if the 'false' block has no other
diff --git a/lib/CodeGen/MachineBasicBlock.cpp b/lib/CodeGen/MachineBasicBlock.cpp
index 5bd47c2f1d..d2df4e5cb8 100644
--- a/lib/CodeGen/MachineBasicBlock.cpp
+++ b/lib/CodeGen/MachineBasicBlock.cpp
@@ -1123,6 +1123,13 @@ uint32_t MachineBasicBlock::getSuccWeight(const_succ_iterator Succ) const {
return *getWeightIterator(Succ);
}
+/// Set successor weight of a given iterator.
+void MachineBasicBlock::setSuccWeight(succ_iterator I, uint32_t weight) {
+ if (Weights.empty())
+ return;
+ *getWeightIterator(I) = weight;
+}
+
/// getWeightIterator - Return wight iterator corresonding to the I successor
/// iterator
MachineBasicBlock::weight_iterator MachineBasicBlock::
diff --git a/test/CodeGen/ARM/ifcvt-branch-weight.ll b/test/CodeGen/ARM/ifcvt-branch-weight.ll
new file mode 100644
index 0000000000..cd8a561386
--- /dev/null
+++ b/test/CodeGen/ARM/ifcvt-branch-weight.ll
@@ -0,0 +1,42 @@
+; RUN: llc < %s -mtriple=thumbv8 -print-machineinstrs=if-converter -o /dev/null 2>&1 | FileCheck %s
+
+%struct.S = type { i8* (i8*)*, [1 x i8] }
+define internal zeroext i8 @bar(%struct.S* %x, %struct.S* nocapture %y) nounwind readonly {
+entry:
+ %0 = getelementptr inbounds %struct.S* %x, i32 0, i32 1, i32 0
+ %1 = load i8* %0, align 1
+ %2 = zext i8 %1 to i32
+ %3 = and i32 %2, 112
+ %4 = icmp eq i32 %3, 0
+ br i1 %4, label %return, label %bb
+
+bb:
+ %5 = getelementptr inbounds %struct.S* %y, i32 0, i32 1, i32 0
+ %6 = load i8* %5, align 1
+ %7 = zext i8 %6 to i32
+ %8 = and i32 %7, 112
+ %9 = icmp eq i32 %8, 0
+ br i1 %9, label %return, label %bb2
+
+; CHECK: BB#2: derived from LLVM BB %bb2
+; CHECK: Successors according to CFG: BB#3(192) BB#4(192)
+
+bb2:
+ %v10 = icmp eq i32 %3, 16
+ br i1 %v10, label %bb4, label %bb3, !prof !0
+
+bb3:
+ %v11 = icmp eq i32 %8, 16
+ br i1 %v11, label %bb4, label %return, !prof !1
+
+bb4:
+ %v12 = ptrtoint %struct.S* %x to i32
+ %phitmp = trunc i32 %v12 to i8
+ ret i8 %phitmp
+
+return:
+ ret i8 1
+}
+
+!0 = metadata !{metadata !"branch_weights", i32 4, i32 12}
+!1 = metadata !{metadata !"branch_weights", i32 8, i32 16}
diff --git a/test/CodeGen/Thumb2/v8_IT_5.ll b/test/CodeGen/Thumb2/v8_IT_5.ll
index 30250c8d02..2f352d6d5f 100644
--- a/test/CodeGen/Thumb2/v8_IT_5.ll
+++ b/test/CodeGen/Thumb2/v8_IT_5.ll
@@ -2,7 +2,7 @@
; RUN: llc < %s -mtriple=thumbv7 -arm-restrict-it | FileCheck %s
; CHECK: it ne
; CHECK-NEXT: cmpne
-; CHECK-NEXT: beq
+; CHECK-NEXT: bne [[JUMPTARGET:.LBB[0-9]+_[0-9]+]]
; CHECK: cmp
; CHECK-NEXT: beq
; CHECK-NEXT: %if.else163
@@ -10,6 +10,7 @@
; CHECK-NEXT: b
; CHECK-NEXT: %if.else145
; CHECK-NEXT: mov.w
+; CHECK: [[JUMPTARGET]]:{{.*}}%if.else173
%struct.hc = type { i32, i32, i32, i32 }
generated by cgit v1.2.3 (git 2.25.1) at 2024-05-02 03:15:22 +0000