64bit CAS on 32bit x86.

git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@47929 91177308-0d34-0410-b5e6-96231b3b80d8

4 files changed, 75 insertions, 8 deletions

diff --git a/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index 71362c5765..c1bf2188b1 100644
--- a/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -6095,6 +6095,17 @@ void SelectionDAGLegalize::ExpandOp(SDOperand Op, SDOperand &Lo, SDOperand &Hi){
     break;
   }
 
+  case ISD::ATOMIC_LCS: {
+    SDOperand Tmp = TLI.LowerOperation(Op, DAG);
+    assert(Tmp.Val && "Node must be custom expanded!");
+    ExpandOp(Tmp.getValue(0), Lo, Hi);
+    AddLegalizedOperand(SDOperand(Node, 1), // Remember we legalized the chain.
+                        LegalizeOp(Tmp.getValue(1)));
+    break;
+  }
+
+
+
     // These operators cannot be expanded directly, emit them as calls to
     // library functions.
   case ISD::FP_TO_SINT: {
diff --git a/lib/Target/X86/X86ISelLowering.cpp b/lib/Target/X86/X86ISelLowering.cpp
index a2143ee168..d0d3f64458 100644
--- a/lib/Target/X86/X86ISelLowering.cpp
+++ b/lib/Target/X86/X86ISelLowering.cpp
@@ -5355,7 +5355,7 @@ SDOperand X86TargetLowering::LowerCTTZ(SDOperand Op, SelectionDAG &DAG) {
   return Op;
 }
 
-SDOperand X86TargetLowering::LowerCAS(SDOperand Op, SelectionDAG &DAG) {
+SDOperand X86TargetLowering::LowerLCS(SDOperand Op, SelectionDAG &DAG) {
   MVT::ValueType T = cast<AtomicSDNode>(Op.Val)->getVT();
   unsigned Reg = 0;
   unsigned size = 0;
@@ -5363,15 +5363,20 @@ SDOperand X86TargetLowering::LowerCAS(SDOperand Op, SelectionDAG &DAG) {
   case MVT::i8:  Reg = X86::AL;  size = 1; break;
   case MVT::i16: Reg = X86::AX;  size = 2; break;
   case MVT::i32: Reg = X86::EAX; size = 4; break;
-  case MVT::i64: Reg = X86::RAX; size = 8; break;
+  case MVT::i64: 
+    if (Subtarget->is64Bit()) {
+      Reg = X86::RAX; size = 8;
+    } else //Should go away when LowerType stuff lands
+      return SDOperand(ExpandATOMIC_LCS(Op.Val, DAG), 0);
+    break;
   };
   SDOperand cpIn = DAG.getCopyToReg(Op.getOperand(0), Reg,
                                     Op.getOperand(3), SDOperand());
   SDOperand Ops[] = { cpIn.getValue(0),
-                       Op.getOperand(1),
-                       Op.getOperand(2),
-                       DAG.getTargetConstant(size, MVT::i8),
-                       cpIn.getValue(1) };
+                      Op.getOperand(1),
+                      Op.getOperand(2),
+                      DAG.getTargetConstant(size, MVT::i8),
+                      cpIn.getValue(1) };
   SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Flag);
   SDOperand Result = DAG.getNode(X86ISD::LCMPXCHG_DAG, Tys, Ops, 5);
   SDOperand cpOut = 
@@ -5379,12 +5384,48 @@ SDOperand X86TargetLowering::LowerCAS(SDOperand Op, SelectionDAG &DAG) {
   return cpOut;
 }
 
+SDNode* X86TargetLowering::ExpandATOMIC_LCS(SDNode* Op, SelectionDAG &DAG) {
+  MVT::ValueType T = cast<AtomicSDNode>(Op)->getVT();
+  assert (T == MVT::i64 && "Only know how to expand i64 CAS");
+  SDOperand cpInL, cpInH;
+  cpInL = DAG.getNode(ISD::EXTRACT_ELEMENT, MVT::i32, Op->getOperand(3),
+                      DAG.getConstant(0, MVT::i32));
+  cpInH = DAG.getNode(ISD::EXTRACT_ELEMENT, MVT::i32, Op->getOperand(3),
+                      DAG.getConstant(1, MVT::i32));
+  cpInL = DAG.getCopyToReg(Op->getOperand(0), X86::EAX,
+                           cpInL, SDOperand());
+  cpInH = DAG.getCopyToReg(cpInL.getValue(0), X86::EDX,
+                           cpInH, cpInL.getValue(1));
+  SDOperand swapInL, swapInH;
+  swapInL = DAG.getNode(ISD::EXTRACT_ELEMENT, MVT::i32, Op->getOperand(2),
+                        DAG.getConstant(0, MVT::i32));
+  swapInH = DAG.getNode(ISD::EXTRACT_ELEMENT, MVT::i32, Op->getOperand(2),
+                        DAG.getConstant(1, MVT::i32));
+  swapInL = DAG.getCopyToReg(cpInH.getValue(0), X86::EBX,
+                             swapInL, cpInH.getValue(1));
+  swapInH = DAG.getCopyToReg(swapInL.getValue(0), X86::ECX,
+                             swapInH, swapInL.getValue(1));
+  SDOperand Ops[] = { swapInH.getValue(0),
+                      Op->getOperand(1),
+                      swapInH.getValue(1)};
+  SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Flag);
+  SDOperand Result = DAG.getNode(X86ISD::LCMPXCHG8_DAG, Tys, Ops, 3);
+  SDOperand cpOutL = DAG.getCopyFromReg(Result.getValue(0), X86::EAX, MVT::i32, 
+                                        Result.getValue(1));
+  SDOperand cpOutH = DAG.getCopyFromReg(cpOutL.getValue(1), X86::EDX, MVT::i32, 
+                                        cpOutL.getValue(2));
+  SDOperand OpsF[] = { cpOutL.getValue(0), cpOutH.getValue(0)};
+  SDOperand ResultVal = DAG.getNode(ISD::BUILD_PAIR, MVT::i64, OpsF, 2);
+  Tys = DAG.getVTList(MVT::i64, MVT::Other);
+  return DAG.getNode(ISD::MERGE_VALUES, Tys, ResultVal, cpOutH.getValue(1)).Val;
+}
+
 /// LowerOperation - Provide custom lowering hooks for some operations.
 ///
 SDOperand X86TargetLowering::LowerOperation(SDOperand Op, SelectionDAG &DAG) {
   switch (Op.getOpcode()) {
   default: assert(0 && "Should not custom lower this!");
-  case ISD::ATOMIC_LCS:         return LowerCAS(Op,DAG);
+  case ISD::ATOMIC_LCS:         return LowerLCS(Op,DAG);
   case ISD::BUILD_VECTOR:       return LowerBUILD_VECTOR(Op, DAG);
   case ISD::VECTOR_SHUFFLE:     return LowerVECTOR_SHUFFLE(Op, DAG);
   case ISD::EXTRACT_VECTOR_ELT: return LowerEXTRACT_VECTOR_ELT(Op, DAG);
@@ -5437,6 +5478,7 @@ SDNode *X86TargetLowering::ExpandOperationResult(SDNode *N, SelectionDAG &DAG) {
   default: assert(0 && "Should not custom lower this!");
   case ISD::FP_TO_SINT:         return ExpandFP_TO_SINT(N, DAG);
   case ISD::READCYCLECOUNTER:   return ExpandREADCYCLECOUNTER(N, DAG);
+  case ISD::ATOMIC_LCS:         return ExpandATOMIC_LCS(N, DAG);
   }
 }
 
@@ -5490,6 +5532,7 @@ const char *X86TargetLowering::getTargetNodeName(unsigned Opcode) const {
   case X86ISD::TC_RETURN:          return "X86ISD::TC_RETURN";
   case X86ISD::FNSTCW16m:          return "X86ISD::FNSTCW16m";
   case X86ISD::LCMPXCHG_DAG:       return "x86ISD::LCMPXCHG_DAG";
+  case X86ISD::LCMPXCHG8_DAG:      return "x86ISD::LCMPXCHG8_DAG";
   }
 }
 
diff --git a/lib/Target/X86/X86ISelLowering.h b/lib/Target/X86/X86ISelLowering.h
index 345aa01e21..d9593a4ef6 100644
--- a/lib/Target/X86/X86ISelLowering.h
+++ b/lib/Target/X86/X86ISelLowering.h
@@ -210,6 +210,7 @@ namespace llvm {
 
       // compare and swap
       LCMPXCHG_DAG,
+      LCMPXCHG8_DAG,
 
       // Store FP control world into i16 memory
       FNSTCW16m
@@ -543,9 +544,10 @@ namespace llvm {
     SDOperand LowerFLT_ROUNDS_(SDOperand Op, SelectionDAG &DAG);
     SDOperand LowerCTLZ(SDOperand Op, SelectionDAG &DAG);
     SDOperand LowerCTTZ(SDOperand Op, SelectionDAG &DAG);
-    SDOperand LowerCAS(SDOperand Op, SelectionDAG &DAG);
+    SDOperand LowerLCS(SDOperand Op, SelectionDAG &DAG);
     SDNode *ExpandFP_TO_SINT(SDNode *N, SelectionDAG &DAG);
     SDNode *ExpandREADCYCLECOUNTER(SDNode *N, SelectionDAG &DAG);
+    SDNode *ExpandATOMIC_LCS(SDNode *N, SelectionDAG &DAG);
   };
 }
 
diff --git a/lib/Target/X86/X86InstrInfo.td b/lib/Target/X86/X86InstrInfo.td
index b62171f558..223889c408 100644
--- a/lib/Target/X86/X86InstrInfo.td
+++ b/lib/Target/X86/X86InstrInfo.td
@@ -37,6 +37,7 @@ def SDTX86SetCC   : SDTypeProfile<1, 2,
 
 def SDTX86cas : SDTypeProfile<0, 3, [SDTCisPtrTy<0>, SDTCisInt<1>, 
                                      SDTCisVT<2, i8>]>;
+def SDTX86cas8 : SDTypeProfile<0, 1, [SDTCisPtrTy<0>]>;
 
 def SDTX86Ret     : SDTypeProfile<0, 1, [SDTCisVT<0, i16>]>;
 
@@ -75,6 +76,9 @@ def X86setcc   : SDNode<"X86ISD::SETCC",    SDTX86SetCC>;
 def X86cas : SDNode<"X86ISD::LCMPXCHG_DAG", SDTX86cas,
                         [SDNPHasChain, SDNPInFlag, SDNPOutFlag, SDNPMayStore,
                          SDNPMayLoad]>;
+def X86cas8 : SDNode<"X86ISD::LCMPXCHG8_DAG", SDTX86cas8,
+                        [SDNPHasChain, SDNPInFlag, SDNPOutFlag, SDNPMayStore,
+                         SDNPMayLoad]>;
 
 def X86retflag : SDNode<"X86ISD::RET_FLAG", SDTX86Ret,
                         [SDNPHasChain, SDNPOptInFlag]>;
@@ -2556,6 +2560,13 @@ def LCMPXCHG32 : I<0xB1, Pseudo, (outs), (ins i32mem:$ptr, GR32:$swap),
                "lock cmpxchgl $swap,$ptr",
                [(X86cas addr:$ptr, GR32:$swap, 4)]>, TB, LOCK;
 }
+let Defs = [EAX, EBX, ECX, EDX, EFLAGS], Uses = [EAX, EBX, ECX, EDX] in {
+def CMPXCHG8B : I<0xC7, Pseudo, (outs), (ins i32mem:$ptr),
+               "cmpxchg8b $ptr", []>, TB;
+def LCMPXCHG8B : I<0xC7, Pseudo, (outs), (ins i32mem:$ptr),
+               "lock cmpxchg8b $ptr",
+               [(X86cas8 addr:$ptr)]>, TB, LOCK;
+}
 
 let Defs = [AX, EFLAGS], Uses = [AX] in {
 def CMPXCHG16 : I<0xB1, Pseudo, (outs), (ins i16mem:$ptr, GR16:$swap),