diff options
61 files changed, 629 insertions, 295 deletions
diff --git a/docs/Atomics.rst b/docs/Atomics.rst index 1243f34548..5f17c6154f 100644 --- a/docs/Atomics.rst +++ b/docs/Atomics.rst @@ -110,8 +110,7 @@ where threads and signals are involved. ``cmpxchg`` and ``atomicrmw`` are essentially like an atomic load followed by an atomic store (where the store is conditional for ``cmpxchg``), but no other -memory operation can happen on any thread between the load and store. Note that -LLVM's cmpxchg does not provide quite as many options as the C++0x version. +memory operation can happen on any thread between the load and store. A ``fence`` provides Acquire and/or Release ordering which is not part of another operation; it is normally used along with Monotonic memory operations. @@ -430,10 +429,9 @@ other ``atomicrmw`` operations generate a loop with ``LOCK CMPXCHG``. Depending on the users of the result, some ``atomicrmw`` operations can be translated into operations like ``LOCK AND``, but that does not work in general. -On ARM, MIPS, and many other RISC architectures, Acquire, Release, and -SequentiallyConsistent semantics require barrier instructions for every such +On ARM (before v8), MIPS, and many other RISC architectures, Acquire, Release, +and SequentiallyConsistent semantics require barrier instructions for every such operation. Loads and stores generate normal instructions. ``cmpxchg`` and ``atomicrmw`` can be represented using a loop with LL/SC-style instructions which take some sort of exclusive lock on a cache line (``LDREX`` and ``STREX`` -on ARM, etc.). At the moment, the IR does not provide any way to represent a -weak ``cmpxchg`` which would not require a loop. +on ARM, etc.). diff --git a/docs/LangRef.rst b/docs/LangRef.rst index ee95cb9f85..8958505e9d 100644 --- a/docs/LangRef.rst +++ b/docs/LangRef.rst @@ -5059,14 +5059,14 @@ Syntax: :: - cmpxchg [volatile] <ty>* <pointer>, <ty> <cmp>, <ty> <new> [singlethread] <success ordering> <failure ordering> ; yields {ty} + cmpxchg [weak] [volatile] <ty>* <pointer>, <ty> <cmp>, <ty> <new> [singlethread] <success ordering> <failure ordering> ; yields { <ty>, i1 } Overview: """"""""" The '``cmpxchg``' instruction is used to atomically modify memory. It loads a value in memory and compares it to a given value. If they are -equal, it stores a new value into the memory. +equal, it tries to store a new value into the memory. Arguments: """""""""" @@ -5099,10 +5099,17 @@ equal to the size in memory of the operand. Semantics: """""""""" -The contents of memory at the location specified by the '``<pointer>``' -operand is read and compared to '``<cmp>``'; if the read value is the -equal, '``<new>``' is written. The original value at the location is -returned. +The contents of memory at the location specified by the '``<pointer>``' operand +is read and compared to '``<cmp>``'; if the read value is the equal, the +'``<new>``' is written. The original value at the location is returned, together +with a flag indicating success (true) or failure (false). + +If the cmpxchg operation is marked as ``weak`` then a spurious failure is +permitted: the operation may not write ``<new>`` even if the comparison +matched. + +If the cmpxchg operation is strong (the default), the i1 value is 1 if and only +if the value loaded equals ``cmp``. A successful ``cmpxchg`` is a read-modify-write instruction for the purpose of identifying release sequences. A failed ``cmpxchg`` is equivalent to an atomic @@ -5114,14 +5121,15 @@ Example: .. code-block:: llvm entry: - %orig = atomic load i32* %ptr unordered ; yields {i32} + %orig = atomic load i32* %ptr unordered ; yields i32 br label %loop loop: %cmp = phi i32 [ %orig, %entry ], [%old, %loop] %squared = mul i32 %cmp, %cmp - %old = cmpxchg i32* %ptr, i32 %cmp, i32 %squared acq_rel monotonic ; yields {i32} - %success = icmp eq i32 %cmp, %old + %val_success = cmpxchg i32* %ptr, i32 %cmp, i32 %squared acq_rel monotonic ; yields { i32, i1 } + %value_loaded = extractvalue { i32, i1 } %val_success, 0 + %success = extractvalue { i32, i1 } %val_success, 1 br i1 %success, label %done, label %loop done: diff --git a/include/llvm/CodeGen/ISDOpcodes.h b/include/llvm/CodeGen/ISDOpcodes.h index 49891b2934..80fb8b2d3a 100644 --- a/include/llvm/CodeGen/ISDOpcodes.h +++ b/include/llvm/CodeGen/ISDOpcodes.h @@ -619,6 +619,12 @@ namespace ISD { /// This corresponds to the cmpxchg instruction. ATOMIC_CMP_SWAP, + /// Val, Success, OUTCHAIN + /// = ATOMIC_CMP_SWAP_WITH_SUCCESS(INCHAIN, ptr, cmp, swap) + /// N.b. this is still a strong cmpxchg operation, so + /// Success == "Val == cmp". + ATOMIC_CMP_SWAP_WITH_SUCCESS, + /// Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) /// Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN, ptr, amt) /// For double-word atomic operations: diff --git a/include/llvm/CodeGen/SelectionDAG.h b/include/llvm/CodeGen/SelectionDAG.h index 610ba25f45..db2e841e17 100644 --- a/include/llvm/CodeGen/SelectionDAG.h +++ b/include/llvm/CodeGen/SelectionDAG.h @@ -700,20 +700,22 @@ public: SDValue getVAArg(EVT VT, SDLoc dl, SDValue Chain, SDValue Ptr, SDValue SV, unsigned Align); - /// getAtomic - Gets a node for an atomic op, produces result and chain and - /// takes 3 operands - SDValue getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT, SDValue Chain, - SDValue Ptr, SDValue Cmp, SDValue Swp, - MachinePointerInfo PtrInfo, unsigned Alignment, - AtomicOrdering SuccessOrdering, - AtomicOrdering FailureOrdering, - SynchronizationScope SynchScope); - SDValue getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT, SDValue Chain, - SDValue Ptr, SDValue Cmp, SDValue Swp, - MachineMemOperand *MMO, - AtomicOrdering SuccessOrdering, - AtomicOrdering FailureOrdering, - SynchronizationScope SynchScope); + /// getAtomicCmpSwap - Gets a node for an atomic cmpxchg op. There are two + /// valid Opcodes. ISD::ATOMIC_CMO_SWAP produces a the value loaded and a + /// chain result. ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS produces the value loaded, + /// a success flag (initially i1), and a chain. + SDValue getAtomicCmpSwap(unsigned Opcode, SDLoc dl, EVT MemVT, SDVTList VTs, + SDValue Chain, SDValue Ptr, SDValue Cmp, SDValue Swp, + MachinePointerInfo PtrInfo, unsigned Alignment, + AtomicOrdering SuccessOrdering, + AtomicOrdering FailureOrdering, + SynchronizationScope SynchScope); + SDValue getAtomicCmpSwap(unsigned Opcode, SDLoc dl, EVT MemVT, SDVTList VTs, + SDValue Chain, SDValue Ptr, SDValue Cmp, SDValue Swp, + MachineMemOperand *MMO, + AtomicOrdering SuccessOrdering, + AtomicOrdering FailureOrdering, + SynchronizationScope SynchScope); /// getAtomic - Gets a node for an atomic op, produces result (if relevant) /// and chain and takes 2 operands. diff --git a/include/llvm/CodeGen/SelectionDAGNodes.h b/include/llvm/CodeGen/SelectionDAGNodes.h index b209268843..a39d35be61 100644 --- a/include/llvm/CodeGen/SelectionDAGNodes.h +++ b/include/llvm/CodeGen/SelectionDAGNodes.h @@ -1126,6 +1126,7 @@ public: N->getOpcode() == ISD::STORE || N->getOpcode() == ISD::PREFETCH || N->getOpcode() == ISD::ATOMIC_CMP_SWAP || + N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS || N->getOpcode() == ISD::ATOMIC_SWAP || N->getOpcode() == ISD::ATOMIC_LOAD_ADD || N->getOpcode() == ISD::ATOMIC_LOAD_SUB || @@ -1234,12 +1235,13 @@ public: bool isCompareAndSwap() const { unsigned Op = getOpcode(); - return Op == ISD::ATOMIC_CMP_SWAP; + return Op == ISD::ATOMIC_CMP_SWAP || Op == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS; } // Methods to support isa and dyn_cast static bool classof(const SDNode *N) { return N->getOpcode() == ISD::ATOMIC_CMP_SWAP || + N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS || N->getOpcode() == ISD::ATOMIC_SWAP || N->getOpcode() == ISD::ATOMIC_LOAD_ADD || N->getOpcode() == ISD::ATOMIC_LOAD_SUB || diff --git a/include/llvm/IR/Instructions.h b/include/llvm/IR/Instructions.h index c6709f4459..e0c829ac98 100644 --- a/include/llvm/IR/Instructions.h +++ b/include/llvm/IR/Instructions.h @@ -500,6 +500,16 @@ public: (unsigned)V); } + /// Return true if this cmpxchg may spuriously fail. + bool isWeak() const { + return getSubclassDataFromInstruction() & 0x100; + } + + void setWeak(bool IsWeak) { + setInstructionSubclassData((getSubclassDataFromInstruction() & ~0x100) | + (IsWeak << 8)); + } + /// Transparently provide more efficient getOperand methods. DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value); diff --git a/lib/AsmParser/LLLexer.cpp b/lib/AsmParser/LLLexer.cpp index b9193e89e7..1334825a7d 100644 --- a/lib/AsmParser/LLLexer.cpp +++ b/lib/AsmParser/LLLexer.cpp @@ -490,7 +490,7 @@ lltok::Kind LLLexer::LexIdentifier() { KEYWORD(available_externally); KEYWORD(linkonce); KEYWORD(linkonce_odr); - KEYWORD(weak); + KEYWORD(weak); // Use as a linkage, and a modifier for "cmpxchg". KEYWORD(weak_odr); KEYWORD(appending); KEYWORD(dllimport); diff --git a/lib/AsmParser/LLParser.cpp b/lib/AsmParser/LLParser.cpp index a528a7e7a5..0c188f983f 100644 --- a/lib/AsmParser/LLParser.cpp +++ b/lib/AsmParser/LLParser.cpp @@ -4256,8 +4256,8 @@ int LLParser::ParseStore(Instruction *&Inst, PerFunctionState &PFS) { } /// ParseCmpXchg -/// ::= 'cmpxchg' 'volatile'? TypeAndValue ',' TypeAndValue ',' TypeAndValue -/// 'singlethread'? AtomicOrdering AtomicOrdering +/// ::= 'cmpxchg' 'weak'? 'volatile'? TypeAndValue ',' TypeAndValue ',' +/// TypeAndValue 'singlethread'? AtomicOrdering AtomicOrdering int LLParser::ParseCmpXchg(Instruction *&Inst, PerFunctionState &PFS) { Value *Ptr, *Cmp, *New; LocTy PtrLoc, CmpLoc, NewLoc; bool AteExtraComma = false; @@ -4265,6 +4265,10 @@ int LLParser::ParseCmpXchg(Instruction *&Inst, PerFunctionState &PFS) { AtomicOrdering FailureOrdering = NotAtomic; SynchronizationScope Scope = CrossThread; bool isVolatile = false; + bool isWeak = false; + + if (EatIfPresent(lltok::kw_weak)) + isWeak = true; if (EatIfPresent(lltok::kw_volatile)) isVolatile = true; @@ -4297,9 +4301,10 @@ int LLParser::ParseCmpXchg(Instruction *&Inst, PerFunctionState &PFS) { return Error(NewLoc, "cmpxchg operand must be power-of-two byte-sized" " integer"); - AtomicCmpXchgInst *CXI = new AtomicCmpXchgInst(Ptr, Cmp, New, SuccessOrdering, - FailureOrdering, Scope); + AtomicCmpXchgInst *CXI = new AtomicCmpXchgInst( + Ptr, Cmp, New, SuccessOrdering, FailureOrdering, Scope); CXI->setVolatile(isVolatile); + CXI->setWeak(isWeak); Inst = CXI; return AteExtraComma ? InstExtraComma : InstNormal; } diff --git a/lib/AsmParser/LLToken.h b/lib/AsmParser/LLToken.h index 3ce09503f5..af8b0da78b 100644 --- a/lib/AsmParser/LLToken.h +++ b/lib/AsmParser/LLToken.h @@ -42,7 +42,8 @@ namespace lltok { kw_linker_private, // NOTE: deprecated, for parser compatibility kw_linker_private_weak, // NOTE: deprecated, for parser compatibility kw_linkonce, kw_linkonce_odr, - kw_weak, kw_weak_odr, kw_appending, + kw_weak, // Used as a linkage, and a modifier for "cmpxchg". + kw_weak_odr, kw_appending, kw_dllimport, kw_dllexport, kw_common, kw_available_externally, kw_default, kw_hidden, kw_protected, kw_unnamed_addr, diff --git a/lib/Bitcode/Reader/BitcodeReader.cpp b/lib/Bitcode/Reader/BitcodeReader.cpp index cd40857e3c..c13eba78c2 100644 --- a/lib/Bitcode/Reader/BitcodeReader.cpp +++ b/lib/Bitcode/Reader/BitcodeReader.cpp @@ -2923,7 +2923,7 @@ std::error_code BitcodeReader::ParseFunctionBody(Function *F) { } case bitc::FUNC_CODE_INST_CMPXCHG: { // CMPXCHG:[ptrty, ptr, cmp, new, vol, successordering, synchscope, - // failureordering] + // failureordering?, isweak?] unsigned OpNum = 0; Value *Ptr, *Cmp, *New; if (getValueTypePair(Record, OpNum, NextValueNo, Ptr) || @@ -2931,7 +2931,7 @@ std::error_code BitcodeReader::ParseFunctionBody(Function *F) { cast<PointerType>(Ptr->getType())->getElementType(), Cmp) || popValue(Record, OpNum, NextValueNo, cast<PointerType>(Ptr->getType())->getElementType(), New) || - (OpNum + 3 != Record.size() && OpNum + 4 != Record.size())) + (Record.size() < OpNum + 3 || Record.size() > OpNum + 5)) return Error(InvalidRecord); AtomicOrdering SuccessOrdering = GetDecodedOrdering(Record[OpNum+1]); if (SuccessOrdering == NotAtomic || SuccessOrdering == Unordered) @@ -2948,6 +2948,17 @@ std::error_code BitcodeReader::ParseFunctionBody(Function *F) { I = new AtomicCmpXchgInst(Ptr, Cmp, New, SuccessOrdering, FailureOrdering, SynchScope); cast<AtomicCmpXchgInst>(I)->setVolatile(Record[OpNum]); + + if (Record.size() < 8) { + // Before weak cmpxchgs existed, the instruction simply returned the + // value loaded from memory, so bitcode files from that era will be + // expecting the first component of a modern cmpxchg. + CurBB->getInstList().push_back(I); + I = ExtractValueInst::Create(I, 0); + } else { + cast<AtomicCmpXchgInst>(I)->setWeak(Record[OpNum+4]); + } + InstructionList.push_back(I); break; } diff --git a/lib/Bitcode/Writer/BitcodeWriter.cpp b/lib/Bitcode/Writer/BitcodeWriter.cpp index 7793d3e69e..3ba7358ae5 100644 --- a/lib/Bitcode/Writer/BitcodeWriter.cpp +++ b/lib/Bitcode/Writer/BitcodeWriter.cpp @@ -1449,6 +1449,7 @@ static void WriteInstruction(const Instruction &I, unsigned InstID, cast<AtomicCmpXchgInst>(I).getSynchScope())); Vals.push_back(GetEncodedOrdering( cast<AtomicCmpXchgInst>(I).getFailureOrdering())); + Vals.push_back(cast<AtomicCmpXchgInst>(I).isWeak()); break; case Instruction::AtomicRMW: Code = bitc::FUNC_CODE_INST_ATOMICRMW; diff --git a/lib/CodeGen/AtomicExpandLoadLinkedPass.cpp b/lib/CodeGen/AtomicExpandLoadLinkedPass.cpp index fbb6e9662d..6f93ced85b 100644 --- a/lib/CodeGen/AtomicExpandLoadLinkedPass.cpp +++ b/lib/CodeGen/AtomicExpandLoadLinkedPass.cpp @@ -299,41 +299,44 @@ bool AtomicExpandLoadLinked::expandAtomicCmpXchg(AtomicCmpXchgInst *CI) { // Setup the builder so we can create any PHIs we need. Builder.SetInsertPoint(FailureBB, FailureBB->begin()); BasicBlock *SuccessBB = FailureOrder == Monotonic ? BarrierBB : TryStoreBB; - PHINode *Success = nullptr, *Failure = nullptr; + PHINode *Success = Builder.CreatePHI(Type::getInt1Ty(Ctx), 2); + Success->addIncoming(ConstantInt::getTrue(Ctx), SuccessBB); + Success->addIncoming(ConstantInt::getFalse(Ctx), LoopBB); // Look for any users of the cmpxchg that are just comparing the loaded value // against the desired one, and replace them with the CFG-derived version. + SmallVector<ExtractValueInst *, 2> PrunedInsts; for (auto User : CI->users()) { - ICmpInst *ICmp = dyn_cast<ICmpInst>(User); - if (!ICmp) + ExtractValueInst *EV = dyn_cast<ExtractValueInst>(User); + if (!EV) continue; - // Because we know ICmp uses CI, we only need one operand to be the old - // value. - if (ICmp->getOperand(0) != CI->getCompareOperand() && - ICmp->getOperand(1) != CI->getCompareOperand()) - continue; + assert(EV->getNumIndices() == 1 && EV->getIndices()[0] <= 1 && + "weird extraction from { iN, i1 }"); - if (ICmp->getPredicate() == CmpInst::ICMP_EQ) { - if (!Success) { - Success = Builder.CreatePHI(Type::getInt1Ty(Ctx), 2); - Success->addIncoming(ConstantInt::getTrue(Ctx), SuccessBB); - Success->addIncoming(ConstantInt::getFalse(Ctx), LoopBB); - } - ICmp->replaceAllUsesWith(Success); - } else if (ICmp->getPredicate() == CmpInst::ICMP_NE) { - if (!Failure) { - Failure = Builder.CreatePHI(Type::getInt1Ty(Ctx), 2); - Failure->addIncoming(ConstantInt::getFalse(Ctx), SuccessBB); - Failure->addIncoming(ConstantInt::getTrue(Ctx), LoopBB); - } - ICmp->replaceAllUsesWith(Failure); - } + if (EV->getIndices()[0] == 0) + EV->replaceAllUsesWith(Loaded); + else + EV->replaceAllUsesWith(Success); + + PrunedInsts.push_back(EV); } - CI->replaceAllUsesWith(Loaded); - CI->eraseFromParent(); + // We can remove the instructions now we're no longer iterating through them. + for (auto EV : PrunedInsts) + EV->eraseFromParent(); + if (!CI->use_empty()) { + // Some use of the full struct return that we don't understand has happened, + // so we've got to reconstruct it properly. + Value *Res; + Res = Builder.CreateInsertValue(UndefValue::get(CI->getType()), Loaded, 0); + Res = Builder.CreateInsertValue(Res, Success, 1); + + CI->replaceAllUsesWith(Res); + } + + CI->eraseFromParent(); return true; } diff --git a/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp index bf7541bf28..c76a52abe6 100644 --- a/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp +++ b/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp @@ -3007,14 +3007,14 @@ void SelectionDAGLegalize::ExpandNode(SDNode *Node) { case ISD::ATOMIC_LOAD: { // There is no libcall for atomic load; fake it with ATOMIC_CMP_SWAP. SDValue Zero = DAG.getConstant(0, Node->getValueType(0)); - SDValue Swap = DAG.getAtomic(ISD::ATOMIC_CMP_SWAP, dl, - cast<AtomicSDNode>(Node)->getMemoryVT(), - Node->getOperand(0), - Node->getOperand(1), Zero, Zero, - cast<AtomicSDNode>(Node)->getMemOperand(), - cast<AtomicSDNode>(Node)->getOrdering(), - cast<AtomicSDNode>(Node)->getOrdering(), - cast<AtomicSDNode>(Node)->getSynchScope()); + SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other); + SDValue Swap = DAG.getAtomicCmpSwap( + ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs, + Node->getOperand(0), Node->getOperand(1), Zero, Zero, + cast<AtomicSDNode>(Node)->getMemOperand(), + cast<AtomicSDNode>(Node)->getOrdering(), + cast<AtomicSDNode>(Node)->getOrdering(), + cast<AtomicSDNode>(Node)->getSynchScope()); Results.push_back(Swap.getValue(0)); Results.push_back(Swap.getValue(1)); break; @@ -3051,6 +3051,27 @@ void SelectionDAGLegalize::ExpandNode(SDNode *Node) { Results.push_back(Tmp.second); break; } + case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: { + // Expanding an ATOMIC_CMP_SWAP_WITH_SUCCESS produces an ATOMIC_CMP_SWAP and + // splits out the success value as a comparison. Expanding the resulting + // ATOMIC_CMP_SWAP will produce a libcall. + SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other); + SDValue Res = DAG.getAtomicCmpSwap( + ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs, + Node->getOperand(0), Node->getOperand(1), Node->getOperand(2), + Node->getOperand(3), cast<MemSDNode>(Node)->getMemOperand(), + cast<AtomicSDNode>(Node)->getSuccessOrdering(), + cast<AtomicSDNode>(Node)->getFailureOrdering(), + cast<AtomicSDNode>(Node)->getSynchScope()); + + SDValue Success = DAG.getSetCC(SDLoc(Node), Node->getValueType(1), + Res, Node->getOperand(2), ISD::SETEQ); + + Results.push_back(Res.getValue(0)); + Results.push_back(Success); + Results.push_back(Res.getValue(1)); + break; + } case ISD::DYNAMIC_STACKALLOC: ExpandDYNAMIC_STACKALLOC(Node, Results); break; diff --git a/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp b/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp index 2483184dee..a8603423e3 100644 --- a/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp +++ b/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp @@ -138,7 +138,9 @@ void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) { Res = PromoteIntRes_Atomic1(cast<AtomicSDNode>(N)); break; case ISD::ATOMIC_CMP_SWAP: - Res = PromoteIntRes_Atomic2(cast<AtomicSDNode>(N)); break; + case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: + Res = PromoteIntRes_AtomicCmpSwap(cast<AtomicSDNode>(N), ResNo); + break; } // If the result is null then the sub-method took care of registering it. @@ -192,16 +194,41 @@ SDValue DAGTypeLegalizer::PromoteIntRes_Atomic1(AtomicSDNode *N) { return Res; } -SDValue DAGTypeLegalizer::PromoteIntRes_Atomic2(AtomicSDNode *N) { +SDValue DAGTypeLegalizer::PromoteIntRes_AtomicCmpSwap(AtomicSDNode *N, + unsigned ResNo) { + if (ResNo == 1) { + assert(N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS); + EVT SVT = getSetCCResultType(N->getOperand(2).getValueType()); + EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1)); + + // Only use the result of getSetCCResultType if it is legal, + // otherwise just use the promoted result type (NVT). + if (!TLI.isTypeLegal(SVT)) + SVT = NVT; + + SDVTList VTs = DAG.getVTList(N->getValueType(0), SVT, MVT::Other); + SDValue Res = DAG.getAtomicCmpSwap( + ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, SDLoc(N), N->getMemoryVT(), VTs, + N->getChain(), N->getBasePtr(), N->getOperand(2), N->getOperand(3), + N->getMemOperand(), N->getSuccessOrdering(), N->getFailureOrdering(), + N->getSynchScope()); + ReplaceValueWith(SDValue(N, 0), Res.getValue(0)); + ReplaceValueWith(SDValue(N, 2), Res.getValue(2)); + return Res.getValue(1); + } + SDValue Op2 = GetPromotedInteger(N->getOperand(2)); SDValue Op3 = GetPromotedInteger(N->getOperand(3)); - SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N), N->getMemoryVT(), - N->getChain(), N->getBasePtr(), Op2, Op3, - N->getMemOperand(), N->getSuccessOrdering(), - N->getFailureOrdering(), N->getSynchScope()); + SDVTList VTs = + DAG.getVTList(Op2.getValueType(), N->getValueType(1), MVT::Other); + SDValue Res = DAG.getAtomicCmpSwap( + N->getOpcode(), SDLoc(N), N->getMemoryVT(), VTs, N->getChain(), + N->getBasePtr(), Op2, Op3, N->getMemOperand(), N->getSuccessOrdering(), + N->getFailureOrdering(), N->getSynchScope()); // Legalized the chain result - switch anything that used the old chain to // use the new one. - ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); + unsigned ChainOp = N->getNumValues() - 1; + ReplaceValueWith(SDValue(N, ChainOp), Res.getValue(ChainOp)); return Res; } @@ -1143,6 +1170,26 @@ void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) { ReplaceValueWith(SDValue(N, 1), Tmp.second); break; } + case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: { + AtomicSDNode *AN = cast<AtomicSDNode>(N); + SDVTList VTs = DAG.getVTList(N->getValueType(0), MVT::Other); + SDValue Tmp = DAG.getAtomicCmpSwap( + ISD::ATOMIC_CMP_SWAP, SDLoc(N), AN->getMemoryVT(), VTs, + N->getOperand(0), N->getOperand(1), N->getOperand(2), N->getOperand(3), + AN->getMemOperand(), AN->getSuccessOrdering(), AN->getFailureOrdering(), + AN->getSynchScope()); + + // Expanding to the strong ATOMIC_CMP_SWAP node means we can determine + // success simply by comparing the loaded value against the ingoing + // comparison. + SDValue Success = DAG.getSetCC(SDLoc(N), N->getValueType(1), Tmp, + N->getOperand(2), ISD::SETEQ); + + SplitInteger(Tmp, Lo, Hi); + ReplaceValueWith(SDValue(N, 1), Success); + ReplaceValueWith(SDValue(N, 2), Tmp.getValue(1)); + break; + } case ISD::AND: case ISD::OR: @@ -2388,16 +2435,18 @@ void DAGTypeLegalizer::ExpandIntRes_ATOMIC_LOAD(SDNode *N, SDValue &Lo, SDValue &Hi) { SDLoc dl(N); EVT VT = cast<AtomicSDNode>(N)->getMemoryVT(); + SDVTList VTs = DAG.getVTList(VT, MVT::i1, MVT::Other); SDValue Zero = DAG.getConstant(0, VT); - SDValue Swap = DAG.getAtomic(ISD::ATOMIC_CMP_SWAP, dl, VT, - N->getOperand(0), - N->getOperand(1), Zero, Zero, - cast<AtomicSDNode>(N)->getMemOperand(), - cast<AtomicSDNode>(N)->getOrdering(), - cast<AtomicSDNode>(N)->getOrdering(), - cast<AtomicSDNode>(N)->getSynchScope()); + SDValue Swap = DAG.getAtomicCmpSwap( + ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, dl, + cast<AtomicSDNode>(N)->getMemoryVT(), VTs, N->getOperand(0), + N->getOperand(1), Zero, Zero, cast<AtomicSDNode>(N)->getMemOperand(), + cast<AtomicSDNode>(N)->getOrdering(), + cast<AtomicSDNode>(N)->getOrdering(), + cast<AtomicSDNode>(N)->getSynchScope()); + ReplaceValueWith(SDValue(N, 0), Swap.getValue(0)); - ReplaceValueWith(SDValue(N, 1), Swap.getValue(1)); + ReplaceValueWith(SDValue(N, 1), Swap.getValue(2)); } //===----------------------------------------------------------------------===// diff --git a/lib/CodeGen/SelectionDAG/LegalizeTypes.h b/lib/CodeGen/SelectionDAG/LegalizeTypes.h index e4bbc78fa7..04c200c952 100644 --- a/lib/CodeGen/SelectionDAG/LegalizeTypes.h +++ b/lib/CodeGen/SelectionDAG/LegalizeTypes.h @@ -220,7 +220,7 @@ private: SDValue PromoteIntRes_AssertZext(SDNode *N); SDValue PromoteIntRes_Atomic0(AtomicSDNode *N); SDValue PromoteIntRes_Atomic1(AtomicSDNode *N); - SDValue PromoteIntRes_Atomic2(AtomicSDNode *N); + SDValue PromoteIntRes_AtomicCmpSwap(AtomicSDNode *N, unsigned ResNo); SDValue PromoteIntRes_EXTRACT_SUBVECTOR(SDNode *N); SDValue PromoteIntRes_VECTOR_SHUFFLE(SDNode *N); SDValue PromoteIntRes_BUILD_VECTOR(SDNode *N); diff --git a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp index 010431e532..639eb462e7 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -502,6 +502,7 @@ static void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N) { break; } case ISD::ATOMIC_CMP_SWAP: + case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: case ISD::ATOMIC_SWAP: case ISD::ATOMIC_LOAD_ADD: case ISD::ATOMIC_LOAD_SUB: @@ -4327,51 +4328,47 @@ SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT, Ordering, SynchScope); } -SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT, - SDValue Chain, SDValue Ptr, SDValue Cmp, - SDValue Swp, MachinePointerInfo PtrInfo, - unsigned Alignment, - AtomicOrdering SuccessOrdering, - AtomicOrdering FailureOrdering, - SynchronizationScope SynchScope) { +SDValue SelectionDAG::getAtomicCmpSwap( + unsigned Opcode, SDLoc dl, EVT MemVT, SDVTList VTs, SDValue Chain, + SDValue Ptr, SDValue Cmp, SDValue Swp, MachinePointerInfo PtrInfo, + unsigned Alignment, AtomicOrdering SuccessOrdering, + AtomicOrdering FailureOrdering, SynchronizationScope SynchScope) { + assert(Opcode == ISD::ATOMIC_CMP_SWAP || + Opcode == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS); + assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types"); + if (Alignment == 0) // Ensure that codegen never sees alignment 0 Alignment = getEVTAlignment(MemVT); MachineFunction &MF = getMachineFunction(); - // All atomics are load and store, except for ATMOIC_LOAD and ATOMIC_STORE. - // For now, atomics are considered to be volatile always. // FIXME: Volatile isn't really correct; we should keep track of atomic // orderings in the memoperand. unsigned Flags = MachineMemOperand::MOVolatile; - if (Opcode != ISD::ATOMIC_STORE) - Flags |= MachineMemOperand::MOLoad; - if (Opcode != ISD::ATOMIC_LOAD) - Flags |= MachineMemOperand::MOStore; + Flags |= MachineMemOperand::MOLoad; + Flags |= MachineMemOperand::MOStore; MachineMemOperand *MMO = MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Alignment); - return getAtomic(Opcode, dl, MemVT, Chain, Ptr, Cmp, Swp, MMO, - SuccessOrdering, FailureOrdering, SynchScope); + return getAtomicCmpSwap(Opcode, dl, MemVT, VTs, Chain, Ptr, Cmp, Swp, MMO, + SuccessOrdering, FailureOrdering, SynchScope); } -SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT, - SDValue Chain, - SDValue Ptr, SDValue Cmp, - SDValue Swp, MachineMemOperand *MMO, - AtomicOrdering SuccessOrdering, - AtomicOrdering FailureOrdering, - SynchronizationScope SynchScope) { - assert(Opcode == ISD::ATOMIC_CMP_SWAP && "Invalid Atomic Op"); +SDValue SelectionDAG::getAtomicCmpSwap(unsigned Opcode, SDLoc dl, EVT MemVT, + SDVTList VTs, SDValue Chain, SDValue Ptr, + SDValue Cmp, SDValue Swp, + MachineMemOperand *MMO, + AtomicOrdering SuccessOrdering, + AtomicOrdering FailureOrdering, + SynchronizationScope SynchScope) { + assert(Opcode == ISD::ATOMIC_CMP_SWAP || + Opcode == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS); assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types"); - EVT VT = Cmp.getValueType(); - - SDVTList VTs = getVTList(VT, MVT::Other); SDValue Ops[] = {Chain, Ptr, Cmp, Swp}; - return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO, SuccessOrdering, - FailureOrdering, SynchScope); + return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO, + SuccessOrdering, FailureOrdering, SynchScope); } SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT, diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp index 92a3670e83..136baf56e8 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -3629,19 +3629,17 @@ void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) { InChain = InsertFenceForAtomic(InChain, SuccessOrder, Scope, true, dl, DAG, *TLI); - SDValue L = - DAG.getAtomic(ISD::ATOMIC_CMP_SWAP, dl, - getValue(I.getCompareOperand()).getSimpleValueType(), - InChain, - getValue(I.getPointerOperand()), - getValue(I.getCompareOperand()), - getValue(I.getNewValOperand()), - MachinePointerInfo(I.getPointerOperand()), 0 /* Alignment */, - TLI->getInsertFencesForAtomic() ? Monotonic : SuccessOrder, - TLI->getInsertFencesForAtomic() ? Monotonic : FailureOrder, - Scope); - - SDValue OutChain = L.getValue(1); + MVT MemVT = getValue(I.getCompareOperand()).getSimpleValueType(); + SDVTList VTs = DAG.getVTList(MemVT, MVT::i1, MVT::Other); + SDValue L = DAG.getAtomicCmpSwap( + ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, dl, MemVT, VTs, InChain, + getValue(I.getPointerOperand()), getValue(I.getCompareOperand()), + getValue(I.getNewValOperand()), MachinePointerInfo(I.getPointerOperand()), + 0 /* Alignment */, + TLI->getInsertFencesForAtomic() ? Monotonic : SuccessOrder, + TLI->getInsertFencesForAtomic() ? Monotonic : FailureOrder, Scope); + + SDValue OutChain = L.getValue(2); if (TLI->getInsertFencesForAtomic()) OutChain = InsertFenceForAtomic(OutChain, SuccessOrder, Scope, false, dl, diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp b/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp index d6b525500a..c92fb2453c 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp +++ b/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp @@ -55,6 +55,7 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const { case ISD::PREFETCH: return "Prefetch"; case ISD::ATOMIC_FENCE: return "AtomicFence"; case ISD::ATOMIC_CMP_SWAP: return "AtomicCmpSwap"; + case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: return "AtomicCmpSwapWithSuccess"; case ISD::ATOMIC_SWAP: return "AtomicSwap"; case ISD::ATOMIC_LOAD_ADD: return "AtomicLoadAdd"; case ISD::ATOMIC_LOAD_SUB: return "AtomicLoadSub"; diff --git a/lib/CodeGen/TargetLoweringBase.cpp b/lib/CodeGen/TargetLoweringBase.cpp index 906ded4810..f530485543 100644 --- a/lib/CodeGen/TargetLoweringBase.cpp +++ b/lib/CodeGen/TargetLoweringBase.cpp @@ -730,6 +730,10 @@ void TargetLoweringBase::initActions() { setIndexedStoreAction(IM, (MVT::SimpleValueType)VT, Expand); } + // Most backends expect to see the node which just returns the value loaded. + setOperationAction(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, + (MVT::SimpleValueType)VT, Expand); + // These operations default to expand. setOperationAction(ISD::FGETSIGN, (MVT::SimpleValueType)VT, Expand); setOperationAction(ISD::CONCAT_VECTORS, (MVT::SimpleValueType)VT, Expand); diff --git a/lib/IR/AsmWriter.cpp b/lib/IR/AsmWriter.cpp index 7afefdc7ac..398e3d5f2a 100644 --- a/lib/IR/AsmWriter.cpp +++ b/lib/IR/AsmWriter.cpp @@ -1786,6 +1786,9 @@ void AssemblyWriter::printInstruction(const Instruction &I) { (isa<StoreInst>(I) && cast<StoreInst>(I).isAtomic())) Out << " atomic"; + if (isa<AtomicCmpXchgInst>(I) && cast<AtomicCmpXchgInst>(I).isWeak()) + Out << " weak"; + // If this is a volatile operation, print out the volatile marker. if ((isa<LoadInst>(I) && cast<LoadInst>(I).isVolatile()) || (isa<StoreInst>(I) && cast<StoreInst>(I).isVolatile()) || diff --git a/lib/IR/Instruction.cpp b/lib/IR/Instruction.cpp index abb4e45576..86421c4ae9 100644 --- a/lib/IR/Instruction.cpp +++ b/lib/IR/Instruction.cpp @@ -300,6 +300,7 @@ static bool haveSameSpecialState(const Instruction *I1, const Instruction *I2, FI->getSynchScope() == cast<FenceInst>(I2)->getSynchScope(); if (const AtomicCmpXchgInst *CXI = dyn_cast<AtomicCmpXchgInst>(I1)) return CXI->isVolatile() == cast<AtomicCmpXchgInst>(I2)->isVolatile() && + CXI->isWeak() == cast<AtomicCmpXchgInst>(I2)->isWeak() && CXI->getSuccessOrdering() == cast<AtomicCmpXchgInst>(I2)->getSuccessOrdering() && CXI->getFailureOrdering() == diff --git a/lib/IR/Instructions.cpp b/lib/IR/Instructions.cpp index 051d63f7fc..a5ceacb563 100644 --- a/lib/IR/Instructions.cpp +++ b/lib/IR/Instructions.cpp @@ -1251,10 +1251,11 @@ AtomicCmpXchgInst::AtomicCmpXchgInst(Value *Ptr, Value *Cmp, Value *NewVal, AtomicOrdering FailureOrdering, SynchronizationScope SynchScope, Instruction *InsertBefore) - : Instruction(Cmp->getType(), AtomicCmpXchg, - OperandTraits<AtomicCmpXchgInst>::op_begin(this), - OperandTraits<AtomicCmpXchgInst>::operands(this), - InsertBefore) { + : Instruction( + StructType::get(Cmp->getType(), Type::getInt1Ty(Cmp->getContext()), + nullptr), + AtomicCmpXchg, OperandTraits<AtomicCmpXchgInst>::op_begin(this), + OperandTraits<AtomicCmpXchgInst>::operands(this), InsertBefore) { Init(Ptr, Cmp, NewVal, SuccessOrdering, FailureOrdering, SynchScope); } @@ -1263,13 +1264,14 @@ AtomicCmpXchgInst::AtomicCmpXchgInst(Value *Ptr, Value *Cmp, Value *NewVal, AtomicOrdering FailureOrdering, SynchronizationScope SynchScope, BasicBlock *InsertAtEnd) - : Instruction(Cmp->getType(), AtomicCmpXchg, - OperandTraits<AtomicCmpXchgInst>::op_begin(this), - OperandTraits<AtomicCmpXchgInst>::operands(this), - InsertAtEnd) { + : Instruction( + StructType::get(Cmp->getType(), Type::getInt1Ty(Cmp->getContext()), + nullptr), + AtomicCmpXchg, OperandTraits<AtomicCmpXchgInst>::op_begin(this), + OperandTraits<AtomicCmpXchgInst>::operands(this), InsertAtEnd) { Init(Ptr, Cmp, NewVal, SuccessOrdering, FailureOrdering, SynchScope); } - + //===----------------------------------------------------------------------===// // AtomicRMWInst Implementation //===----------------------------------------------------------------------===// @@ -3604,6 +3606,7 @@ AtomicCmpXchgInst *AtomicCmpXchgInst::clone_impl() const { getSuccessOrdering(), getFailureOrdering(), getSynchScope()); Result->setVolatile(isVolatile()); + Result->setWeak(isWeak()); return Result; } diff --git a/lib/Target/CppBackend/CPPBackend.cpp b/lib/Target/CppBackend/CPPBackend.cpp index 7f822b6794..f610fbb969 100644 --- a/lib/Target/CppBackend/CPPBackend.cpp +++ b/lib/Target/CppBackend/CPPBackend.cpp @@ -1579,6 +1579,8 @@ void CppWriter::printInstruction(const Instruction *I, Out << "\");"; nl(Out) << iName << "->setVolatile(" << (cxi->isVolatile() ? "true" : "false") << ");"; + nl(Out) << iName << "->setWeak(" + << (cxi->isWeak() ? "true" : "false") << ");"; break; } case Instruction::AtomicRMW: { diff --git a/lib/Target/X86/X86ISelLowering.cpp b/lib/Target/X86/X86ISelLowering.cpp index af9e1d9187..38a0f06bc4 100644 --- a/lib/Target/X86/X86ISelLowering.cpp +++ b/lib/Target/X86/X86ISelLowering.cpp @@ -14813,13 +14813,14 @@ static void ReplaceATOMIC_LOAD(SDNode *Node, // (The only way to get a 16-byte load is cmpxchg16b) // FIXME: 16-byte ATOMIC_CMP_SWAP isn't actually hooked up at the moment. SDValue Zero = DAG.getConstant(0, VT); - SDValue Swap = DAG.getAtomic(ISD::ATOMIC_CMP_SWAP, dl, VT, - Node->getOperand(0), - Node->getOperand(1), Zero, Zero, - cast<AtomicSDNode>(Node)->getMemOperand(), - cast<AtomicSDNode>(Node)->getOrdering(), - cast<AtomicSDNode>(Node)->getOrdering(), - cast<AtomicSDNode>(Node)->getSynchScope()); + SDVTList VTs = DAG.getVTList(VT, MVT::Other); + SDValue Swap = + DAG.getAtomicCmpSwap(ISD::ATOMIC_CMP_SWAP, dl, VT, VTs, + Node->getOperand(0), Node->getOperand(1), Zero, Zero, + cast<AtomicSDNode>(Node)->getMemOperand(), + cast<AtomicSDNode>(Node)->getOrdering(), + cast<AtomicSDNode>(Node)->getOrdering(), + cast<AtomicSDNode>(Node)->getSynchScope()); Results.push_back(Swap.getValue(0)); Results.push_back(Swap.getValue(1)); } diff --git a/lib/Transforms/IPO/MergeFunctions.cpp b/lib/Transforms/IPO/MergeFunctions.cpp index c3a2b1205c..c9b4af7691 100644 --- a/lib/Transforms/IPO/MergeFunctions.cpp +++ b/lib/Transforms/IPO/MergeFunctions.cpp @@ -847,6 +847,9 @@ int FunctionComparator::cmpOperation(const Instruction *L, if (int Res = cmpNumbers(CXI->isVolatile(), cast<AtomicCmpXchgInst>(R)->isVolatile())) return Res; + if (int Res = cmpNumbers(CXI->isWeak(), + cast<AtomicCmpXchgInst>(R)->isWeak())) + return Res; if (int Res = cmpNumbers(CXI->getSuccessOrdering(), cast<AtomicCmpXchgInst>(R)->getSuccessOrdering())) return Res; diff --git a/lib/Transforms/Instrumentation/ThreadSanitizer.cpp b/lib/Transforms/Instrumentation/ThreadSanitizer.cpp index 957afc4e9c..f3bc36f04c 100644 --- a/lib/Transforms/Instrumentation/ThreadSanitizer.cpp +++ b/lib/Transforms/Instrumentation/ThreadSanitizer.cpp @@ -532,8 +532,14 @@ bool ThreadSanitizer::instrumentAtomic(Instruction *I) { IRB.CreateIntCast(CASI->getNewValOperand(), Ty, false), createOrdering(&IRB, CASI->getSuccessOrdering()), createOrdering(&IRB, CASI->getFailureOrdering())}; - CallInst *C = CallInst::Create(TsanAtomicCAS[Idx], ArrayRef<Value*>(Args)); - ReplaceInstWithInst(I, C); + CallInst *C = IRB.CreateCall(TsanAtomicCAS[Idx], Args); + Value *Success = IRB.CreateICmpEQ(C, CASI->getCompareOperand()); + + Value *Res = IRB.CreateInsertValue(UndefValue::get(CASI->getType()), C, 0); + Res = IRB.CreateInsertValue(Res, Success, 1); + + I->replaceAllUsesWith(Res); + I->eraseFromParent(); } else if (FenceInst *FI = dyn_cast<FenceInst>(I)) { Value *Args[] = {createOrdering(&IRB, FI->getOrdering())}; Function *F = FI->getSynchScope() == SingleThread ? diff --git a/lib/Transforms/Scalar/LowerAtomic.cpp b/lib/Transforms/Scalar/LowerAtomic.cpp index 4251ac47ed..3314e1ed41 100644 --- a/lib/Transforms/Scalar/LowerAtomic.cpp +++ b/lib/Transforms/Scalar/LowerAtomic.cpp @@ -32,7 +32,10 @@ static bool LowerAtomicCmpXchgInst(AtomicCmpXchgInst *CXI) { Value *Res = Builder.CreateSelect(Equal, Val, Orig); Builder.CreateStore(Res, Ptr); - CXI->replaceAllUsesWith(Orig); + Res = Builder.CreateInsertValue(UndefValue::get(CXI->getType()), Orig, 0); + Res = Builder.CreateInsertValue(Res, Equal, 1); + + CXI->replaceAllUsesWith(Res); CXI->eraseFromParent(); return true; } diff --git a/test/Assembler/atomic.ll b/test/Assembler/atomic.ll index a2ae58e296..d7ccd9900b 100644 --- a/test/Assembler/atomic.ll +++ b/test/Assembler/atomic.ll @@ -16,6 +16,8 @@ define void @f(i32* %x) { cmpxchg volatile i32* %x, i32 0, i32 1 acq_rel acquire ; CHECK: cmpxchg i32* %x, i32 42, i32 0 acq_rel monotonic cmpxchg i32* %x, i32 42, i32 0 acq_rel monotonic + ; CHECK: cmpxchg weak i32* %x, i32 13, i32 0 seq_cst monotonic + cmpxchg weak i32* %x, i32 13, i32 0 seq_cst monotonic ; CHECK: atomicrmw add i32* %x, i32 10 seq_cst atomicrmw add i32* %x, i32 10 seq_cst ; CHECK: atomicrmw volatile xchg i32* %x, i32 10 monotonic diff --git a/test/Bitcode/atomic.ll b/test/Bitcode/atomic.ll new file mode 100644 index 0000000000..37815a749b --- /dev/null +++ b/test/Bitcode/atomic.ll @@ -0,0 +1,17 @@ +; RUN: llvm-as %s -o - | llvm-dis | FileCheck %s + +define void @test_cmpxchg(i32* %addr, i32 %desired, i32 %new) { + cmpxchg i32* %addr, i32 %desired, i32 %new seq_cst seq_cst + ; CHECK: cmpxchg i32* %addr, i32 %desired, i32 %new seq_cst seq_cst + + cmpxchg volatile i32* %addr, i32 %desired, i32 %new seq_cst monotonic + ; CHECK: cmpxchg volatile i32* %addr, i32 %desired, i32 %new seq_cst monotonic + + cmpxchg weak i32* %addr, i32 %desired, i32 %new acq_rel acquire + ; CHECK: cmpxchg weak i32* %addr, i32 %desired, i32 %new acq_rel acquire + + cmpxchg weak volatile i32* %addr, i32 %desired, i32 %new singlethread release monotonic + ; CHECK: cmpxchg weak volatile i32* %addr, i32 %desired, i32 %new singlethread release monotonic + + ret void +}
\ No newline at end of file diff --git a/test/Bitcode/memInstructions.3.2.ll b/test/Bitcode/memInstructions.3.2.ll index 21c3deb8a5..e4cb6bdbe9 100644 --- a/test/Bitcode/memInstructions.3.2.ll +++ b/test/Bitcode/memInstructions.3.2.ll @@ -223,68 +223,88 @@ define void @cmpxchg(i32* %ptr,i32 %cmp,i32 %new){ entry:
;cmpxchg [volatile] <ty>* <pointer>, <ty> <cmp>, <ty> <new> [singlethread] <ordering>
-; CHECK: %res1 = cmpxchg i32* %ptr, i32 %cmp, i32 %new monotonic monotonic
+; CHECK: [[TMP:%[a-z0-9]+]] = cmpxchg i32* %ptr, i32 %cmp, i32 %new monotonic monotonic
+; CHECK-NEXT: %res1 = extractvalue { i32, i1 } [[TMP]], 0
%res1 = cmpxchg i32* %ptr, i32 %cmp, i32 %new monotonic monotonic
-; CHECK-NEXT: %res2 = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new monotonic monotonic
+; CHECK-NEXT: [[TMP:%[a-z0-9]+]] = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new monotonic monotonic
+; CHECK-NEXT: %res2 = extractvalue { i32, i1 } [[TMP]], 0
%res2 = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new monotonic monotonic
-; CHECK-NEXT: %res3 = cmpxchg i32* %ptr, i32 %cmp, i32 %new singlethread monotonic monotonic
+; CHECK-NEXT: [[TMP:%[a-z0-9]+]] = cmpxchg i32* %ptr, i32 %cmp, i32 %new singlethread monotonic monotonic
+; CHECK-NEXT: %res3 = extractvalue { i32, i1 } [[TMP]], 0
%res3 = cmpxchg i32* %ptr, i32 %cmp, i32 %new singlethread monotonic monotonic
-; CHECK-NEXT: %res4 = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new singlethread monotonic monotonic
+; CHECK-NEXT: [[TMP:%[a-z0-9]+]] = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new singlethread monotonic monotonic
+; CHECK-NEXT: %res4 = extractvalue { i32, i1 } [[TMP]], 0
%res4 = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new singlethread monotonic monotonic
-; CHECK-NEXT: %res5 = cmpxchg i32* %ptr, i32 %cmp, i32 %new acquire acquire
+; CHECK-NEXT: [[TMP:%[a-z0-9]+]] = cmpxchg i32* %ptr, i32 %cmp, i32 %new acquire acquire
+; CHECK-NEXT: %res5 = extractvalue { i32, i1 } [[TMP]], 0
%res5 = cmpxchg i32* %ptr, i32 %cmp, i32 %new acquire acquire
-; CHECK-NEXT: %res6 = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new acquire acquire
+; CHECK-NEXT: [[TMP:%[a-z0-9]+]] = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new acquire acquire
+; CHECK-NEXT: %res6 = extractvalue { i32, i1 } [[TMP]], 0
%res6 = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new acquire acquire
-; CHECK-NEXT: %res7 = cmpxchg i32* %ptr, i32 %cmp, i32 %new singlethread acquire acquire
+; CHECK-NEXT: [[TMP:%[a-z0-9]+]] = cmpxchg i32* %ptr, i32 %cmp, i32 %new singlethread acquire acquire
+; CHECK-NEXT: %res7 = extractvalue { i32, i1 } [[TMP]], 0
%res7 = cmpxchg i32* %ptr, i32 %cmp, i32 %new singlethread acquire acquire
-; CHECK-NEXT: %res8 = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new singlethread acquire acquire
+; CHECK-NEXT: [[TMP:%[a-z0-9]+]] = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new singlethread acquire acquire
+; CHECK-NEXT: %res8 = extractvalue { i32, i1 } [[TMP]], 0
%res8 = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new singlethread acquire acquire
-; CHECK-NEXT: %res9 = cmpxchg i32* %ptr, i32 %cmp, i32 %new release monotonic
+; CHECK-NEXT: [[TMP:%[a-z0-9]+]] = cmpxchg i32* %ptr, i32 %cmp, i32 %new release monotonic
+; CHECK-NEXT: %res9 = extractvalue { i32, i1 } [[TMP]], 0
%res9 = cmpxchg i32* %ptr, i32 %cmp, i32 %new release monotonic
-; CHECK-NEXT: %res10 = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new release monotonic
+; CHECK-NEXT: [[TMP:%[a-z0-9]+]] = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new release monotonic
+; CHECK-NEXT: %res10 = extractvalue { i32, i1 } [[TMP]], 0
%res10 = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new release monotonic
-; CHECK-NEXT: %res11 = cmpxchg i32* %ptr, i32 %cmp, i32 %new singlethread release monotonic
+; CHECK-NEXT: [[TMP:%[a-z0-9]+]] = cmpxchg i32* %ptr, i32 %cmp, i32 %new singlethread release monotonic
+; CHECK-NEXT: %res11 = extractvalue { i32, i1 } [[TMP]], 0
%res11 = cmpxchg i32* %ptr, i32 %cmp, i32 %new singlethread release monotonic
-; CHECK-NEXT: %res12 = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new singlethread release monotonic
+; CHECK-NEXT: [[TMP:%[a-z0-9]+]] = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new singlethread release monotonic
+; CHECK-NEXT: %res12 = extractvalue { i32, i1 } [[TMP]], 0
%res12 = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new singlethread release monotonic
-; CHECK-NEXT: %res13 = cmpxchg i32* %ptr, i32 %cmp, i32 %new acq_rel acquire
+; CHECK-NEXT: [[TMP:%[a-z0-9]+]] = cmpxchg i32* %ptr, i32 %cmp, i32 %new acq_rel acquire
+; CHECK-NEXT: %res13 = extractvalue { i32, i1 } [[TMP]], 0
%res13 = cmpxchg i32* %ptr, i32 %cmp, i32 %new acq_rel acquire
-; CHECK-NEXT: %res14 = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new acq_rel acquire
+; CHECK-NEXT: [[TMP:%[a-z0-9]+]] = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new acq_rel acquire
+; CHECK-NEXT: %res14 = extractvalue { i32, i1 } [[TMP]], 0
%res14 = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new acq_rel acquire
-; CHECK-NEXT: %res15 = cmpxchg i32* %ptr, i32 %cmp, i32 %new singlethread acq_rel acquire
+; CHECK-NEXT: [[TMP:%[a-z0-9]+]] = cmpxchg i32* %ptr, i32 %cmp, i32 %new singlethread acq_rel acquire
+; CHECK-NEXT: %res15 = extractvalue { i32, i1 } [[TMP]], 0
%res15 = cmpxchg i32* %ptr, i32 %cmp, i32 %new singlethread acq_rel acquire
-; CHECK-NEXT: %res16 = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new singlethread acq_rel acquire
+; CHECK-NEXT: [[TMP:%[a-z0-9]+]] = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new singlethread acq_rel acquire
+; CHECK-NEXT: %res16 = extractvalue { i32, i1 } [[TMP]], 0
%res16 = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new singlethread acq_rel acquire
-; CHECK-NEXT: %res17 = cmpxchg i32* %ptr, i32 %cmp, i32 %new seq_cst seq_cst
+; CHECK-NEXT: [[TMP:%[a-z0-9]+]] = cmpxchg i32* %ptr, i32 %cmp, i32 %new seq_cst seq_cst
+; CHECK-NEXT: %res17 = extractvalue { i32, i1 } [[TMP]], 0
%res17 = cmpxchg i32* %ptr, i32 %cmp, i32 %new seq_cst seq_cst
-; CHECK-NEXT: %res18 = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new seq_cst seq_cst
+; CHECK-NEXT: [[TMP:%[a-z0-9]+]] = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new seq_cst seq_cst
+; CHECK-NEXT: %res18 = extractvalue { i32, i1 } [[TMP]], 0
%res18 = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new seq_cst seq_cst
-; CHECK-NEXT: %res19 = cmpxchg i32* %ptr, i32 %cmp, i32 %new singlethread seq_cst seq_cst
+; CHECK-NEXT: [[TMP:%[a-z0-9]+]] = cmpxchg i32* %ptr, i32 %cmp, i32 %new singlethread seq_cst seq_cst
+; CHECK-NEXT: %res19 = extractvalue { i32, i1 } [[TMP]], 0
%res19 = cmpxchg i32* %ptr, i32 %cmp, i32 %new singlethread seq_cst seq_cst
-; CHECK-NEXT: %res20 = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new singlethread seq_cst seq_cst
+; CHECK-NEXT: [[TMP:%[a-z0-9]+]] = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new singlethread seq_cst seq_cst
+; CHECK-NEXT: %res20 = extractvalue { i32, i1 } [[TMP]], 0
%res20 = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new singlethread seq_cst seq_cst
ret void
diff --git a/test/Bitcode/weak-cmpxchg-upgrade.ll b/test/Bitcode/weak-cmpxchg-upgrade.ll new file mode 100644 index 0000000000..dbcd150633 --- /dev/null +++ b/test/Bitcode/weak-cmpxchg-upgrade.ll @@ -0,0 +1,15 @@ +; RUN: llvm-dis < %s.bc | FileCheck %s + +; cmpxchg-upgrade.ll.bc was produced by running a version of llvm-as from just +; before the IR change on this file. + +define i32 @test(i32* %addr, i32 %old, i32 %new) { +; CHECK: [[TMP:%.*]] = cmpxchg i32* %addr, i32 %old, i32 %new seq_cst monotonic +; CHECK: %val = extractvalue { i32, i1 } [[TMP]], 0 + %val = cmpxchg i32* %addr, i32 %old, i32 %new seq_cst monotonic + ret i32 %val +} + +define i32 @test(i32* %addr, i32 %old, i32 %new) { + ret i1 %val +} diff --git a/test/Bitcode/weak-cmpxchg-upgrade.ll.bc b/test/Bitcode/weak-cmpxchg-upgrade.ll.bc Binary files differnew file mode 100644 index 0000000000..f713c317d4 --- /dev/null +++ b/test/Bitcode/weak-cmpxchg-upgrade.ll.bc diff --git a/test/CodeGen/AArch64/arm64-atomic-128.ll b/test/CodeGen/AArch64/arm64-atomic-128.ll index 3b43aa16d2..0f5b23998e 100644 --- a/test/CodeGen/AArch64/arm64-atomic-128.ll +++ b/test/CodeGen/AArch64/arm64-atomic-128.ll @@ -13,7 +13,8 @@ define i128 @val_compare_and_swap(i128* %p, i128 %oldval, i128 %newval) { ; CHECK: stxp [[SCRATCH_RES:w[0-9]+]], x4, x5, [x[[ADDR]]] ; CHECK: cbnz [[SCRATCH_RES]], [[LABEL]] ; CHECK: [[DONE]]: - %val = cmpxchg i128* %p, i128 %oldval, i128 %newval acquire acquire + %pair = cmpxchg i128* %p, i128 %oldval, i128 %newval acquire acquire + %val = extractvalue { i128, i1 } %pair, 0 ret i128 %val } diff --git a/test/CodeGen/AArch64/arm64-atomic.ll b/test/CodeGen/AArch64/arm64-atomic.ll index aa9b284410..aef79cb386 100644 --- a/test/CodeGen/AArch64/arm64-atomic.ll +++ b/test/CodeGen/AArch64/arm64-atomic.ll @@ -10,7 +10,8 @@ define i32 @val_compare_and_swap(i32* %p) { ; CHECK: stxr [[SCRATCH_REG:w[0-9]+]], [[NEWVAL_REG]], [x0] ; CHECK: cbnz [[SCRATCH_REG]], [[LABEL]] ; CHECK: [[LABEL2]]: - %val = cmpxchg i32* %p, i32 7, i32 4 acquire acquire + %pair = cmpxchg i32* %p, i32 7, i32 4 acquire acquire + %val = extractvalue { i32, i1 } %pair, 0 ret i32 %val } @@ -25,7 +26,8 @@ define i64 @val_compare_and_swap_64(i64* %p) { ; CHECK: stxr [[SCRATCH_REG:w[0-9]+]], x[[NEWVAL_REG]], [x0] ; CHECK: cbnz [[SCRATCH_REG]], [[LABEL]] ; CHECK: [[LABEL2]]: - %val = cmpxchg i64* %p, i64 7, i64 4 monotonic monotonic + %pair = cmpxchg i64* %p, i64 7, i64 4 monotonic monotonic + %val = extractvalue { i64, i1 } %pair, 0 ret i64 %val } diff --git a/test/CodeGen/AArch64/atomic-ops.ll b/test/CodeGen/AArch64/atomic-ops.ll index 36d0eda1e1..26301b92f9 100644 --- a/test/CodeGen/AArch64/atomic-ops.ll +++ b/test/CodeGen/AArch64/atomic-ops.ll @@ -878,7 +878,9 @@ define i64 @test_atomic_load_umax_i64(i64 %offset) nounwind { define i8 @test_atomic_cmpxchg_i8(i8 %wanted, i8 %new) nounwind { ; CHECK-LABEL: test_atomic_cmpxchg_i8: - %old = cmpxchg i8* @var8, i8 %wanted, i8 %new acquire acquire + %pair = cmpxchg i8* @var8, i8 %wanted, i8 %new acquire acquire + %old = extractvalue { i8, i1 } %pair, 0 + ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 @@ -899,7 +901,9 @@ define i8 @test_atomic_cmpxchg_i8(i8 %wanted, i8 %new) nounwind { define i16 @test_atomic_cmpxchg_i16(i16 %wanted, i16 %new) nounwind { ; CHECK-LABEL: test_atomic_cmpxchg_i16: - %old = cmpxchg i16* @var16, i16 %wanted, i16 %new seq_cst seq_cst + %pair = cmpxchg i16* @var16, i16 %wanted, i16 %new seq_cst seq_cst + %old = extractvalue { i16, i1 } %pair, 0 + ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 @@ -920,7 +924,9 @@ define i16 @test_atomic_cmpxchg_i16(i16 %wanted, i16 %new) nounwind { define i32 @test_atomic_cmpxchg_i32(i32 %wanted, i32 %new) nounwind { ; CHECK-LABEL: test_atomic_cmpxchg_i32: - %old = cmpxchg i32* @var32, i32 %wanted, i32 %new release monotonic + %pair = cmpxchg i32* @var32, i32 %wanted, i32 %new release monotonic + %old = extractvalue { i32, i1 } %pair, 0 + ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 @@ -941,7 +947,9 @@ define i32 @test_atomic_cmpxchg_i32(i32 %wanted, i32 %new) nounwind { define void @test_atomic_cmpxchg_i64(i64 %wanted, i64 %new) nounwind { ; CHECK-LABEL: test_atomic_cmpxchg_i64: - %old = cmpxchg i64* @var64, i64 %wanted, i64 %new monotonic monotonic + %pair = cmpxchg i64* @var64, i64 %wanted, i64 %new monotonic monotonic + %old = extractvalue { i64, i1 } %pair, 0 + ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 diff --git a/test/CodeGen/AArch64/cmpxchg-idioms.ll b/test/CodeGen/AArch64/cmpxchg-idioms.ll index a6b96af99d..0c008c2697 100644 --- a/test/CodeGen/AArch64/cmpxchg-idioms.ll +++ b/test/CodeGen/AArch64/cmpxchg-idioms.ll @@ -20,8 +20,8 @@ define i32 @test_return(i32* %p, i32 %oldval, i32 %newval) { ; CHECK: mov w0, wzr ; CHECK: ret - %loaded = cmpxchg i32* %p, i32 %oldval, i32 %newval seq_cst seq_cst - %success = icmp eq i32 %loaded, %oldval + %pair = cmpxchg i32* %p, i32 %oldval, i32 %newval seq_cst seq_cst + %success = extractvalue { i32, i1 } %pair, 1 %conv = zext i1 %success to i32 ret i32 %conv } @@ -38,16 +38,20 @@ define i1 @test_return_bool(i8* %value, i8 %oldValue, i8 %newValue) { ; CHECK: cbnz [[STATUS]], [[LOOP]] ; CHECK-NOT: cmp {{w[0-9]+}}, {{w[0-9]+}} -; CHECK: mov w0, wzr + ; FIXME: DAG combine should be able to deal with this. +; CHECK: orr [[TMP:w[0-9]+]], wzr, #0x1 +; CHECK: eor w0, [[TMP]], #0x1 ; CHECK: ret ; CHECK: [[FAILED]]: ; CHECK-NOT: cmp {{w[0-9]+}}, {{w[0-9]+}} -; CHECK: orr w0, wzr, #0x1 +; CHECK: mov [[TMP:w[0-9]+]], wzr +; CHECK: eor w0, [[TMP]], #0x1 ; CHECK: ret - %loaded = cmpxchg i8* %value, i8 %oldValue, i8 %newValue acq_rel monotonic - %failure = icmp ne i8 %loaded, %oldValue + %pair = cmpxchg i8* %value, i8 %oldValue, i8 %newValue acq_rel monotonic + %success = extractvalue { i8, i1 } %pair, 1 + %failure = xor i1 %success, 1 ret i1 %failure } @@ -69,8 +73,8 @@ define void @test_conditional(i32* %p, i32 %oldval, i32 %newval) { ; CHECK-NOT: cmp {{w[0-9]+}}, {{w[0-9]+}} ; CHECK: b _baz - %loaded = cmpxchg i32* %p, i32 %oldval, i32 %newval seq_cst seq_cst - %success = icmp eq i32 %loaded, %oldval + %pair = cmpxchg i32* %p, i32 %oldval, i32 %newval seq_cst seq_cst + %success = extractvalue { i32, i1 } %pair, 1 br i1 %success, label %true, label %false true: diff --git a/test/CodeGen/ARM/atomic-64bit.ll b/test/CodeGen/ARM/atomic-64bit.ll index db092de387..f18cf22cd6 100644 --- a/test/CodeGen/ARM/atomic-64bit.ll +++ b/test/CodeGen/ARM/atomic-64bit.ll @@ -198,7 +198,8 @@ define i64 @test7(i64* %ptr, i64 %val1, i64 %val2) { ; CHECK-THUMB: bne ; CHECK-THUMB: dmb {{ish$}} - %r = cmpxchg i64* %ptr, i64 %val1, i64 %val2 seq_cst seq_cst + %pair = cmpxchg i64* %ptr, i64 %val1, i64 %val2 seq_cst seq_cst + %r = extractvalue { i64, i1 } %pair, 0 ret i64 %r } diff --git a/test/CodeGen/ARM/atomic-cmp.ll b/test/CodeGen/ARM/atomic-cmp.ll index a4738077b1..629b16d86a 100644 --- a/test/CodeGen/ARM/atomic-cmp.ll +++ b/test/CodeGen/ARM/atomic-cmp.ll @@ -11,5 +11,6 @@ define i8 @t(i8* %a, i8 %b, i8 %c) nounwind { ; T2: ldrexb ; T2: strexb %tmp0 = cmpxchg i8* %a, i8 %b, i8 %c monotonic monotonic - ret i8 %tmp0 + %tmp1 = extractvalue { i8, i1 } %tmp0, 0 + ret i8 %tmp1 } diff --git a/test/CodeGen/ARM/atomic-op.ll b/test/CodeGen/ARM/atomic-op.ll index ac8e949cf1..b988242ae5 100644 --- a/test/CodeGen/ARM/atomic-op.ll +++ b/test/CodeGen/ARM/atomic-op.ll @@ -198,7 +198,8 @@ entry: define i32 @test_cmpxchg_fail_order(i32 *%addr, i32 %desired, i32 %new) { ; CHECK-LABEL: test_cmpxchg_fail_order: - %oldval = cmpxchg i32* %addr, i32 %desired, i32 %new seq_cst monotonic + %pair = cmpxchg i32* %addr, i32 %desired, i32 %new seq_cst monotonic + %oldval = extractvalue { i32, i1 } %pair, 0 ; CHECK: dmb ish ; CHECK: [[LOOP_BB:\.?LBB[0-9]+_1]]: ; CHECK: ldrex [[OLDVAL:r[0-9]+]], [r[[ADDR:[0-9]+]]] @@ -216,7 +217,8 @@ define i32 @test_cmpxchg_fail_order(i32 *%addr, i32 %desired, i32 %new) { define i32 @test_cmpxchg_fail_order1(i32 *%addr, i32 %desired, i32 %new) { ; CHECK-LABEL: test_cmpxchg_fail_order1: - %oldval = cmpxchg i32* %addr, i32 %desired, i32 %new acquire acquire + %pair = cmpxchg i32* %addr, i32 %desired, i32 %new acquire acquire + %oldval = extractvalue { i32, i1 } %pair, 0 ; CHECK-NOT: dmb ish ; CHECK: [[LOOP_BB:\.?LBB[0-9]+_1]]: ; CHECK: ldrex [[OLDVAL:r[0-9]+]], [r[[ADDR:[0-9]+]]] diff --git a/test/CodeGen/ARM/atomic-ops-v8.ll b/test/CodeGen/ARM/atomic-ops-v8.ll index a39565e515..7072aaaf73 100644 --- a/test/CodeGen/ARM/atomic-ops-v8.ll +++ b/test/CodeGen/ARM/atomic-ops-v8.ll @@ -1051,7 +1051,8 @@ define void @test_atomic_load_umax_i64(i64 %offset) nounwind { define i8 @test_atomic_cmpxchg_i8(i8 zeroext %wanted, i8 zeroext %new) nounwind { ; CHECK-LABEL: test_atomic_cmpxchg_i8: - %old = cmpxchg i8* @var8, i8 %wanted, i8 %new acquire acquire + %pair = cmpxchg i8* @var8, i8 %wanted, i8 %new acquire acquire + %old = extractvalue { i8, i1 } %pair, 0 ; CHECK-NOT: dmb ; CHECK-NOT: mcr ; CHECK: movw r[[ADDR:[0-9]+]], :lower16:var8 @@ -1077,7 +1078,8 @@ define i8 @test_atomic_cmpxchg_i8(i8 zeroext %wanted, i8 zeroext %new) nounwind define i16 @test_atomic_cmpxchg_i16(i16 zeroext %wanted, i16 zeroext %new) nounwind { ; CHECK-LABEL: test_atomic_cmpxchg_i16: - %old = cmpxchg i16* @var16, i16 %wanted, i16 %new seq_cst seq_cst + %pair = cmpxchg i16* @var16, i16 %wanted, i16 %new seq_cst seq_cst + %old = extractvalue { i16, i1 } %pair, 0 ; CHECK-NOT: dmb ; CHECK-NOT: mcr ; CHECK: movw r[[ADDR:[0-9]+]], :lower16:var16 @@ -1103,7 +1105,8 @@ define i16 @test_atomic_cmpxchg_i16(i16 zeroext %wanted, i16 zeroext %new) nounw define void @test_atomic_cmpxchg_i32(i32 %wanted, i32 %new) nounwind { ; CHECK-LABEL: test_atomic_cmpxchg_i32: - %old = cmpxchg i32* @var32, i32 %wanted, i32 %new release monotonic + %pair = cmpxchg i32* @var32, i32 %wanted, i32 %new release monotonic + %old = extractvalue { i32, i1 } %pair, 0 store i32 %old, i32* @var32 ; CHECK-NOT: dmb ; CHECK-NOT: mcr @@ -1130,7 +1133,8 @@ define void @test_atomic_cmpxchg_i32(i32 %wanted, i32 %new) nounwind { define void @test_atomic_cmpxchg_i64(i64 %wanted, i64 %new) nounwind { ; CHECK-LABEL: test_atomic_cmpxchg_i64: - %old = cmpxchg i64* @var64, i64 %wanted, i64 %new monotonic monotonic + %pair = cmpxchg i64* @var64, i64 %wanted, i64 %new monotonic monotonic + %old = extractvalue { i64, i1 } %pair, 0 ; CHECK-NOT: dmb ; CHECK-NOT: mcr ; CHECK: movw r[[ADDR:[0-9]+]], :lower16:var64 diff --git a/test/CodeGen/ARM/cmpxchg-idioms.ll b/test/CodeGen/ARM/cmpxchg-idioms.ll index a4f853450b..fb88575cab 100644 --- a/test/CodeGen/ARM/cmpxchg-idioms.ll +++ b/test/CodeGen/ARM/cmpxchg-idioms.ll @@ -25,8 +25,8 @@ define i32 @test_return(i32* %p, i32 %oldval, i32 %newval) { ; CHECK: dmb ish ; CHECK: bx lr - %loaded = cmpxchg i32* %p, i32 %oldval, i32 %newval seq_cst seq_cst - %success = icmp eq i32 %loaded, %oldval + %pair = cmpxchg i32* %p, i32 %oldval, i32 %newval seq_cst seq_cst + %success = extractvalue { i32, i1 } %pair, 1 %conv = zext i1 %success to i32 ret i32 %conv } @@ -40,21 +40,27 @@ define i1 @test_return_bool(i8* %value, i8 %oldValue, i8 %newValue) { ; CHECK: [[LOOP:LBB[0-9]+_[0-9]+]]: ; CHECK: ldrexb [[LOADED:r[0-9]+]], [r0] ; CHECK: cmp [[LOADED]], [[OLDBYTE]] - -; CHECK: itt ne -; CHECK: movne r0, #1 -; CHECK: bxne lr +; CHECK: bne [[FAIL:LBB[0-9]+_[0-9]+]] ; CHECK: strexb [[STATUS:r[0-9]+]], {{r[0-9]+}}, [r0] ; CHECK: cmp [[STATUS]], #0 ; CHECK: bne [[LOOP]] + ; FIXME: this eor is redundant. Need to teach DAG combine that. ; CHECK-NOT: cmp {{r[0-9]+}}, {{r[0-9]+}} -; CHECK: movs r0, #0 +; CHECK: movs [[TMP:r[0-9]+]], #1 +; CHECK: eor r0, [[TMP]], #1 ; CHECK: bx lr - %loaded = cmpxchg i8* %value, i8 %oldValue, i8 %newValue acq_rel monotonic - %failure = icmp ne i8 %loaded, %oldValue +; CHECK: [[FAIL]]: +; CHECK: movs [[TMP:r[0-9]+]], #0 +; CHECK: eor r0, [[TMP]], #1 +; CHECK: bx lr + + + %pair = cmpxchg i8* %value, i8 %oldValue, i8 %newValue acq_rel monotonic + %success = extractvalue { i8, i1 } %pair, 1 + %failure = xor i1 %success, 1 ret i1 %failure } @@ -81,8 +87,8 @@ define void @test_conditional(i32* %p, i32 %oldval, i32 %newval) { ; CHECK: dmb ish ; CHECK: b.w _baz - %loaded = cmpxchg i32* %p, i32 %oldval, i32 %newval seq_cst seq_cst - %success = icmp eq i32 %loaded, %oldval + %pair = cmpxchg i32* %p, i32 %oldval, i32 %newval seq_cst seq_cst + %success = extractvalue { i32, i1 } %pair, 1 br i1 %success, label %true, label %false true: diff --git a/test/CodeGen/CPP/atomic.ll b/test/CodeGen/CPP/atomic.ll index 476370cae7..e79c45d166 100644 --- a/test/CodeGen/CPP/atomic.ll +++ b/test/CodeGen/CPP/atomic.ll @@ -65,11 +65,25 @@ define void @test_cmpxchg(i32* %addr, i32 %desired, i32 %new) { ; CHECK: AtomicCmpXchgInst* [[INST:[a-zA-Z0-9_]+]] = new AtomicCmpXchgInst({{.*}}, SequentiallyConsistent, Monotonic, CrossThread ; CHECK: [[INST]]->setName("inst0"); ; CHECK: [[INST]]->setVolatile(false); + ; CHECK: [[INST]]->setWeak(false); %inst1 = cmpxchg volatile i32* %addr, i32 %desired, i32 %new singlethread acq_rel acquire ; CHECK: AtomicCmpXchgInst* [[INST:[a-zA-Z0-9_]+]] = new AtomicCmpXchgInst({{.*}}, AcquireRelease, Acquire, SingleThread ; CHECK: [[INST]]->setName("inst1"); ; CHECK: [[INST]]->setVolatile(true); + ; CHECK: [[INST]]->setWeak(false); + + %inst2 = cmpxchg weak i32* %addr, i32 %desired, i32 %new seq_cst monotonic + ; CHECK: AtomicCmpXchgInst* [[INST:[a-zA-Z0-9_]+]] = new AtomicCmpXchgInst({{.*}}, SequentiallyConsistent, Monotonic, CrossThread + ; CHECK: [[INST]]->setName("inst2"); + ; CHECK: [[INST]]->setVolatile(false); + ; CHECK: [[INST]]->setWeak(true); + + %inst3 = cmpxchg weak volatile i32* %addr, i32 %desired, i32 %new singlethread acq_rel acquire + ; CHECK: AtomicCmpXchgInst* [[INST:[a-zA-Z0-9_]+]] = new AtomicCmpXchgInst({{.*}}, AcquireRelease, Acquire, SingleThread + ; CHECK: [[INST]]->setName("inst3"); + ; CHECK: [[INST]]->setVolatile(true); + ; CHECK: [[INST]]->setWeak(true); ret void } diff --git a/test/CodeGen/Mips/atomic.ll b/test/CodeGen/Mips/atomic.ll index 77d7bf3154..2907961091 100644 --- a/test/CodeGen/Mips/atomic.ll +++ b/test/CodeGen/Mips/atomic.ll @@ -78,7 +78,8 @@ entry: store i32 %newval, i32* %newval.addr, align 4 %tmp = load i32* %newval.addr, align 4 %0 = cmpxchg i32* @x, i32 %oldval, i32 %tmp monotonic monotonic - ret i32 %0 + %1 = extractvalue { i32, i1 } %0, 0 + ret i32 %1 ; CHECK-EL-LABEL: AtomicCmpSwap32: ; CHECK-EL: lw $[[R0:[0-9]+]], %got(x) @@ -333,7 +334,8 @@ entry: define signext i8 @AtomicCmpSwap8(i8 signext %oldval, i8 signext %newval) nounwind { entry: - %0 = cmpxchg i8* @y, i8 %oldval, i8 %newval monotonic monotonic + %pair0 = cmpxchg i8* @y, i8 %oldval, i8 %newval monotonic monotonic + %0 = extractvalue { i8, i1 } %pair0, 0 ret i8 %0 ; CHECK-EL-LABEL: AtomicCmpSwap8: @@ -429,7 +431,8 @@ entry: define i32 @zeroreg() nounwind { entry: - %0 = cmpxchg i32* @a, i32 1, i32 0 seq_cst seq_cst + %pair0 = cmpxchg i32* @a, i32 1, i32 0 seq_cst seq_cst + %0 = extractvalue { i32, i1 } %pair0, 0 %1 = icmp eq i32 %0, 1 %conv = zext i1 %1 to i32 ret i32 %conv diff --git a/test/CodeGen/Mips/atomicops.ll b/test/CodeGen/Mips/atomicops.ll index dc07c63741..c26415233d 100644 --- a/test/CodeGen/Mips/atomicops.ll +++ b/test/CodeGen/Mips/atomicops.ll @@ -20,7 +20,8 @@ entry: %add.i = add nsw i32 %0, 2 %1 = load volatile i32* %x, align 4 %call1 = call i32 (i8*, ...)* @printf(i8* getelementptr inbounds ([8 x i8]* @.str, i32 0, i32 0), i32 %add.i, i32 %1) nounwind - %2 = cmpxchg i32* %x, i32 1, i32 2 seq_cst seq_cst + %pair = cmpxchg i32* %x, i32 1, i32 2 seq_cst seq_cst + %2 = extractvalue { i32, i1 } %pair, 0 %3 = load volatile i32* %x, align 4 %call2 = call i32 (i8*, ...)* @printf(i8* getelementptr inbounds ([8 x i8]* @.str, i32 0, i32 0), i32 %2, i32 %3) nounwind %4 = atomicrmw xchg i32* %x, i32 1 seq_cst diff --git a/test/CodeGen/PowerPC/Atomics-32.ll b/test/CodeGen/PowerPC/Atomics-32.ll index b5c03e2b20..b7f23b1dd8 100644 --- a/test/CodeGen/PowerPC/Atomics-32.ll +++ b/test/CodeGen/PowerPC/Atomics-32.ll @@ -529,63 +529,73 @@ define void @test_compare_and_swap() nounwind { entry: %0 = load i8* @uc, align 1 %1 = load i8* @sc, align 1 - %2 = cmpxchg i8* @sc, i8 %0, i8 %1 monotonic monotonic + %pair2 = cmpxchg i8* @sc, i8 %0, i8 %1 monotonic monotonic + %2 = extractvalue { i8, i1 } %pair2, 0 store i8 %2, i8* @sc, align 1 %3 = load i8* @uc, align 1 %4 = load i8* @sc, align 1 - %5 = cmpxchg i8* @uc, i8 %3, i8 %4 monotonic monotonic + %pair5 = cmpxchg i8* @uc, i8 %3, i8 %4 monotonic monotonic + %5 = extractvalue { i8, i1 } %pair5, 0 store i8 %5, i8* @uc, align 1 %6 = load i8* @uc, align 1 %7 = zext i8 %6 to i16 %8 = load i8* @sc, align 1 %9 = sext i8 %8 to i16 %10 = bitcast i8* bitcast (i16* @ss to i8*) to i16* - %11 = cmpxchg i16* %10, i16 %7, i16 %9 monotonic monotonic + %pair11 = cmpxchg i16* %10, i16 %7, i16 %9 monotonic monotonic + %11 = extractvalue { i16, i1 } %pair11, 0 store i16 %11, i16* @ss, align 2 %12 = load i8* @uc, align 1 %13 = zext i8 %12 to i16 %14 = load i8* @sc, align 1 %15 = sext i8 %14 to i16 %16 = bitcast i8* bitcast (i16* @us to i8*) to i16* - %17 = cmpxchg i16* %16, i16 %13, i16 %15 monotonic monotonic + %pair17 = cmpxchg i16* %16, i16 %13, i16 %15 monotonic monotonic + %17 = extractvalue { i16, i1 } %pair17, 0 store i16 %17, i16* @us, align 2 %18 = load i8* @uc, align 1 %19 = zext i8 %18 to i32 %20 = load i8* @sc, align 1 %21 = sext i8 %20 to i32 %22 = bitcast i8* bitcast (i32* @si to i8*) to i32* - %23 = cmpxchg i32* %22, i32 %19, i32 %21 monotonic monotonic + %pair23 = cmpxchg i32* %22, i32 %19, i32 %21 monotonic monotonic + %23 = extractvalue { i32, i1 } %pair23, 0 store i32 %23, i32* @si, align 4 %24 = load i8* @uc, align 1 %25 = zext i8 %24 to i32 %26 = load i8* @sc, align 1 %27 = sext i8 %26 to i32 %28 = bitcast i8* bitcast (i32* @ui to i8*) to i32* - %29 = cmpxchg i32* %28, i32 %25, i32 %27 monotonic monotonic + %pair29 = cmpxchg i32* %28, i32 %25, i32 %27 monotonic monotonic + %29 = extractvalue { i32, i1 } %pair29, 0 store i32 %29, i32* @ui, align 4 %30 = load i8* @uc, align 1 %31 = zext i8 %30 to i32 %32 = load i8* @sc, align 1 %33 = sext i8 %32 to i32 %34 = bitcast i8* bitcast (i32* @sl to i8*) to i32* - %35 = cmpxchg i32* %34, i32 %31, i32 %33 monotonic monotonic + %pair35 = cmpxchg i32* %34, i32 %31, i32 %33 monotonic monotonic + %35 = extractvalue { i32, i1 } %pair35, 0 store i32 %35, i32* @sl, align 4 %36 = load i8* @uc, align 1 %37 = zext i8 %36 to i32 %38 = load i8* @sc, align 1 %39 = sext i8 %38 to i32 %40 = bitcast i8* bitcast (i32* @ul to i8*) to i32* - %41 = cmpxchg i32* %40, i32 %37, i32 %39 monotonic monotonic + %pair41 = cmpxchg i32* %40, i32 %37, i32 %39 monotonic monotonic + %41 = extractvalue { i32, i1 } %pair41, 0 store i32 %41, i32* @ul, align 4 %42 = load i8* @uc, align 1 %43 = load i8* @sc, align 1 - %44 = cmpxchg i8* @sc, i8 %42, i8 %43 monotonic monotonic + %pair44 = cmpxchg i8* @sc, i8 %42, i8 %43 monotonic monotonic + %44 = extractvalue { i8, i1 } %pair44, 0 %45 = icmp eq i8 %44, %42 %46 = zext i1 %45 to i32 store i32 %46, i32* @ui, align 4 %47 = load i8* @uc, align 1 %48 = load i8* @sc, align 1 - %49 = cmpxchg i8* @uc, i8 %47, i8 %48 monotonic monotonic + %pair49 = cmpxchg i8* @uc, i8 %47, i8 %48 monotonic monotonic + %49 = extractvalue { i8, i1 } %pair49, 0 %50 = icmp eq i8 %49, %47 %51 = zext i1 %50 to i32 store i32 %51, i32* @ui, align 4 @@ -594,7 +604,8 @@ entry: %54 = load i8* @sc, align 1 %55 = sext i8 %54 to i16 %56 = bitcast i8* bitcast (i16* @ss to i8*) to i16* - %57 = cmpxchg i16* %56, i16 %53, i16 %55 monotonic monotonic + %pair57 = cmpxchg i16* %56, i16 %53, i16 %55 monotonic monotonic + %57 = extractvalue { i16, i1 } %pair57, 0 %58 = icmp eq i16 %57, %53 %59 = zext i1 %58 to i32 store i32 %59, i32* @ui, align 4 @@ -603,7 +614,8 @@ entry: %62 = load i8* @sc, align 1 %63 = sext i8 %62 to i16 %64 = bitcast i8* bitcast (i16* @us to i8*) to i16* - %65 = cmpxchg i16* %64, i16 %61, i16 %63 monotonic monotonic + %pair65 = cmpxchg i16* %64, i16 %61, i16 %63 monotonic monotonic + %65 = extractvalue { i16, i1 } %pair65, 0 %66 = icmp eq i16 %65, %61 %67 = zext i1 %66 to i32 store i32 %67, i32* @ui, align 4 @@ -612,7 +624,8 @@ entry: %70 = load i8* @sc, align 1 %71 = sext i8 %70 to i32 %72 = bitcast i8* bitcast (i32* @si to i8*) to i32* - %73 = cmpxchg i32* %72, i32 %69, i32 %71 monotonic monotonic + %pair73 = cmpxchg i32* %72, i32 %69, i32 %71 monotonic monotonic + %73 = extractvalue { i32, i1 } %pair73, 0 %74 = icmp eq i32 %73, %69 %75 = zext i1 %74 to i32 store i32 %75, i32* @ui, align 4 @@ -621,7 +634,8 @@ entry: %78 = load i8* @sc, align 1 %79 = sext i8 %78 to i32 %80 = bitcast i8* bitcast (i32* @ui to i8*) to i32* - %81 = cmpxchg i32* %80, i32 %77, i32 %79 monotonic monotonic + %pair81 = cmpxchg i32* %80, i32 %77, i32 %79 monotonic monotonic + %81 = extractvalue { i32, i1 } %pair81, 0 %82 = icmp eq i32 %81, %77 %83 = zext i1 %82 to i32 store i32 %83, i32* @ui, align 4 @@ -630,7 +644,8 @@ entry: %86 = load i8* @sc, align 1 %87 = sext i8 %86 to i32 %88 = bitcast i8* bitcast (i32* @sl to i8*) to i32* - %89 = cmpxchg i32* %88, i32 %85, i32 %87 monotonic monotonic + %pair89 = cmpxchg i32* %88, i32 %85, i32 %87 monotonic monotonic + %89 = extractvalue { i32, i1 } %pair89, 0 %90 = icmp eq i32 %89, %85 %91 = zext i1 %90 to i32 store i32 %91, i32* @ui, align 4 @@ -639,7 +654,8 @@ entry: %94 = load i8* @sc, align 1 %95 = sext i8 %94 to i32 %96 = bitcast i8* bitcast (i32* @ul to i8*) to i32* - %97 = cmpxchg i32* %96, i32 %93, i32 %95 monotonic monotonic + %pair97 = cmpxchg i32* %96, i32 %93, i32 %95 monotonic monotonic + %97 = extractvalue { i32, i1 } %pair97, 0 %98 = icmp eq i32 %97, %93 %99 = zext i1 %98 to i32 store i32 %99, i32* @ui, align 4 diff --git a/test/CodeGen/PowerPC/atomic-1.ll b/test/CodeGen/PowerPC/atomic-1.ll index 083df47e56..997a016a5d 100644 --- a/test/CodeGen/PowerPC/atomic-1.ll +++ b/test/CodeGen/PowerPC/atomic-1.ll @@ -11,7 +11,8 @@ define i32 @exchange_and_add(i32* %mem, i32 %val) nounwind { define i32 @exchange_and_cmp(i32* %mem) nounwind { ; CHECK-LABEL: exchange_and_cmp: ; CHECK: lwarx - %tmp = cmpxchg i32* %mem, i32 0, i32 1 monotonic monotonic + %tmppair = cmpxchg i32* %mem, i32 0, i32 1 monotonic monotonic + %tmp = extractvalue { i32, i1 } %tmppair, 0 ; CHECK: stwcx. ; CHECK: stwcx. ret i32 %tmp diff --git a/test/CodeGen/PowerPC/atomic-2.ll b/test/CodeGen/PowerPC/atomic-2.ll index 261335e81d..843250f10b 100644 --- a/test/CodeGen/PowerPC/atomic-2.ll +++ b/test/CodeGen/PowerPC/atomic-2.ll @@ -11,7 +11,8 @@ define i64 @exchange_and_add(i64* %mem, i64 %val) nounwind { define i64 @exchange_and_cmp(i64* %mem) nounwind { ; CHECK-LABEL: exchange_and_cmp: ; CHECK: ldarx - %tmp = cmpxchg i64* %mem, i64 0, i64 1 monotonic monotonic + %tmppair = cmpxchg i64* %mem, i64 0, i64 1 monotonic monotonic + %tmp = extractvalue { i64, i1 } %tmppair, 0 ; CHECK: stdcx. ; CHECK: stdcx. ret i64 %tmp diff --git a/test/CodeGen/R600/atomic_cmp_swap_local.ll b/test/CodeGen/R600/atomic_cmp_swap_local.ll index fd5ca64ac9..eb9539eec5 100644 --- a/test/CodeGen/R600/atomic_cmp_swap_local.ll +++ b/test/CodeGen/R600/atomic_cmp_swap_local.ll @@ -10,7 +10,8 @@ ; SI: S_ENDPGM define void @lds_atomic_cmpxchg_ret_i32_offset(i32 addrspace(1)* %out, i32 addrspace(3)* %ptr, i32 %swap) nounwind { %gep = getelementptr i32 addrspace(3)* %ptr, i32 4 - %result = cmpxchg i32 addrspace(3)* %gep, i32 7, i32 %swap seq_cst monotonic + %pair = cmpxchg i32 addrspace(3)* %gep, i32 7, i32 %swap seq_cst monotonic + %result = extractvalue { i32, i1 } %pair, 0 store i32 %result, i32 addrspace(1)* %out, align 4 ret void } @@ -29,7 +30,8 @@ define void @lds_atomic_cmpxchg_ret_i32_offset(i32 addrspace(1)* %out, i32 addrs ; SI: S_ENDPGM define void @lds_atomic_cmpxchg_ret_i64_offset(i64 addrspace(1)* %out, i64 addrspace(3)* %ptr, i64 %swap) nounwind { %gep = getelementptr i64 addrspace(3)* %ptr, i32 4 - %result = cmpxchg i64 addrspace(3)* %gep, i64 7, i64 %swap seq_cst monotonic + %pair = cmpxchg i64 addrspace(3)* %gep, i64 7, i64 %swap seq_cst monotonic + %result = extractvalue { i64, i1 } %pair, 0 store i64 %result, i64 addrspace(1)* %out, align 8 ret void } diff --git a/test/CodeGen/SPARC/atomics.ll b/test/CodeGen/SPARC/atomics.ll index 5e41300596..ee6c1f8999 100644 --- a/test/CodeGen/SPARC/atomics.ll +++ b/test/CodeGen/SPARC/atomics.ll @@ -38,7 +38,8 @@ entry: define i32 @test_cmpxchg_i32(i32 %a, i32* %ptr) { entry: - %b = cmpxchg i32* %ptr, i32 %a, i32 123 monotonic monotonic + %pair = cmpxchg i32* %ptr, i32 %a, i32 123 monotonic monotonic + %b = extractvalue { i32, i1 } %pair, 0 ret i32 %b } @@ -48,7 +49,8 @@ entry: define i64 @test_cmpxchg_i64(i64 %a, i64* %ptr) { entry: - %b = cmpxchg i64* %ptr, i64 %a, i64 123 monotonic monotonic + %pair = cmpxchg i64* %ptr, i64 %a, i64 123 monotonic monotonic + %b = extractvalue { i64, i1 } %pair, 0 ret i64 %b } diff --git a/test/CodeGen/SystemZ/cmpxchg-01.ll b/test/CodeGen/SystemZ/cmpxchg-01.ll index bb0b18ad57..5118aadcf2 100644 --- a/test/CodeGen/SystemZ/cmpxchg-01.ll +++ b/test/CodeGen/SystemZ/cmpxchg-01.ll @@ -32,7 +32,8 @@ define i8 @f1(i8 %dummy, i8 *%src, i8 %cmp, i8 %swap) { ; CHECK-SHIFT: lcr [[NEGSHIFT:%r[1-9]+]], [[SHIFT]] ; CHECK-SHIFT: rll ; CHECK-SHIFT: rll {{%r[0-9]+}}, %r5, -8([[NEGSHIFT]]) - %res = cmpxchg i8 *%src, i8 %cmp, i8 %swap seq_cst seq_cst + %pair = cmpxchg i8 *%src, i8 %cmp, i8 %swap seq_cst seq_cst + %res = extractvalue { i8, i1 } %pair, 0 ret i8 %res } @@ -50,6 +51,7 @@ define i8 @f2(i8 *%src) { ; CHECK-SHIFT: risbg ; CHECK-SHIFT: risbg [[SWAP]], {{%r[0-9]+}}, 32, 55, 0 ; CHECK-SHIFT: br %r14 - %res = cmpxchg i8 *%src, i8 42, i8 88 seq_cst seq_cst + %pair = cmpxchg i8 *%src, i8 42, i8 88 seq_cst seq_cst + %res = extractvalue { i8, i1 } %pair, 0 ret i8 %res } diff --git a/test/CodeGen/SystemZ/cmpxchg-02.ll b/test/CodeGen/SystemZ/cmpxchg-02.ll index 8d46a8c073..9eb0628b5a 100644 --- a/test/CodeGen/SystemZ/cmpxchg-02.ll +++ b/test/CodeGen/SystemZ/cmpxchg-02.ll @@ -32,7 +32,8 @@ define i16 @f1(i16 %dummy, i16 *%src, i16 %cmp, i16 %swap) { ; CHECK-SHIFT: lcr [[NEGSHIFT:%r[1-9]+]], [[SHIFT]] ; CHECK-SHIFT: rll ; CHECK-SHIFT: rll {{%r[0-9]+}}, %r5, -16([[NEGSHIFT]]) - %res = cmpxchg i16 *%src, i16 %cmp, i16 %swap seq_cst seq_cst + %pair = cmpxchg i16 *%src, i16 %cmp, i16 %swap seq_cst seq_cst + %res = extractvalue { i16, i1 } %pair, 0 ret i16 %res } @@ -50,6 +51,7 @@ define i16 @f2(i16 *%src) { ; CHECK-SHIFT: risbg ; CHECK-SHIFT: risbg [[SWAP]], {{%r[0-9]+}}, 32, 47, 0 ; CHECK-SHIFT: br %r14 - %res = cmpxchg i16 *%src, i16 42, i16 88 seq_cst seq_cst + %pair = cmpxchg i16 *%src, i16 42, i16 88 seq_cst seq_cst + %res = extractvalue { i16, i1 } %pair, 0 ret i16 %res } diff --git a/test/CodeGen/SystemZ/cmpxchg-03.ll b/test/CodeGen/SystemZ/cmpxchg-03.ll index f6a2ad0b69..c5fab4dc04 100644 --- a/test/CodeGen/SystemZ/cmpxchg-03.ll +++ b/test/CodeGen/SystemZ/cmpxchg-03.ll @@ -7,7 +7,8 @@ define i32 @f1(i32 %cmp, i32 %swap, i32 *%src) { ; CHECK-LABEL: f1: ; CHECK: cs %r2, %r3, 0(%r4) ; CHECK: br %r14 - %val = cmpxchg i32 *%src, i32 %cmp, i32 %swap seq_cst seq_cst + %pair = cmpxchg i32 *%src, i32 %cmp, i32 %swap seq_cst seq_cst + %val = extractvalue { i32, i1 } %pair, 0 ret i32 %val } @@ -17,7 +18,8 @@ define i32 @f2(i32 %cmp, i32 %swap, i32 *%src) { ; CHECK: cs %r2, %r3, 4092(%r4) ; CHECK: br %r14 %ptr = getelementptr i32 *%src, i64 1023 - %val = cmpxchg i32 *%ptr, i32 %cmp, i32 %swap seq_cst seq_cst + %pair = cmpxchg i32 *%ptr, i32 %cmp, i32 %swap seq_cst seq_cst + %val = extractvalue { i32, i1 } %pair, 0 ret i32 %val } @@ -27,7 +29,8 @@ define i32 @f3(i32 %cmp, i32 %swap, i32 *%src) { ; CHECK: csy %r2, %r3, 4096(%r4) ; CHECK: br %r14 %ptr = getelementptr i32 *%src, i64 1024 - %val = cmpxchg i32 *%ptr, i32 %cmp, i32 %swap seq_cst seq_cst + %pair = cmpxchg i32 *%ptr, i32 %cmp, i32 %swap seq_cst seq_cst + %val = extractvalue { i32, i1 } %pair, 0 ret i32 %val } @@ -37,7 +40,8 @@ define i32 @f4(i32 %cmp, i32 %swap, i32 *%src) { ; CHECK: csy %r2, %r3, 524284(%r4) ; CHECK: br %r14 %ptr = getelementptr i32 *%src, i64 131071 - %val = cmpxchg i32 *%ptr, i32 %cmp, i32 %swap seq_cst seq_cst + %pair = cmpxchg i32 *%ptr, i32 %cmp, i32 %swap seq_cst seq_cst + %val = extractvalue { i32, i1 } %pair, 0 ret i32 %val } @@ -49,7 +53,8 @@ define i32 @f5(i32 %cmp, i32 %swap, i32 *%src) { ; CHECK: cs %r2, %r3, 0(%r4) ; CHECK: br %r14 %ptr = getelementptr i32 *%src, i64 131072 - %val = cmpxchg i32 *%ptr, i32 %cmp, i32 %swap seq_cst seq_cst + %pair = cmpxchg i32 *%ptr, i32 %cmp, i32 %swap seq_cst seq_cst + %val = extractvalue { i32, i1 } %pair, 0 ret i32 %val } @@ -59,7 +64,8 @@ define i32 @f6(i32 %cmp, i32 %swap, i32 *%src) { ; CHECK: csy %r2, %r3, -4(%r4) ; CHECK: br %r14 %ptr = getelementptr i32 *%src, i64 -1 - %val = cmpxchg i32 *%ptr, i32 %cmp, i32 %swap seq_cst seq_cst + %pair = cmpxchg i32 *%ptr, i32 %cmp, i32 %swap seq_cst seq_cst + %val = extractvalue { i32, i1 } %pair, 0 ret i32 %val } @@ -69,7 +75,8 @@ define i32 @f7(i32 %cmp, i32 %swap, i32 *%src) { ; CHECK: csy %r2, %r3, -524288(%r4) ; CHECK: br %r14 %ptr = getelementptr i32 *%src, i64 -131072 - %val = cmpxchg i32 *%ptr, i32 %cmp, i32 %swap seq_cst seq_cst + %pair = cmpxchg i32 *%ptr, i32 %cmp, i32 %swap seq_cst seq_cst + %val = extractvalue { i32, i1 } %pair, 0 ret i32 %val } @@ -81,7 +88,8 @@ define i32 @f8(i32 %cmp, i32 %swap, i32 *%src) { ; CHECK: cs %r2, %r3, 0(%r4) ; CHECK: br %r14 %ptr = getelementptr i32 *%src, i64 -131073 - %val = cmpxchg i32 *%ptr, i32 %cmp, i32 %swap seq_cst seq_cst + %pair = cmpxchg i32 *%ptr, i32 %cmp, i32 %swap seq_cst seq_cst + %val = extractvalue { i32, i1 } %pair, 0 ret i32 %val } @@ -93,7 +101,8 @@ define i32 @f9(i32 %cmp, i32 %swap, i64 %src, i64 %index) { ; CHECK: br %r14 %add1 = add i64 %src, %index %ptr = inttoptr i64 %add1 to i32 * - %val = cmpxchg i32 *%ptr, i32 %cmp, i32 %swap seq_cst seq_cst + %pair = cmpxchg i32 *%ptr, i32 %cmp, i32 %swap seq_cst seq_cst + %val = extractvalue { i32, i1 } %pair, 0 ret i32 %val } @@ -106,7 +115,8 @@ define i32 @f10(i32 %cmp, i32 %swap, i64 %src, i64 %index) { %add1 = add i64 %src, %index %add2 = add i64 %add1, 4096 %ptr = inttoptr i64 %add2 to i32 * - %val = cmpxchg i32 *%ptr, i32 %cmp, i32 %swap seq_cst seq_cst + %pair = cmpxchg i32 *%ptr, i32 %cmp, i32 %swap seq_cst seq_cst + %val = extractvalue { i32, i1 } %pair, 0 ret i32 %val } @@ -116,7 +126,8 @@ define i32 @f11(i32 %dummy, i32 %swap, i32 *%ptr) { ; CHECK: lhi %r2, 1001 ; CHECK: cs %r2, %r3, 0(%r4) ; CHECK: br %r14 - %val = cmpxchg i32 *%ptr, i32 1001, i32 %swap seq_cst seq_cst + %pair = cmpxchg i32 *%ptr, i32 1001, i32 %swap seq_cst seq_cst + %val = extractvalue { i32, i1 } %pair, 0 ret i32 %val } @@ -126,6 +137,7 @@ define i32 @f12(i32 %cmp, i32 *%ptr) { ; CHECK: lhi [[SWAP:%r[0-9]+]], 1002 ; CHECK: cs %r2, [[SWAP]], 0(%r3) ; CHECK: br %r14 - %val = cmpxchg i32 *%ptr, i32 %cmp, i32 1002 seq_cst seq_cst + %pair = cmpxchg i32 *%ptr, i32 %cmp, i32 1002 seq_cst seq_cst + %val = extractvalue { i32, i1 } %pair, 0 ret i32 %val } diff --git a/test/CodeGen/SystemZ/cmpxchg-04.ll b/test/CodeGen/SystemZ/cmpxchg-04.ll index 069bad6514..ba1493e185 100644 --- a/test/CodeGen/SystemZ/cmpxchg-04.ll +++ b/test/CodeGen/SystemZ/cmpxchg-04.ll @@ -7,7 +7,8 @@ define i64 @f1(i64 %cmp, i64 %swap, i64 *%src) { ; CHECK-LABEL: f1: ; CHECK: csg %r2, %r3, 0(%r4) ; CHECK: br %r14 - %val = cmpxchg i64 *%src, i64 %cmp, i64 %swap seq_cst seq_cst + %pairval = cmpxchg i64 *%src, i64 %cmp, i64 %swap seq_cst seq_cst + %val = extractvalue { i64, i1 } %pairval, 0 ret i64 %val } @@ -17,7 +18,8 @@ define i64 @f2(i64 %cmp, i64 %swap, i64 *%src) { ; CHECK: csg %r2, %r3, 524280(%r4) ; CHECK: br %r14 %ptr = getelementptr i64 *%src, i64 65535 - %val = cmpxchg i64 *%ptr, i64 %cmp, i64 %swap seq_cst seq_cst + %pairval = cmpxchg i64 *%ptr, i64 %cmp, i64 %swap seq_cst seq_cst + %val = extractvalue { i64, i1 } %pairval, 0 ret i64 %val } @@ -29,7 +31,8 @@ define i64 @f3(i64 %cmp, i64 %swap, i64 *%src) { ; CHECK: csg %r2, %r3, 0(%r4) ; CHECK: br %r14 %ptr = getelementptr i64 *%src, i64 65536 - %val = cmpxchg i64 *%ptr, i64 %cmp, i64 %swap seq_cst seq_cst + %pairval = cmpxchg i64 *%ptr, i64 %cmp, i64 %swap seq_cst seq_cst + %val = extractvalue { i64, i1 } %pairval, 0 ret i64 %val } @@ -39,7 +42,8 @@ define i64 @f4(i64 %cmp, i64 %swap, i64 *%src) { ; CHECK: csg %r2, %r3, -8(%r4) ; CHECK: br %r14 %ptr = getelementptr i64 *%src, i64 -1 - %val = cmpxchg i64 *%ptr, i64 %cmp, i64 %swap seq_cst seq_cst + %pairval = cmpxchg i64 *%ptr, i64 %cmp, i64 %swap seq_cst seq_cst + %val = extractvalue { i64, i1 } %pairval, 0 ret i64 %val } @@ -49,7 +53,8 @@ define i64 @f5(i64 %cmp, i64 %swap, i64 *%src) { ; CHECK: csg %r2, %r3, -524288(%r4) ; CHECK: br %r14 %ptr = getelementptr i64 *%src, i64 -65536 - %val = cmpxchg i64 *%ptr, i64 %cmp, i64 %swap seq_cst seq_cst + %pairval = cmpxchg i64 *%ptr, i64 %cmp, i64 %swap seq_cst seq_cst + %val = extractvalue { i64, i1 } %pairval, 0 ret i64 %val } @@ -61,7 +66,8 @@ define i64 @f6(i64 %cmp, i64 %swap, i64 *%src) { ; CHECK: csg %r2, %r3, 0(%r4) ; CHECK: br %r14 %ptr = getelementptr i64 *%src, i64 -65537 - %val = cmpxchg i64 *%ptr, i64 %cmp, i64 %swap seq_cst seq_cst + %pairval = cmpxchg i64 *%ptr, i64 %cmp, i64 %swap seq_cst seq_cst + %val = extractvalue { i64, i1 } %pairval, 0 ret i64 %val } @@ -73,7 +79,8 @@ define i64 @f7(i64 %cmp, i64 %swap, i64 %src, i64 %index) { ; CHECK: br %r14 %add1 = add i64 %src, %index %ptr = inttoptr i64 %add1 to i64 * - %val = cmpxchg i64 *%ptr, i64 %cmp, i64 %swap seq_cst seq_cst + %pairval = cmpxchg i64 *%ptr, i64 %cmp, i64 %swap seq_cst seq_cst + %val = extractvalue { i64, i1 } %pairval, 0 ret i64 %val } @@ -83,7 +90,8 @@ define i64 @f8(i64 %dummy, i64 %swap, i64 *%ptr) { ; CHECK: lghi %r2, 1001 ; CHECK: csg %r2, %r3, 0(%r4) ; CHECK: br %r14 - %val = cmpxchg i64 *%ptr, i64 1001, i64 %swap seq_cst seq_cst + %pairval = cmpxchg i64 *%ptr, i64 1001, i64 %swap seq_cst seq_cst + %val = extractvalue { i64, i1 } %pairval, 0 ret i64 %val } @@ -93,6 +101,7 @@ define i64 @f9(i64 %cmp, i64 *%ptr) { ; CHECK: lghi [[SWAP:%r[0-9]+]], 1002 ; CHECK: csg %r2, [[SWAP]], 0(%r3) ; CHECK: br %r14 - %val = cmpxchg i64 *%ptr, i64 %cmp, i64 1002 seq_cst seq_cst + %pairval = cmpxchg i64 *%ptr, i64 %cmp, i64 1002 seq_cst seq_cst + %val = extractvalue { i64, i1 } %pairval, 0 ret i64 %val } diff --git a/test/CodeGen/X86/2010-10-08-cmpxchg8b.ll b/test/CodeGen/X86/2010-10-08-cmpxchg8b.ll index f69cedc4d3..ebf51a5d66 100644 --- a/test/CodeGen/X86/2010-10-08-cmpxchg8b.ll +++ b/test/CodeGen/X86/2010-10-08-cmpxchg8b.ll @@ -18,7 +18,8 @@ entry: loop: ; CHECK: lock ; CHECK-NEXT: cmpxchg8b - %r = cmpxchg i64* %ptr, i64 0, i64 1 monotonic monotonic + %pair = cmpxchg i64* %ptr, i64 0, i64 1 monotonic monotonic + %r = extractvalue { i64, i1 } %pair, 0 %stored1 = icmp eq i64 %r, 0 br i1 %stored1, label %loop, label %continue continue: diff --git a/test/CodeGen/X86/Atomics-64.ll b/test/CodeGen/X86/Atomics-64.ll index c274688504..f9c25fc822 100644 --- a/test/CodeGen/X86/Atomics-64.ll +++ b/test/CodeGen/X86/Atomics-64.ll @@ -704,7 +704,8 @@ entry: %3 = zext i8 %2 to i32 %4 = trunc i32 %3 to i8 %5 = trunc i32 %1 to i8 - %6 = cmpxchg i8* @sc, i8 %4, i8 %5 monotonic monotonic + %pair6 = cmpxchg i8* @sc, i8 %4, i8 %5 monotonic monotonic + %6 = extractvalue { i8, i1 } %pair6, 0 store i8 %6, i8* @sc, align 1 %7 = load i8* @sc, align 1 %8 = zext i8 %7 to i32 @@ -712,7 +713,8 @@ entry: %10 = zext i8 %9 to i32 %11 = trunc i32 %10 to i8 %12 = trunc i32 %8 to i8 - %13 = cmpxchg i8* @uc, i8 %11, i8 %12 monotonic monotonic + %pair13 = cmpxchg i8* @uc, i8 %11, i8 %12 monotonic monotonic + %13 = extractvalue { i8, i1 } %pair13, 0 store i8 %13, i8* @uc, align 1 %14 = load i8* @sc, align 1 %15 = sext i8 %14 to i16 @@ -722,7 +724,8 @@ entry: %19 = bitcast i8* bitcast (i16* @ss to i8*) to i16* %20 = trunc i32 %18 to i16 %21 = trunc i32 %16 to i16 - %22 = cmpxchg i16* %19, i16 %20, i16 %21 monotonic monotonic + %pair22 = cmpxchg i16* %19, i16 %20, i16 %21 monotonic monotonic + %22 = extractvalue { i16, i1 } %pair22, 0 store i16 %22, i16* @ss, align 2 %23 = load i8* @sc, align 1 %24 = sext i8 %23 to i16 @@ -732,49 +735,56 @@ entry: %28 = bitcast i8* bitcast (i16* @us to i8*) to i16* %29 = trunc i32 %27 to i16 %30 = trunc i32 %25 to i16 - %31 = cmpxchg i16* %28, i16 %29, i16 %30 monotonic monotonic + %pair31 = cmpxchg i16* %28, i16 %29, i16 %30 monotonic monotonic + %31 = extractvalue { i16, i1 } %pair31, 0 store i16 %31, i16* @us, align 2 %32 = load i8* @sc, align 1 %33 = sext i8 %32 to i32 %34 = load i8* @uc, align 1 %35 = zext i8 %34 to i32 %36 = bitcast i8* bitcast (i32* @si to i8*) to i32* - %37 = cmpxchg i32* %36, i32 %35, i32 %33 monotonic monotonic + %pair37 = cmpxchg i32* %36, i32 %35, i32 %33 monotonic monotonic + %37 = extractvalue { i32, i1 } %pair37, 0 store i32 %37, i32* @si, align 4 %38 = load i8* @sc, align 1 %39 = sext i8 %38 to i32 %40 = load i8* @uc, align 1 %41 = zext i8 %40 to i32 %42 = bitcast i8* bitcast (i32* @ui to i8*) to i32* - %43 = cmpxchg i32* %42, i32 %41, i32 %39 monotonic monotonic + %pair43 = cmpxchg i32* %42, i32 %41, i32 %39 monotonic monotonic + %43 = extractvalue { i32, i1 } %pair43, 0 store i32 %43, i32* @ui, align 4 %44 = load i8* @sc, align 1 %45 = sext i8 %44 to i64 %46 = load i8* @uc, align 1 %47 = zext i8 %46 to i64 %48 = bitcast i8* bitcast (i64* @sl to i8*) to i64* - %49 = cmpxchg i64* %48, i64 %47, i64 %45 monotonic monotonic + %pair49 = cmpxchg i64* %48, i64 %47, i64 %45 monotonic monotonic + %49 = extractvalue { i64, i1 } %pair49, 0 store i64 %49, i64* @sl, align 8 %50 = load i8* @sc, align 1 %51 = sext i8 %50 to i64 %52 = load i8* @uc, align 1 %53 = zext i8 %52 to i64 %54 = bitcast i8* bitcast (i64* @ul to i8*) to i64* - %55 = cmpxchg i64* %54, i64 %53, i64 %51 monotonic monotonic + %pair55 = cmpxchg i64* %54, i64 %53, i64 %51 monotonic monotonic + %55 = extractvalue { i64, i1 } %pair55, 0 store i64 %55, i64* @ul, align 8 %56 = load i8* @sc, align 1 %57 = sext i8 %56 to i64 %58 = load i8* @uc, align 1 %59 = zext i8 %58 to i64 %60 = bitcast i8* bitcast (i64* @sll to i8*) to i64* - %61 = cmpxchg i64* %60, i64 %59, i64 %57 monotonic monotonic + %pair61 = cmpxchg i64* %60, i64 %59, i64 %57 monotonic monotonic + %61 = extractvalue { i64, i1 } %pair61, 0 store i64 %61, i64* @sll, align 8 %62 = load i8* @sc, align 1 %63 = sext i8 %62 to i64 %64 = load i8* @uc, align 1 %65 = zext i8 %64 to i64 %66 = bitcast i8* bitcast (i64* @ull to i8*) to i64* - %67 = cmpxchg i64* %66, i64 %65, i64 %63 monotonic monotonic + %pair67 = cmpxchg i64* %66, i64 %65, i64 %63 monotonic monotonic + %67 = extractvalue { i64, i1 } %pair67, 0 store i64 %67, i64* @ull, align 8 %68 = load i8* @sc, align 1 %69 = zext i8 %68 to i32 @@ -782,7 +792,8 @@ entry: %71 = zext i8 %70 to i32 %72 = trunc i32 %71 to i8 %73 = trunc i32 %69 to i8 - %74 = cmpxchg i8* @sc, i8 %72, i8 %73 monotonic monotonic + %pair74 = cmpxchg i8* @sc, i8 %72, i8 %73 monotonic monotonic + %74 = extractvalue { i8, i1 } %pair74, 0 %75 = icmp eq i8 %74, %72 %76 = zext i1 %75 to i8 %77 = zext i8 %76 to i32 @@ -793,7 +804,8 @@ entry: %81 = zext i8 %80 to i32 %82 = trunc i32 %81 to i8 %83 = trunc i32 %79 to i8 - %84 = cmpxchg i8* @uc, i8 %82, i8 %83 monotonic monotonic + %pair84 = cmpxchg i8* @uc, i8 %82, i8 %83 monotonic monotonic + %84 = extractvalue { i8, i1 } %pair84, 0 %85 = icmp eq i8 %84, %82 %86 = zext i1 %85 to i8 %87 = zext i8 %86 to i32 @@ -805,7 +817,8 @@ entry: %92 = zext i8 %91 to i32 %93 = trunc i32 %92 to i8 %94 = trunc i32 %90 to i8 - %95 = cmpxchg i8* bitcast (i16* @ss to i8*), i8 %93, i8 %94 monotonic monotonic + %pair95 = cmpxchg i8* bitcast (i16* @ss to i8*), i8 %93, i8 %94 monotonic monotonic + %95 = extractvalue { i8, i1 } %pair95, 0 %96 = icmp eq i8 %95, %93 %97 = zext i1 %96 to i8 %98 = zext i8 %97 to i32 @@ -817,7 +830,8 @@ entry: %103 = zext i8 %102 to i32 %104 = trunc i32 %103 to i8 %105 = trunc i32 %101 to i8 - %106 = cmpxchg i8* bitcast (i16* @us to i8*), i8 %104, i8 %105 monotonic monotonic + %pair106 = cmpxchg i8* bitcast (i16* @us to i8*), i8 %104, i8 %105 monotonic monotonic + %106 = extractvalue { i8, i1 } %pair106, 0 %107 = icmp eq i8 %106, %104 %108 = zext i1 %107 to i8 %109 = zext i8 %108 to i32 @@ -828,7 +842,8 @@ entry: %113 = zext i8 %112 to i32 %114 = trunc i32 %113 to i8 %115 = trunc i32 %111 to i8 - %116 = cmpxchg i8* bitcast (i32* @si to i8*), i8 %114, i8 %115 monotonic monotonic + %pair116 = cmpxchg i8* bitcast (i32* @si to i8*), i8 %114, i8 %115 monotonic monotonic + %116 = extractvalue { i8, i1 } %pair116, 0 %117 = icmp eq i8 %116, %114 %118 = zext i1 %117 to i8 %119 = zext i8 %118 to i32 @@ -839,7 +854,8 @@ entry: %123 = zext i8 %122 to i32 %124 = trunc i32 %123 to i8 %125 = trunc i32 %121 to i8 - %126 = cmpxchg i8* bitcast (i32* @ui to i8*), i8 %124, i8 %125 monotonic monotonic + %pair126 = cmpxchg i8* bitcast (i32* @ui to i8*), i8 %124, i8 %125 monotonic monotonic + %126 = extractvalue { i8, i1 } %pair126, 0 %127 = icmp eq i8 %126, %124 %128 = zext i1 %127 to i8 %129 = zext i8 %128 to i32 @@ -850,7 +866,8 @@ entry: %133 = zext i8 %132 to i64 %134 = trunc i64 %133 to i8 %135 = trunc i64 %131 to i8 - %136 = cmpxchg i8* bitcast (i64* @sl to i8*), i8 %134, i8 %135 monotonic monotonic + %pair136 = cmpxchg i8* bitcast (i64* @sl to i8*), i8 %134, i8 %135 monotonic monotonic + %136 = extractvalue { i8, i1 } %pair136, 0 %137 = icmp eq i8 %136, %134 %138 = zext i1 %137 to i8 %139 = zext i8 %138 to i32 @@ -861,7 +878,8 @@ entry: %143 = zext i8 %142 to i64 %144 = trunc i64 %143 to i8 %145 = trunc i64 %141 to i8 - %146 = cmpxchg i8* bitcast (i64* @ul to i8*), i8 %144, i8 %145 monotonic monotonic + %pair146 = cmpxchg i8* bitcast (i64* @ul to i8*), i8 %144, i8 %145 monotonic monotonic + %146 = extractvalue { i8, i1 } %pair146, 0 %147 = icmp eq i8 %146, %144 %148 = zext i1 %147 to i8 %149 = zext i8 %148 to i32 @@ -872,7 +890,8 @@ entry: %153 = zext i8 %152 to i64 %154 = trunc i64 %153 to i8 %155 = trunc i64 %151 to i8 - %156 = cmpxchg i8* bitcast (i64* @sll to i8*), i8 %154, i8 %155 monotonic monotonic + %pair156 = cmpxchg i8* bitcast (i64* @sll to i8*), i8 %154, i8 %155 monotonic monotonic + %156 = extractvalue { i8, i1 } %pair156, 0 %157 = icmp eq i8 %156, %154 %158 = zext i1 %157 to i8 %159 = zext i8 %158 to i32 @@ -883,7 +902,8 @@ entry: %163 = zext i8 %162 to i64 %164 = trunc i64 %163 to i8 %165 = trunc i64 %161 to i8 - %166 = cmpxchg i8* bitcast (i64* @ull to i8*), i8 %164, i8 %165 monotonic monotonic + %pair166 = cmpxchg i8* bitcast (i64* @ull to i8*), i8 %164, i8 %165 monotonic monotonic + %166 = extractvalue { i8, i1 } %pair166, 0 %167 = icmp eq i8 %166, %164 %168 = zext i1 %167 to i8 %169 = zext i8 %168 to i32 diff --git a/test/CodeGen/X86/atomic_op.ll b/test/CodeGen/X86/atomic_op.ll index b3045ed645..cb639abadd 100644 --- a/test/CodeGen/X86/atomic_op.ll +++ b/test/CodeGen/X86/atomic_op.ll @@ -101,11 +101,13 @@ entry: %neg1 = sub i32 0, 10 ; <i32> [#uses=1] ; CHECK: lock ; CHECK: cmpxchgl - %16 = cmpxchg i32* %val2, i32 %neg1, i32 1 monotonic monotonic + %pair16 = cmpxchg i32* %val2, i32 %neg1, i32 1 monotonic monotonic + %16 = extractvalue { i32, i1 } %pair16, 0 store i32 %16, i32* %old ; CHECK: lock ; CHECK: cmpxchgl - %17 = cmpxchg i32* %val2, i32 1976, i32 1 monotonic monotonic + %pair17 = cmpxchg i32* %val2, i32 1976, i32 1 monotonic monotonic + %17 = extractvalue { i32, i1 } %pair17, 0 store i32 %17, i32* %old ; CHECK: movl [[R17atomic:.*]], %eax ; CHECK: movl $1401, %[[R17mask:[a-z]*]] @@ -133,6 +135,7 @@ entry: ; CHECK: lock ; CHECK: cmpxchgl %{{.*}}, %gs:(%{{.*}}) - %0 = cmpxchg i32 addrspace(256)* %P, i32 0, i32 1 monotonic monotonic + %pair0 = cmpxchg i32 addrspace(256)* %P, i32 0, i32 1 monotonic monotonic + %0 = extractvalue { i32, i1 } %pair0, 0 ret void } diff --git a/test/CodeGen/X86/coalescer-remat.ll b/test/CodeGen/X86/coalescer-remat.ll index 468b70bdc8..bb08a0ec52 100644 --- a/test/CodeGen/X86/coalescer-remat.ll +++ b/test/CodeGen/X86/coalescer-remat.ll @@ -5,7 +5,8 @@ define i32 @main() nounwind { entry: - %0 = cmpxchg i64* @val, i64 0, i64 1 monotonic monotonic + %t0 = cmpxchg i64* @val, i64 0, i64 1 monotonic monotonic + %0 = extractvalue { i64, i1 } %t0, 0 %1 = tail call i32 (i8*, ...)* @printf(i8* getelementptr ([7 x i8]* @"\01LC", i32 0, i64 0), i64 %0) nounwind ret i32 0 } diff --git a/test/Instrumentation/MemorySanitizer/atomics.ll b/test/Instrumentation/MemorySanitizer/atomics.ll index 98697d7038..c8f3b88815 100644 --- a/test/Instrumentation/MemorySanitizer/atomics.ll +++ b/test/Instrumentation/MemorySanitizer/atomics.ll @@ -37,12 +37,13 @@ entry: define i32 @Cmpxchg(i32* %p, i32 %a, i32 %b) sanitize_memory { entry: - %0 = cmpxchg i32* %p, i32 %a, i32 %b seq_cst seq_cst + %pair = cmpxchg i32* %p, i32 %a, i32 %b seq_cst seq_cst + %0 = extractvalue { i32, i1 } %pair, 0 ret i32 %0 } ; CHECK: @Cmpxchg -; CHECK: store i32 0, +; CHECK: store { i32, i1 } zeroinitializer, ; CHECK: icmp ; CHECK: br ; CHECK: @__msan_warning @@ -55,12 +56,13 @@ entry: define i32 @CmpxchgMonotonic(i32* %p, i32 %a, i32 %b) sanitize_memory { entry: - %0 = cmpxchg i32* %p, i32 %a, i32 %b monotonic monotonic + %pair = cmpxchg i32* %p, i32 %a, i32 %b monotonic monotonic + %0 = extractvalue { i32, i1 } %pair, 0 ret i32 %0 } ; CHECK: @CmpxchgMonotonic -; CHECK: store i32 0, +; CHECK: store { i32, i1 } zeroinitializer, ; CHECK: icmp ; CHECK: br ; CHECK: @__msan_warning diff --git a/test/Transforms/AtomicExpandLoadLinked/ARM/atomic-expansion-v7.ll b/test/Transforms/AtomicExpandLoadLinked/ARM/atomic-expansion-v7.ll index ac9fc1f586..c307d544b6 100644 --- a/test/Transforms/AtomicExpandLoadLinked/ARM/atomic-expansion-v7.ll +++ b/test/Transforms/AtomicExpandLoadLinked/ARM/atomic-expansion-v7.ll @@ -238,13 +238,15 @@ define i8 @test_cmpxchg_i8_seqcst_seqcst(i8* %ptr, i8 %desired, i8 %newval) { ; CHECK: br i1 [[TST]], label %[[LOOP]], label %[[BARRIER:.*]] ; CHECK: [[BARRIER]]: +; CHECK: [[SUCCESS:%.*]] = phi i1 [ true, %[[TRY_STORE]] ], [ false, %[[LOOP]] ] ; CHECK: fence seq_cst ; CHECK: br label %[[DONE:.*]] ; CHECK: [[DONE]]: ; CHECK: ret i8 [[OLDVAL]] - %old = cmpxchg i8* %ptr, i8 %desired, i8 %newval seq_cst seq_cst + %pairold = cmpxchg i8* %ptr, i8 %desired, i8 %newval seq_cst seq_cst + %old = extractvalue { i8, i1 } %pairold, 0 ret i8 %old } @@ -270,9 +272,11 @@ define i16 @test_cmpxchg_i16_seqcst_monotonic(i16* %ptr, i16 %desired, i16 %newv ; CHECK: br label %[[DONE:.*]] ; CHECK: [[DONE]]: +; CHECK: [[SUCCESS:%.*]] = phi i1 [ true, %[[BARRIER]] ], [ false, %[[LOOP]] ] ; CHECK: ret i16 [[OLDVAL]] - %old = cmpxchg i16* %ptr, i16 %desired, i16 %newval seq_cst monotonic + %pairold = cmpxchg i16* %ptr, i16 %desired, i16 %newval seq_cst monotonic + %old = extractvalue { i16, i1 } %pairold, 0 ret i16 %old } @@ -292,13 +296,15 @@ define i32 @test_cmpxchg_i32_acquire_acquire(i32* %ptr, i32 %desired, i32 %newva ; CHECK: br i1 [[TST]], label %[[LOOP]], label %[[BARRIER:.*]] ; CHECK: [[BARRIER]]: +; CHECK: [[SUCCESS:%.*]] = phi i1 [ true, %[[TRY_STORE]] ], [ false, %[[LOOP]] ] ; CHECK: fence acquire ; CHECK: br label %[[DONE:.*]] ; CHECK: [[DONE]]: ; CHECK: ret i32 [[OLDVAL]] - %old = cmpxchg i32* %ptr, i32 %desired, i32 %newval acquire acquire + %pairold = cmpxchg i32* %ptr, i32 %desired, i32 %newval acquire acquire + %old = extractvalue { i32, i1 } %pairold, 0 ret i32 %old } @@ -333,8 +339,10 @@ define i64 @test_cmpxchg_i64_monotonic_monotonic(i64* %ptr, i64 %desired, i64 %n ; CHECK: br label %[[DONE:.*]] ; CHECK: [[DONE]]: +; CHECK: [[SUCCESS:%.*]] = phi i1 [ true, %[[BARRIER]] ], [ false, %[[LOOP]] ] ; CHECK: ret i64 [[OLDVAL]] - %old = cmpxchg i64* %ptr, i64 %desired, i64 %newval monotonic monotonic + %pairold = cmpxchg i64* %ptr, i64 %desired, i64 %newval monotonic monotonic + %old = extractvalue { i64, i1 } %pairold, 0 ret i64 %old }
\ No newline at end of file diff --git a/test/Transforms/AtomicExpandLoadLinked/ARM/atomic-expansion-v8.ll b/test/Transforms/AtomicExpandLoadLinked/ARM/atomic-expansion-v8.ll index bec5befaab..cf641b9f70 100644 --- a/test/Transforms/AtomicExpandLoadLinked/ARM/atomic-expansion-v8.ll +++ b/test/Transforms/AtomicExpandLoadLinked/ARM/atomic-expansion-v8.ll @@ -100,13 +100,15 @@ define i8 @test_cmpxchg_i8_seqcst_seqcst(i8* %ptr, i8 %desired, i8 %newval) { ; CHECK: br i1 [[TST]], label %[[LOOP]], label %[[BARRIER:.*]] ; CHECK: [[BARRIER]]: +; CHECK: [[SUCCESS:%.*]] = phi i1 [ true, %[[TRY_STORE]] ], [ false, %[[LOOP]] ] ; CHECK-NOT: fence ; CHECK: br label %[[DONE:.*]] ; CHECK: [[DONE]]: ; CHECK: ret i8 [[OLDVAL]] - %old = cmpxchg i8* %ptr, i8 %desired, i8 %newval seq_cst seq_cst + %pairold = cmpxchg i8* %ptr, i8 %desired, i8 %newval seq_cst seq_cst + %old = extractvalue { i8, i1 } %pairold, 0 ret i8 %old } @@ -132,9 +134,11 @@ define i16 @test_cmpxchg_i16_seqcst_monotonic(i16* %ptr, i16 %desired, i16 %newv ; CHECK: br label %[[DONE:.*]] ; CHECK: [[DONE]]: +; CHECK: [[SUCCESS:%.*]] = phi i1 [ true, %[[BARRIER]] ], [ false, %[[LOOP]] ] ; CHECK: ret i16 [[OLDVAL]] - %old = cmpxchg i16* %ptr, i16 %desired, i16 %newval seq_cst monotonic + %pairold = cmpxchg i16* %ptr, i16 %desired, i16 %newval seq_cst monotonic + %old = extractvalue { i16, i1 } %pairold, 0 ret i16 %old } @@ -154,13 +158,15 @@ define i32 @test_cmpxchg_i32_acquire_acquire(i32* %ptr, i32 %desired, i32 %newva ; CHECK: br i1 [[TST]], label %[[LOOP]], label %[[BARRIER:.*]] ; CHECK: [[BARRIER]]: +; CHECK: [[SUCCESS:%.*]] = phi i1 [ true, %[[TRY_STORE]] ], [ false, %[[LOOP]] ] ; CHECK-NOT: fence ; CHECK: br label %[[DONE:.*]] ; CHECK: [[DONE]]: ; CHECK: ret i32 [[OLDVAL]] - %old = cmpxchg i32* %ptr, i32 %desired, i32 %newval acquire acquire + %pairold = cmpxchg i32* %ptr, i32 %desired, i32 %newval acquire acquire + %old = extractvalue { i32, i1 } %pairold, 0 ret i32 %old } @@ -197,6 +203,7 @@ define i64 @test_cmpxchg_i64_monotonic_monotonic(i64* %ptr, i64 %desired, i64 %n ; CHECK: [[DONE]]: ; CHECK: ret i64 [[OLDVAL]] - %old = cmpxchg i64* %ptr, i64 %desired, i64 %newval monotonic monotonic + %pairold = cmpxchg i64* %ptr, i64 %desired, i64 %newval monotonic monotonic + %old = extractvalue { i64, i1 } %pairold, 0 ret i64 %old }
\ No newline at end of file diff --git a/test/Transforms/LowerAtomic/atomic-swap.ll b/test/Transforms/LowerAtomic/atomic-swap.ll index c319834b62..cb1124136f 100644 --- a/test/Transforms/LowerAtomic/atomic-swap.ll +++ b/test/Transforms/LowerAtomic/atomic-swap.ll @@ -3,15 +3,20 @@ define i8 @cmpswap() { ; CHECK-LABEL: @cmpswap( %i = alloca i8 - %j = cmpxchg i8* %i, i8 0, i8 42 monotonic monotonic -; CHECK: [[INST:%[a-z0-9]+]] = load -; CHECK-NEXT: icmp -; CHECK-NEXT: select -; CHECK-NEXT: store + %pair = cmpxchg i8* %i, i8 0, i8 42 monotonic monotonic + %j = extractvalue { i8, i1 } %pair, 0 +; CHECK: [[OLDVAL:%[a-z0-9]+]] = load i8* [[ADDR:%[a-z0-9]+]] +; CHECK-NEXT: [[SAME:%[a-z0-9]+]] = icmp eq i8 [[OLDVAL]], 0 +; CHECK-NEXT: [[TO_STORE:%[a-z0-9]+]] = select i1 [[SAME]], i8 42, i8 [[OLDVAL]] +; CHECK-NEXT: store i8 [[TO_STORE]], i8* [[ADDR]] +; CHECK-NEXT: [[TMP:%[a-z0-9]+]] = insertvalue { i8, i1 } undef, i8 [[OLDVAL]], 0 +; CHECK-NEXT: [[RES:%[a-z0-9]+]] = insertvalue { i8, i1 } [[TMP]], i1 [[SAME]], 1 +; CHECK-NEXT: [[VAL:%[a-z0-9]+]] = extractvalue { i8, i1 } [[RES]], 0 ret i8 %j -; CHECK: ret i8 [[INST]] +; CHECK: ret i8 [[VAL]] } + define i8 @swap() { ; CHECK-LABEL: @swap( %i = alloca i8 |