diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst --- a/llvm/docs/LangRef.rst +++ b/llvm/docs/LangRef.rst @@ -9410,7 +9410,7 @@ :: - cmpxchg [weak] [volatile] * , , [syncscope("")] ; yields { ty, i1 } + cmpxchg [weak] [volatile] * , , [syncscope("")] [, align ] ; yields { ty, i1 } Overview: """"""""" @@ -9442,6 +9442,13 @@ A ``cmpxchg`` instruction can also take an optional ":ref:`syncscope `" argument. +The instruction can take an optional ``align`` attribute. +The alignment must be a power of two greater or equal to the size of the +`` type. If unspecified, the alignment is assumed to be equal to the + size of the '' type. Note that this default alignment assumption is + different from the alignment used for the load/store instructions when align + isn't specified. + The pointer passed into cmpxchg must have alignment greater than or equal to the size in memory of the operand. diff --git a/llvm/lib/AsmParser/LLParser.cpp b/llvm/lib/AsmParser/LLParser.cpp --- a/llvm/lib/AsmParser/LLParser.cpp +++ b/llvm/lib/AsmParser/LLParser.cpp @@ -7225,7 +7225,8 @@ /// ParseCmpXchg /// ::= 'cmpxchg' 'weak'? 'volatile'? TypeAndValue ',' TypeAndValue ',' -/// TypeAndValue 'singlethread'? AtomicOrdering AtomicOrdering +/// TypeAndValue 'singlethread'? AtomicOrdering AtomicOrdering ',' +/// 'Align'? int LLParser::ParseCmpXchg(Instruction *&Inst, PerFunctionState &PFS) { Value *Ptr, *Cmp, *New; LocTy PtrLoc, CmpLoc, NewLoc; bool AteExtraComma = false; @@ -7234,6 +7235,7 @@ SyncScope::ID SSID = SyncScope::System; bool isVolatile = false; bool isWeak = false; + MaybeAlign Alignment; if (EatIfPresent(lltok::kw_weak)) isWeak = true; @@ -7247,7 +7249,8 @@ ParseToken(lltok::comma, "expected ',' after cmpxchg cmp operand") || ParseTypeAndValue(New, NewLoc, PFS) || ParseScopeAndOrdering(true /*Always atomic*/, SSID, SuccessOrdering) || - ParseOrdering(FailureOrdering)) + ParseOrdering(FailureOrdering) || + ParseOptionalCommaAlign(Alignment, AteExtraComma)) return true; if (SuccessOrdering == AtomicOrdering::Unordered || @@ -7269,14 +7272,16 @@ if (!New->getType()->isFirstClassType()) return Error(NewLoc, "cmpxchg operand must be a first class value"); - Align Alignment( + const Align DefaultAlignment( PFS.getFunction().getParent()->getDataLayout().getTypeStoreSize( Cmp->getType())); AtomicCmpXchgInst *CXI = new AtomicCmpXchgInst( - Ptr, Cmp, New, Alignment, SuccessOrdering, FailureOrdering, SSID); + Ptr, Cmp, New, Alignment.getValueOr(DefaultAlignment), SuccessOrdering, + FailureOrdering, SSID); CXI->setVolatile(isVolatile); CXI->setWeak(isWeak); + Inst = CXI; return AteExtraComma ? InstExtraComma : InstNormal; } diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp --- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp +++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp @@ -5053,7 +5053,7 @@ } case bitc::FUNC_CODE_INST_CMPXCHG: { // CMPXCHG: [ptrty, ptr, cmp, val, vol, success_ordering, synchscope, - // failure_ordering, weak] + // failure_ordering, weak, align?] const size_t NumRecords = Record.size(); unsigned OpNum = 0; Value *Ptr = nullptr; @@ -5068,10 +5068,14 @@ return error("Invalid record"); Value *Val = nullptr; - if (popValue(Record, OpNum, NextValueNo, Cmp->getType(), Val) || - NumRecords < OpNum + 3 || NumRecords > OpNum + 5) + if (popValue(Record, OpNum, NextValueNo, Cmp->getType(), Val)) + return error("Invalid record"); + + if (NumRecords < OpNum + 3 || NumRecords > OpNum + 6) return error("Invalid record"); + const bool IsVol = Record[OpNum]; + const AtomicOrdering SuccessOrdering = getDecodedOrdering(Record[OpNum + 1]); if (SuccessOrdering == AtomicOrdering::NotAtomic || @@ -5086,14 +5090,23 @@ const AtomicOrdering FailureOrdering = getDecodedOrdering(Record[OpNum + 3]); - const Align Alignment( - TheModule->getDataLayout().getTypeStoreSize(Cmp->getType())); + const bool IsWeak = Record[OpNum + 4]; - I = new AtomicCmpXchgInst(Ptr, Cmp, Val, Alignment, SuccessOrdering, + MaybeAlign Alignment; + + if (NumRecords == (OpNum + 6)) { + if (Error Err = parseAlignmentValue(Record[OpNum + 5], Alignment)) + return Err; + } + if (!Alignment) + Alignment = + Align(TheModule->getDataLayout().getTypeStoreSize(Cmp->getType())); + + I = new AtomicCmpXchgInst(Ptr, Cmp, Val, *Alignment, SuccessOrdering, FailureOrdering, SSID); FullTy = StructType::get(Context, {FullTy, Type::getInt1Ty(Context)}); - cast(I)->setVolatile(Record[OpNum]); - cast(I)->setWeak(Record[OpNum + 4]); + cast(I)->setVolatile(IsVol); + cast(I)->setWeak(IsWeak); InstructionList.push_back(I); break; diff --git a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp --- a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp +++ b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp @@ -3025,6 +3025,7 @@ Vals.push_back( getEncodedOrdering(cast(I).getFailureOrdering())); Vals.push_back(cast(I).isWeak()); + Vals.push_back(getEncodedAlign(cast(I).getAlign())); break; case Instruction::AtomicRMW: Code = bitc::FUNC_CODE_INST_ATOMICRMW; diff --git a/llvm/lib/IR/AsmWriter.cpp b/llvm/lib/IR/AsmWriter.cpp --- a/llvm/lib/IR/AsmWriter.cpp +++ b/llvm/lib/IR/AsmWriter.cpp @@ -4219,6 +4219,7 @@ } else if (const AtomicCmpXchgInst *CXI = dyn_cast(&I)) { writeAtomicCmpXchg(CXI->getContext(), CXI->getSuccessOrdering(), CXI->getFailureOrdering(), CXI->getSyncScopeID()); + Out << ", align " << CXI->getAlign().value(); } else if (const AtomicRMWInst *RMWI = dyn_cast(&I)) { writeAtomic(RMWI->getContext(), RMWI->getOrdering(), RMWI->getSyncScopeID()); diff --git a/llvm/test/Bitcode/compatibility.ll b/llvm/test/Bitcode/compatibility.ll --- a/llvm/test/Bitcode/compatibility.ll +++ b/llvm/test/Bitcode/compatibility.ll @@ -716,22 +716,42 @@ ;; Atomic Memory Ordering Constraints define void @atomics(i32* %word) { - %cmpxchg.0 = cmpxchg i32* %word, i32 0, i32 4 monotonic monotonic - ; CHECK: %cmpxchg.0 = cmpxchg i32* %word, i32 0, i32 4 monotonic monotonic - %cmpxchg.1 = cmpxchg i32* %word, i32 0, i32 5 acq_rel monotonic - ; CHECK: %cmpxchg.1 = cmpxchg i32* %word, i32 0, i32 5 acq_rel monotonic - %cmpxchg.2 = cmpxchg i32* %word, i32 0, i32 6 acquire monotonic - ; CHECK: %cmpxchg.2 = cmpxchg i32* %word, i32 0, i32 6 acquire monotonic - %cmpxchg.3 = cmpxchg i32* %word, i32 0, i32 7 release monotonic - ; CHECK: %cmpxchg.3 = cmpxchg i32* %word, i32 0, i32 7 release monotonic - %cmpxchg.4 = cmpxchg i32* %word, i32 0, i32 8 seq_cst monotonic - ; CHECK: %cmpxchg.4 = cmpxchg i32* %word, i32 0, i32 8 seq_cst monotonic - %cmpxchg.5 = cmpxchg weak i32* %word, i32 0, i32 9 seq_cst monotonic - ; CHECK: %cmpxchg.5 = cmpxchg weak i32* %word, i32 0, i32 9 seq_cst monotonic - %cmpxchg.6 = cmpxchg volatile i32* %word, i32 0, i32 10 seq_cst monotonic - ; CHECK: %cmpxchg.6 = cmpxchg volatile i32* %word, i32 0, i32 10 seq_cst monotonic - %cmpxchg.7 = cmpxchg weak volatile i32* %word, i32 0, i32 11 syncscope("singlethread") seq_cst monotonic - ; CHECK: %cmpxchg.7 = cmpxchg weak volatile i32* %word, i32 0, i32 11 syncscope("singlethread") seq_cst monotonic + ;; Atomic Compare And Exchange w/o alignment + %cmpxchg_no_align.0 = cmpxchg i32* %word, i32 0, i32 4 monotonic monotonic + ; CHECK: %cmpxchg_no_align.0 = cmpxchg i32* %word, i32 0, i32 4 monotonic monotonic + %cmpxchg_no_align.1 = cmpxchg i32* %word, i32 0, i32 5 acq_rel monotonic + ; CHECK: %cmpxchg_no_align.1 = cmpxchg i32* %word, i32 0, i32 5 acq_rel monotonic + %cmpxchg_no_align.2 = cmpxchg i32* %word, i32 0, i32 6 acquire monotonic + ; CHECK: %cmpxchg_no_align.2 = cmpxchg i32* %word, i32 0, i32 6 acquire monotonic + %cmpxchg_no_align.3 = cmpxchg i32* %word, i32 0, i32 7 release monotonic + ; CHECK: %cmpxchg_no_align.3 = cmpxchg i32* %word, i32 0, i32 7 release monotonic + %cmpxchg_no_align.4 = cmpxchg i32* %word, i32 0, i32 8 seq_cst monotonic + ; CHECK: %cmpxchg_no_align.4 = cmpxchg i32* %word, i32 0, i32 8 seq_cst monotonic + %cmpxchg_no_align.5 = cmpxchg weak i32* %word, i32 0, i32 9 seq_cst monotonic + ; CHECK: %cmpxchg_no_align.5 = cmpxchg weak i32* %word, i32 0, i32 9 seq_cst monotonic + %cmpxchg_no_align.6 = cmpxchg volatile i32* %word, i32 0, i32 10 seq_cst monotonic + ; CHECK: %cmpxchg_no_align.6 = cmpxchg volatile i32* %word, i32 0, i32 10 seq_cst monotonic + %cmpxchg_no_align.7 = cmpxchg weak volatile i32* %word, i32 0, i32 11 syncscope("singlethread") seq_cst monotonic + ; CHECK: %cmpxchg_no_align.7 = cmpxchg weak volatile i32* %word, i32 0, i32 11 syncscope("singlethread") seq_cst monotonic + + ;; Atomic Compare And Exchange w/ alignment + %cmpxchg.0 = cmpxchg i32* %word, i32 0, i32 4 monotonic monotonic, align 16 + ; CHECK: %cmpxchg.0 = cmpxchg i32* %word, i32 0, i32 4 monotonic monotonic, align 16 + %cmpxchg.1 = cmpxchg i32* %word, i32 0, i32 5 acq_rel monotonic, align 16 + ; CHECK: %cmpxchg.1 = cmpxchg i32* %word, i32 0, i32 5 acq_rel monotonic, align 16 + %cmpxchg.2 = cmpxchg i32* %word, i32 0, i32 6 acquire monotonic, align 16 + ; CHECK: %cmpxchg.2 = cmpxchg i32* %word, i32 0, i32 6 acquire monotonic, align 16 + %cmpxchg.3 = cmpxchg i32* %word, i32 0, i32 7 release monotonic, align 16 + ; CHECK: %cmpxchg.3 = cmpxchg i32* %word, i32 0, i32 7 release monotonic, align 16 + %cmpxchg.4 = cmpxchg i32* %word, i32 0, i32 8 seq_cst monotonic, align 16 + ; CHECK: %cmpxchg.4 = cmpxchg i32* %word, i32 0, i32 8 seq_cst monotonic, align 16 + %cmpxchg.5 = cmpxchg weak i32* %word, i32 0, i32 9 seq_cst monotonic, align 16 + ; CHECK: %cmpxchg.5 = cmpxchg weak i32* %word, i32 0, i32 9 seq_cst monotonic, align 16 + %cmpxchg.6 = cmpxchg volatile i32* %word, i32 0, i32 10 seq_cst monotonic, align 16 + ; CHECK: %cmpxchg.6 = cmpxchg volatile i32* %word, i32 0, i32 10 seq_cst monotonic, align 16 + %cmpxchg.7 = cmpxchg weak volatile i32* %word, i32 0, i32 11 syncscope("singlethread") seq_cst monotonic, align 16 + ; CHECK: %cmpxchg.7 = cmpxchg weak volatile i32* %word, i32 0, i32 11 syncscope("singlethread") seq_cst monotonic, align 16 + %atomicrmw.xchg = atomicrmw xchg i32* %word, i32 12 monotonic ; CHECK: %atomicrmw.xchg = atomicrmw xchg i32* %word, i32 12 monotonic %atomicrmw.add = atomicrmw add i32* %word, i32 13 monotonic