Index: llvm/include/llvm/CodeGen/MachinePassRegistry.def =================================================================== --- llvm/include/llvm/CodeGen/MachinePassRegistry.def +++ llvm/include/llvm/CodeGen/MachinePassRegistry.def @@ -208,4 +208,5 @@ DUMMY_MACHINE_FUNCTION_PASS("machine-sanmd", MachineSanitizerBinaryMetadata, ()) DUMMY_MACHINE_FUNCTION_PASS("machine-uniformity", MachineUniformityInfoWrapperPass, ()) DUMMY_MACHINE_FUNCTION_PASS("print-machine-uniformity", MachineUniformityInfoPrinterPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("dupconstphiusers", DupConstPhiUsersPass, ()) #undef DUMMY_MACHINE_FUNCTION_PASS Index: llvm/include/llvm/CodeGen/Passes.h =================================================================== --- llvm/include/llvm/CodeGen/Passes.h +++ llvm/include/llvm/CodeGen/Passes.h @@ -375,6 +375,9 @@ /// This pass implements the "patchable-function" attribute. extern char &PatchableFunctionID; + /// This pass duplicate constant phi's user. + extern char &DupConstPhiUsersID; + /// createStackProtectorPass - This pass adds stack protectors to functions. /// FunctionPass *createStackProtectorPass(); Index: llvm/include/llvm/CodeGen/TargetInstrInfo.h =================================================================== --- llvm/include/llvm/CodeGen/TargetInstrInfo.h +++ llvm/include/llvm/CodeGen/TargetInstrInfo.h @@ -1665,6 +1665,18 @@ return false; } + /// Check if the \p ImmVal can be folded into the \p UseMI instruction, and + /// replace \p UseReg in \p UseMI. + /// This function must be synchronizated with FoldImmediate. + /// TTI has similar isLegalXXXImmediate functions. But they are mainly used + /// in IR phase, and limited to small number of operations. In backend we have + /// more and precise knowledge of immediate usage. + virtual bool canFoldImmediate(MachineInstr &UseMI, Register UseReg, + int64_t ImmVal, + MachineRegisterInfo *MRI) const { + return false; + } + /// Return the number of u-operations the given machine /// instruction will be decoded to on the target cpu. The itinerary's /// IssueWidth is the number of microops that can be dispatched each Index: llvm/include/llvm/InitializePasses.h =================================================================== --- llvm/include/llvm/InitializePasses.h +++ llvm/include/llvm/InitializePasses.h @@ -101,6 +101,7 @@ void initializeDominanceFrontierWrapperPassPass(PassRegistry&); void initializeDominatorTreeWrapperPassPass(PassRegistry&); void initializeDwarfEHPrepareLegacyPassPass(PassRegistry &); +void initializeDupConstPhiUsersPass(PassRegistry &); void initializeEarlyCSELegacyPassPass(PassRegistry&); void initializeEarlyCSEMemSSALegacyPassPass(PassRegistry&); void initializeEarlyIfConverterPass(PassRegistry&); Index: llvm/lib/CodeGen/CMakeLists.txt =================================================================== --- llvm/lib/CodeGen/CMakeLists.txt +++ llvm/lib/CodeGen/CMakeLists.txt @@ -64,6 +64,7 @@ DetectDeadLanes.cpp DFAPacketizer.cpp DwarfEHPrepare.cpp + DupConstPhiUsers.cpp EarlyIfConversion.cpp EdgeBundles.cpp EHContGuardCatchret.cpp Index: llvm/lib/CodeGen/CodeGen.cpp =================================================================== --- llvm/lib/CodeGen/CodeGen.cpp +++ llvm/lib/CodeGen/CodeGen.cpp @@ -34,6 +34,7 @@ initializeDebugifyMachineModulePass(Registry); initializeDetectDeadLanesPass(Registry); initializeDwarfEHPrepareLegacyPassPass(Registry); + initializeDupConstPhiUsersPass(Registry); initializeEarlyIfConverterPass(Registry); initializeEarlyIfPredicatorPass(Registry); initializeEarlyMachineLICMPass(Registry); Index: llvm/lib/CodeGen/DupConstPhiUsers.cpp =================================================================== --- /dev/null +++ llvm/lib/CodeGen/DupConstPhiUsers.cpp @@ -0,0 +1,730 @@ +//===- DupConstPhiUsers.cpp -----------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// \file This pass finds out PHIs whose operands are constants, duplicates its +// user to predecessors and folds in the constants. So we can achieve both code +// size and performance improvements. +// +// bb.0: +// ... +// %3:gr32 = MOV32ri 7 +// JCC_1 %bb.2, 5, implicit $eflags +// +// bb.1: +// %5:gr32 = MOV32ri 11 +// +// bb.2: +// %0:gr32 = PHI %3:gr32, %bb.0, %5:gr32, %bb.1 +// %6:gr32 = ADD32rr %2:gr32(tied-def 0), %0:gr32, implicit-def dead $eflags +// ... +// +// => +// +// bb.0: +// ... +// %7:gr32 = ADD32ri8 %2:gr32(tied-def 0), 7, implicit-def dead $eflags +// JCC_1 %bb.2, 5, implicit $eflags +// +// bb.1: +// %8:gr32 = ADD32ri8 %2:gr32(tied-def 0), 11, implicit-def dead $eflags +// +// bb.2: +// %6:gr32 = PHI %7:gr32, %bb.0, %8:gr32, %bb.1 +// ... +// +//===----------------------------------------------------------------------===// + +#include "PHIEliminationUtils.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/SetVector.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/CodeGen/LiveRegUnits.h" +#include "llvm/CodeGen/MachineBasicBlock.h" +#include "llvm/CodeGen/MachineDominators.h" +#include "llvm/CodeGen/MachineFunction.h" +#include "llvm/CodeGen/MachineInstr.h" +#include "llvm/CodeGen/MachineOperand.h" +#include "llvm/CodeGen/MachineRegisterInfo.h" +#include "llvm/CodeGen/Passes.h" +#include "llvm/CodeGen/TargetInstrInfo.h" +#include "llvm/CodeGen/TargetRegisterInfo.h" +#include "llvm/InitializePasses.h" +#include "llvm/Pass.h" + +using namespace llvm; + +#define DEBUG_TYPE "dupconstphiusers" + +#define MAX_ROUNDS 10 + +namespace { + +class DupConstPhiUsers : public MachineFunctionPass { +public: + static char ID; + DupConstPhiUsers() : MachineFunctionPass(ID) { + initializeDupConstPhiUsersPass(*PassRegistry::getPassRegistry()); + } + + void getAnalysisUsage(AnalysisUsage &AU) const override { + AU.setPreservesAll(); + AU.addRequired(); + MachineFunctionPass::getAnalysisUsage(AU); + } + + bool runOnMachineFunction(MachineFunction &MF) override; + +private: + struct CandidateInfo { + // PHI instruction whose incoming registers are mostly constants. + MachineInstr *Phi; + // The user of PHI dest register, will be duplicated into predecessors. + MachineInstr *UserMI; + + // Some instruction accepts either fixed physical register or immediate, + // like + // %6:gr8 = PHI + // $cl = COPY %6:gr8 + // %7:gr32 = SHL32rCL %1:gr32(tied-def 0), implicit-def dead $eflags, implicit $cl + // In this case we can still duplicate SHL into predecessors and fold + // constant. + MachineInstr *PhyCopy; + + // PHI dest register, usually used by UserMI except when there is a PhyReg. + Register Reg; + // When set it is the dest physical register of PhyCopy and used by UserMI. + Register PhyReg; + + // Dependent instructions, their results are used by UserMI. They are in the + // same MBB as UserMI, will be moved to dominator. + SmallSetVector DepMIs; + + // The first Register is PHI dest register, which is used by UserMI. A + // single UserMI may have multiple operands come from PHI instructions. We + // must record all of them and replace them with appropriate incoming + // registers after duplication. + // The second Register is PHI incoming register from the corresponding + // predecessor. + DenseMap> PhiRegs; + + // Insert position in predecessors for duplicated instructions. Also include + // insert position in dominator for dependent instructions if necessary. + DenseMap InsertPos; + + CandidateInfo(MachineInstr *P) { + Phi = P; + UserMI = PhyCopy = nullptr; + } + }; + + SmallSetVector collectConstPhis(MachineBasicBlock &MBB); + MachineInstr *findSinglePhyRegUser(Register PhyReg, MachineInstr *CopyMI); + bool findCandidateInfo(CandidateInfo &Candidate); + bool isMISafeToMove(MachineInstr *MI, CandidateInfo &Candidate); + void collectPhiRegs(CandidateInfo &Candidate); + bool checkMIAtPos(MachineInstr *MI, MachineBasicBlock *Pred, + MachineBasicBlock::iterator Pos, LiveRegUnits &RegUnits, + CandidateInfo &Candidate); + bool findInsertPosMBB(MachineBasicBlock *PredBB, + SmallSetVector &Instrs, + MachineBasicBlock::iterator &Pos, + CandidateInfo &Candidate); + bool findInsertPos(CandidateInfo &Candidate); + void moveDepMIs(CandidateInfo &Candidate); + MachineInstr *duplicatePhiUser(CandidateInfo &Candidate); + MachineInstr *tryToDupPhiUser(MachineInstr *Phi); + + SmallSetVector Worklist; + SmallSetVector HelperList; + + MachineDominatorTree *MDT; + MachineRegisterInfo *MRI; + const TargetInstrInfo *TII; + const TargetRegisterInfo *TRI; +}; + +} // end anonymous namespace + +char DupConstPhiUsers::ID; + +char &llvm::DupConstPhiUsersID = DupConstPhiUsers::ID; + +INITIALIZE_PASS_BEGIN(DupConstPhiUsers, DEBUG_TYPE, "Duplicate PHI Users", + false, false) +INITIALIZE_PASS_DEPENDENCY(MachineDominatorTree) +INITIALIZE_PASS_END(DupConstPhiUsers, DEBUG_TYPE, "Duplicate PHI Users", false, + false) + +// Collect a list of PHI instructions whose operands are mostly constants. There +// can be at most 1 non-const incoming value. +SmallSetVector +DupConstPhiUsers::collectConstPhis(MachineBasicBlock &MBB) { + SmallSetVector Phis; + for (MachineBasicBlock::iterator PhiIt : MBB.phis()) { + int NonConst = 0; + for (unsigned I = 1, E = PhiIt->getNumOperands(); I != E; I += 2) { + Register Reg = PhiIt->getOperand(I).getReg(); + if (Reg.isPhysical()) { + // We don't handle physical PHI incoming register. + NonConst = 2; + break; + } + int64_t ImmVal; + MachineInstr *Def = MRI->getVRegDef(Reg); + if (!TII->getConstValDefinedInReg(*Def, Reg, ImmVal)) + NonConst++; + } + if (NonConst <= 1) + Phis.insert(&*PhiIt); + } + return Phis; +} + +// PhyReg is defined by CopyMI. This function returns the single user of PhyReg. +MachineInstr *DupConstPhiUsers::findSinglePhyRegUser(Register PhyReg, + MachineInstr *CopyMI) { + MachineBasicBlock *MBB = CopyMI->getParent(); + MachineInstr *UserMI = nullptr; + for (auto I = std::next(MachineBasicBlock::iterator(CopyMI)); I != MBB->end(); + I++) { + if (I->findRegisterUseOperandIdx(PhyReg, false, TRI) != -1) { + if (UserMI) + return nullptr; + UserMI = &*I; + } + if (I->findRegisterDefOperandIdx(PhyReg, false, true, TRI) != -1) + return UserMI; + } + + if (!UserMI) + return nullptr; + // We have found a single user of CopyReg in current MBB. But we still need to + // prove CopyReg is not live out of MBB. + for (MachineBasicBlock *Succ : MBB->successors()) + if (Succ->isLiveIn(PhyReg)) + return nullptr; + + return UserMI; +} + +// Check if Reg is only used by PHI instructions in MBB. Otherwise it can't be +// deleted after folded into duplicated ALU instructions. +static bool hasOnlyPhiUser(Register Reg, MachineBasicBlock *MBB, + MachineRegisterInfo *MRI) { + for (MachineInstr &UseMI : MRI->use_nodbg_instructions(Reg)) + if (!UseMI.isPHI() || UseMI.getParent() != MBB) + return false; + return true; +} + +// Check and fill in basic candidate info. +bool DupConstPhiUsers::findCandidateInfo(CandidateInfo &Candidate) { + MachineInstr *Phi = Candidate.Phi; + + // Some quick checking. + unsigned NumPHIOperands = Phi->getNumOperands(); + if (NumPHIOperands <= 3) + return false; + Register PhiReg = Phi->getOperand(0).getReg(); + if (!MRI->hasOneNonDBGUse(PhiReg)) + return false; + MachineInstr *UserMI = &*MRI->use_instr_nodbg_begin(PhiReg); + if (UserMI->getParent() != Phi->getParent()) + return false; + + // Check for the special case: + // %6:gr8 = PHI ... + // $cl = COPY %6:gr8 + // %7:gr32 = SHL32rCL %1:gr32(tied-def 0), implicit-def dead $eflags, implicit $cl + Register UseReg = PhiReg; + if (UserMI->getOpcode() == TargetOpcode::COPY) { + Register CopyReg = UserMI->getOperand(0).getReg(); + if (CopyReg.isPhysical()) { + // We need to check if there is a single user of the CopyReg in current + // MBB. + MachineInstr *CopyMI = UserMI; + UserMI = findSinglePhyRegUser(CopyReg, CopyMI); + if (!UserMI) + return false; + Candidate.PhyReg = CopyReg; + Candidate.PhyCopy = CopyMI; + UseReg = CopyReg; + } + } + + // Check if most of the operands of the PHI are simple constants and can be + // folded into PHI user. There can be at most one value can't be folded. + MachineBasicBlock *NonFoldableBB = nullptr; + for (unsigned I = 1; I != NumPHIOperands; I += 2) { + MachineBasicBlock *PredBB = Phi->getOperand(I + 1).getMBB(); + Register InReg = Phi->getOperand(I).getReg(); + MachineInstr *Def = MRI->getVRegDef(InReg); + int64_t ImmVal; + if (!TII->getConstValDefinedInReg(*Def, InReg, ImmVal) || + !TII->canFoldImmediate(*UserMI, UseReg, ImmVal, MRI) || + !hasOnlyPhiUser(InReg, Phi->getParent(), MRI)) { + if (NonFoldableBB) + return false; + // We'll move UserMI to NonFoldableBB. If (NonFoldableBB, MBB) is a + // critical edge, it will add cost to other path. So do the move when + // the current MBB is the only successor of NonFoldableBB. + // This is very conservative. In future we may compare the increased cost + // on the critical edge with the decreased cost on other edges. + if (PredBB->succ_size() > 1) + return false; + NonFoldableBB = PredBB; + } + } + + Candidate.UserMI = UserMI; + Candidate.Reg = PhiReg; + return true; +} + +// Check if it is safe to move/fold MI to predecessors. +// MI may depend on other instructions, we can move them into DominatorBB if it +// does not break dependence. All dependent instructions are pushed into +// Candidate.DepMIs. +bool DupConstPhiUsers::isMISafeToMove(MachineInstr *MI, + CandidateInfo &Candidate) { + bool DontMoveAcrossStore = true; + if (!MI->isSafeToMove(nullptr, DontMoveAcrossStore)) + return false; + + for (const MachineOperand &MO : MI->operands()) { + if (!MO.isReg()) + continue; + Register MOReg = MO.getReg(); + if (!MOReg) + continue; + if (MO.isDef()) { + if (MOReg.isVirtual()) + continue; + // MO is a physical reg def, we handle dead def only. Non-dead def is + // also possible, but need more checking. + if (MO.isDead()) + continue; + return false; + } + if (MOReg.isPhysical()) { + if (MOReg != Candidate.PhyReg) + return false; + continue; + } + MachineInstr *Def = MRI->getVRegDef(MOReg); + if (Def->getParent() != MI->getParent()) + continue; + // UserMI's operands can be PHI value. Dependent instructions' operands + // can't be PHI because they will be moved to domintor BB. + if (Def->isPHI()) { + if (MI == Candidate.UserMI) + continue; + else + return false; + } + + // Def is a dependent instruction in the same BB. We also need to check if + // it is safe to move before adding it to DepMIs. + if (Candidate.DepMIs.contains(Def)) + continue; + if (!isMISafeToMove(Def, Candidate)) + return false; + Candidate.DepMIs.insert(Def); + } + return true; +} + +// Collect all incoming registers for PHIs used by UserMI. +void DupConstPhiUsers::collectPhiRegs(CandidateInfo &Candidate) { + for (const MachineOperand &MO : Candidate.UserMI->operands()) { + if (!MO.isReg() || MO.isDef()) + continue; + Register PhiReg = MO.getReg(); + if (PhiReg.isPhysical()) + continue; + MachineInstr *Phi = MRI->getVRegDef(PhiReg); + if (!Phi->isPHI() || Phi->getParent() != Candidate.UserMI->getParent()) + continue; + for (unsigned I = 1, E = Phi->getNumOperands(); I != E; I += 2) { + Register InReg = Phi->getOperand(I).getReg(); + MachineBasicBlock *PredBB = Phi->getOperand(I + 1).getMBB(); + Candidate.PhiRegs[PhiReg].insert(std::make_pair(PredBB, InReg)); + } + } + + // If UserMI uses a physical register copied from PHI reg, we need to add the + // PHI reg. + if (Candidate.PhyReg) { + Register PhiReg = Candidate.Reg; + MachineInstr *Phi = Candidate.Phi; + for (unsigned I = 1, E = Phi->getNumOperands(); I != E; I += 2) { + Register InReg = Phi->getOperand(I).getReg(); + MachineBasicBlock *PredBB = Phi->getOperand(I + 1).getMBB(); + Candidate.PhiRegs[PhiReg].insert(std::make_pair(PredBB, InReg)); + } + } +} + +// Check if we can move MI before Pos in Pred. +bool DupConstPhiUsers::checkMIAtPos(MachineInstr *MI, MachineBasicBlock *Pred, + MachineBasicBlock::iterator Pos, + LiveRegUnits &RegUnits, + CandidateInfo &Candidate) { + for (const MachineOperand &MO : MI->operands()) { + if (!MO.isReg()) + continue; + Register Reg = MO.getReg(); + + if (Reg.isPhysical()) { + // In isMISafeToMove we allow dead def physical register access only, or a + // physical register defined by COPY and used by UserMI. So here we check + // if the physical register is live. + if (RegUnits.available(Reg)) + continue; + return false; + } + + if (MO.isDef()) + continue; + + // For a virtual register reference in MI, it is either defined in the same + // MBB as MI and added as dependent instruction, or come from predecessors, + // then it is always available at the end of predecessor. + if (Pos == Pred->end()) + continue; + + // Check if Reg is defined by a PHI. If so we need to check the + // corresponding register in PredBB. + auto PRI = Candidate.PhiRegs.find(Reg); + if (PRI != Candidate.PhiRegs.end()) + Reg = PRI->second[Pred]; + + // If the definition of Reg is in the same MBB as MI, it's a dependent + // instruction, will be moved to dominator. + MachineInstr *Def = MRI->getVRegDef(Reg); + if (Def->getParent() == MI->getParent()) + continue; + + // Otherwise Def must properly dominate Pos. + if (!(MDT->dominates(Def, &*Pos) && Def != &*Pos)) + return false; + } + return true; +} + +// Find a position in PredBB to insert instructions from Instrs. +bool DupConstPhiUsers::findInsertPosMBB( + MachineBasicBlock *PredBB, SmallSetVector &Instrs, + MachineBasicBlock::iterator &Pos, CandidateInfo &Candidate) { + MachineBasicBlock::iterator ITerm = + findPHICopyInsertPoint(PredBB, Candidate.Phi->getParent(), + Candidate.PhiRegs[Candidate.Reg][PredBB]); + + // Collect live physical regs from PredBB->end() to first terminator. + LiveRegUnits PhyRegUnits(*TRI); + PhyRegUnits.addLiveOuts(*PredBB); + auto I = PredBB->end(); + while (I != ITerm) { + I--; + PhyRegUnits.stepBackward(*I); + }; + + // Check each position to see if dependence can be satisfied. + bool Found = false; + while (true) { + bool Fail = false; + for (MachineInstr *MI : Instrs) { + if (checkMIAtPos(MI, PredBB, I, PhyRegUnits, Candidate)) + continue; + Fail = true; + break; + } + + if (!Fail) { + Pos = I; + Found = true; + // Found a possible insert position. But we still need to check earlier + // positions to avoid the following situation: + // + // %5:gr32 = SUB32ri %1:gr32(tied-def 0), 1025, implicit-def $eflags + // %9:gr8 = MOV8ri 3 + // JCC_1 %bb.2, 2, implicit $eflags + // + // The MOV8ri is inserted between SUB32ri and JCC. When the user of %9 is + // also duplicated into this BB, it can only be placed between MOV/JCC, + // but most X86 ALU instructions clobber $eflags, so there is no possible + // position for %9's user ALU. + // Insert the MOV instruction to an earlier position can solve the + // problem. + } + + if (I == PredBB->begin()) + return Found; + + // Move the position one instruction earlier. + I--; + PhyRegUnits.stepBackward(*I); + if (I->isPHI()) + return Found; + } +} + +// Find positions in predecessors to insert the duplicated PHI user. +bool DupConstPhiUsers::findInsertPos(CandidateInfo &Candidate) { + MachineBasicBlock::iterator Pos; + MachineInstr *MI = Candidate.UserMI; + MachineBasicBlock *MBB = MI->getParent(); + MachineBasicBlock *DominatorBB = (*MDT)[MBB]->getIDom()->getBlock(); + + // Find insert position in predecessors. + SmallSetVector MIs; + MIs.insert(MI); + if (Candidate.PhyCopy) + MIs.insert(Candidate.PhyCopy); + for (MachineBasicBlock *Pred : MBB->predecessors()) { + bool FoundPos; + // If PredBB is also dominator, we'll move all instructions from DepMIs and + // MI to PredBB. + if (Pred == DominatorBB && !Candidate.DepMIs.empty()) { + // Temporarily add UserMI and COPY into DepMIs. Remove them later. + Candidate.DepMIs.insert(MI); + if (Candidate.PhyCopy) + Candidate.DepMIs.insert(Candidate.PhyCopy); + FoundPos = findInsertPosMBB(Pred, Candidate.DepMIs, Pos, Candidate); + if (Candidate.PhyCopy) + Candidate.DepMIs.pop_back(); + Candidate.DepMIs.pop_back(); + } else + // PredBB is not dominator. + FoundPos = findInsertPosMBB(Pred, MIs, Pos, Candidate); + if (!FoundPos) + return false; + Candidate.InsertPos[Pred] = Pos; + } + + // If we have instructions to be moved into dominator, and dominator is not a + // predecessor of MBB, we need to find insert position in dominator. + if (!Candidate.DepMIs.empty() && !MBB->isPredecessor(DominatorBB)) { + if (!findInsertPosMBB(DominatorBB, Candidate.DepMIs, Pos, Candidate)) + return false; + Candidate.InsertPos[DominatorBB] = Pos; + } + return true; +} + +// Move Dependent instructions to immediate dominator. +void DupConstPhiUsers::moveDepMIs(CandidateInfo &Candidate) { + if (!Candidate.DepMIs.empty()) { + MachineBasicBlock *MBB = Candidate.Phi->getParent(); + MachineBasicBlock *DominatorBB = (*MDT)[MBB]->getIDom()->getBlock(); + MachineBasicBlock::iterator ToPos = Candidate.InsertPos[DominatorBB]; + for (MachineInstr *MI : Candidate.DepMIs) { + MBB->remove_instr(MI); + DominatorBB->insert(ToPos, MI); + MI->clearKillInfo(); + } + } +} + +// Duplicate UserMI into predecessors and try to fold in immediate operand. +MachineInstr *DupConstPhiUsers::duplicatePhiUser(CandidateInfo &Candidate) { + LLVM_DEBUG(dbgs() << "Duplicating PHI user: " << *Candidate.UserMI); + LLVM_DEBUG(dbgs() << "PHI: " << *Candidate.Phi); + if (Candidate.PhyCopy) + LLVM_DEBUG(dbgs() << "COPY: " << *Candidate.PhyCopy); + + // MI will be duplicated to predecessors, create a new PHI instruction for + // them. + MachineInstr *UserMI = Candidate.UserMI; + MachineBasicBlock *MBB = UserMI->getParent(); + Register DstReg = UserMI->getOperand(0).getReg(); + const TargetRegisterClass *RegRC = MRI->getRegClass(DstReg); + MachineInstrBuilder MIB = BuildMI(*MBB, MBB->begin(), UserMI->getDebugLoc(), + TII->get(TargetOpcode::PHI), DstReg); + + // COPY and UserMI are duplicated to predecessors, the attached Kill info may + // not be correct. + if (Candidate.PhyCopy) + Candidate.PhyCopy->clearKillInfo(); + UserMI->clearKillInfo(); + + // Now we can duplicate COPY and UserMI into predecessors. + for (MachineBasicBlock *Pred : MBB->predecessors()) { + MachineInstr *ImmDef; + MachineInstr *NewCopy; + MachineBasicBlock::iterator InsertPos = Candidate.InsertPos[Pred]; + + if (Candidate.PhyCopy) { + NewCopy = MBB->getParent()->CloneMachineInstr(Candidate.PhyCopy); + Pred->insert(InsertPos, NewCopy); + // Replace the COPY src with predecessor register. + Register PredReg = Candidate.PhiRegs[Candidate.Reg][Pred]; + assert(NewCopy->getOperand(1).getReg() == Candidate.Reg); + NewCopy->getOperand(1).setReg(PredReg); + // If PredReg is an immediate, try to fold it. + int64_t ImmVal; + ImmDef = MRI->getVRegDef(PredReg); + if (TII->getConstValDefinedInReg(*ImmDef, PredReg, ImmVal)) + TII->FoldImmediate(*NewCopy, *ImmDef, PredReg, MRI); + } + + MachineInstr *NewMI = MBB->getParent()->CloneMachineInstr(UserMI); + Pred->insert(InsertPos, NewMI); + + // Replace PHI registers with predecessor registers. + // FoldImmediate may commute operands. So we don't have a good method to + // visit each operand exactly once. Repeat checking operands until there is + // no change. + bool Changed = true; + while (Changed) { + Changed = false; + for (MachineOperand &MO : NewMI->operands()) { + if (!MO.isReg() || MO.isDef()) + continue; + Register PredReg, Reg = MO.getReg(); + if (Reg == Candidate.PhyReg) { + PredReg = Reg; + ImmDef = NewCopy; + } else { + if (!Candidate.PhiRegs.count(Reg)) + continue; + PredReg = Candidate.PhiRegs[Reg][Pred]; + MO.setReg(PredReg); + ImmDef = MRI->getVRegDef(PredReg); + } + // If PredReg is an immediate, try to fold it. + int64_t ImmVal; + if (TII->getConstValDefinedInReg(*ImmDef, PredReg, ImmVal)) { + if (TII->FoldImmediate(*NewMI, *ImmDef, PredReg, MRI)) { + Changed = true; + // Physical copy instruction can't be easily deleted by + // FoldImmediate. But we know it's dead after folding because + // FindSinglePhyRegUser has checked that UserMI is its only user. + // So we need to manually delete it now. + if (Candidate.PhyCopy) + NewCopy->eraseFromParent(); + } + } + } + } + + // Create a new dst register for duplicated UserMI, add it to the new PHI. + Register NewVR = MRI->createVirtualRegister(RegRC); + NewMI->getOperand(0).setReg(NewVR); + MIB.addReg(NewVR); + MIB.addMBB(Pred); + } + + LLVM_DEBUG(dbgs() << "New PHI: " << *MIB); + + if (Candidate.PhyCopy) + Candidate.PhyCopy->eraseFromParent(); + UserMI->eraseFromParent(); + + // Delete the PHI instructions which are used by UserMI only, and incoming + // register define instructions if possible. + // Don't do this before duplication, we may have + // + // %6:gr64 = MOV64ri 0 + // bb.2: + // %2:gr64 = PHI %6:gr64, %bb.0, %1:gr64, %bb.1 + // %3:gr64 = PHI %6:gr64, %bb.0, %0:gr64, %bb.1 + // %13:gr64 = OR64rr %3:gr64(tied-def 0), %2:gr64, implicit-def dead $eflags + // + // Both %3 and %2 use %6, after duplicating the OR instruction and replacing + // %3 with %6, OR will be the single user of %6, FoldImmediate will delete + // the MOV instruction. Later when %2 is replaced by %6, we can't find the + // define instruction of %6. + for (auto P : Candidate.PhiRegs) { + Register PhiReg = P.first; + if (!MRI->use_nodbg_empty(PhiReg)) + continue; + MachineInstr *Phi = MRI->getVRegDef(PhiReg); + Phi->eraseFromParent(); + Worklist.remove(Phi); + HelperList.remove(Phi); + for (auto RP : P.second) { + auto InReg = RP.second; + if (!MRI->use_nodbg_empty(InReg)) + continue; + MachineInstr *Def = MRI->getVRegDef(InReg); + if (Def) + Def->eraseFromParent(); + } + } + + return MIB; +} + +// Try to duplicate the single user of Phi into its predecessors. +// Most of Phi's incoming registers are constants. +MachineInstr *DupConstPhiUsers::tryToDupPhiUser(MachineInstr *Phi) { + CandidateInfo Candidate(Phi); + if (!findCandidateInfo(Candidate)) + return nullptr; + + // Check if UserMI has other dependent instructions, and if they can be moved + // to dominator. + if (!isMISafeToMove(Candidate.UserMI, Candidate)) + return nullptr; + + collectPhiRegs(Candidate); + + if (!findInsertPos(Candidate)) + return nullptr; + + // Move the dependent instructions to immediate dominator. + moveDepMIs(Candidate); + + // Duplicate UserMI to predecessors. + return duplicatePhiUser(Candidate); +} + +bool DupConstPhiUsers::runOnMachineFunction(MachineFunction &MF) { + if (skipFunction(MF.getFunction())) + return false; + + MDT = &getAnalysis(); + MRI = &MF.getRegInfo(); + TII = MF.getSubtarget().getInstrInfo(); + TRI = MF.getSubtarget().getRegisterInfo(); + + bool Changed = false; + + for (auto &MBB : MF) { + Worklist = collectConstPhis(MBB); + + // Transformation of one PHI can cause another PHI eligible for the + // optimization. So we iteratively work on the PHI list until no changes + // made. Theoretically it may have high cost, but we don't expect it + // triggers many times in a single MBB in practice. We also limit it in a + // arbitrary defined small number of rounds for pathological cases. + int Rounds = 0; + bool LocalChanged = true; + while (LocalChanged && Rounds < MAX_ROUNDS) { + LocalChanged = false; + HelperList.clear(); + + while (!Worklist.empty()) { + MachineInstr *Phi = Worklist.pop_back_val(); + MachineInstr *NewPhi = tryToDupPhiUser(Phi); + if (NewPhi) { + HelperList.insert(NewPhi); + LocalChanged = true; + } else + HelperList.insert(Phi); + } + + Changed |= LocalChanged; + Worklist = HelperList; + Rounds++; + } + } + + return Changed; +} Index: llvm/lib/CodeGen/PeepholeOptimizer.cpp =================================================================== --- llvm/lib/CodeGen/PeepholeOptimizer.cpp +++ llvm/lib/CodeGen/PeepholeOptimizer.cpp @@ -202,7 +202,8 @@ bool isMoveImmediate(MachineInstr &MI, SmallSet &ImmDefRegs, DenseMap &ImmDefMIs); bool foldImmediate(MachineInstr &MI, SmallSet &ImmDefRegs, - DenseMap &ImmDefMIs); + DenseMap &ImmDefMIs, + bool &Deleted); /// Finds recurrence cycles, but only ones that formulated around /// a def operand and a use operand that are tied. If there is a use @@ -218,7 +219,8 @@ /// copy, replace the uses of this copy with the previously seen copy's /// destination register. bool foldRedundantCopy(MachineInstr &MI, - DenseMap &CopyMIs); + DenseMap &CopyMIs, + SmallPtrSetImpl &LocalMIs); /// Is the register \p Reg a non-allocatable physical register? bool isNAPhysCopy(Register Reg); @@ -1370,7 +1372,8 @@ /// and only if the def and use are in the same BB. bool PeepholeOptimizer::foldImmediate( MachineInstr &MI, SmallSet &ImmDefRegs, - DenseMap &ImmDefMIs) { + DenseMap &ImmDefMIs, bool &Deleted) { + Deleted = false; for (unsigned i = 0, e = MI.getDesc().getNumOperands(); i != e; ++i) { MachineOperand &MO = MI.getOperand(i); if (!MO.isReg() || MO.isDef()) @@ -1384,6 +1387,17 @@ assert(II != ImmDefMIs.end() && "couldn't find immediate definition"); if (TII->FoldImmediate(MI, *II->second, Reg, MRI)) { ++NumImmFold; + // If ImmDefMI is not deleted, try to see if MI can be deleted. + if (MRI->getVRegDef(Reg) && + MI.isIdenticalTo(*II->second, MachineInstr::IgnoreVRegDefs)) { + Register DstReg = MI.getOperand(0).getReg(); + if (DstReg.isVirtual() && + MRI->getRegClass(DstReg) == MRI->getRegClass(Reg)) { + MRI->replaceRegWith(DstReg, Reg); + MI.eraseFromParent(); + Deleted = true; + } + } return true; } } @@ -1405,7 +1419,8 @@ // // Should replace %2 uses with %1:sub1 bool PeepholeOptimizer::foldRedundantCopy( - MachineInstr &MI, DenseMap &CopyMIs) { + MachineInstr &MI, DenseMap &CopyMIs, + SmallPtrSetImpl &LocalMIs) { assert(MI.isCopy() && "expected a COPY machine instruction"); Register SrcReg = MI.getOperand(1).getReg(); @@ -1425,6 +1440,8 @@ } MachineInstr *PrevCopy = CopyMIs.find(SrcPair)->second; + if (!LocalMIs.count(PrevCopy)) + return false; assert(SrcSubReg == PrevCopy->getOperand(1).getSubReg() && "Unexpected mismatching subreg!"); @@ -1732,7 +1749,7 @@ continue; } - if (MI->isCopy() && (foldRedundantCopy(*MI, CopySrcMIs) || + if (MI->isCopy() && (foldRedundantCopy(*MI, CopySrcMIs, LocalMIs) || foldRedundantNAPhysCopy(*MI, NAPhysToVirtMIs))) { LocalMIs.erase(MI); LLVM_DEBUG(dbgs() << "Deleting redundant copy: " << *MI << "\n"); @@ -1750,8 +1767,14 @@ // next iteration sees the new instructions. MII = MI; ++MII; - if (SeenMoveImm) - Changed |= foldImmediate(*MI, ImmDefRegs, ImmDefMIs); + if (SeenMoveImm) { + bool Deleted; + Changed |= foldImmediate(*MI, ImmDefRegs, ImmDefMIs, Deleted); + if (Deleted) { + LocalMIs.erase(MI); + continue; + } + } } // Check whether MI is a load candidate for folding into a later Index: llvm/lib/CodeGen/TargetPassConfig.cpp =================================================================== --- llvm/lib/CodeGen/TargetPassConfig.cpp +++ llvm/lib/CodeGen/TargetPassConfig.cpp @@ -1293,6 +1293,8 @@ /// Add passes that optimize machine instructions in SSA form. void TargetPassConfig::addMachineSSAOptimization() { + addPass(&DupConstPhiUsersID); + // Pre-ra tail duplication. addPass(&EarlyTailDuplicateID); Index: llvm/lib/Target/X86/X86InstrInfo.h =================================================================== --- llvm/lib/Target/X86/X86InstrInfo.h +++ llvm/lib/Target/X86/X86InstrInfo.h @@ -547,6 +547,20 @@ Register &FoldAsLoadDefReg, MachineInstr *&DefMI) const override; + bool FoldImmediateImpl(MachineInstr &UseMI, MachineInstr *DefMI, Register Reg, + int64_t ImmVal, MachineRegisterInfo *MRI, + bool MakeChange) const; + + /// FoldImmediate - 'Reg' is known to be defined by a move immediate + /// instruction, try to fold the immediate into the use instruction. + bool FoldImmediate(MachineInstr &UseMI, MachineInstr &DefMI, Register Reg, + MachineRegisterInfo *MRI) const override; + + /// Check if FoldImmediate can fold ImmVal into the given instruction. + /// This function must be synchronizated with FoldImmediate. + bool canFoldImmediate(MachineInstr &UseMI, Register UseReg, int64_t ImmVal, + MachineRegisterInfo *MRI) const override; + std::pair decomposeMachineOperandsTargetFlags(unsigned TF) const override; Index: llvm/lib/Target/X86/X86InstrInfo.cpp =================================================================== --- llvm/lib/Target/X86/X86InstrInfo.cpp +++ llvm/lib/Target/X86/X86InstrInfo.cpp @@ -3861,12 +3861,35 @@ bool X86InstrInfo::getConstValDefinedInReg(const MachineInstr &MI, const Register Reg, int64_t &ImmVal) const { - if (MI.getOpcode() != X86::MOV32ri && MI.getOpcode() != X86::MOV64ri) + Register MovReg = Reg; + const MachineInstr *MovMI = &MI; + if (MI.isSubregToReg()) { + // We use following pattern to setup 64b immediate. + // %8:gr32 = MOV32r0 implicit-def dead $eflags + // %6:gr64 = SUBREG_TO_REG 0, killed %8:gr32, %subreg.sub_32bit + unsigned FillBits = MI.getOperand(1).getImm(); + unsigned SubIdx = MI.getOperand(3).getImm(); + MovReg = MI.getOperand(2).getReg(); + if (SubIdx != X86::sub_32bit || FillBits != 0) + return false; + const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo(); + MovMI = MRI.getVRegDef(MovReg); + } + + if (MovMI->getOpcode() == X86::MOV32r0 && + MovMI->getOperand(0).getReg() == MovReg) { + ImmVal = 0; + return true; + } + + if (MovMI->getOpcode() != X86::MOV32ri && + MovMI->getOpcode() != X86::MOV64ri && + MovMI->getOpcode() != X86::MOV32ri64 && MovMI->getOpcode() != X86::MOV8ri) return false; // Mov Src can be a global address. if (!MI.getOperand(1).isImm() || MI.getOperand(0).getReg() != Reg) return false; - ImmVal = MI.getOperand(1).getImm(); + ImmVal = MovMI->getOperand(1).getImm(); return true; } @@ -4763,6 +4786,230 @@ return nullptr; } +/// Conservatively check if there is live EFLAGS reaching MI. +static bool LiveEFLAGS(MachineInstr *MI, const TargetRegisterInfo *TRI) { + MachineBasicBlock *MBB = MI->getParent(); + MachineBasicBlock::iterator I(MI); + while (true) { + if (I == MBB->begin()) + break; + --I; + if (I->registerDefIsDead(X86::EFLAGS, TRI)) + return false; + if (I->readsRegister(X86::EFLAGS, TRI)) + return true; + if (I->modifiesRegister(X86::EFLAGS, TRI)) + return true; + } + return MBB->isLiveIn(X86::EFLAGS); +} + +/// Real implementation of FoldImmediate and canFoldImmediate. +bool X86InstrInfo::FoldImmediateImpl(MachineInstr &UseMI, MachineInstr *DefMI, + Register Reg, int64_t ImmVal, + MachineRegisterInfo *MRI, + bool MakeChange) const { + bool Modified = false; + bool ShiftRotate = false; + // When ImmVal is 0, some instructions can be changed to COPY. + bool CanChangeToCopy = false; + + if (!isInt<32>(ImmVal)) + return false; + + unsigned Opc = UseMI.getOpcode(); + unsigned NewOpc; + switch (Opc) { + case TargetOpcode::COPY: { + Register ToReg = UseMI.getOperand(0).getReg(); + const TargetRegisterClass *RC = nullptr; + if (ToReg.isVirtual()) + RC = MRI->getRegClass(ToReg); + bool GR32Reg = (ToReg.isVirtual() && X86::GR32RegClass.hasSubClassEq(RC)) || + (ToReg.isPhysical() && X86::GR32RegClass.contains(ToReg)); + bool GR64Reg = (ToReg.isVirtual() && X86::GR64RegClass.hasSubClassEq(RC)) || + (ToReg.isPhysical() && X86::GR64RegClass.contains(ToReg)); + bool GR8Reg = (ToReg.isVirtual() && X86::GR8RegClass.hasSubClassEq(RC)) || + (ToReg.isPhysical() && X86::GR8RegClass.contains(ToReg)); + + if (ImmVal == 0) { + // We have MOV32r0 only. + if (!GR32Reg) + return false; + } + + if (GR64Reg) + NewOpc = X86::MOV64ri; + else if (GR32Reg) { + NewOpc = X86::MOV32ri; + if (ImmVal == 0) { + // MOV32r0 is different than other cases because it doesn't encode the + // immediate in the instruction. So we directly modify it here. + if (!MakeChange) + return true; + + // MOV32r0 clobbers EFLAGS. + if (LiveEFLAGS(&UseMI, MRI->getTargetRegisterInfo())) + return false; + UseMI.setDesc(get(X86::MOV32r0)); + UseMI.removeOperand(UseMI.findRegisterUseOperandIdx(Reg)); + UseMI.addOperand(MachineOperand::CreateReg(X86::EFLAGS, /*isDef*/ true, + /*isImp*/ true, + /*isKill*/ false, + /*isDead*/ true)); + Modified = true; + } + } else if (GR8Reg) + NewOpc = X86::MOV8ri; + else + return false; + break; + } + + case X86::ADD64rr: + NewOpc = X86::ADD64ri32; + CanChangeToCopy = true; + break; + case X86::SUB64rr: + NewOpc = X86::SUB64ri32; + CanChangeToCopy = true; + if (UseMI.findRegisterUseOperandIdx(Reg) != 2) + return false; + break; + case X86::AND64rr: + NewOpc = X86::AND64ri32; + break; + case X86::OR64rr: + NewOpc = X86::OR64ri32; + CanChangeToCopy = true; + break; + case X86::XOR64rr: + NewOpc = X86::XOR64ri32; + CanChangeToCopy = true; + break; + case X86::SHR64rCL: + NewOpc = X86::SHR64ri; + ShiftRotate = true; + break; + case X86::SHL64rCL: + NewOpc = X86::SHL64ri; + ShiftRotate = true; + break; + case X86::SAR64rCL: + NewOpc = X86::SAR64ri; + ShiftRotate = true; + break; + + default: + switch (Opc) { + case X86::ADD32rr: + NewOpc = X86::ADD32ri; + CanChangeToCopy = true; + break; + case X86::SUB32rr: + NewOpc = X86::SUB32ri; + CanChangeToCopy = true; + if (UseMI.findRegisterUseOperandIdx(Reg) != 2) + return false; + break; + case X86::AND32rr: + NewOpc = X86::AND32ri; + break; + case X86::OR32rr: + NewOpc = X86::OR32ri; + CanChangeToCopy = true; + break; + case X86::XOR32rr: + NewOpc = X86::XOR32ri; + CanChangeToCopy = true; + break; + case X86::SHR32rCL: + NewOpc = X86::SHR32ri; + ShiftRotate = true; + break; + case X86::SHL32rCL: + NewOpc = X86::SHL32ri; + ShiftRotate = true; + break; + case X86::SAR32rCL: + NewOpc = X86::SAR32ri; + ShiftRotate = true; + break; + default: + return false; + } + } + + if (ShiftRotate) { + unsigned RegIdx = UseMI.findRegisterUseOperandIdx(Reg); + if (RegIdx < 2) + return false; + if (!isInt<8>(ImmVal)) + return false; + assert(Reg == X86::CL); + + if (!MakeChange) + return true; + UseMI.setDesc(get(NewOpc)); + UseMI.removeOperand(RegIdx); + UseMI.addOperand(MachineOperand::CreateImm(ImmVal)); + // Reg is physical register $cl, so we don't know if DefMI is dead through + // MRI. Let the caller handle it, or pass dead-mi-elimination can delete + // the dead physical register define instruction. + return true; + } + + if (!MakeChange) + return true; + + if (!Modified) { + // Modify the instruction. + if (ImmVal == 0 && CanChangeToCopy && + UseMI.registerDefIsDead(X86::EFLAGS)) { + // %100 = add %101, 0 + // ==> + // %100 = COPY %101 + UseMI.setDesc(get(TargetOpcode::COPY)); + UseMI.removeOperand(UseMI.findRegisterUseOperandIdx(Reg)); + UseMI.removeOperand(UseMI.findRegisterDefOperandIdx(X86::EFLAGS)); + UseMI.untieRegOperand(0); + UseMI.clearFlag(MachineInstr::MIFlag::NoSWrap); + UseMI.clearFlag(MachineInstr::MIFlag::NoUWrap); + } else { + unsigned Op1 = 1, Op2 = CommuteAnyOperandIndex; + if (findCommutedOpIndices(UseMI, Op1, Op2) && + UseMI.getOperand(1).getReg() == Reg) + commuteInstruction(UseMI); + UseMI.setDesc(get(NewOpc)); + UseMI.findRegisterUseOperand(Reg)->ChangeToImmediate(ImmVal); + } + } + + if (Reg.isVirtual() && MRI->use_nodbg_empty(Reg)) + DefMI->eraseFromBundle(); + + return true; +} + +/// FoldImmediate - 'Reg' is known to be defined by a move immediate +/// instruction, try to fold the immediate into the use instruction. +bool X86InstrInfo::FoldImmediate(MachineInstr &UseMI, MachineInstr &DefMI, + Register Reg, MachineRegisterInfo *MRI) const { + int64_t ImmVal; + if (!getConstValDefinedInReg(DefMI, Reg, ImmVal)) + return false; + + return FoldImmediateImpl(UseMI, &DefMI, Reg, ImmVal, MRI, true); +} + +/// Check if FoldImmediate can fold ImmVal into the given instruction. +/// This function must be synchronizated with FoldImmediate. +bool X86InstrInfo::canFoldImmediate(MachineInstr &UseMI, Register UseReg, + int64_t ImmVal, + MachineRegisterInfo *MRI) const { + return FoldImmediateImpl(UseMI, nullptr, UseReg, ImmVal, MRI, false); +} + /// Expand a single-def pseudo instruction to a two-addr /// instruction with two undef reads of the register being defined. /// This is used for mapping: Index: llvm/test/CodeGen/AArch64/O3-pipeline.ll =================================================================== --- llvm/test/CodeGen/AArch64/O3-pipeline.ll +++ llvm/test/CodeGen/AArch64/O3-pipeline.ll @@ -117,6 +117,8 @@ ; CHECK-NEXT: MachineDominator Tree Construction ; CHECK-NEXT: AArch64 Local Dynamic TLS Access Clean-up ; CHECK-NEXT: Finalize ISel and expand pseudo-instructions +; CHECK-NEXT: MachineDominator Tree Construction +; CHECK-NEXT: Duplicate PHI Users ; CHECK-NEXT: Lazy Machine Block Frequency Analysis ; CHECK-NEXT: Early Tail Duplication ; CHECK-NEXT: Optimize machine instruction PHIs Index: llvm/test/CodeGen/AMDGPU/llc-pipeline.ll =================================================================== --- llvm/test/CodeGen/AMDGPU/llc-pipeline.ll +++ llvm/test/CodeGen/AMDGPU/llc-pipeline.ll @@ -297,6 +297,8 @@ ; GCN-O1-NEXT: MachinePostDominator Tree Construction ; GCN-O1-NEXT: SI Lower i1 Copies ; GCN-O1-NEXT: Finalize ISel and expand pseudo-instructions +; GCN-O1-NEXT: MachineDominator Tree Construction +; GCN-O1-NEXT: Duplicate PHI Users ; GCN-O1-NEXT: Lazy Machine Block Frequency Analysis ; GCN-O1-NEXT: Early Tail Duplication ; GCN-O1-NEXT: Optimize machine instruction PHIs @@ -583,6 +585,8 @@ ; GCN-O1-OPTS-NEXT: MachinePostDominator Tree Construction ; GCN-O1-OPTS-NEXT: SI Lower i1 Copies ; GCN-O1-OPTS-NEXT: Finalize ISel and expand pseudo-instructions +; GCN-O1-OPTS-NEXT: MachineDominator Tree Construction +; GCN-O1-OPTS-NEXT: Duplicate PHI Users ; GCN-O1-OPTS-NEXT: Lazy Machine Block Frequency Analysis ; GCN-O1-OPTS-NEXT: Early Tail Duplication ; GCN-O1-OPTS-NEXT: Optimize machine instruction PHIs @@ -886,6 +890,8 @@ ; GCN-O2-NEXT: MachinePostDominator Tree Construction ; GCN-O2-NEXT: SI Lower i1 Copies ; GCN-O2-NEXT: Finalize ISel and expand pseudo-instructions +; GCN-O2-NEXT: MachineDominator Tree Construction +; GCN-O2-NEXT: Duplicate PHI Users ; GCN-O2-NEXT: Lazy Machine Block Frequency Analysis ; GCN-O2-NEXT: Early Tail Duplication ; GCN-O2-NEXT: Optimize machine instruction PHIs @@ -1202,6 +1208,8 @@ ; GCN-O3-NEXT: MachinePostDominator Tree Construction ; GCN-O3-NEXT: SI Lower i1 Copies ; GCN-O3-NEXT: Finalize ISel and expand pseudo-instructions +; GCN-O3-NEXT: MachineDominator Tree Construction +; GCN-O3-NEXT: Duplicate PHI Users ; GCN-O3-NEXT: Lazy Machine Block Frequency Analysis ; GCN-O3-NEXT: Early Tail Duplication ; GCN-O3-NEXT: Optimize machine instruction PHIs Index: llvm/test/CodeGen/AMDGPU/peephole-fold-imm.mir =================================================================== --- llvm/test/CodeGen/AMDGPU/peephole-fold-imm.mir +++ llvm/test/CodeGen/AMDGPU/peephole-fold-imm.mir @@ -8,7 +8,6 @@ ; GCN-LABEL: name: fold_simm_virtual ; GCN: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0 - ; GCN-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 0 ; GCN-NEXT: SI_RETURN_TO_EPILOG %0:sreg_32 = S_MOV_B32 0 %1:sreg_32 = COPY killed %0 Index: llvm/test/CodeGen/ARM/O3-pipeline.ll =================================================================== --- llvm/test/CodeGen/ARM/O3-pipeline.ll +++ llvm/test/CodeGen/ARM/O3-pipeline.ll @@ -80,6 +80,8 @@ ; CHECK-NEXT: Lazy Block Frequency Analysis ; CHECK-NEXT: ARM Instruction Selection ; CHECK-NEXT: Finalize ISel and expand pseudo-instructions +; CHECK-NEXT: MachineDominator Tree Construction +; CHECK-NEXT: Duplicate PHI Users ; CHECK-NEXT: Lazy Machine Block Frequency Analysis ; CHECK-NEXT: Early Tail Duplication ; CHECK-NEXT: Optimize machine instruction PHIs Index: llvm/test/CodeGen/LoongArch/opt-pipeline.ll =================================================================== --- llvm/test/CodeGen/LoongArch/opt-pipeline.ll +++ llvm/test/CodeGen/LoongArch/opt-pipeline.ll @@ -84,6 +84,8 @@ ; CHECK-NEXT: Lazy Block Frequency Analysis ; CHECK-NEXT: LoongArch DAG->DAG Pattern Instruction Selection ; CHECK-NEXT: Finalize ISel and expand pseudo-instructions +; CHECK-NEXT: MachineDominator Tree Construction +; CHECK-NEXT: Duplicate PHI Users ; CHECK-NEXT: Lazy Machine Block Frequency Analysis ; CHECK-NEXT: Early Tail Duplication ; CHECK-NEXT: Optimize machine instruction PHIs Index: llvm/test/CodeGen/PowerPC/O3-pipeline.ll =================================================================== --- llvm/test/CodeGen/PowerPC/O3-pipeline.ll +++ llvm/test/CodeGen/PowerPC/O3-pipeline.ll @@ -99,6 +99,8 @@ ; CHECK-NEXT: MachineDominator Tree Construction ; CHECK-NEXT: Machine Natural Loop Construction ; CHECK-NEXT: PowerPC CTR loops generation +; CHECK-NEXT: MachineDominator Tree Construction +; CHECK-NEXT: Duplicate PHI Users ; CHECK-NEXT: Lazy Machine Block Frequency Analysis ; CHECK-NEXT: Early Tail Duplication ; CHECK-NEXT: Optimize machine instruction PHIs Index: llvm/test/CodeGen/RISCV/O3-pipeline.ll =================================================================== --- llvm/test/CodeGen/RISCV/O3-pipeline.ll +++ llvm/test/CodeGen/RISCV/O3-pipeline.ll @@ -82,6 +82,8 @@ ; CHECK-NEXT: Lazy Block Frequency Analysis ; CHECK-NEXT: RISC-V DAG->DAG Pattern Instruction Selection ; CHECK-NEXT: Finalize ISel and expand pseudo-instructions +; CHECK-NEXT: MachineDominator Tree Construction +; CHECK-NEXT: Duplicate PHI Users ; CHECK-NEXT: Lazy Machine Block Frequency Analysis ; CHECK-NEXT: Early Tail Duplication ; CHECK-NEXT: Optimize machine instruction PHIs Index: llvm/test/CodeGen/X86/bit_ceil.ll =================================================================== --- llvm/test/CodeGen/X86/bit_ceil.ll +++ llvm/test/CodeGen/X86/bit_ceil.ll @@ -16,7 +16,7 @@ ; NOBMI-NEXT: xorl $31, %ecx ; NOBMI-NEXT: jmp .LBB0_3 ; NOBMI-NEXT: .LBB0_1: -; NOBMI-NEXT: movl $32, %ecx +; NOBMI-NEXT: movb $32, %cl ; NOBMI-NEXT: .LBB0_3: # %cond.end ; NOBMI-NEXT: negb %cl ; NOBMI-NEXT: movl $1, %edx @@ -58,7 +58,7 @@ ; NOBMI-NEXT: xorl $31, %ecx ; NOBMI-NEXT: jmp .LBB1_3 ; NOBMI-NEXT: .LBB1_1: -; NOBMI-NEXT: movl $32, %ecx +; NOBMI-NEXT: movb $32, %cl ; NOBMI-NEXT: .LBB1_3: # %cond.end ; NOBMI-NEXT: negb %cl ; NOBMI-NEXT: movl $1, %edx @@ -102,7 +102,7 @@ ; NOBMI-NEXT: xorq $63, %rcx ; NOBMI-NEXT: jmp .LBB2_3 ; NOBMI-NEXT: .LBB2_1: -; NOBMI-NEXT: movl $64, %ecx +; NOBMI-NEXT: movb $64, %cl ; NOBMI-NEXT: .LBB2_3: # %cond.end ; NOBMI-NEXT: negb %cl ; NOBMI-NEXT: movl $1, %edx @@ -143,7 +143,7 @@ ; NOBMI-NEXT: xorq $63, %rcx ; NOBMI-NEXT: jmp .LBB3_3 ; NOBMI-NEXT: .LBB3_1: -; NOBMI-NEXT: movl $64, %ecx +; NOBMI-NEXT: movb $64, %cl ; NOBMI-NEXT: .LBB3_3: # %cond.end ; NOBMI-NEXT: negb %cl ; NOBMI-NEXT: movl $1, %edx Index: llvm/test/CodeGen/X86/div-rem-pair-recomposition-signed.ll =================================================================== --- llvm/test/CodeGen/X86/div-rem-pair-recomposition-signed.ll +++ llvm/test/CodeGen/X86/div-rem-pair-recomposition-signed.ll @@ -273,63 +273,62 @@ ; X86-NEXT: movl %ebp, %esi ; X86-NEXT: orl %edi, %esi ; X86-NEXT: cmovnel %ecx, %edx -; X86-NEXT: xorl %ebp, %ebp +; X86-NEXT: xorl %esi, %esi ; X86-NEXT: subl %edx, %ebx -; X86-NEXT: movl $0, %eax -; X86-NEXT: sbbl %eax, %eax +; X86-NEXT: movl $0, %ebp +; X86-NEXT: sbbl %ebp, %ebp ; X86-NEXT: movl $0, %edx ; X86-NEXT: sbbl %edx, %edx -; X86-NEXT: movl $0, %esi -; X86-NEXT: sbbl %esi, %esi +; X86-NEXT: movl $0, %eax +; X86-NEXT: sbbl %eax, %eax ; X86-NEXT: movl $127, %ecx ; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X86-NEXT: cmpl %ebx, %ecx ; X86-NEXT: movl $0, %ecx -; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X86-NEXT: sbbl %eax, %ecx +; X86-NEXT: sbbl %ebp, %ecx ; X86-NEXT: movl $0, %ecx ; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X86-NEXT: sbbl %edx, %ecx ; X86-NEXT: movl $0, %ecx -; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X86-NEXT: sbbl %esi, %ecx +; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill +; X86-NEXT: sbbl %eax, %ecx ; X86-NEXT: setb %cl ; X86-NEXT: orb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Folded Reload -; X86-NEXT: movl %edi, %ebx -; X86-NEXT: cmovnel %ebp, %ebx +; X86-NEXT: movl %edi, %edx +; X86-NEXT: cmovnel %esi, %edx ; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload -; X86-NEXT: cmovnel %ebp, %edi -; X86-NEXT: movl (%esp), %esi # 4-byte Reload -; X86-NEXT: cmovnel %ebp, %esi -; X86-NEXT: cmovel {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload -; X86-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X86-NEXT: movl {{[0-9]+}}(%esp), %edx +; X86-NEXT: cmovnel %esi, %edi +; X86-NEXT: movl (%esp), %ebx # 4-byte Reload +; X86-NEXT: cmovnel %esi, %ebx +; X86-NEXT: cmovel {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload ; X86-NEXT: jne .LBB4_1 ; X86-NEXT: # %bb.8: # %_udiv-special-cases ; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload ; X86-NEXT: xorl $127, %eax ; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload -; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload +; X86-NEXT: movl %ebp, %ecx ; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload ; X86-NEXT: orl %eax, %ecx -; X86-NEXT: movl %esi, %ebp +; X86-NEXT: movl %ebp, %eax +; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload ; X86-NEXT: je .LBB4_9 ; X86-NEXT: # %bb.5: # %udiv-bb1 -; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload -; X86-NEXT: movl %eax, {{[0-9]+}}(%esp) -; X86-NEXT: movl (%esp), %eax # 4-byte Reload -; X86-NEXT: movl %eax, {{[0-9]+}}(%esp) -; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload -; X86-NEXT: movl %eax, {{[0-9]+}}(%esp) -; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload -; X86-NEXT: movl %eax, {{[0-9]+}}(%esp) +; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload +; X86-NEXT: movl %ecx, {{[0-9]+}}(%esp) +; X86-NEXT: movl (%esp), %ecx # 4-byte Reload +; X86-NEXT: movl %ecx, {{[0-9]+}}(%esp) +; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload +; X86-NEXT: movl %ecx, {{[0-9]+}}(%esp) +; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload +; X86-NEXT: movl %ecx, {{[0-9]+}}(%esp) ; X86-NEXT: movl $0, {{[0-9]+}}(%esp) ; X86-NEXT: movl $0, {{[0-9]+}}(%esp) ; X86-NEXT: movl $0, {{[0-9]+}}(%esp) ; X86-NEXT: movl $0, {{[0-9]+}}(%esp) ; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload +; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X86-NEXT: movl %ecx, %eax -; X86-NEXT: movl %ecx, %esi +; X86-NEXT: movl %ecx, %edi ; X86-NEXT: xorb $127, %al ; X86-NEXT: movb %al, %ch ; X86-NEXT: andb $7, %ch @@ -338,46 +337,47 @@ ; X86-NEXT: negb %al ; X86-NEXT: movsbl %al, %eax ; X86-NEXT: movl 148(%esp,%eax), %edx -; X86-NEXT: movl 152(%esp,%eax), %ebx +; X86-NEXT: movl 152(%esp,%eax), %esi ; X86-NEXT: movb %ch, %cl -; X86-NEXT: shldl %cl, %edx, %ebx +; X86-NEXT: shldl %cl, %edx, %esi +; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X86-NEXT: shll %cl, %edx ; X86-NEXT: notb %cl -; X86-NEXT: movl 144(%esp,%eax), %ebp -; X86-NEXT: movl %ebp, %edi -; X86-NEXT: shrl %edi -; X86-NEXT: shrl %cl, %edi -; X86-NEXT: orl %edx, %edi +; X86-NEXT: movl 144(%esp,%eax), %ebx +; X86-NEXT: movl %ebx, %esi +; X86-NEXT: shrl %esi +; X86-NEXT: shrl %cl, %esi +; X86-NEXT: orl %edx, %esi +; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X86-NEXT: movl 140(%esp,%eax), %eax ; X86-NEXT: movb %ch, %cl -; X86-NEXT: shldl %cl, %eax, %ebp +; X86-NEXT: shldl %cl, %eax, %ebx ; X86-NEXT: shll %cl, %eax ; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X86-NEXT: addl $1, %esi -; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill +; X86-NEXT: addl $1, %edi +; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill +; X86-NEXT: adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill ; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload ; X86-NEXT: adcl $0, %eax ; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload ; X86-NEXT: adcl $0, %ecx -; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload -; X86-NEXT: adcl $0, %edx ; X86-NEXT: jae .LBB4_2 ; X86-NEXT: # %bb.6: ; X86-NEXT: xorl %ecx, %ecx ; X86-NEXT: xorl %eax, %eax ; X86-NEXT: jmp .LBB4_7 ; X86-NEXT: .LBB4_1: -; X86-NEXT: movl %esi, %ebp +; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload ; X86-NEXT: jmp .LBB4_9 ; X86-NEXT: .LBB4_2: # %udiv-preheader -; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload -; X86-NEXT: movl %esi, {{[0-9]+}}(%esp) -; X86-NEXT: movl (%esp), %esi # 4-byte Reload -; X86-NEXT: movl %esi, {{[0-9]+}}(%esp) -; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload -; X86-NEXT: movl %esi, {{[0-9]+}}(%esp) -; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload -; X86-NEXT: movl %esi, {{[0-9]+}}(%esp) +; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload +; X86-NEXT: movl %edx, {{[0-9]+}}(%esp) +; X86-NEXT: movl (%esp), %edx # 4-byte Reload +; X86-NEXT: movl %edx, {{[0-9]+}}(%esp) +; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload +; X86-NEXT: movl %edx, {{[0-9]+}}(%esp) +; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload +; X86-NEXT: movl %edx, {{[0-9]+}}(%esp) ; X86-NEXT: movl $0, {{[0-9]+}}(%esp) ; X86-NEXT: movl $0, {{[0-9]+}}(%esp) ; X86-NEXT: movl $0, {{[0-9]+}}(%esp) @@ -387,31 +387,30 @@ ; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X86-NEXT: movb %al, %ch ; X86-NEXT: andb $7, %ch +; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X86-NEXT: # kill: def $al killed $al killed $eax ; X86-NEXT: shrb $3, %al ; X86-NEXT: andb $15, %al ; X86-NEXT: movzbl %al, %eax -; X86-NEXT: movl 104(%esp,%eax), %esi -; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X86-NEXT: movl 100(%esp,%eax), %ebx -; X86-NEXT: movl %ebx, (%esp) # 4-byte Spill +; X86-NEXT: movl 104(%esp,%eax), %edx +; X86-NEXT: movl 100(%esp,%eax), %esi +; X86-NEXT: movl %esi, %ebp ; X86-NEXT: movb %ch, %cl -; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload -; X86-NEXT: shrdl %cl, %esi, (%esp) # 4-byte Folded Spill -; X86-NEXT: movl 92(%esp,%eax), %esi -; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X86-NEXT: movl 96(%esp,%eax), %esi -; X86-NEXT: movl %esi, %eax +; X86-NEXT: shrdl %cl, %edx, %ebp +; X86-NEXT: movl 92(%esp,%eax), %edi +; X86-NEXT: movl 96(%esp,%eax), %ebx +; X86-NEXT: movl %ebx, %eax ; X86-NEXT: shrl %cl, %eax ; X86-NEXT: notb %cl -; X86-NEXT: addl %ebx, %ebx -; X86-NEXT: shll %cl, %ebx -; X86-NEXT: orl %eax, %ebx -; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill +; X86-NEXT: addl %esi, %esi +; X86-NEXT: shll %cl, %esi +; X86-NEXT: orl %eax, %esi +; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X86-NEXT: movb %ch, %cl -; X86-NEXT: shrl %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill -; X86-NEXT: shrdl %cl, %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill +; X86-NEXT: shrl %cl, %edx +; X86-NEXT: movl %edx, (%esp) # 4-byte Spill +; X86-NEXT: shrdl %cl, %ebx, %edi +; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload ; X86-NEXT: addl $-1, %eax ; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill @@ -426,47 +425,49 @@ ; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill ; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill +; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload +; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload ; X86-NEXT: .p2align 4, 0x90 ; X86-NEXT: .LBB4_3: # %udiv-do-while ; X86-NEXT: # =>This Inner Loop Header: Depth=1 ; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill +; X86-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X86-NEXT: movl (%esp), %eax # 4-byte Reload -; X86-NEXT: shldl $1, %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill -; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload -; X86-NEXT: shldl $1, %esi, %eax +; X86-NEXT: shldl $1, %ebp, %eax ; X86-NEXT: movl %eax, (%esp) # 4-byte Spill +; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload +; X86-NEXT: shldl $1, %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill ; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload -; X86-NEXT: shldl $1, %edx, %esi -; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload -; X86-NEXT: shldl $1, %ecx, %edx -; X86-NEXT: shldl $1, %edi, %ecx -; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload -; X86-NEXT: orl %ebx, %ecx -; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X86-NEXT: shldl $1, %ebp, %edi -; X86-NEXT: orl %ebx, %edi -; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill +; X86-NEXT: shldl $1, %edx, %ebp +; X86-NEXT: shldl $1, %ebx, %edx +; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload +; X86-NEXT: shldl $1, %esi, %ebx +; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload +; X86-NEXT: orl %edi, %ebx ; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload -; X86-NEXT: shldl $1, %ecx, %ebp -; X86-NEXT: orl %ebx, %ebp -; X86-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X86-NEXT: addl %ecx, %ecx -; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload +; X86-NEXT: shldl $1, %ecx, %esi +; X86-NEXT: orl %edi, %esi +; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill +; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload +; X86-NEXT: shldl $1, %esi, %ecx +; X86-NEXT: orl %edi, %ecx ; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill +; X86-NEXT: addl %esi, %esi +; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload +; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X86-NEXT: cmpl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload ; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload -; X86-NEXT: sbbl %esi, %ecx +; X86-NEXT: sbbl %ebp, %ecx ; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload -; X86-NEXT: sbbl %eax, %ecx +; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload ; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload -; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload -; X86-NEXT: sbbl %ebp, %ecx +; X86-NEXT: sbbl %eax, %ecx ; X86-NEXT: sarl $31, %ecx -; X86-NEXT: movl %ecx, %edi -; X86-NEXT: andl $1, %edi -; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X86-NEXT: movl %ecx, %ebx -; X86-NEXT: andl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload +; X86-NEXT: movl %ecx, %esi +; X86-NEXT: andl $1, %esi +; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill +; X86-NEXT: movl %ecx, %esi +; X86-NEXT: andl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload ; X86-NEXT: movl %ecx, %edi ; X86-NEXT: andl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload ; X86-NEXT: movl %ecx, %eax @@ -474,110 +475,109 @@ ; X86-NEXT: andl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload ; X86-NEXT: subl %ecx, %edx ; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X86-NEXT: sbbl %eax, %esi -; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload -; X86-NEXT: sbbl %edi, (%esp) # 4-byte Folded Spill -; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload -; X86-NEXT: movl %ebp, %eax -; X86-NEXT: sbbl %ebx, %eax -; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill +; X86-NEXT: sbbl %eax, %ebp +; X86-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill +; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload +; X86-NEXT: sbbl %edi, %ebp +; X86-NEXT: sbbl %esi, (%esp) # 4-byte Folded Spill ; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload ; X86-NEXT: addl $-1, %ecx -; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload -; X86-NEXT: adcl $-1, %eax +; X86-NEXT: adcl $-1, %edx ; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload ; X86-NEXT: adcl $-1, %esi -; X86-NEXT: adcl $-1, %edx -; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X86-NEXT: orl %edx, %eax +; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload +; X86-NEXT: adcl $-1, %edi +; X86-NEXT: movl %edx, %eax +; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill +; X86-NEXT: orl %edi, %eax ; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X86-NEXT: orl %esi, %ecx -; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload ; X86-NEXT: orl %eax, %ecx ; X86-NEXT: jne .LBB4_3 ; X86-NEXT: # %bb.4: +; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill +; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload ; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload ; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload ; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload ; X86-NEXT: .LBB4_7: # %udiv-loop-exit -; X86-NEXT: shldl $1, %edi, %ebx -; X86-NEXT: orl %eax, %ebx -; X86-NEXT: shldl $1, %ebp, %edi -; X86-NEXT: orl %eax, %edi ; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload -; X86-NEXT: shldl $1, %edx, %ebp -; X86-NEXT: orl %eax, %ebp -; X86-NEXT: addl %edx, %edx -; X86-NEXT: orl %ecx, %edx -; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X86-NEXT: movl {{[0-9]+}}(%esp), %edx -; X86-NEXT: .LBB4_9: # %udiv-end -; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload -; X86-NEXT: xorl %eax, %ebx -; X86-NEXT: xorl %eax, %edi -; X86-NEXT: xorl %eax, %ebp +; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload +; X86-NEXT: shldl $1, %edi, %edx +; X86-NEXT: orl %eax, %edx +; X86-NEXT: shldl $1, %ebx, %edi +; X86-NEXT: orl %eax, %edi ; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload -; X86-NEXT: xorl %eax, %esi -; X86-NEXT: subl %eax, %esi +; X86-NEXT: shldl $1, %esi, %ebx +; X86-NEXT: orl %eax, %ebx +; X86-NEXT: addl %esi, %esi +; X86-NEXT: orl %ecx, %esi +; X86-NEXT: .LBB4_9: # %udiv-end +; X86-NEXT: xorl %ebp, %edx +; X86-NEXT: movl %edi, %ecx +; X86-NEXT: xorl %ebp, %ecx +; X86-NEXT: xorl %ebp, %ebx +; X86-NEXT: xorl %ebp, %esi +; X86-NEXT: subl %ebp, %esi ; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X86-NEXT: sbbl %eax, %ebp -; X86-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X86-NEXT: sbbl %eax, %edi -; X86-NEXT: sbbl %eax, %ebx +; X86-NEXT: sbbl %ebp, %ebx +; X86-NEXT: sbbl %ebp, %ecx +; X86-NEXT: sbbl %ebp, %edx +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: movl %esi, (%eax) +; X86-NEXT: movl %ebx, 4(%eax) +; X86-NEXT: movl %ecx, 8(%eax) +; X86-NEXT: movl %edx, 12(%eax) +; X86-NEXT: movl %ebx, %eax ; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X86-NEXT: movl %esi, (%edx) -; X86-NEXT: movl %ebp, 4(%edx) -; X86-NEXT: movl %edi, 8(%edx) -; X86-NEXT: movl %ebx, 12(%edx) -; X86-NEXT: movl %ebp, %eax -; X86-NEXT: movl %edi, %ecx +; X86-NEXT: movl %ecx, %ebp ; X86-NEXT: movl {{[0-9]+}}(%esp), %edi +; X86-NEXT: movl %edx, %ebx ; X86-NEXT: mull %edi -; X86-NEXT: movl %edx, %ebp +; X86-NEXT: movl %edx, %ecx ; X86-NEXT: movl %eax, (%esp) # 4-byte Spill ; X86-NEXT: movl %esi, %eax ; X86-NEXT: mull %edi ; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X86-NEXT: movl %edx, %ebx -; X86-NEXT: addl (%esp), %ebx # 4-byte Folded Reload -; X86-NEXT: adcl $0, %ebp +; X86-NEXT: movl %edx, %edi +; X86-NEXT: addl (%esp), %edi # 4-byte Folded Reload +; X86-NEXT: adcl $0, %ecx ; X86-NEXT: movl %esi, %eax -; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload ; X86-NEXT: movl {{[0-9]+}}(%esp), %esi ; X86-NEXT: mull %esi -; X86-NEXT: addl %ebx, %eax +; X86-NEXT: addl %edi, %eax ; X86-NEXT: movl %eax, (%esp) # 4-byte Spill -; X86-NEXT: adcl %ebp, %edx -; X86-NEXT: movl %edx, %ebp -; X86-NEXT: setb %bl +; X86-NEXT: adcl %ecx, %edx +; X86-NEXT: movl %edx, %ecx +; X86-NEXT: setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill +; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload ; X86-NEXT: movl %edi, %eax ; X86-NEXT: mull %esi -; X86-NEXT: addl %ebp, %eax +; X86-NEXT: addl %ecx, %eax ; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X86-NEXT: movzbl %bl, %eax +; X86-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload ; X86-NEXT: adcl %eax, %edx ; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X86-NEXT: movl {{[0-9]+}}(%esp), %eax -; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload ; X86-NEXT: imull %eax, %ebx -; X86-NEXT: mull %ecx +; X86-NEXT: mull %ebp ; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X86-NEXT: imull %esi, %ecx -; X86-NEXT: addl %edx, %ecx -; X86-NEXT: addl %ebx, %ecx +; X86-NEXT: imull %esi, %ebp +; X86-NEXT: addl %edx, %ebp +; X86-NEXT: addl %ebx, %ebp ; X86-NEXT: movl {{[0-9]+}}(%esp), %eax -; X86-NEXT: movl %eax, %esi -; X86-NEXT: imull %edi, %esi +; X86-NEXT: movl %eax, %ecx +; X86-NEXT: imull %edi, %ecx ; X86-NEXT: movl {{[0-9]+}}(%esp), %ebx ; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload ; X86-NEXT: imull %edx, %ebx ; X86-NEXT: mull %edx ; X86-NEXT: addl %edx, %ebx -; X86-NEXT: addl %esi, %ebx +; X86-NEXT: addl %ecx, %ebx ; X86-NEXT: addl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload -; X86-NEXT: adcl %ecx, %ebx +; X86-NEXT: adcl %ebp, %ebx ; X86-NEXT: addl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload ; X86-NEXT: adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload ; X86-NEXT: movl {{[0-9]+}}(%esp), %edx Index: llvm/test/CodeGen/X86/div-rem-pair-recomposition-unsigned.ll =================================================================== --- llvm/test/CodeGen/X86/div-rem-pair-recomposition-unsigned.ll +++ llvm/test/CodeGen/X86/div-rem-pair-recomposition-unsigned.ll @@ -177,118 +177,121 @@ ; X86-NEXT: pushl %ebx ; X86-NEXT: pushl %edi ; X86-NEXT: pushl %esi -; X86-NEXT: subl $132, %esp +; X86-NEXT: subl $136, %esp ; X86-NEXT: movl {{[0-9]+}}(%esp), %ebp -; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx -; X86-NEXT: movl {{[0-9]+}}(%esp), %esi +; X86-NEXT: movl {{[0-9]+}}(%esp), %ebx ; X86-NEXT: movl {{[0-9]+}}(%esp), %edi ; X86-NEXT: movl %edi, %eax -; X86-NEXT: orl %esi, %eax -; X86-NEXT: orl %ebp, %ecx +; X86-NEXT: orl %ebx, %eax +; X86-NEXT: movl %ebp, %ecx +; X86-NEXT: orl {{[0-9]+}}(%esp), %ecx ; X86-NEXT: orl %eax, %ecx ; X86-NEXT: movl {{[0-9]+}}(%esp), %eax -; X86-NEXT: sete %bl +; X86-NEXT: sete %cl ; X86-NEXT: orl {{[0-9]+}}(%esp), %eax +; X86-NEXT: movl {{[0-9]+}}(%esp), %esi ; X86-NEXT: movl {{[0-9]+}}(%esp), %edx -; X86-NEXT: orl {{[0-9]+}}(%esp), %edx +; X86-NEXT: orl %esi, %edx ; X86-NEXT: orl %eax, %edx ; X86-NEXT: sete %al -; X86-NEXT: orb %bl, %al +; X86-NEXT: orb %cl, %al ; X86-NEXT: movb %al, (%esp) # 1-byte Spill -; X86-NEXT: bsrl %esi, %edx +; X86-NEXT: bsrl %ebx, %edx ; X86-NEXT: xorl $31, %edx -; X86-NEXT: bsrl %ebp, %ecx +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: bsrl %eax, %ecx ; X86-NEXT: xorl $31, %ecx ; X86-NEXT: addl $32, %ecx -; X86-NEXT: testl %esi, %esi +; X86-NEXT: testl %ebx, %ebx ; X86-NEXT: cmovnel %edx, %ecx ; X86-NEXT: bsrl %edi, %edx ; X86-NEXT: xorl $31, %edx -; X86-NEXT: bsrl {{[0-9]+}}(%esp), %eax -; X86-NEXT: xorl $31, %eax -; X86-NEXT: addl $32, %eax +; X86-NEXT: bsrl %ebp, %ebp +; X86-NEXT: xorl $31, %ebp +; X86-NEXT: addl $32, %ebp ; X86-NEXT: testl %edi, %edi -; X86-NEXT: cmovnel %edx, %eax -; X86-NEXT: addl $64, %eax -; X86-NEXT: orl %esi, %ebp -; X86-NEXT: cmovnel %ecx, %eax -; X86-NEXT: movl {{[0-9]+}}(%esp), %ebp -; X86-NEXT: bsrl %ebp, %edx +; X86-NEXT: cmovnel %edx, %ebp +; X86-NEXT: addl $64, %ebp +; X86-NEXT: movl %eax, %edx +; X86-NEXT: orl %ebx, %edx +; X86-NEXT: movl {{[0-9]+}}(%esp), %edi +; X86-NEXT: cmovnel %ecx, %ebp +; X86-NEXT: bsrl %edi, %edx ; X86-NEXT: xorl $31, %edx -; X86-NEXT: movl {{[0-9]+}}(%esp), %ebx -; X86-NEXT: bsrl %ebx, %ecx +; X86-NEXT: bsrl %esi, %ecx +; X86-NEXT: movl %esi, %ebx ; X86-NEXT: xorl $31, %ecx ; X86-NEXT: addl $32, %ecx -; X86-NEXT: testl %ebp, %ebp +; X86-NEXT: testl %edi, %edi ; X86-NEXT: cmovnel %edx, %ecx -; X86-NEXT: movl {{[0-9]+}}(%esp), %edi -; X86-NEXT: bsrl %edi, %esi +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: bsrl %eax, %esi ; X86-NEXT: xorl $31, %esi ; X86-NEXT: bsrl {{[0-9]+}}(%esp), %edx ; X86-NEXT: xorl $31, %edx ; X86-NEXT: addl $32, %edx -; X86-NEXT: testl %edi, %edi +; X86-NEXT: testl %eax, %eax ; X86-NEXT: cmovnel %esi, %edx ; X86-NEXT: addl $64, %edx ; X86-NEXT: movl %ebx, %esi -; X86-NEXT: orl %ebp, %esi +; X86-NEXT: orl %edi, %esi ; X86-NEXT: cmovnel %ecx, %edx ; X86-NEXT: xorl %ecx, %ecx -; X86-NEXT: subl %edx, %eax +; X86-NEXT: subl %edx, %ebp ; X86-NEXT: movl $0, %esi ; X86-NEXT: sbbl %esi, %esi +; X86-NEXT: movl $0, %eax +; X86-NEXT: sbbl %eax, %eax ; X86-NEXT: movl $0, %ebx ; X86-NEXT: sbbl %ebx, %ebx -; X86-NEXT: movl $0, %edi -; X86-NEXT: sbbl %edi, %edi ; X86-NEXT: movl $127, %edx -; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X86-NEXT: cmpl %eax, %edx -; X86-NEXT: movl %edi, %eax +; X86-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill +; X86-NEXT: cmpl %ebp, %edx ; X86-NEXT: movl $0, %edx ; X86-NEXT: sbbl %esi, %edx ; X86-NEXT: movl $0, %edx -; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X86-NEXT: sbbl %ebx, %edx +; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill +; X86-NEXT: sbbl %eax, %edx ; X86-NEXT: movl $0, %edx -; X86-NEXT: sbbl %edi, %edx +; X86-NEXT: sbbl %ebx, %edx ; X86-NEXT: setb %dl ; X86-NEXT: orb (%esp), %dl # 1-byte Folded Reload -; X86-NEXT: cmovnel %ecx, %ebp +; X86-NEXT: movl %edi, %edx +; X86-NEXT: cmovnel %ecx, %edx ; X86-NEXT: movl {{[0-9]+}}(%esp), %edi ; X86-NEXT: cmovnel %ecx, %edi -; X86-NEXT: movl {{[0-9]+}}(%esp), %edx -; X86-NEXT: cmovnel %ecx, %edx -; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X86-NEXT: movl {{[0-9]+}}(%esp), %edx -; X86-NEXT: cmovnel %ecx, %edx -; X86-NEXT: jne .LBB4_8 -; X86-NEXT: # %bb.1: # %_udiv-special-cases -; X86-NEXT: movl %eax, %ebx +; X86-NEXT: movl {{[0-9]+}}(%esp), %ebp +; X86-NEXT: cmovnel %ecx, %ebp +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: cmovnel %ecx, %eax +; X86-NEXT: jne .LBB4_1 +; X86-NEXT: # %bb.8: # %_udiv-special-cases +; X86-NEXT: movl %esi, %ecx +; X86-NEXT: movl %eax, %esi ; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload ; X86-NEXT: xorl $127, %eax ; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload -; X86-NEXT: movl %esi, %ecx -; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill +; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X86-NEXT: orl %ebx, %ecx ; X86-NEXT: orl %eax, %ecx -; X86-NEXT: je .LBB4_8 -; X86-NEXT: # %bb.2: # %udiv-bb1 +; X86-NEXT: movl %esi, %eax +; X86-NEXT: movl {{[0-9]+}}(%esp), %esi +; X86-NEXT: je .LBB4_9 +; X86-NEXT: # %bb.5: # %udiv-bb1 +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: movl %eax, {{[0-9]+}}(%esp) ; X86-NEXT: movl {{[0-9]+}}(%esp), %eax ; X86-NEXT: movl %eax, {{[0-9]+}}(%esp) ; X86-NEXT: movl {{[0-9]+}}(%esp), %eax ; X86-NEXT: movl %eax, {{[0-9]+}}(%esp) ; X86-NEXT: movl {{[0-9]+}}(%esp), %eax ; X86-NEXT: movl %eax, {{[0-9]+}}(%esp) -; X86-NEXT: movl {{[0-9]+}}(%esp), %ebp -; X86-NEXT: movl %ebp, {{[0-9]+}}(%esp) ; X86-NEXT: movl $0, {{[0-9]+}}(%esp) ; X86-NEXT: movl $0, {{[0-9]+}}(%esp) ; X86-NEXT: movl $0, {{[0-9]+}}(%esp) ; X86-NEXT: movl $0, {{[0-9]+}}(%esp) -; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload -; X86-NEXT: movl %ecx, %eax -; X86-NEXT: movl %ecx, %ebp +; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload +; X86-NEXT: # kill: def $al killed $al killed $eax ; X86-NEXT: xorb $127, %al ; X86-NEXT: movb %al, %ch ; X86-NEXT: andb $7, %ch @@ -296,70 +299,72 @@ ; X86-NEXT: andb $15, %al ; X86-NEXT: negb %al ; X86-NEXT: movsbl %al, %eax -; X86-NEXT: movl 124(%esp,%eax), %edx -; X86-NEXT: movl 128(%esp,%eax), %edi +; X86-NEXT: movl 128(%esp,%eax), %edx +; X86-NEXT: movl 132(%esp,%eax), %esi ; X86-NEXT: movb %ch, %cl -; X86-NEXT: shldl %cl, %edx, %edi -; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill +; X86-NEXT: shldl %cl, %edx, %esi +; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X86-NEXT: shll %cl, %edx ; X86-NEXT: notb %cl -; X86-NEXT: movl 120(%esp,%eax), %ebx -; X86-NEXT: movl %ebx, %edi +; X86-NEXT: movl 124(%esp,%eax), %ebp +; X86-NEXT: movl %ebp, %edi ; X86-NEXT: shrl %edi ; X86-NEXT: shrl %cl, %edi ; X86-NEXT: orl %edx, %edi -; X86-NEXT: movl 116(%esp,%eax), %edx +; X86-NEXT: movl 120(%esp,%eax), %eax ; X86-NEXT: movb %ch, %cl -; X86-NEXT: shldl %cl, %edx, %ebx -; X86-NEXT: shll %cl, %edx -; X86-NEXT: addl $1, %ebp -; X86-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X86-NEXT: adcl $0, %esi -; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill +; X86-NEXT: shldl %cl, %eax, %ebp +; X86-NEXT: shll %cl, %eax +; X86-NEXT: addl $1, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill +; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload +; X86-NEXT: adcl $0, %edx ; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload ; X86-NEXT: adcl $0, %ecx -; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload -; X86-NEXT: adcl $0, %eax -; X86-NEXT: jae .LBB4_3 +; X86-NEXT: adcl $0, %ebx +; X86-NEXT: jae .LBB4_2 ; X86-NEXT: # %bb.6: +; X86-NEXT: xorl %ebx, %ebx ; X86-NEXT: xorl %ecx, %ecx -; X86-NEXT: xorl %eax, %eax +; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload +; X86-NEXT: movl {{[0-9]+}}(%esp), %esi ; X86-NEXT: jmp .LBB4_7 -; X86-NEXT: .LBB4_3: # %udiv-preheader +; X86-NEXT: .LBB4_1: ; X86-NEXT: movl {{[0-9]+}}(%esp), %esi -; X86-NEXT: movl %esi, {{[0-9]+}}(%esp) +; X86-NEXT: jmp .LBB4_9 +; X86-NEXT: .LBB4_2: # %udiv-preheader ; X86-NEXT: movl {{[0-9]+}}(%esp), %esi ; X86-NEXT: movl %esi, {{[0-9]+}}(%esp) ; X86-NEXT: movl {{[0-9]+}}(%esp), %esi ; X86-NEXT: movl %esi, {{[0-9]+}}(%esp) ; X86-NEXT: movl {{[0-9]+}}(%esp), %esi ; X86-NEXT: movl %esi, {{[0-9]+}}(%esp) +; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: movl %eax, {{[0-9]+}}(%esp) ; X86-NEXT: movl $0, {{[0-9]+}}(%esp) ; X86-NEXT: movl $0, {{[0-9]+}}(%esp) ; X86-NEXT: movl $0, {{[0-9]+}}(%esp) ; X86-NEXT: movl $0, {{[0-9]+}}(%esp) -; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload ; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill +; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X86-NEXT: movb %al, %ch ; X86-NEXT: andb $7, %ch ; X86-NEXT: # kill: def $al killed $al killed $eax ; X86-NEXT: shrb $3, %al ; X86-NEXT: andb $15, %al ; X86-NEXT: movzbl %al, %eax -; X86-NEXT: movl 80(%esp,%eax), %esi -; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X86-NEXT: movl 76(%esp,%eax), %edi +; X86-NEXT: movl 84(%esp,%eax), %esi ; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X86-NEXT: movl %edi, %edx +; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill +; X86-NEXT: movl 80(%esp,%eax), %edi +; X86-NEXT: movl %edi, %ebx ; X86-NEXT: movb %ch, %cl -; X86-NEXT: shrdl %cl, %esi, %edx -; X86-NEXT: movl %ebx, %ebp -; X86-NEXT: movl %edx, %ebx -; X86-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X86-NEXT: movl 68(%esp,%eax), %edx -; X86-NEXT: movl 72(%esp,%eax), %ebp -; X86-NEXT: movl %ebp, %eax +; X86-NEXT: shrdl %cl, %esi, %ebx +; X86-NEXT: movl 72(%esp,%eax), %edx +; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill +; X86-NEXT: movl 76(%esp,%eax), %edx +; X86-NEXT: movl %edx, %eax ; X86-NEXT: shrl %cl, %eax ; X86-NEXT: notb %cl ; X86-NEXT: addl %edi, %edi @@ -368,9 +373,9 @@ ; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X86-NEXT: movb %ch, %cl ; X86-NEXT: shrl %cl, %esi -; X86-NEXT: movl %esi, (%esp) # 4-byte Spill -; X86-NEXT: shrdl %cl, %ebp, %edx -; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill +; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill +; X86-NEXT: shrdl %cl, %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill +; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload ; X86-NEXT: movl {{[0-9]+}}(%esp), %eax ; X86-NEXT: addl $-1, %eax ; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill @@ -383,126 +388,124 @@ ; X86-NEXT: movl {{[0-9]+}}(%esp), %eax ; X86-NEXT: adcl $-1, %eax ; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X86-NEXT: xorl %eax, %eax ; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill -; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload -; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload +; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill ; X86-NEXT: .p2align 4, 0x90 -; X86-NEXT: .LBB4_4: # %udiv-do-while +; X86-NEXT: .LBB4_3: # %udiv-do-while ; X86-NEXT: # =>This Inner Loop Header: Depth=1 -; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X86-NEXT: movl %ebx, %edx -; X86-NEXT: shldl $1, %ebx, (%esp) # 4-byte Folded Spill -; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload -; X86-NEXT: shldl $1, %ebx, %edx +; X86-NEXT: movl %ebx, (%esp) # 4-byte Spill +; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload -; X86-NEXT: shldl $1, %edi, %ebx -; X86-NEXT: shldl $1, %ebp, %edi +; X86-NEXT: shldl $1, %ebx, %edi +; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload +; X86-NEXT: shldl $1, %ebx, (%esp) # 4-byte Folded Spill +; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload +; X86-NEXT: shldl $1, %edx, %ebx ; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload -; X86-NEXT: shldl $1, %esi, %ebp -; X86-NEXT: orl %eax, %ebp +; X86-NEXT: shldl $1, %esi, %edx +; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload +; X86-NEXT: shldl $1, %eax, %esi ; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload -; X86-NEXT: shldl $1, %ecx, %esi -; X86-NEXT: orl %eax, %esi +; X86-NEXT: orl %ecx, %esi ; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill +; X86-NEXT: shldl $1, %ebp, %eax +; X86-NEXT: orl %ecx, %eax +; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload -; X86-NEXT: shldl $1, %eax, %ecx -; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload -; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill +; X86-NEXT: shldl $1, %eax, %ebp +; X86-NEXT: orl %ecx, %ebp +; X86-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X86-NEXT: addl %eax, %eax ; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload ; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X86-NEXT: cmpl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload +; X86-NEXT: cmpl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload ; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload ; X86-NEXT: sbbl %ebx, %ecx ; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload -; X86-NEXT: sbbl %edx, %ecx -; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload ; X86-NEXT: sbbl (%esp), %ecx # 4-byte Folded Reload +; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload +; X86-NEXT: sbbl %edi, %ecx ; X86-NEXT: sarl $31, %ecx ; X86-NEXT: movl %ecx, %eax ; X86-NEXT: andl $1, %eax ; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X86-NEXT: movl %ecx, %eax -; X86-NEXT: andl {{[0-9]+}}(%esp), %eax -; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill +; X86-NEXT: movl %ecx, %ebp +; X86-NEXT: andl {{[0-9]+}}(%esp), %ebp ; X86-NEXT: movl %ecx, %esi ; X86-NEXT: andl {{[0-9]+}}(%esp), %esi ; X86-NEXT: movl %ecx, %eax ; X86-NEXT: andl {{[0-9]+}}(%esp), %eax ; X86-NEXT: andl {{[0-9]+}}(%esp), %ecx -; X86-NEXT: subl %ecx, %edi -; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload +; X86-NEXT: subl %ecx, %edx +; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X86-NEXT: sbbl %eax, %ebx ; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X86-NEXT: sbbl %esi, %edx -; X86-NEXT: movl %edx, %ebx -; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload -; X86-NEXT: sbbl %eax, (%esp) # 4-byte Folded Spill +; X86-NEXT: movl (%esp), %ebx # 4-byte Reload +; X86-NEXT: sbbl %esi, %ebx +; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload +; X86-NEXT: sbbl %ebp, %edi +; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill +; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload ; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload ; X86-NEXT: addl $-1, %ecx -; X86-NEXT: adcl $-1, %edi -; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload ; X86-NEXT: adcl $-1, %edx ; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload ; X86-NEXT: adcl $-1, %esi -; X86-NEXT: movl %edi, %eax -; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X86-NEXT: orl %esi, %eax +; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload +; X86-NEXT: adcl $-1, %edi +; X86-NEXT: movl %edx, %eax +; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill +; X86-NEXT: orl %edi, %eax ; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X86-NEXT: orl %edx, %ecx +; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill +; X86-NEXT: orl %esi, %ecx ; X86-NEXT: orl %eax, %ecx -; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload -; X86-NEXT: jne .LBB4_4 -; X86-NEXT: # %bb.5: -; X86-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill +; X86-NEXT: jne .LBB4_3 +; X86-NEXT: # %bb.4: +; X86-NEXT: movl {{[0-9]+}}(%esp), %esi ; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload -; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload ; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload +; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload ; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload +; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload ; X86-NEXT: .LBB4_7: # %udiv-loop-exit -; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload -; X86-NEXT: shldl $1, %edi, %ebp -; X86-NEXT: orl %eax, %ebp -; X86-NEXT: shldl $1, %ebx, %edi -; X86-NEXT: orl %eax, %edi -; X86-NEXT: shldl $1, %edx, %ebx -; X86-NEXT: orl %eax, %ebx -; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X86-NEXT: addl %edx, %edx +; X86-NEXT: shldl $1, %edi, %edx ; X86-NEXT: orl %ecx, %edx -; X86-NEXT: .LBB4_8: # %udiv-end -; X86-NEXT: movl {{[0-9]+}}(%esp), %eax -; X86-NEXT: movl %edx, (%eax) -; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload -; X86-NEXT: movl %ecx, 4(%eax) -; X86-NEXT: movl %edi, 8(%eax) -; X86-NEXT: movl %ebp, 12(%eax) -; X86-NEXT: movl {{[0-9]+}}(%esp), %eax -; X86-NEXT: movl %eax, %esi -; X86-NEXT: imull %ecx, %esi +; X86-NEXT: shldl $1, %ebp, %edi +; X86-NEXT: orl %ecx, %edi +; X86-NEXT: shldl $1, %eax, %ebp +; X86-NEXT: orl %ecx, %ebp +; X86-NEXT: addl %eax, %eax +; X86-NEXT: orl %ebx, %eax +; X86-NEXT: .LBB4_9: # %udiv-end +; X86-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill +; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl %eax, (%ecx) +; X86-NEXT: movl %ebp, 4(%ecx) +; X86-NEXT: movl %edi, 8(%ecx) +; X86-NEXT: movl %edx, 12(%ecx) ; X86-NEXT: movl %edx, %ecx -; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X86-NEXT: mull %edx +; X86-NEXT: movl %eax, %ebx +; X86-NEXT: movl %esi, %eax +; X86-NEXT: imull %ebp, %esi +; X86-NEXT: mull %ebx +; X86-NEXT: movl %ebx, %ebp +; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X86-NEXT: movl %eax, (%esp) # 4-byte Spill ; X86-NEXT: addl %esi, %edx ; X86-NEXT: movl {{[0-9]+}}(%esp), %ebx -; X86-NEXT: imull %ecx, %ebx +; X86-NEXT: imull %ebp, %ebx ; X86-NEXT: addl %edx, %ebx ; X86-NEXT: movl {{[0-9]+}}(%esp), %esi ; X86-NEXT: movl %esi, %eax ; X86-NEXT: mull %edi -; X86-NEXT: movl %eax, %ecx -; X86-NEXT: imull %esi, %ebp -; X86-NEXT: addl %edx, %ebp -; X86-NEXT: movl {{[0-9]+}}(%esp), %eax -; X86-NEXT: imull %eax, %edi -; X86-NEXT: addl %ebp, %edi -; X86-NEXT: addl (%esp), %ecx # 4-byte Folded Reload -; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill +; X86-NEXT: imull %esi, %ecx +; X86-NEXT: addl %edx, %ecx +; X86-NEXT: movl {{[0-9]+}}(%esp), %ebp +; X86-NEXT: imull %ebp, %edi +; X86-NEXT: addl %ecx, %edi +; X86-NEXT: addl (%esp), %eax # 4-byte Folded Reload +; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X86-NEXT: adcl %ebx, %edi ; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload @@ -517,13 +520,12 @@ ; X86-NEXT: addl %edi, %esi ; X86-NEXT: adcl $0, %ecx ; X86-NEXT: movl %ebx, %eax -; X86-NEXT: movl {{[0-9]+}}(%esp), %edx -; X86-NEXT: mull %edx +; X86-NEXT: mull %ebp ; X86-NEXT: movl {{[0-9]+}}(%esp), %ebx ; X86-NEXT: movl {{[0-9]+}}(%esp), %edi ; X86-NEXT: movl %edx, %ebp ; X86-NEXT: addl %esi, %eax -; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill +; X86-NEXT: movl %eax, %esi ; X86-NEXT: adcl %ecx, %ebp ; X86-NEXT: setb %cl ; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload @@ -534,17 +536,17 @@ ; X86-NEXT: addl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload ; X86-NEXT: adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload ; X86-NEXT: subl (%esp), %ebx # 4-byte Folded Reload -; X86-NEXT: movl {{[0-9]+}}(%esp), %esi -; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload -; X86-NEXT: sbbl %eax, %edi ; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx -; X86-NEXT: sbbl %edx, %ecx +; X86-NEXT: sbbl %esi, %ecx +; X86-NEXT: sbbl %eax, %edi +; X86-NEXT: movl {{[0-9]+}}(%esp), %esi +; X86-NEXT: sbbl %edx, %esi ; X86-NEXT: movl {{[0-9]+}}(%esp), %eax ; X86-NEXT: movl %ebx, (%eax) -; X86-NEXT: movl %esi, 4(%eax) +; X86-NEXT: movl %ecx, 4(%eax) ; X86-NEXT: movl %edi, 8(%eax) -; X86-NEXT: movl %ecx, 12(%eax) -; X86-NEXT: addl $132, %esp +; X86-NEXT: movl %esi, 12(%eax) +; X86-NEXT: addl $136, %esp ; X86-NEXT: popl %esi ; X86-NEXT: popl %edi ; X86-NEXT: popl %ebx Index: llvm/test/CodeGen/X86/dup-phi-users1.ll =================================================================== --- /dev/null +++ llvm/test/CodeGen/X86/dup-phi-users1.ll @@ -0,0 +1,358 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s + + +; Base test case. +define zeroext i1 @test1(i32 %s, i32* %ptr) { +; CHECK-LABEL: test1: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: cmpl $1025, %edi # imm = 0x401 +; CHECK-NEXT: jae .LBB0_2 +; CHECK-NEXT: # %bb.1: +; CHECK-NEXT: andl $3, %edi +; CHECK-NEXT: jmp .LBB0_4 +; CHECK-NEXT: .LBB0_2: # %if.else +; CHECK-NEXT: cmpl $4096, %edi # imm = 0x1000 +; CHECK-NEXT: ja .LBB0_6 +; CHECK-NEXT: # %bb.3: +; CHECK-NEXT: andl $7, %edi +; CHECK-NEXT: .LBB0_4: # %return.sink.split +; CHECK-NEXT: movl %edi, (%rsi) +; CHECK-NEXT: movb $1, %al +; CHECK-NEXT: # kill: def $al killed $al killed $eax +; CHECK-NEXT: retq +; CHECK-NEXT: .LBB0_6: +; CHECK-NEXT: xorl %eax, %eax +; CHECK-NEXT: # kill: def $al killed $al killed $eax +; CHECK-NEXT: retq +entry: + %cmp = icmp ult i32 %s, 1025 + br i1 %cmp, label %return.sink.split, label %if.else + +if.else: + %cmp2 = icmp ult i32 %s, 4097 + br i1 %cmp2, label %return.sink.split, label %return + +return.sink.split: + %.sink = phi i32 [ 3, %entry ], [ 7, %if.else ] + %and0 = and i32 %s, %.sink + store i32 %and0, i32* %ptr, align 4 + br label %return + +return: + %retval = phi i1 [ false, %if.else ], [ true, %return.sink.split ] + ret i1 %retval +} + + +; The PHI user XOR has a dependent instruction TRUNC, which can also be moved +; dominator. +define zeroext i1 @test2(i64 %s, i32* %ptr) { +; CHECK-LABEL: test2: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: cmpq $1025, %rdi # imm = 0x401 +; CHECK-NEXT: jae .LBB1_2 +; CHECK-NEXT: # %bb.1: +; CHECK-NEXT: xorl $3, %edi +; CHECK-NEXT: jmp .LBB1_4 +; CHECK-NEXT: .LBB1_2: # %if.else +; CHECK-NEXT: cmpq $4096, %rdi # imm = 0x1000 +; CHECK-NEXT: ja .LBB1_6 +; CHECK-NEXT: # %bb.3: +; CHECK-NEXT: xorl $7, %edi +; CHECK-NEXT: .LBB1_4: # %return.sink.split +; CHECK-NEXT: movl %edi, (%rsi) +; CHECK-NEXT: movb $1, %al +; CHECK-NEXT: # kill: def $al killed $al killed $eax +; CHECK-NEXT: retq +; CHECK-NEXT: .LBB1_6: +; CHECK-NEXT: xorl %eax, %eax +; CHECK-NEXT: # kill: def $al killed $al killed $eax +; CHECK-NEXT: retq +entry: + %cmp = icmp ult i64 %s, 1025 + br i1 %cmp, label %return.sink.split, label %if.else + +if.else: + %cmp2 = icmp ult i64 %s, 4097 + br i1 %cmp2, label %return.sink.split, label %return + +return.sink.split: + %.sink = phi i32 [ 3, %entry ], [ 7, %if.else ] + %t = trunc i64 %s to i32 + %xor0 = xor i32 %t, %.sink + store i32 %xor0, i32* %ptr, align 4 + br label %return + +return: + %retval = phi i1 [ false, %if.else ], [ true, %return.sink.split ] + ret i1 %retval +} + + +; SHL has a physical register operand, which is set up by previous COPY +; instruction. Both instructions should be duplicated together. +define zeroext i1 @test3(i32 %s, i32 %shm, i32* %ptr) { +; CHECK-LABEL: test3: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: cmpl $1025, %edi # imm = 0x401 +; CHECK-NEXT: jae .LBB2_2 +; CHECK-NEXT: # %bb.1: +; CHECK-NEXT: shll $3, %edi +; CHECK-NEXT: jmp .LBB2_3 +; CHECK-NEXT: .LBB2_2: # %if.else +; CHECK-NEXT: movl %esi, %ecx +; CHECK-NEXT: shll %cl, %edi +; CHECK-NEXT: .LBB2_3: # %return.sink.split +; CHECK-NEXT: movl %edi, (%rdx) +; CHECK-NEXT: movb $1, %al +; CHECK-NEXT: retq +entry: + %cmp = icmp ult i32 %s, 1025 + br i1 %cmp, label %return.sink.split, label %if.else + +if.else: + br label %return.sink.split + +return.sink.split: + %.sink = phi i32 [ 3, %entry ], [ %shm, %if.else ] + %shl0 = shl i32 %s, %.sink + store i32 %shl0, i32* %ptr, align 4 + ret i1 true +} + + +; Two PHI instructions, one PHI user depends on another PHI user. Both +; instructions can be duplicated. +define zeroext i1 @test4(i32 %s, i32* %ptr) { +; CHECK-LABEL: test4: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: cmpl $1025, %edi # imm = 0x401 +; CHECK-NEXT: jae .LBB3_2 +; CHECK-NEXT: # %bb.1: +; CHECK-NEXT: addl $7, %edi +; CHECK-NEXT: shrl $3, %edi +; CHECK-NEXT: jmp .LBB3_4 +; CHECK-NEXT: .LBB3_2: # %if.else +; CHECK-NEXT: cmpl $4096, %edi # imm = 0x1000 +; CHECK-NEXT: ja .LBB3_6 +; CHECK-NEXT: # %bb.3: +; CHECK-NEXT: addl $100, %edi +; CHECK-NEXT: shrl $7, %edi +; CHECK-NEXT: .LBB3_4: # %return.sink.split +; CHECK-NEXT: movl %edi, (%rsi) +; CHECK-NEXT: movb $1, %al +; CHECK-NEXT: # kill: def $al killed $al killed $eax +; CHECK-NEXT: retq +; CHECK-NEXT: .LBB3_6: +; CHECK-NEXT: xorl %eax, %eax +; CHECK-NEXT: # kill: def $al killed $al killed $eax +; CHECK-NEXT: retq +entry: + %cmp = icmp ult i32 %s, 1025 + br i1 %cmp, label %return.sink.split, label %if.else + +if.else: + %cmp2 = icmp ult i32 %s, 4097 + br i1 %cmp2, label %return.sink.split, label %return + +return.sink.split: + %.sink5 = phi i32 [ 7, %entry ], [ 100, %if.else ] + %.sink = phi i32 [ 3, %entry ], [ 7, %if.else ] + %add6 = add nuw nsw i32 %.sink5, %s + %shr7 = lshr i32 %add6, %.sink + store i32 %shr7, i32* %ptr, align 4 + br label %return + +return: + %retval = phi i1 [ false, %if.else ], [ true, %return.sink.split ] + ret i1 %retval +} + + +; Two dependent PHI users, with movable dependent instructions and physical +; register usage. +define zeroext i1 @test5(i64 %s, i32* %ptr) { +; CHECK-LABEL: test5: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: cmpq $1025, %rdi # imm = 0x401 +; CHECK-NEXT: jae .LBB4_2 +; CHECK-NEXT: # %bb.1: +; CHECK-NEXT: leal 7(%rdi), %eax +; CHECK-NEXT: sarl $3, %eax +; CHECK-NEXT: jmp .LBB4_4 +; CHECK-NEXT: .LBB4_2: # %if.else +; CHECK-NEXT: cmpq $4096, %rdi # imm = 0x1000 +; CHECK-NEXT: ja .LBB4_6 +; CHECK-NEXT: # %bb.3: +; CHECK-NEXT: leal 100(%rdi), %eax +; CHECK-NEXT: sarl $7, %eax +; CHECK-NEXT: .LBB4_4: # %return.sink.split +; CHECK-NEXT: movl %eax, (%rsi) +; CHECK-NEXT: movb $1, %al +; CHECK-NEXT: # kill: def $al killed $al killed $eax +; CHECK-NEXT: retq +; CHECK-NEXT: .LBB4_6: +; CHECK-NEXT: xorl %eax, %eax +; CHECK-NEXT: # kill: def $al killed $al killed $eax +; CHECK-NEXT: retq +entry: + %cmp = icmp ult i64 %s, 1025 + br i1 %cmp, label %return.sink.split, label %if.else + +if.else: + %cmp2 = icmp ult i64 %s, 4097 + br i1 %cmp2, label %return.sink.split, label %return + +return.sink.split: + %.sink5 = phi i32 [ 7, %entry ], [ 100, %if.else ] + %.sink = phi i32 [ 3, %entry ], [ 7, %if.else ] + %conv = trunc i64 %s to i32 + %add6 = add nuw nsw i32 %.sink5, %conv + %shr7 = ashr i32 %add6, %.sink + store i32 %shr7, i32* %ptr, align 4 + br label %return + +return: + %retval = phi i1 [ false, %if.else ], [ true, %return.sink.split ] + ret i1 %retval +} + + +; One user instruction with two PHI operands. +define i64 @test6(i1 %cond, i64 ()* %p) { +; CHECK-LABEL: test6: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: pushq %r14 +; CHECK-NEXT: .cfi_def_cfa_offset 16 +; CHECK-NEXT: pushq %rbx +; CHECK-NEXT: .cfi_def_cfa_offset 24 +; CHECK-NEXT: pushq %rax +; CHECK-NEXT: .cfi_def_cfa_offset 32 +; CHECK-NEXT: .cfi_offset %rbx, -24 +; CHECK-NEXT: .cfi_offset %r14, -16 +; CHECK-NEXT: movq %rsi, %rbx +; CHECK-NEXT: testb $1, %dil +; CHECK-NEXT: je .LBB5_2 +; CHECK-NEXT: # %bb.1: +; CHECK-NEXT: movl $5, %r14d +; CHECK-NEXT: orq $4, %r14 +; CHECK-NEXT: jmp .LBB5_3 +; CHECK-NEXT: .LBB5_2: # %if.end +; CHECK-NEXT: callq *%rbx +; CHECK-NEXT: movabsq $-4294967296, %rcx # imm = 0xFFFFFFFF00000000 +; CHECK-NEXT: andq %rax, %rcx +; CHECK-NEXT: movl %eax, %r14d +; CHECK-NEXT: orq %rcx, %r14 +; CHECK-NEXT: .LBB5_3: # %return +; CHECK-NEXT: callq *%rbx +; CHECK-NEXT: movq %r14, %rax +; CHECK-NEXT: addq $8, %rsp +; CHECK-NEXT: .cfi_def_cfa_offset 24 +; CHECK-NEXT: popq %rbx +; CHECK-NEXT: .cfi_def_cfa_offset 16 +; CHECK-NEXT: popq %r14 +; CHECK-NEXT: .cfi_def_cfa_offset 8 +; CHECK-NEXT: retq +entry: + br i1 %cond, label %return, label %if.end + +if.end: + %call = tail call i64 %p() + %high = and i64 %call, -4294967296 + %low = and i64 %call, 4294967295 + br label %return + +return: + %p1 = phi i64 [ %low, %if.end ], [ 5, %entry ] + %p2 = phi i64 [ %high, %if.end ], [ 4, %entry ] + %v = or i64 %p2, %p1 + %call2 = tail call i64 %p() + ret i64 %v +} + + +define i64 @test7(i1 %cond, i8* %Ptr, i64 %Val) { +; CHECK-LABEL: test7: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: movq %rsi, %rax +; CHECK-NEXT: testb $1, %dil +; CHECK-NEXT: jne .LBB6_2 +; CHECK-NEXT: # %bb.1: # %two +; CHECK-NEXT: addq %rdx, %rax +; CHECK-NEXT: .LBB6_2: # %end +; CHECK-NEXT: retq +entry: + %t41 = ptrtoint i8* %Ptr to i64 + %t42 = zext i64 %t41 to i128 + br i1 %cond, label %end, label %two + +two: + %t36 = zext i64 %Val to i128 + %t37 = shl i128 %t36, 64 + %ins39 = or i128 %t42, %t37 + br label %end + +end: + %t869.0 = phi i128 [ %t42, %entry ], [ %ins39, %two ] + %t32 = trunc i128 %t869.0 to i64 + %t29 = lshr i128 %t869.0, 64 + %t30 = trunc i128 %t29 to i64 + + %t2 = add i64 %t32, %t30 + ret i64 %t2 +} + + +define i32 @test8(i1 %c1, i32 %v1, i32 %v2, i32 %v3) { +; CHECK-LABEL: test8: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: movl %ecx, %eax +; CHECK-NEXT: testb $1, %dil +; CHECK-NEXT: je .LBB7_2 +; CHECK-NEXT: # %bb.1: # %cond.true +; CHECK-NEXT: incl %esi +; CHECK-NEXT: xorl %edx, %esi +; CHECK-NEXT: incl %esi +; CHECK-NEXT: subl %esi, %eax +; CHECK-NEXT: .LBB7_2: # %cond.end +; CHECK-NEXT: retq +entry: + br i1 %c1, label %cond.true, label %cond.end + +cond.true: + %add = add nsw i32 1, %v1 + %xor = xor i32 %add, %v2 + %add1 = add nsw i32 1, %xor + br label %cond.end + +cond.end: + %cond = phi i32 [ %add1, %cond.true ], [ 0, %entry ] + %sub = sub nsw i32 %v3, %cond + ret i32 %sub +} + + +define i32 @test9(i8 %x, i1 %cond) { +; CHECK-LABEL: test9: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: testb $1, %sil +; CHECK-NEXT: je .LBB8_1 +; CHECK-NEXT: # %bb.2: # %if +; CHECK-NEXT: xorl $33, %eax +; CHECK-NEXT: retq +; CHECK-NEXT: .LBB8_1: +; CHECK-NEXT: xorl $21, %eax +; CHECK-NEXT: retq +entry: + br i1 %cond, label %if, label %endif +if: + br label %endif +endif: + %phi = phi i32 [ 21, %entry], [ 33, %if ] + %zext = zext i8 %x to i32 + %logic = xor i32 %zext, %phi + ret i32 %logic +} + Index: llvm/test/CodeGen/X86/dup-phi-users2.ll =================================================================== --- /dev/null +++ llvm/test/CodeGen/X86/dup-phi-users2.ll @@ -0,0 +1,627 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s + +define i32 @test_basic(i1 %flag, i32 %arg) { +; CHECK-LABEL: test_basic: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: movl %esi, %eax +; CHECK-NEXT: testb $1, %dil +; CHECK-NEXT: je .LBB0_2 +; CHECK-NEXT: # %bb.1: +; CHECK-NEXT: addl $7, %eax +; CHECK-NEXT: retq +; CHECK-NEXT: .LBB0_2: # %b +; CHECK-NEXT: addl $11, %eax +; CHECK-NEXT: retq +entry: + br i1 %flag, label %a, label %b + +a: + br label %exit + +b: + br label %exit + +exit: + %p = phi i32 [ 7, %a ], [ 11, %b ] + %sum = add i32 %arg, %p + ret i32 %sum +} + +; Check that we handle commuted operands and get the constant onto the RHS. +define i32 @test_commuted(i1 %flag, i32 %arg) { +; CHECK-LABEL: test_commuted: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: movl %esi, %eax +; CHECK-NEXT: testb $1, %dil +; CHECK-NEXT: je .LBB1_2 +; CHECK-NEXT: # %bb.1: +; CHECK-NEXT: addl $7, %eax +; CHECK-NEXT: retq +; CHECK-NEXT: .LBB1_2: # %b +; CHECK-NEXT: addl $11, %eax +; CHECK-NEXT: retq +entry: + br i1 %flag, label %a, label %b + +a: + br label %exit + +b: + br label %exit + +exit: + %p = phi i32 [ 7, %a ], [ 11, %b ] + %sum = add i32 %p, %arg + ret i32 %sum +} + +; We don't split critical edge. But we can still replace the MOV immediate +; instruction with ALU instruction without any extra cost. +define i32 @test_simple_crit_edge(i1 %flag, i32 %arg) { +; CHECK-LABEL: test_simple_crit_edge: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: movl %esi, %eax +; CHECK-NEXT: testb $1, %dil +; CHECK-NEXT: je .LBB2_2 +; CHECK-NEXT: # %bb.1: +; CHECK-NEXT: addl $7, %eax +; CHECK-NEXT: retq +; CHECK-NEXT: .LBB2_2: # %a +; CHECK-NEXT: addl $11, %eax +; CHECK-NEXT: retq +entry: + br i1 %flag, label %exit, label %a + +a: + br label %exit + +exit: + %p = phi i32 [ 7, %entry ], [ 11, %a ] + %sum = add i32 %arg, %p + ret i32 %sum +} + +define i32 @test_no_spec_dominating_inst(i1 %flag, i32* %ptr) { +; CHECK-LABEL: test_no_spec_dominating_inst: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: movl (%rsi), %eax +; CHECK-NEXT: testb $1, %dil +; CHECK-NEXT: je .LBB3_2 +; CHECK-NEXT: # %bb.1: +; CHECK-NEXT: addl $7, %eax +; CHECK-NEXT: retq +; CHECK-NEXT: .LBB3_2: # %b +; CHECK-NEXT: addl $11, %eax +; CHECK-NEXT: retq +entry: + %load = load i32, i32* %ptr + br i1 %flag, label %a, label %b + +a: + br label %exit + +b: + br label %exit + +exit: + %p = phi i32 [ 7, %a ], [ 11, %b ] + %sum = add i32 %load, %p + ret i32 %sum +} + +; We have special logic handling PHI nodes, make sure it doesn't get confused +; by a dominating PHI. +define i32 @test_no_spec_dominating_phi(i1 %flag1, i1 %flag2, i32 %x, i32 %y) { +; CHECK-LABEL: test_no_spec_dominating_phi: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: movl %edx, %eax +; CHECK-NEXT: testb $1, %dil +; CHECK-NEXT: jne .LBB4_2 +; CHECK-NEXT: # %bb.1: # %y.block +; CHECK-NEXT: movl %ecx, %eax +; CHECK-NEXT: .LBB4_2: # %merge +; CHECK-NEXT: testb $1, %sil +; CHECK-NEXT: je .LBB4_4 +; CHECK-NEXT: # %bb.3: +; CHECK-NEXT: addl $7, %eax +; CHECK-NEXT: retq +; CHECK-NEXT: .LBB4_4: # %b +; CHECK-NEXT: addl $11, %eax +; CHECK-NEXT: retq +entry: + br i1 %flag1, label %x.block, label %y.block + +x.block: + br label %merge + +y.block: + br label %merge + +merge: + %xy.phi = phi i32 [ %x, %x.block ], [ %y, %y.block ] + br i1 %flag2, label %a, label %b + +a: + br label %exit + +b: + br label %exit + +exit: + %p = phi i32 [ 7, %a ], [ 11, %b ] + %sum = add i32 %xy.phi, %p + ret i32 %sum +} + +; Ensure that we will speculate some number of "free" instructions on the given +; architecture even though they are unrelated to the PHI itself. +define i32 @test_speculate_free_insts(i1 %flag, i64 %arg) { +; CHECK-LABEL: test_speculate_free_insts: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: movq %rsi, %rax +; CHECK-NEXT: testb $1, %dil +; CHECK-NEXT: je .LBB5_2 +; CHECK-NEXT: # %bb.1: +; CHECK-NEXT: addl $7, %eax +; CHECK-NEXT: # kill: def $eax killed $eax killed $rax +; CHECK-NEXT: retq +; CHECK-NEXT: .LBB5_2: # %b +; CHECK-NEXT: addl $11, %eax +; CHECK-NEXT: # kill: def $eax killed $eax killed $rax +; CHECK-NEXT: retq +entry: + br i1 %flag, label %a, label %b + +a: + br label %exit + +b: + br label %exit + +exit: + %p = phi i32 [ 7, %a ], [ 11, %b ] + %t1 = trunc i64 %arg to i48 + %t2 = trunc i48 %t1 to i32 + %sum = add i32 %t2, %p + ret i32 %sum +} + +define i32 @test_speculate_free_phis(i1 %flag, i32 %arg1, i32 %arg2) { +; CHECK-LABEL: test_speculate_free_phis: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: testb $1, %dil +; CHECK-NEXT: je .LBB6_2 +; CHECK-NEXT: # %bb.1: +; CHECK-NEXT: movl %esi, %eax +; CHECK-NEXT: addl $7, %eax +; CHECK-NEXT: retq +; CHECK-NEXT: .LBB6_2: # %b +; CHECK-NEXT: addl $11, %edx +; CHECK-NEXT: movl %edx, %eax +; CHECK-NEXT: retq +entry: + br i1 %flag, label %a, label %b + +a: + br label %exit + +b: + br label %exit + +exit: + %p1 = phi i32 [ 7, %a ], [ 11, %b ] + %p2 = phi i32 [ %arg1, %a ], [ %arg2, %b ] + %sum = add i32 %p2, %p1 + ret i32 %sum +} + +; We shouldn't speculate multiple uses even if each individually looks +; profitable because of the total cost. +define i32 @test_no_spec_multi_uses(i1 %flag, i32 %arg1, i32 %arg2, i32 %arg3) { +; CHECK-LABEL: test_no_spec_multi_uses: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: movl %ecx, %eax +; CHECK-NEXT: movl $7, %ecx +; CHECK-NEXT: testb $1, %dil +; CHECK-NEXT: jne .LBB7_2 +; CHECK-NEXT: # %bb.1: # %b +; CHECK-NEXT: movl $11, %ecx +; CHECK-NEXT: .LBB7_2: # %exit +; CHECK-NEXT: addl %ecx, %esi +; CHECK-NEXT: addl %ecx, %edx +; CHECK-NEXT: addl %esi, %edx +; CHECK-NEXT: addl %ecx, %eax +; CHECK-NEXT: addl %edx, %eax +; CHECK-NEXT: retq +entry: + br i1 %flag, label %a, label %b + +a: + br label %exit + +b: + br label %exit + +exit: + %p = phi i32 [ 7, %a ], [ 11, %b ] + %add1 = add i32 %arg1, %p + %add2 = add i32 %arg2, %p + %add3 = add i32 %arg3, %p + %sum1 = add i32 %add1, %add2 + %sum2 = add i32 %sum1, %add3 + ret i32 %sum2 +} + +define i32 @test_multi_phis1(i1 %flag, i32 %arg) { +; CHECK-LABEL: test_multi_phis1: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: movl %esi, %eax +; CHECK-NEXT: testb $1, %dil +; CHECK-NEXT: je .LBB8_2 +; CHECK-NEXT: # %bb.1: +; CHECK-NEXT: addl $1, %eax +; CHECK-NEXT: addl $3, %eax +; CHECK-NEXT: addl $5, %eax +; CHECK-NEXT: retq +; CHECK-NEXT: .LBB8_2: # %b +; CHECK-NEXT: addl $2, %eax +; CHECK-NEXT: addl $4, %eax +; CHECK-NEXT: addl $6, %eax +; CHECK-NEXT: retq +entry: + br i1 %flag, label %a, label %b + +a: + br label %exit + +b: + br label %exit + +exit: + %p1 = phi i32 [ 1, %a ], [ 2, %b ] + %p2 = phi i32 [ 3, %a ], [ 4, %b ] + %p3 = phi i32 [ 5, %a ], [ 6, %b ] + %sum1 = add i32 %arg, %p1 + %sum2 = add i32 %sum1, %p2 + %sum3 = add i32 %sum2, %p3 + ret i32 %sum3 +} + +; Check that the order of the PHIs doesn't impact the behavior. +define i32 @test_multi_phis2(i1 %flag, i32 %arg) { +; CHECK-LABEL: test_multi_phis2: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: movl %esi, %eax +; CHECK-NEXT: testb $1, %dil +; CHECK-NEXT: je .LBB9_2 +; CHECK-NEXT: # %bb.1: +; CHECK-NEXT: addl $1, %eax +; CHECK-NEXT: addl $3, %eax +; CHECK-NEXT: addl $5, %eax +; CHECK-NEXT: retq +; CHECK-NEXT: .LBB9_2: # %b +; CHECK-NEXT: addl $2, %eax +; CHECK-NEXT: addl $4, %eax +; CHECK-NEXT: addl $6, %eax +; CHECK-NEXT: retq +entry: + br i1 %flag, label %a, label %b + +a: + br label %exit + +b: + br label %exit + +exit: + %p3 = phi i32 [ 5, %a ], [ 6, %b ] + %p2 = phi i32 [ 3, %a ], [ 4, %b ] + %p1 = phi i32 [ 1, %a ], [ 2, %b ] + %sum1 = add i32 %arg, %p1 + %sum2 = add i32 %sum1, %p2 + %sum3 = add i32 %sum2, %p3 + ret i32 %sum3 +} + +define i32 @test_no_spec_indirectbr(i1 %flag, i32 %arg) { +; CHECK-LABEL: test_no_spec_indirectbr: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: movl %esi, %eax +; CHECK-NEXT: testb $1, %dil +; CHECK-NEXT: je .LBB10_2 +; CHECK-NEXT: # %bb.1: # %a +; CHECK-NEXT: addl $7, %eax +; CHECK-NEXT: jmpq *%rax +; CHECK-NEXT: .LBB10_2: # %b +; CHECK-NEXT: addl $11, %eax +; CHECK-NEXT: jmpq *%rax +; CHECK-NEXT: .LBB10_3: # %exit +; CHECK-NEXT: retq +entry: + br i1 %flag, label %a, label %b + +a: + indirectbr i8* undef, [label %exit] + +b: + indirectbr i8* undef, [label %exit] + +exit: + %p = phi i32 [ 7, %a ], [ 11, %b ] + %sum = add i32 %arg, %p + ret i32 %sum +} + + +define i32 @test_no_spec_invoke_continue(i1 %flag, i32 %arg) personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) { +; CHECK-LABEL: test_no_spec_invoke_continue: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: pushq %rbx +; CHECK-NEXT: .cfi_def_cfa_offset 16 +; CHECK-NEXT: .cfi_offset %rbx, -16 +; CHECK-NEXT: movl %esi, %ebx +; CHECK-NEXT: testb $1, %dil +; CHECK-NEXT: je .LBB11_3 +; CHECK-NEXT: # %bb.1: # %a +; CHECK-NEXT: .Ltmp2: +; CHECK-NEXT: callq g@PLT +; CHECK-NEXT: .Ltmp3: +; CHECK-NEXT: # %bb.2: +; CHECK-NEXT: addl $7, %ebx +; CHECK-NEXT: jmp .LBB11_5 +; CHECK-NEXT: .LBB11_3: # %b +; CHECK-NEXT: .Ltmp0: +; CHECK-NEXT: callq g@PLT +; CHECK-NEXT: .Ltmp1: +; CHECK-NEXT: # %bb.4: +; CHECK-NEXT: addl $11, %ebx +; CHECK-NEXT: .LBB11_5: # %exit +; CHECK-NEXT: movl %ebx, %eax +; CHECK-NEXT: popq %rbx +; CHECK-NEXT: .cfi_def_cfa_offset 8 +; CHECK-NEXT: retq +; CHECK-NEXT: .LBB11_6: # %lpad +; CHECK-NEXT: .cfi_def_cfa_offset 16 +; CHECK-NEXT: .Ltmp4: +; CHECK-NEXT: callq _Unwind_Resume@PLT +entry: + br i1 %flag, label %a, label %b + +a: + invoke void @g() + to label %exit unwind label %lpad + +b: + invoke void @g() + to label %exit unwind label %lpad + +exit: + %p = phi i32 [ 7, %a ], [ 11, %b ] + %sum = add i32 %arg, %p + ret i32 %sum + +lpad: + %lp = landingpad { i8*, i32 } + cleanup + resume { i8*, i32 } undef +} + +define i32 @test_no_spec_landingpad(i32 %arg, i32* %ptr) personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) { +; CHECK-LABEL: test_no_spec_landingpad: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: pushq %rbp +; CHECK-NEXT: .cfi_def_cfa_offset 16 +; CHECK-NEXT: pushq %r14 +; CHECK-NEXT: .cfi_def_cfa_offset 24 +; CHECK-NEXT: pushq %rbx +; CHECK-NEXT: .cfi_def_cfa_offset 32 +; CHECK-NEXT: .cfi_offset %rbx, -32 +; CHECK-NEXT: .cfi_offset %r14, -24 +; CHECK-NEXT: .cfi_offset %rbp, -16 +; CHECK-NEXT: movq %rsi, %rbx +; CHECK-NEXT: movl %edi, %r14d +; CHECK-NEXT: leal 7(%r14), %ebp +; CHECK-NEXT: .Ltmp5: +; CHECK-NEXT: callq g@PLT +; CHECK-NEXT: .Ltmp6: +; CHECK-NEXT: # %bb.1: # %invoke.cont +; CHECK-NEXT: addl $11, %r14d +; CHECK-NEXT: .Ltmp7: +; CHECK-NEXT: movl %r14d, %ebp +; CHECK-NEXT: callq g@PLT +; CHECK-NEXT: .Ltmp8: +; CHECK-NEXT: # %bb.2: # %exit +; CHECK-NEXT: xorl %eax, %eax +; CHECK-NEXT: popq %rbx +; CHECK-NEXT: .cfi_def_cfa_offset 24 +; CHECK-NEXT: popq %r14 +; CHECK-NEXT: .cfi_def_cfa_offset 16 +; CHECK-NEXT: popq %rbp +; CHECK-NEXT: .cfi_def_cfa_offset 8 +; CHECK-NEXT: retq +; CHECK-NEXT: .LBB12_3: # %lpad +; CHECK-NEXT: .cfi_def_cfa_offset 32 +; CHECK-NEXT: .Ltmp9: +; CHECK-NEXT: movl %ebp, (%rbx) +; CHECK-NEXT: callq _Unwind_Resume@PLT +entry: + invoke void @g() + to label %invoke.cont unwind label %lpad + +invoke.cont: + invoke void @g() + to label %exit unwind label %lpad + +lpad: + %p = phi i32 [ 7, %entry ], [ 11, %invoke.cont ] + %lp = landingpad { i8*, i32 } + cleanup + %sum = add i32 %arg, %p + store i32 %sum, i32* %ptr + resume { i8*, i32 } undef + +exit: + ret i32 0 +} + + +define i32 @test_no_spec_cleanuppad(i32 %arg, i32* %ptr) personality i32 (...)* @__CxxFrameHandler3 { +; CHECK-LABEL: test_no_spec_cleanuppad: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: pushq %rbp +; CHECK-NEXT: .cfi_def_cfa_offset 16 +; CHECK-NEXT: .cfi_offset %rbp, -16 +; CHECK-NEXT: movq %rsp, %rbp +; CHECK-NEXT: .cfi_def_cfa_register %rbp +; CHECK-NEXT: subq $32, %rsp +; CHECK-NEXT: movq $-2, -8(%rbp) +; CHECK-NEXT: movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill +; CHECK-NEXT: movl %edi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill +; CHECK-NEXT: movl $7, -12(%rbp) +; CHECK-NEXT: .Ltmp10: +; CHECK-NEXT: callq g@PLT +; CHECK-NEXT: .Ltmp11: +; CHECK-NEXT: # %bb.1: # %invoke.cont +; CHECK-NEXT: movl $11, -12(%rbp) +; CHECK-NEXT: .Ltmp12: +; CHECK-NEXT: callq g@PLT +; CHECK-NEXT: .Ltmp13: +; CHECK-NEXT: # %bb.2: # %exit +; CHECK-NEXT: xorl %eax, %eax +; CHECK-NEXT: addq $32, %rsp +; CHECK-NEXT: popq %rbp +; CHECK-NEXT: .cfi_def_cfa %rsp, 8 +; CHECK-NEXT: retq +; CHECK-NEXT: .LBB13_3: # %lpad +; CHECK-NEXT: .cfi_def_cfa %rbp, 16 +; CHECK-NEXT: pushq %rbp +; CHECK-NEXT: .cfi_def_cfa_offset 16 +; CHECK-NEXT: .cfi_offset %rbp, -16 +; CHECK-NEXT: movl {{[-0-9]+}}(%r{{[sb]}}p), %ecx # 4-byte Reload +; CHECK-NEXT: addl -12(%rbp), %ecx +; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload +; CHECK-NEXT: movl %ecx, (%rax) +; CHECK-NEXT: popq %rbp +; CHECK-NEXT: .cfi_def_cfa %rsp, 8 +; CHECK-NEXT: retq # CLEANUPRET +entry: + %p.wineh.spillslot = alloca i32, align 4 + store i32 7, i32* %p.wineh.spillslot, align 4 + invoke void @g() + to label %invoke.cont unwind label %lpad + +invoke.cont: ; preds = %entry + store i32 11, i32* %p.wineh.spillslot, align 4 + invoke void @g() + to label %exit unwind label %lpad + +lpad: ; preds = %invoke.cont, %entry + %cp = cleanuppad within none [] + %p.wineh.reload = load i32, i32* %p.wineh.spillslot, align 4 + %sum = add i32 %arg, %p.wineh.reload + store i32 %sum, i32* %ptr, align 4 + cleanupret from %cp unwind to caller + +exit: ; preds = %invoke.cont + ret i32 0 +} + +; Check that we don't speculate in the face of an expensive immediate. A large +; immediate can't be folded into ALU instructions, we still need an extra MOV +; instruction. So the duplication of PHI user doesn't bring us benefit. +define i64 @test_expensive_imm(i32 %flag, i64 %arg) { +; CHECK-LABEL: test_expensive_imm: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: movq %rsi, %rax +; CHECK-NEXT: # kill: def $edi killed $edi def $rdi +; CHECK-NEXT: movl $1, %ecx +; CHECK-NEXT: leal -2(%rdi), %edx +; CHECK-NEXT: cmpl $2, %edx +; CHECK-NEXT: jb .LBB14_4 +; CHECK-NEXT: # %bb.1: # %entry +; CHECK-NEXT: cmpl $1, %edi +; CHECK-NEXT: jne .LBB14_2 +; CHECK-NEXT: # %bb.3: # %b +; CHECK-NEXT: movabsq $29496729640, %rcx # imm = 0x6DE246028 +; CHECK-NEXT: .LBB14_4: # %exit +; CHECK-NEXT: addq %rcx, %rax +; CHECK-NEXT: retq +; CHECK-NEXT: .LBB14_2: # %a +; CHECK-NEXT: movabsq $42949672960, %rcx # imm = 0xA00000000 +; CHECK-NEXT: addq %rcx, %rax +; CHECK-NEXT: retq +entry: + switch i32 %flag, label %a [ + i32 1, label %b + i32 2, label %c + i32 3, label %d + ] + +a: + br label %exit + +b: + br label %exit + +c: + br label %exit + +d: + br label %exit + +exit: + %p = phi i64 [ 42949672960, %a ], [ 29496729640, %b ], [ 1, %c ], [ 1, %d ] + %sum1 = add i64 %arg, %p + ret i64 %sum1 +} + +define i32 @test_no_spec_non_postdominating_uses(i1 %flag1, i1 %flag2, i32 %arg) { +; CHECK-LABEL: test_no_spec_non_postdominating_uses: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: # kill: def $edx killed $edx def $rdx +; CHECK-NEXT: testb $1, %dil +; CHECK-NEXT: je .LBB15_2 +; CHECK-NEXT: # %bb.1: +; CHECK-NEXT: movl $13, %ecx +; CHECK-NEXT: leal 7(%rdx), %eax +; CHECK-NEXT: testb $1, %sil +; CHECK-NEXT: jne .LBB15_4 +; CHECK-NEXT: .LBB15_5: # %exit2 +; CHECK-NEXT: addl %ecx, %edx +; CHECK-NEXT: movl %edx, %eax +; CHECK-NEXT: retq +; CHECK-NEXT: .LBB15_2: # %b +; CHECK-NEXT: movl $42, %ecx +; CHECK-NEXT: leal 11(%rdx), %eax +; CHECK-NEXT: testb $1, %sil +; CHECK-NEXT: je .LBB15_5 +; CHECK-NEXT: .LBB15_4: # %exit1 +; CHECK-NEXT: retq +entry: + br i1 %flag1, label %a, label %b + +a: + br label %merge + +b: + br label %merge + +merge: + %p1 = phi i32 [ 7, %a ], [ 11, %b ] + %p2 = phi i32 [ 13, %a ], [ 42, %b ] + %sum1 = add i32 %arg, %p1 + br i1 %flag2, label %exit1, label %exit2 + +exit1: + ret i32 %sum1 + +exit2: + %sum2 = add i32 %arg, %p2 + ret i32 %sum2 +} + +declare void @g() +declare i32 @__gxx_personality_v0(...) +declare i32 @__CxxFrameHandler3(...) + Index: llvm/test/CodeGen/X86/fast-isel-freeze.ll =================================================================== --- llvm/test/CodeGen/X86/fast-isel-freeze.ll +++ llvm/test/CodeGen/X86/fast-isel-freeze.ll @@ -11,8 +11,8 @@ ; ; FAST-LABEL: freeze: ; FAST: # %bb.0: -; FAST-NEXT: movl $10, %eax -; FAST-NEXT: xorl %edi, %eax +; FAST-NEXT: movl %edi, %eax +; FAST-NEXT: xorl $10, %eax ; FAST-NEXT: retq %1 = freeze i32 %t %2 = freeze i32 10 Index: llvm/test/CodeGen/X86/lrshrink.ll =================================================================== --- llvm/test/CodeGen/X86/lrshrink.ll +++ llvm/test/CodeGen/X86/lrshrink.ll @@ -16,18 +16,20 @@ ; CHECK-NEXT: .cfi_offset %rbx, -32 ; CHECK-NEXT: .cfi_offset %r14, -24 ; CHECK-NEXT: .cfi_offset %r15, -16 -; CHECK-NEXT: movq %rcx, %rbx -; CHECK-NEXT: movl $4, %r14d ; CHECK-NEXT: testb $1, %dil -; CHECK-NEXT: je .LBB0_2 -; CHECK-NEXT: # %bb.1: # %then +; CHECK-NEXT: je .LBB0_1 +; CHECK-NEXT: # %bb.2: # %then ; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %r9 -; CHECK-NEXT: movl $10, %r14d -; CHECK-NEXT: movq %rdx, %rsi +; CHECK-NEXT: addq $10, %rdx +; CHECK-NEXT: movq %rdx, %r14 ; CHECK-NEXT: movq %r8, %rbx -; CHECK-NEXT: .LBB0_2: # %else +; CHECK-NEXT: jmp .LBB0_3 +; CHECK-NEXT: .LBB0_1: +; CHECK-NEXT: movq %rcx, %rbx +; CHECK-NEXT: movq %rsi, %r14 +; CHECK-NEXT: addq $4, %r14 +; CHECK-NEXT: .LBB0_3: # %else ; CHECK-NEXT: addq %r9, %rbx -; CHECK-NEXT: addq %rsi, %r14 ; CHECK-NEXT: callq _Z3foov@PLT ; CHECK-NEXT: movl %eax, %r15d ; CHECK-NEXT: addq %r14, %r15 Index: llvm/test/CodeGen/X86/opt-pipeline.ll =================================================================== --- llvm/test/CodeGen/X86/opt-pipeline.ll +++ llvm/test/CodeGen/X86/opt-pipeline.ll @@ -91,6 +91,8 @@ ; CHECK-NEXT: Argument Stack Rebase ; CHECK-NEXT: Finalize ISel and expand pseudo-instructions ; CHECK-NEXT: X86 Domain Reassignment Pass +; CHECK-NEXT: MachineDominator Tree Construction +; CHECK-NEXT: Duplicate PHI Users ; CHECK-NEXT: Lazy Machine Block Frequency Analysis ; CHECK-NEXT: Early Tail Duplication ; CHECK-NEXT: Optimize machine instruction PHIs Index: llvm/test/CodeGen/X86/pcsections-atomics.ll =================================================================== --- llvm/test/CodeGen/X86/pcsections-atomics.ll +++ llvm/test/CodeGen/X86/pcsections-atomics.ll @@ -2148,14 +2148,17 @@ ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax ; O1-NEXT: movb $1, %cl -; O1-NEXT: movb $42, %al ; O1-NEXT: .Lpcsection65: -; O1-NEXT: lock cmpxchgb %cl, (%rdi) ; O1-NEXT: movb $42, %al ; O1-NEXT: .Lpcsection66: ; O1-NEXT: lock cmpxchgb %cl, (%rdi) -; O1-NEXT: movb $42, %al ; O1-NEXT: .Lpcsection67: +; O1-NEXT: movb $42, %al +; O1-NEXT: .Lpcsection68: +; O1-NEXT: lock cmpxchgb %cl, (%rdi) +; O1-NEXT: .Lpcsection69: +; O1-NEXT: movb $42, %al +; O1-NEXT: .Lpcsection70: ; O1-NEXT: lock cmpxchgb %cl, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -2164,14 +2167,17 @@ ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax ; O2-NEXT: movb $1, %cl -; O2-NEXT: movb $42, %al ; O2-NEXT: .Lpcsection65: -; O2-NEXT: lock cmpxchgb %cl, (%rdi) ; O2-NEXT: movb $42, %al ; O2-NEXT: .Lpcsection66: ; O2-NEXT: lock cmpxchgb %cl, (%rdi) -; O2-NEXT: movb $42, %al ; O2-NEXT: .Lpcsection67: +; O2-NEXT: movb $42, %al +; O2-NEXT: .Lpcsection68: +; O2-NEXT: lock cmpxchgb %cl, (%rdi) +; O2-NEXT: .Lpcsection69: +; O2-NEXT: movb $42, %al +; O2-NEXT: .Lpcsection70: ; O2-NEXT: lock cmpxchgb %cl, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -2180,14 +2186,17 @@ ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax ; O3-NEXT: movb $1, %cl -; O3-NEXT: movb $42, %al ; O3-NEXT: .Lpcsection65: -; O3-NEXT: lock cmpxchgb %cl, (%rdi) ; O3-NEXT: movb $42, %al ; O3-NEXT: .Lpcsection66: ; O3-NEXT: lock cmpxchgb %cl, (%rdi) -; O3-NEXT: movb $42, %al ; O3-NEXT: .Lpcsection67: +; O3-NEXT: movb $42, %al +; O3-NEXT: .Lpcsection68: +; O3-NEXT: lock cmpxchgb %cl, (%rdi) +; O3-NEXT: .Lpcsection69: +; O3-NEXT: movb $42, %al +; O3-NEXT: .Lpcsection70: ; O3-NEXT: lock cmpxchgb %cl, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -2226,14 +2235,17 @@ ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax ; O1-NEXT: movb $1, %cl +; O1-NEXT: .Lpcsection71: ; O1-NEXT: movb $42, %al -; O1-NEXT: .Lpcsection68: +; O1-NEXT: .Lpcsection72: ; O1-NEXT: lock cmpxchgb %cl, (%rdi) +; O1-NEXT: .Lpcsection73: ; O1-NEXT: movb $42, %al -; O1-NEXT: .Lpcsection69: +; O1-NEXT: .Lpcsection74: ; O1-NEXT: lock cmpxchgb %cl, (%rdi) +; O1-NEXT: .Lpcsection75: ; O1-NEXT: movb $42, %al -; O1-NEXT: .Lpcsection70: +; O1-NEXT: .Lpcsection76: ; O1-NEXT: lock cmpxchgb %cl, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -2242,14 +2254,17 @@ ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax ; O2-NEXT: movb $1, %cl +; O2-NEXT: .Lpcsection71: ; O2-NEXT: movb $42, %al -; O2-NEXT: .Lpcsection68: +; O2-NEXT: .Lpcsection72: ; O2-NEXT: lock cmpxchgb %cl, (%rdi) +; O2-NEXT: .Lpcsection73: ; O2-NEXT: movb $42, %al -; O2-NEXT: .Lpcsection69: +; O2-NEXT: .Lpcsection74: ; O2-NEXT: lock cmpxchgb %cl, (%rdi) +; O2-NEXT: .Lpcsection75: ; O2-NEXT: movb $42, %al -; O2-NEXT: .Lpcsection70: +; O2-NEXT: .Lpcsection76: ; O2-NEXT: lock cmpxchgb %cl, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -2258,14 +2273,17 @@ ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax ; O3-NEXT: movb $1, %cl +; O3-NEXT: .Lpcsection71: ; O3-NEXT: movb $42, %al -; O3-NEXT: .Lpcsection68: +; O3-NEXT: .Lpcsection72: ; O3-NEXT: lock cmpxchgb %cl, (%rdi) +; O3-NEXT: .Lpcsection73: ; O3-NEXT: movb $42, %al -; O3-NEXT: .Lpcsection69: +; O3-NEXT: .Lpcsection74: ; O3-NEXT: lock cmpxchgb %cl, (%rdi) +; O3-NEXT: .Lpcsection75: ; O3-NEXT: movb $42, %al -; O3-NEXT: .Lpcsection70: +; O3-NEXT: .Lpcsection76: ; O3-NEXT: lock cmpxchgb %cl, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -2304,14 +2322,17 @@ ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax ; O1-NEXT: movb $1, %cl +; O1-NEXT: .Lpcsection77: ; O1-NEXT: movb $42, %al -; O1-NEXT: .Lpcsection71: +; O1-NEXT: .Lpcsection78: ; O1-NEXT: lock cmpxchgb %cl, (%rdi) +; O1-NEXT: .Lpcsection79: ; O1-NEXT: movb $42, %al -; O1-NEXT: .Lpcsection72: +; O1-NEXT: .Lpcsection80: ; O1-NEXT: lock cmpxchgb %cl, (%rdi) +; O1-NEXT: .Lpcsection81: ; O1-NEXT: movb $42, %al -; O1-NEXT: .Lpcsection73: +; O1-NEXT: .Lpcsection82: ; O1-NEXT: lock cmpxchgb %cl, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -2320,14 +2341,17 @@ ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax ; O2-NEXT: movb $1, %cl +; O2-NEXT: .Lpcsection77: ; O2-NEXT: movb $42, %al -; O2-NEXT: .Lpcsection71: +; O2-NEXT: .Lpcsection78: ; O2-NEXT: lock cmpxchgb %cl, (%rdi) +; O2-NEXT: .Lpcsection79: ; O2-NEXT: movb $42, %al -; O2-NEXT: .Lpcsection72: +; O2-NEXT: .Lpcsection80: ; O2-NEXT: lock cmpxchgb %cl, (%rdi) +; O2-NEXT: .Lpcsection81: ; O2-NEXT: movb $42, %al -; O2-NEXT: .Lpcsection73: +; O2-NEXT: .Lpcsection82: ; O2-NEXT: lock cmpxchgb %cl, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -2336,14 +2360,17 @@ ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax ; O3-NEXT: movb $1, %cl +; O3-NEXT: .Lpcsection77: ; O3-NEXT: movb $42, %al -; O3-NEXT: .Lpcsection71: +; O3-NEXT: .Lpcsection78: ; O3-NEXT: lock cmpxchgb %cl, (%rdi) +; O3-NEXT: .Lpcsection79: ; O3-NEXT: movb $42, %al -; O3-NEXT: .Lpcsection72: +; O3-NEXT: .Lpcsection80: ; O3-NEXT: lock cmpxchgb %cl, (%rdi) +; O3-NEXT: .Lpcsection81: ; O3-NEXT: movb $42, %al -; O3-NEXT: .Lpcsection73: +; O3-NEXT: .Lpcsection82: ; O3-NEXT: lock cmpxchgb %cl, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -2382,14 +2409,17 @@ ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax ; O1-NEXT: movb $1, %cl +; O1-NEXT: .Lpcsection83: ; O1-NEXT: movb $42, %al -; O1-NEXT: .Lpcsection74: +; O1-NEXT: .Lpcsection84: ; O1-NEXT: lock cmpxchgb %cl, (%rdi) +; O1-NEXT: .Lpcsection85: ; O1-NEXT: movb $42, %al -; O1-NEXT: .Lpcsection75: +; O1-NEXT: .Lpcsection86: ; O1-NEXT: lock cmpxchgb %cl, (%rdi) +; O1-NEXT: .Lpcsection87: ; O1-NEXT: movb $42, %al -; O1-NEXT: .Lpcsection76: +; O1-NEXT: .Lpcsection88: ; O1-NEXT: lock cmpxchgb %cl, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -2398,14 +2428,17 @@ ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax ; O2-NEXT: movb $1, %cl +; O2-NEXT: .Lpcsection83: ; O2-NEXT: movb $42, %al -; O2-NEXT: .Lpcsection74: +; O2-NEXT: .Lpcsection84: ; O2-NEXT: lock cmpxchgb %cl, (%rdi) +; O2-NEXT: .Lpcsection85: ; O2-NEXT: movb $42, %al -; O2-NEXT: .Lpcsection75: +; O2-NEXT: .Lpcsection86: ; O2-NEXT: lock cmpxchgb %cl, (%rdi) +; O2-NEXT: .Lpcsection87: ; O2-NEXT: movb $42, %al -; O2-NEXT: .Lpcsection76: +; O2-NEXT: .Lpcsection88: ; O2-NEXT: lock cmpxchgb %cl, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -2414,14 +2447,17 @@ ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax ; O3-NEXT: movb $1, %cl +; O3-NEXT: .Lpcsection83: ; O3-NEXT: movb $42, %al -; O3-NEXT: .Lpcsection74: +; O3-NEXT: .Lpcsection84: ; O3-NEXT: lock cmpxchgb %cl, (%rdi) +; O3-NEXT: .Lpcsection85: ; O3-NEXT: movb $42, %al -; O3-NEXT: .Lpcsection75: +; O3-NEXT: .Lpcsection86: ; O3-NEXT: lock cmpxchgb %cl, (%rdi) +; O3-NEXT: .Lpcsection87: ; O3-NEXT: movb $42, %al -; O3-NEXT: .Lpcsection76: +; O3-NEXT: .Lpcsection88: ; O3-NEXT: lock cmpxchgb %cl, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -2460,14 +2496,17 @@ ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax ; O1-NEXT: movb $1, %cl +; O1-NEXT: .Lpcsection89: ; O1-NEXT: movb $42, %al -; O1-NEXT: .Lpcsection77: +; O1-NEXT: .Lpcsection90: ; O1-NEXT: lock cmpxchgb %cl, (%rdi) +; O1-NEXT: .Lpcsection91: ; O1-NEXT: movb $42, %al -; O1-NEXT: .Lpcsection78: +; O1-NEXT: .Lpcsection92: ; O1-NEXT: lock cmpxchgb %cl, (%rdi) +; O1-NEXT: .Lpcsection93: ; O1-NEXT: movb $42, %al -; O1-NEXT: .Lpcsection79: +; O1-NEXT: .Lpcsection94: ; O1-NEXT: lock cmpxchgb %cl, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -2476,14 +2515,17 @@ ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax ; O2-NEXT: movb $1, %cl +; O2-NEXT: .Lpcsection89: ; O2-NEXT: movb $42, %al -; O2-NEXT: .Lpcsection77: +; O2-NEXT: .Lpcsection90: ; O2-NEXT: lock cmpxchgb %cl, (%rdi) +; O2-NEXT: .Lpcsection91: ; O2-NEXT: movb $42, %al -; O2-NEXT: .Lpcsection78: +; O2-NEXT: .Lpcsection92: ; O2-NEXT: lock cmpxchgb %cl, (%rdi) +; O2-NEXT: .Lpcsection93: ; O2-NEXT: movb $42, %al -; O2-NEXT: .Lpcsection79: +; O2-NEXT: .Lpcsection94: ; O2-NEXT: lock cmpxchgb %cl, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -2492,14 +2534,17 @@ ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax ; O3-NEXT: movb $1, %cl +; O3-NEXT: .Lpcsection89: ; O3-NEXT: movb $42, %al -; O3-NEXT: .Lpcsection77: +; O3-NEXT: .Lpcsection90: ; O3-NEXT: lock cmpxchgb %cl, (%rdi) +; O3-NEXT: .Lpcsection91: ; O3-NEXT: movb $42, %al -; O3-NEXT: .Lpcsection78: +; O3-NEXT: .Lpcsection92: ; O3-NEXT: lock cmpxchgb %cl, (%rdi) +; O3-NEXT: .Lpcsection93: ; O3-NEXT: movb $42, %al -; O3-NEXT: .Lpcsection79: +; O3-NEXT: .Lpcsection94: ; O3-NEXT: lock cmpxchgb %cl, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -2524,7 +2569,7 @@ ; O1-LABEL: atomic16_load_unordered: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection80: +; O1-NEXT: .Lpcsection95: ; O1-NEXT: movzwl (%rdi), %eax ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -2532,7 +2577,7 @@ ; O2-LABEL: atomic16_load_unordered: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection80: +; O2-NEXT: .Lpcsection95: ; O2-NEXT: movzwl (%rdi), %eax ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -2540,7 +2585,7 @@ ; O3-LABEL: atomic16_load_unordered: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection80: +; O3-NEXT: .Lpcsection95: ; O3-NEXT: movzwl (%rdi), %eax ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -2563,7 +2608,7 @@ ; O1-LABEL: atomic16_load_monotonic: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection81: +; O1-NEXT: .Lpcsection96: ; O1-NEXT: movzwl (%rdi), %eax ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -2571,7 +2616,7 @@ ; O2-LABEL: atomic16_load_monotonic: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection81: +; O2-NEXT: .Lpcsection96: ; O2-NEXT: movzwl (%rdi), %eax ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -2579,7 +2624,7 @@ ; O3-LABEL: atomic16_load_monotonic: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection81: +; O3-NEXT: .Lpcsection96: ; O3-NEXT: movzwl (%rdi), %eax ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -2602,7 +2647,7 @@ ; O1-LABEL: atomic16_load_acquire: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection82: +; O1-NEXT: .Lpcsection97: ; O1-NEXT: movzwl (%rdi), %eax ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -2610,7 +2655,7 @@ ; O2-LABEL: atomic16_load_acquire: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection82: +; O2-NEXT: .Lpcsection97: ; O2-NEXT: movzwl (%rdi), %eax ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -2618,7 +2663,7 @@ ; O3-LABEL: atomic16_load_acquire: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection82: +; O3-NEXT: .Lpcsection97: ; O3-NEXT: movzwl (%rdi), %eax ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -2641,7 +2686,7 @@ ; O1-LABEL: atomic16_load_seq_cst: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection83: +; O1-NEXT: .Lpcsection98: ; O1-NEXT: movzwl (%rdi), %eax ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -2649,7 +2694,7 @@ ; O2-LABEL: atomic16_load_seq_cst: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection83: +; O2-NEXT: .Lpcsection98: ; O2-NEXT: movzwl (%rdi), %eax ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -2657,7 +2702,7 @@ ; O3-LABEL: atomic16_load_seq_cst: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection83: +; O3-NEXT: .Lpcsection98: ; O3-NEXT: movzwl (%rdi), %eax ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -2680,7 +2725,7 @@ ; O1-LABEL: atomic16_store_unordered: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection84: +; O1-NEXT: .Lpcsection99: ; O1-NEXT: movw $42, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -2688,7 +2733,7 @@ ; O2-LABEL: atomic16_store_unordered: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection84: +; O2-NEXT: .Lpcsection99: ; O2-NEXT: movw $42, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -2696,7 +2741,7 @@ ; O3-LABEL: atomic16_store_unordered: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection84: +; O3-NEXT: .Lpcsection99: ; O3-NEXT: movw $42, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -2719,7 +2764,7 @@ ; O1-LABEL: atomic16_store_monotonic: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection85: +; O1-NEXT: .Lpcsection100: ; O1-NEXT: movw $42, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -2727,7 +2772,7 @@ ; O2-LABEL: atomic16_store_monotonic: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection85: +; O2-NEXT: .Lpcsection100: ; O2-NEXT: movw $42, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -2735,7 +2780,7 @@ ; O3-LABEL: atomic16_store_monotonic: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection85: +; O3-NEXT: .Lpcsection100: ; O3-NEXT: movw $42, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -2758,7 +2803,7 @@ ; O1-LABEL: atomic16_store_release: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection86: +; O1-NEXT: .Lpcsection101: ; O1-NEXT: movw $42, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -2766,7 +2811,7 @@ ; O2-LABEL: atomic16_store_release: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection86: +; O2-NEXT: .Lpcsection101: ; O2-NEXT: movw $42, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -2774,7 +2819,7 @@ ; O3-LABEL: atomic16_store_release: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection86: +; O3-NEXT: .Lpcsection101: ; O3-NEXT: movw $42, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -2799,7 +2844,7 @@ ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax ; O1-NEXT: movw $42, %ax -; O1-NEXT: .Lpcsection87: +; O1-NEXT: .Lpcsection102: ; O1-NEXT: xchgw %ax, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -2808,7 +2853,7 @@ ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax ; O2-NEXT: movw $42, %ax -; O2-NEXT: .Lpcsection87: +; O2-NEXT: .Lpcsection102: ; O2-NEXT: xchgw %ax, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -2817,7 +2862,7 @@ ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax ; O3-NEXT: movw $42, %ax -; O3-NEXT: .Lpcsection87: +; O3-NEXT: .Lpcsection102: ; O3-NEXT: xchgw %ax, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -2842,7 +2887,7 @@ ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax ; O1-NEXT: movw $42, %ax -; O1-NEXT: .Lpcsection88: +; O1-NEXT: .Lpcsection103: ; O1-NEXT: xchgw %ax, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -2851,7 +2896,7 @@ ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax ; O2-NEXT: movw $42, %ax -; O2-NEXT: .Lpcsection88: +; O2-NEXT: .Lpcsection103: ; O2-NEXT: xchgw %ax, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -2860,7 +2905,7 @@ ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax ; O3-NEXT: movw $42, %ax -; O3-NEXT: .Lpcsection88: +; O3-NEXT: .Lpcsection103: ; O3-NEXT: xchgw %ax, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -2883,7 +2928,7 @@ ; O1-LABEL: atomic16_add_monotonic: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection89: +; O1-NEXT: .Lpcsection104: ; O1-NEXT: lock addw $42, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -2891,7 +2936,7 @@ ; O2-LABEL: atomic16_add_monotonic: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection89: +; O2-NEXT: .Lpcsection104: ; O2-NEXT: lock addw $42, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -2899,7 +2944,7 @@ ; O3-LABEL: atomic16_add_monotonic: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection89: +; O3-NEXT: .Lpcsection104: ; O3-NEXT: lock addw $42, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -2922,7 +2967,7 @@ ; O1-LABEL: atomic16_sub_monotonic: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection90: +; O1-NEXT: .Lpcsection105: ; O1-NEXT: lock subw $42, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -2930,7 +2975,7 @@ ; O2-LABEL: atomic16_sub_monotonic: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection90: +; O2-NEXT: .Lpcsection105: ; O2-NEXT: lock subw $42, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -2938,7 +2983,7 @@ ; O3-LABEL: atomic16_sub_monotonic: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection90: +; O3-NEXT: .Lpcsection105: ; O3-NEXT: lock subw $42, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -2961,7 +3006,7 @@ ; O1-LABEL: atomic16_and_monotonic: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection91: +; O1-NEXT: .Lpcsection106: ; O1-NEXT: lock andw $42, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -2969,7 +3014,7 @@ ; O2-LABEL: atomic16_and_monotonic: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection91: +; O2-NEXT: .Lpcsection106: ; O2-NEXT: lock andw $42, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -2977,7 +3022,7 @@ ; O3-LABEL: atomic16_and_monotonic: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection91: +; O3-NEXT: .Lpcsection106: ; O3-NEXT: lock andw $42, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -3000,7 +3045,7 @@ ; O1-LABEL: atomic16_or_monotonic: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection92: +; O1-NEXT: .Lpcsection107: ; O1-NEXT: lock orw $42, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -3008,7 +3053,7 @@ ; O2-LABEL: atomic16_or_monotonic: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection92: +; O2-NEXT: .Lpcsection107: ; O2-NEXT: lock orw $42, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -3016,7 +3061,7 @@ ; O3-LABEL: atomic16_or_monotonic: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection92: +; O3-NEXT: .Lpcsection107: ; O3-NEXT: lock orw $42, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -3039,7 +3084,7 @@ ; O1-LABEL: atomic16_xor_monotonic: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection93: +; O1-NEXT: .Lpcsection108: ; O1-NEXT: lock xorw $42, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -3047,7 +3092,7 @@ ; O2-LABEL: atomic16_xor_monotonic: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection93: +; O2-NEXT: .Lpcsection108: ; O2-NEXT: lock xorw $42, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -3055,7 +3100,7 @@ ; O3-LABEL: atomic16_xor_monotonic: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection93: +; O3-NEXT: .Lpcsection108: ; O3-NEXT: lock xorw $42, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -3104,23 +3149,23 @@ ; O1-LABEL: atomic16_nand_monotonic: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection94: +; O1-NEXT: .Lpcsection109: ; O1-NEXT: movzwl (%rdi), %eax ; O1-NEXT: .p2align 4, 0x90 ; O1-NEXT: .LBB64_1: # %atomicrmw.start ; O1-NEXT: # =>This Inner Loop Header: Depth=1 ; O1-NEXT: movl %eax, %ecx -; O1-NEXT: .Lpcsection95: +; O1-NEXT: .Lpcsection110: ; O1-NEXT: notl %ecx -; O1-NEXT: .Lpcsection96: +; O1-NEXT: .Lpcsection111: ; O1-NEXT: orl $65493, %ecx # imm = 0xFFD5 -; O1-NEXT: .Lpcsection97: +; O1-NEXT: .Lpcsection112: ; O1-NEXT: # kill: def $ax killed $ax killed $eax -; O1-NEXT: .Lpcsection98: +; O1-NEXT: .Lpcsection113: ; O1-NEXT: lock cmpxchgw %cx, (%rdi) -; O1-NEXT: .Lpcsection99: +; O1-NEXT: .Lpcsection114: ; O1-NEXT: # kill: def $ax killed $ax def $eax -; O1-NEXT: .Lpcsection100: +; O1-NEXT: .Lpcsection115: ; O1-NEXT: jne .LBB64_1 ; O1-NEXT: # %bb.2: # %atomicrmw.end ; O1-NEXT: movq $1, foo(%rip) @@ -3129,23 +3174,23 @@ ; O2-LABEL: atomic16_nand_monotonic: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection94: +; O2-NEXT: .Lpcsection109: ; O2-NEXT: movzwl (%rdi), %eax ; O2-NEXT: .p2align 4, 0x90 ; O2-NEXT: .LBB64_1: # %atomicrmw.start ; O2-NEXT: # =>This Inner Loop Header: Depth=1 ; O2-NEXT: movl %eax, %ecx -; O2-NEXT: .Lpcsection95: +; O2-NEXT: .Lpcsection110: ; O2-NEXT: notl %ecx -; O2-NEXT: .Lpcsection96: +; O2-NEXT: .Lpcsection111: ; O2-NEXT: orl $65493, %ecx # imm = 0xFFD5 -; O2-NEXT: .Lpcsection97: +; O2-NEXT: .Lpcsection112: ; O2-NEXT: # kill: def $ax killed $ax killed $eax -; O2-NEXT: .Lpcsection98: +; O2-NEXT: .Lpcsection113: ; O2-NEXT: lock cmpxchgw %cx, (%rdi) -; O2-NEXT: .Lpcsection99: +; O2-NEXT: .Lpcsection114: ; O2-NEXT: # kill: def $ax killed $ax def $eax -; O2-NEXT: .Lpcsection100: +; O2-NEXT: .Lpcsection115: ; O2-NEXT: jne .LBB64_1 ; O2-NEXT: # %bb.2: # %atomicrmw.end ; O2-NEXT: movq $1, foo(%rip) @@ -3154,23 +3199,23 @@ ; O3-LABEL: atomic16_nand_monotonic: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection94: +; O3-NEXT: .Lpcsection109: ; O3-NEXT: movzwl (%rdi), %eax ; O3-NEXT: .p2align 4, 0x90 ; O3-NEXT: .LBB64_1: # %atomicrmw.start ; O3-NEXT: # =>This Inner Loop Header: Depth=1 ; O3-NEXT: movl %eax, %ecx -; O3-NEXT: .Lpcsection95: +; O3-NEXT: .Lpcsection110: ; O3-NEXT: notl %ecx -; O3-NEXT: .Lpcsection96: +; O3-NEXT: .Lpcsection111: ; O3-NEXT: orl $65493, %ecx # imm = 0xFFD5 -; O3-NEXT: .Lpcsection97: +; O3-NEXT: .Lpcsection112: ; O3-NEXT: # kill: def $ax killed $ax killed $eax -; O3-NEXT: .Lpcsection98: +; O3-NEXT: .Lpcsection113: ; O3-NEXT: lock cmpxchgw %cx, (%rdi) -; O3-NEXT: .Lpcsection99: +; O3-NEXT: .Lpcsection114: ; O3-NEXT: # kill: def $ax killed $ax def $eax -; O3-NEXT: .Lpcsection100: +; O3-NEXT: .Lpcsection115: ; O3-NEXT: jne .LBB64_1 ; O3-NEXT: # %bb.2: # %atomicrmw.end ; O3-NEXT: movq $1, foo(%rip) @@ -3196,7 +3241,7 @@ ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax ; O1-NEXT: movw $42, %ax -; O1-NEXT: .Lpcsection101: +; O1-NEXT: .Lpcsection116: ; O1-NEXT: xchgw %ax, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -3205,7 +3250,7 @@ ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax ; O2-NEXT: movw $42, %ax -; O2-NEXT: .Lpcsection101: +; O2-NEXT: .Lpcsection116: ; O2-NEXT: xchgw %ax, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -3214,7 +3259,7 @@ ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax ; O3-NEXT: movw $42, %ax -; O3-NEXT: .Lpcsection101: +; O3-NEXT: .Lpcsection116: ; O3-NEXT: xchgw %ax, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -3237,7 +3282,7 @@ ; O1-LABEL: atomic16_add_acquire: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection102: +; O1-NEXT: .Lpcsection117: ; O1-NEXT: lock addw $42, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -3245,7 +3290,7 @@ ; O2-LABEL: atomic16_add_acquire: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection102: +; O2-NEXT: .Lpcsection117: ; O2-NEXT: lock addw $42, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -3253,7 +3298,7 @@ ; O3-LABEL: atomic16_add_acquire: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection102: +; O3-NEXT: .Lpcsection117: ; O3-NEXT: lock addw $42, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -3276,7 +3321,7 @@ ; O1-LABEL: atomic16_sub_acquire: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection103: +; O1-NEXT: .Lpcsection118: ; O1-NEXT: lock subw $42, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -3284,7 +3329,7 @@ ; O2-LABEL: atomic16_sub_acquire: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection103: +; O2-NEXT: .Lpcsection118: ; O2-NEXT: lock subw $42, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -3292,7 +3337,7 @@ ; O3-LABEL: atomic16_sub_acquire: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection103: +; O3-NEXT: .Lpcsection118: ; O3-NEXT: lock subw $42, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -3315,7 +3360,7 @@ ; O1-LABEL: atomic16_and_acquire: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection104: +; O1-NEXT: .Lpcsection119: ; O1-NEXT: lock andw $42, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -3323,7 +3368,7 @@ ; O2-LABEL: atomic16_and_acquire: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection104: +; O2-NEXT: .Lpcsection119: ; O2-NEXT: lock andw $42, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -3331,7 +3376,7 @@ ; O3-LABEL: atomic16_and_acquire: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection104: +; O3-NEXT: .Lpcsection119: ; O3-NEXT: lock andw $42, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -3354,7 +3399,7 @@ ; O1-LABEL: atomic16_or_acquire: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection105: +; O1-NEXT: .Lpcsection120: ; O1-NEXT: lock orw $42, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -3362,7 +3407,7 @@ ; O2-LABEL: atomic16_or_acquire: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection105: +; O2-NEXT: .Lpcsection120: ; O2-NEXT: lock orw $42, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -3370,7 +3415,7 @@ ; O3-LABEL: atomic16_or_acquire: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection105: +; O3-NEXT: .Lpcsection120: ; O3-NEXT: lock orw $42, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -3393,7 +3438,7 @@ ; O1-LABEL: atomic16_xor_acquire: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection106: +; O1-NEXT: .Lpcsection121: ; O1-NEXT: lock xorw $42, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -3401,7 +3446,7 @@ ; O2-LABEL: atomic16_xor_acquire: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection106: +; O2-NEXT: .Lpcsection121: ; O2-NEXT: lock xorw $42, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -3409,7 +3454,7 @@ ; O3-LABEL: atomic16_xor_acquire: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection106: +; O3-NEXT: .Lpcsection121: ; O3-NEXT: lock xorw $42, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -3458,23 +3503,23 @@ ; O1-LABEL: atomic16_nand_acquire: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection107: +; O1-NEXT: .Lpcsection122: ; O1-NEXT: movzwl (%rdi), %eax ; O1-NEXT: .p2align 4, 0x90 ; O1-NEXT: .LBB71_1: # %atomicrmw.start ; O1-NEXT: # =>This Inner Loop Header: Depth=1 ; O1-NEXT: movl %eax, %ecx -; O1-NEXT: .Lpcsection108: +; O1-NEXT: .Lpcsection123: ; O1-NEXT: notl %ecx -; O1-NEXT: .Lpcsection109: +; O1-NEXT: .Lpcsection124: ; O1-NEXT: orl $65493, %ecx # imm = 0xFFD5 -; O1-NEXT: .Lpcsection110: +; O1-NEXT: .Lpcsection125: ; O1-NEXT: # kill: def $ax killed $ax killed $eax -; O1-NEXT: .Lpcsection111: +; O1-NEXT: .Lpcsection126: ; O1-NEXT: lock cmpxchgw %cx, (%rdi) -; O1-NEXT: .Lpcsection112: +; O1-NEXT: .Lpcsection127: ; O1-NEXT: # kill: def $ax killed $ax def $eax -; O1-NEXT: .Lpcsection113: +; O1-NEXT: .Lpcsection128: ; O1-NEXT: jne .LBB71_1 ; O1-NEXT: # %bb.2: # %atomicrmw.end ; O1-NEXT: movq $1, foo(%rip) @@ -3483,23 +3528,23 @@ ; O2-LABEL: atomic16_nand_acquire: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection107: +; O2-NEXT: .Lpcsection122: ; O2-NEXT: movzwl (%rdi), %eax ; O2-NEXT: .p2align 4, 0x90 ; O2-NEXT: .LBB71_1: # %atomicrmw.start ; O2-NEXT: # =>This Inner Loop Header: Depth=1 ; O2-NEXT: movl %eax, %ecx -; O2-NEXT: .Lpcsection108: +; O2-NEXT: .Lpcsection123: ; O2-NEXT: notl %ecx -; O2-NEXT: .Lpcsection109: +; O2-NEXT: .Lpcsection124: ; O2-NEXT: orl $65493, %ecx # imm = 0xFFD5 -; O2-NEXT: .Lpcsection110: +; O2-NEXT: .Lpcsection125: ; O2-NEXT: # kill: def $ax killed $ax killed $eax -; O2-NEXT: .Lpcsection111: +; O2-NEXT: .Lpcsection126: ; O2-NEXT: lock cmpxchgw %cx, (%rdi) -; O2-NEXT: .Lpcsection112: +; O2-NEXT: .Lpcsection127: ; O2-NEXT: # kill: def $ax killed $ax def $eax -; O2-NEXT: .Lpcsection113: +; O2-NEXT: .Lpcsection128: ; O2-NEXT: jne .LBB71_1 ; O2-NEXT: # %bb.2: # %atomicrmw.end ; O2-NEXT: movq $1, foo(%rip) @@ -3508,23 +3553,23 @@ ; O3-LABEL: atomic16_nand_acquire: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection107: +; O3-NEXT: .Lpcsection122: ; O3-NEXT: movzwl (%rdi), %eax ; O3-NEXT: .p2align 4, 0x90 ; O3-NEXT: .LBB71_1: # %atomicrmw.start ; O3-NEXT: # =>This Inner Loop Header: Depth=1 ; O3-NEXT: movl %eax, %ecx -; O3-NEXT: .Lpcsection108: +; O3-NEXT: .Lpcsection123: ; O3-NEXT: notl %ecx -; O3-NEXT: .Lpcsection109: +; O3-NEXT: .Lpcsection124: ; O3-NEXT: orl $65493, %ecx # imm = 0xFFD5 -; O3-NEXT: .Lpcsection110: +; O3-NEXT: .Lpcsection125: ; O3-NEXT: # kill: def $ax killed $ax killed $eax -; O3-NEXT: .Lpcsection111: +; O3-NEXT: .Lpcsection126: ; O3-NEXT: lock cmpxchgw %cx, (%rdi) -; O3-NEXT: .Lpcsection112: +; O3-NEXT: .Lpcsection127: ; O3-NEXT: # kill: def $ax killed $ax def $eax -; O3-NEXT: .Lpcsection113: +; O3-NEXT: .Lpcsection128: ; O3-NEXT: jne .LBB71_1 ; O3-NEXT: # %bb.2: # %atomicrmw.end ; O3-NEXT: movq $1, foo(%rip) @@ -3550,7 +3595,7 @@ ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax ; O1-NEXT: movw $42, %ax -; O1-NEXT: .Lpcsection114: +; O1-NEXT: .Lpcsection129: ; O1-NEXT: xchgw %ax, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -3559,7 +3604,7 @@ ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax ; O2-NEXT: movw $42, %ax -; O2-NEXT: .Lpcsection114: +; O2-NEXT: .Lpcsection129: ; O2-NEXT: xchgw %ax, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -3568,7 +3613,7 @@ ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax ; O3-NEXT: movw $42, %ax -; O3-NEXT: .Lpcsection114: +; O3-NEXT: .Lpcsection129: ; O3-NEXT: xchgw %ax, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -3591,7 +3636,7 @@ ; O1-LABEL: atomic16_add_release: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection115: +; O1-NEXT: .Lpcsection130: ; O1-NEXT: lock addw $42, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -3599,7 +3644,7 @@ ; O2-LABEL: atomic16_add_release: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection115: +; O2-NEXT: .Lpcsection130: ; O2-NEXT: lock addw $42, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -3607,7 +3652,7 @@ ; O3-LABEL: atomic16_add_release: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection115: +; O3-NEXT: .Lpcsection130: ; O3-NEXT: lock addw $42, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -3630,7 +3675,7 @@ ; O1-LABEL: atomic16_sub_release: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection116: +; O1-NEXT: .Lpcsection131: ; O1-NEXT: lock subw $42, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -3638,7 +3683,7 @@ ; O2-LABEL: atomic16_sub_release: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection116: +; O2-NEXT: .Lpcsection131: ; O2-NEXT: lock subw $42, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -3646,7 +3691,7 @@ ; O3-LABEL: atomic16_sub_release: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection116: +; O3-NEXT: .Lpcsection131: ; O3-NEXT: lock subw $42, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -3669,7 +3714,7 @@ ; O1-LABEL: atomic16_and_release: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection117: +; O1-NEXT: .Lpcsection132: ; O1-NEXT: lock andw $42, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -3677,7 +3722,7 @@ ; O2-LABEL: atomic16_and_release: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection117: +; O2-NEXT: .Lpcsection132: ; O2-NEXT: lock andw $42, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -3685,7 +3730,7 @@ ; O3-LABEL: atomic16_and_release: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection117: +; O3-NEXT: .Lpcsection132: ; O3-NEXT: lock andw $42, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -3708,7 +3753,7 @@ ; O1-LABEL: atomic16_or_release: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection118: +; O1-NEXT: .Lpcsection133: ; O1-NEXT: lock orw $42, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -3716,7 +3761,7 @@ ; O2-LABEL: atomic16_or_release: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection118: +; O2-NEXT: .Lpcsection133: ; O2-NEXT: lock orw $42, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -3724,7 +3769,7 @@ ; O3-LABEL: atomic16_or_release: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection118: +; O3-NEXT: .Lpcsection133: ; O3-NEXT: lock orw $42, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -3747,7 +3792,7 @@ ; O1-LABEL: atomic16_xor_release: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection119: +; O1-NEXT: .Lpcsection134: ; O1-NEXT: lock xorw $42, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -3755,7 +3800,7 @@ ; O2-LABEL: atomic16_xor_release: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection119: +; O2-NEXT: .Lpcsection134: ; O2-NEXT: lock xorw $42, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -3763,7 +3808,7 @@ ; O3-LABEL: atomic16_xor_release: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection119: +; O3-NEXT: .Lpcsection134: ; O3-NEXT: lock xorw $42, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -3812,23 +3857,23 @@ ; O1-LABEL: atomic16_nand_release: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection120: +; O1-NEXT: .Lpcsection135: ; O1-NEXT: movzwl (%rdi), %eax ; O1-NEXT: .p2align 4, 0x90 ; O1-NEXT: .LBB78_1: # %atomicrmw.start ; O1-NEXT: # =>This Inner Loop Header: Depth=1 ; O1-NEXT: movl %eax, %ecx -; O1-NEXT: .Lpcsection121: +; O1-NEXT: .Lpcsection136: ; O1-NEXT: notl %ecx -; O1-NEXT: .Lpcsection122: +; O1-NEXT: .Lpcsection137: ; O1-NEXT: orl $65493, %ecx # imm = 0xFFD5 -; O1-NEXT: .Lpcsection123: +; O1-NEXT: .Lpcsection138: ; O1-NEXT: # kill: def $ax killed $ax killed $eax -; O1-NEXT: .Lpcsection124: +; O1-NEXT: .Lpcsection139: ; O1-NEXT: lock cmpxchgw %cx, (%rdi) -; O1-NEXT: .Lpcsection125: +; O1-NEXT: .Lpcsection140: ; O1-NEXT: # kill: def $ax killed $ax def $eax -; O1-NEXT: .Lpcsection126: +; O1-NEXT: .Lpcsection141: ; O1-NEXT: jne .LBB78_1 ; O1-NEXT: # %bb.2: # %atomicrmw.end ; O1-NEXT: movq $1, foo(%rip) @@ -3837,23 +3882,23 @@ ; O2-LABEL: atomic16_nand_release: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection120: +; O2-NEXT: .Lpcsection135: ; O2-NEXT: movzwl (%rdi), %eax ; O2-NEXT: .p2align 4, 0x90 ; O2-NEXT: .LBB78_1: # %atomicrmw.start ; O2-NEXT: # =>This Inner Loop Header: Depth=1 ; O2-NEXT: movl %eax, %ecx -; O2-NEXT: .Lpcsection121: +; O2-NEXT: .Lpcsection136: ; O2-NEXT: notl %ecx -; O2-NEXT: .Lpcsection122: +; O2-NEXT: .Lpcsection137: ; O2-NEXT: orl $65493, %ecx # imm = 0xFFD5 -; O2-NEXT: .Lpcsection123: +; O2-NEXT: .Lpcsection138: ; O2-NEXT: # kill: def $ax killed $ax killed $eax -; O2-NEXT: .Lpcsection124: +; O2-NEXT: .Lpcsection139: ; O2-NEXT: lock cmpxchgw %cx, (%rdi) -; O2-NEXT: .Lpcsection125: +; O2-NEXT: .Lpcsection140: ; O2-NEXT: # kill: def $ax killed $ax def $eax -; O2-NEXT: .Lpcsection126: +; O2-NEXT: .Lpcsection141: ; O2-NEXT: jne .LBB78_1 ; O2-NEXT: # %bb.2: # %atomicrmw.end ; O2-NEXT: movq $1, foo(%rip) @@ -3862,23 +3907,23 @@ ; O3-LABEL: atomic16_nand_release: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection120: +; O3-NEXT: .Lpcsection135: ; O3-NEXT: movzwl (%rdi), %eax ; O3-NEXT: .p2align 4, 0x90 ; O3-NEXT: .LBB78_1: # %atomicrmw.start ; O3-NEXT: # =>This Inner Loop Header: Depth=1 ; O3-NEXT: movl %eax, %ecx -; O3-NEXT: .Lpcsection121: +; O3-NEXT: .Lpcsection136: ; O3-NEXT: notl %ecx -; O3-NEXT: .Lpcsection122: +; O3-NEXT: .Lpcsection137: ; O3-NEXT: orl $65493, %ecx # imm = 0xFFD5 -; O3-NEXT: .Lpcsection123: +; O3-NEXT: .Lpcsection138: ; O3-NEXT: # kill: def $ax killed $ax killed $eax -; O3-NEXT: .Lpcsection124: +; O3-NEXT: .Lpcsection139: ; O3-NEXT: lock cmpxchgw %cx, (%rdi) -; O3-NEXT: .Lpcsection125: +; O3-NEXT: .Lpcsection140: ; O3-NEXT: # kill: def $ax killed $ax def $eax -; O3-NEXT: .Lpcsection126: +; O3-NEXT: .Lpcsection141: ; O3-NEXT: jne .LBB78_1 ; O3-NEXT: # %bb.2: # %atomicrmw.end ; O3-NEXT: movq $1, foo(%rip) @@ -3904,7 +3949,7 @@ ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax ; O1-NEXT: movw $42, %ax -; O1-NEXT: .Lpcsection127: +; O1-NEXT: .Lpcsection142: ; O1-NEXT: xchgw %ax, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -3913,7 +3958,7 @@ ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax ; O2-NEXT: movw $42, %ax -; O2-NEXT: .Lpcsection127: +; O2-NEXT: .Lpcsection142: ; O2-NEXT: xchgw %ax, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -3922,7 +3967,7 @@ ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax ; O3-NEXT: movw $42, %ax -; O3-NEXT: .Lpcsection127: +; O3-NEXT: .Lpcsection142: ; O3-NEXT: xchgw %ax, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -3945,7 +3990,7 @@ ; O1-LABEL: atomic16_add_acq_rel: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection128: +; O1-NEXT: .Lpcsection143: ; O1-NEXT: lock addw $42, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -3953,7 +3998,7 @@ ; O2-LABEL: atomic16_add_acq_rel: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection128: +; O2-NEXT: .Lpcsection143: ; O2-NEXT: lock addw $42, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -3961,7 +4006,7 @@ ; O3-LABEL: atomic16_add_acq_rel: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection128: +; O3-NEXT: .Lpcsection143: ; O3-NEXT: lock addw $42, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -3984,7 +4029,7 @@ ; O1-LABEL: atomic16_sub_acq_rel: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection129: +; O1-NEXT: .Lpcsection144: ; O1-NEXT: lock subw $42, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -3992,7 +4037,7 @@ ; O2-LABEL: atomic16_sub_acq_rel: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection129: +; O2-NEXT: .Lpcsection144: ; O2-NEXT: lock subw $42, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -4000,7 +4045,7 @@ ; O3-LABEL: atomic16_sub_acq_rel: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection129: +; O3-NEXT: .Lpcsection144: ; O3-NEXT: lock subw $42, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -4023,7 +4068,7 @@ ; O1-LABEL: atomic16_and_acq_rel: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection130: +; O1-NEXT: .Lpcsection145: ; O1-NEXT: lock andw $42, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -4031,7 +4076,7 @@ ; O2-LABEL: atomic16_and_acq_rel: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection130: +; O2-NEXT: .Lpcsection145: ; O2-NEXT: lock andw $42, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -4039,7 +4084,7 @@ ; O3-LABEL: atomic16_and_acq_rel: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection130: +; O3-NEXT: .Lpcsection145: ; O3-NEXT: lock andw $42, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -4062,7 +4107,7 @@ ; O1-LABEL: atomic16_or_acq_rel: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection131: +; O1-NEXT: .Lpcsection146: ; O1-NEXT: lock orw $42, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -4070,7 +4115,7 @@ ; O2-LABEL: atomic16_or_acq_rel: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection131: +; O2-NEXT: .Lpcsection146: ; O2-NEXT: lock orw $42, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -4078,7 +4123,7 @@ ; O3-LABEL: atomic16_or_acq_rel: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection131: +; O3-NEXT: .Lpcsection146: ; O3-NEXT: lock orw $42, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -4101,7 +4146,7 @@ ; O1-LABEL: atomic16_xor_acq_rel: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection132: +; O1-NEXT: .Lpcsection147: ; O1-NEXT: lock xorw $42, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -4109,7 +4154,7 @@ ; O2-LABEL: atomic16_xor_acq_rel: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection132: +; O2-NEXT: .Lpcsection147: ; O2-NEXT: lock xorw $42, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -4117,7 +4162,7 @@ ; O3-LABEL: atomic16_xor_acq_rel: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection132: +; O3-NEXT: .Lpcsection147: ; O3-NEXT: lock xorw $42, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -4166,23 +4211,23 @@ ; O1-LABEL: atomic16_nand_acq_rel: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection133: +; O1-NEXT: .Lpcsection148: ; O1-NEXT: movzwl (%rdi), %eax ; O1-NEXT: .p2align 4, 0x90 ; O1-NEXT: .LBB85_1: # %atomicrmw.start ; O1-NEXT: # =>This Inner Loop Header: Depth=1 ; O1-NEXT: movl %eax, %ecx -; O1-NEXT: .Lpcsection134: +; O1-NEXT: .Lpcsection149: ; O1-NEXT: notl %ecx -; O1-NEXT: .Lpcsection135: +; O1-NEXT: .Lpcsection150: ; O1-NEXT: orl $65493, %ecx # imm = 0xFFD5 -; O1-NEXT: .Lpcsection136: +; O1-NEXT: .Lpcsection151: ; O1-NEXT: # kill: def $ax killed $ax killed $eax -; O1-NEXT: .Lpcsection137: +; O1-NEXT: .Lpcsection152: ; O1-NEXT: lock cmpxchgw %cx, (%rdi) -; O1-NEXT: .Lpcsection138: +; O1-NEXT: .Lpcsection153: ; O1-NEXT: # kill: def $ax killed $ax def $eax -; O1-NEXT: .Lpcsection139: +; O1-NEXT: .Lpcsection154: ; O1-NEXT: jne .LBB85_1 ; O1-NEXT: # %bb.2: # %atomicrmw.end ; O1-NEXT: movq $1, foo(%rip) @@ -4191,23 +4236,23 @@ ; O2-LABEL: atomic16_nand_acq_rel: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection133: +; O2-NEXT: .Lpcsection148: ; O2-NEXT: movzwl (%rdi), %eax ; O2-NEXT: .p2align 4, 0x90 ; O2-NEXT: .LBB85_1: # %atomicrmw.start ; O2-NEXT: # =>This Inner Loop Header: Depth=1 ; O2-NEXT: movl %eax, %ecx -; O2-NEXT: .Lpcsection134: +; O2-NEXT: .Lpcsection149: ; O2-NEXT: notl %ecx -; O2-NEXT: .Lpcsection135: +; O2-NEXT: .Lpcsection150: ; O2-NEXT: orl $65493, %ecx # imm = 0xFFD5 -; O2-NEXT: .Lpcsection136: +; O2-NEXT: .Lpcsection151: ; O2-NEXT: # kill: def $ax killed $ax killed $eax -; O2-NEXT: .Lpcsection137: +; O2-NEXT: .Lpcsection152: ; O2-NEXT: lock cmpxchgw %cx, (%rdi) -; O2-NEXT: .Lpcsection138: +; O2-NEXT: .Lpcsection153: ; O2-NEXT: # kill: def $ax killed $ax def $eax -; O2-NEXT: .Lpcsection139: +; O2-NEXT: .Lpcsection154: ; O2-NEXT: jne .LBB85_1 ; O2-NEXT: # %bb.2: # %atomicrmw.end ; O2-NEXT: movq $1, foo(%rip) @@ -4216,23 +4261,23 @@ ; O3-LABEL: atomic16_nand_acq_rel: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection133: +; O3-NEXT: .Lpcsection148: ; O3-NEXT: movzwl (%rdi), %eax ; O3-NEXT: .p2align 4, 0x90 ; O3-NEXT: .LBB85_1: # %atomicrmw.start ; O3-NEXT: # =>This Inner Loop Header: Depth=1 ; O3-NEXT: movl %eax, %ecx -; O3-NEXT: .Lpcsection134: +; O3-NEXT: .Lpcsection149: ; O3-NEXT: notl %ecx -; O3-NEXT: .Lpcsection135: +; O3-NEXT: .Lpcsection150: ; O3-NEXT: orl $65493, %ecx # imm = 0xFFD5 -; O3-NEXT: .Lpcsection136: +; O3-NEXT: .Lpcsection151: ; O3-NEXT: # kill: def $ax killed $ax killed $eax -; O3-NEXT: .Lpcsection137: +; O3-NEXT: .Lpcsection152: ; O3-NEXT: lock cmpxchgw %cx, (%rdi) -; O3-NEXT: .Lpcsection138: +; O3-NEXT: .Lpcsection153: ; O3-NEXT: # kill: def $ax killed $ax def $eax -; O3-NEXT: .Lpcsection139: +; O3-NEXT: .Lpcsection154: ; O3-NEXT: jne .LBB85_1 ; O3-NEXT: # %bb.2: # %atomicrmw.end ; O3-NEXT: movq $1, foo(%rip) @@ -4258,7 +4303,7 @@ ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax ; O1-NEXT: movw $42, %ax -; O1-NEXT: .Lpcsection140: +; O1-NEXT: .Lpcsection155: ; O1-NEXT: xchgw %ax, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -4267,7 +4312,7 @@ ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax ; O2-NEXT: movw $42, %ax -; O2-NEXT: .Lpcsection140: +; O2-NEXT: .Lpcsection155: ; O2-NEXT: xchgw %ax, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -4276,7 +4321,7 @@ ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax ; O3-NEXT: movw $42, %ax -; O3-NEXT: .Lpcsection140: +; O3-NEXT: .Lpcsection155: ; O3-NEXT: xchgw %ax, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -4299,7 +4344,7 @@ ; O1-LABEL: atomic16_add_seq_cst: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection141: +; O1-NEXT: .Lpcsection156: ; O1-NEXT: lock addw $42, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -4307,7 +4352,7 @@ ; O2-LABEL: atomic16_add_seq_cst: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection141: +; O2-NEXT: .Lpcsection156: ; O2-NEXT: lock addw $42, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -4315,7 +4360,7 @@ ; O3-LABEL: atomic16_add_seq_cst: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection141: +; O3-NEXT: .Lpcsection156: ; O3-NEXT: lock addw $42, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -4338,7 +4383,7 @@ ; O1-LABEL: atomic16_sub_seq_cst: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection142: +; O1-NEXT: .Lpcsection157: ; O1-NEXT: lock subw $42, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -4346,7 +4391,7 @@ ; O2-LABEL: atomic16_sub_seq_cst: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection142: +; O2-NEXT: .Lpcsection157: ; O2-NEXT: lock subw $42, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -4354,7 +4399,7 @@ ; O3-LABEL: atomic16_sub_seq_cst: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection142: +; O3-NEXT: .Lpcsection157: ; O3-NEXT: lock subw $42, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -4377,7 +4422,7 @@ ; O1-LABEL: atomic16_and_seq_cst: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection143: +; O1-NEXT: .Lpcsection158: ; O1-NEXT: lock andw $42, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -4385,7 +4430,7 @@ ; O2-LABEL: atomic16_and_seq_cst: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection143: +; O2-NEXT: .Lpcsection158: ; O2-NEXT: lock andw $42, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -4393,7 +4438,7 @@ ; O3-LABEL: atomic16_and_seq_cst: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection143: +; O3-NEXT: .Lpcsection158: ; O3-NEXT: lock andw $42, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -4416,7 +4461,7 @@ ; O1-LABEL: atomic16_or_seq_cst: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection144: +; O1-NEXT: .Lpcsection159: ; O1-NEXT: lock orw $42, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -4424,7 +4469,7 @@ ; O2-LABEL: atomic16_or_seq_cst: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection144: +; O2-NEXT: .Lpcsection159: ; O2-NEXT: lock orw $42, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -4432,7 +4477,7 @@ ; O3-LABEL: atomic16_or_seq_cst: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection144: +; O3-NEXT: .Lpcsection159: ; O3-NEXT: lock orw $42, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -4455,7 +4500,7 @@ ; O1-LABEL: atomic16_xor_seq_cst: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection145: +; O1-NEXT: .Lpcsection160: ; O1-NEXT: lock xorw $42, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -4463,7 +4508,7 @@ ; O2-LABEL: atomic16_xor_seq_cst: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection145: +; O2-NEXT: .Lpcsection160: ; O2-NEXT: lock xorw $42, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -4471,7 +4516,7 @@ ; O3-LABEL: atomic16_xor_seq_cst: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection145: +; O3-NEXT: .Lpcsection160: ; O3-NEXT: lock xorw $42, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -4520,23 +4565,23 @@ ; O1-LABEL: atomic16_nand_seq_cst: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection146: +; O1-NEXT: .Lpcsection161: ; O1-NEXT: movzwl (%rdi), %eax ; O1-NEXT: .p2align 4, 0x90 ; O1-NEXT: .LBB92_1: # %atomicrmw.start ; O1-NEXT: # =>This Inner Loop Header: Depth=1 ; O1-NEXT: movl %eax, %ecx -; O1-NEXT: .Lpcsection147: +; O1-NEXT: .Lpcsection162: ; O1-NEXT: notl %ecx -; O1-NEXT: .Lpcsection148: +; O1-NEXT: .Lpcsection163: ; O1-NEXT: orl $65493, %ecx # imm = 0xFFD5 -; O1-NEXT: .Lpcsection149: +; O1-NEXT: .Lpcsection164: ; O1-NEXT: # kill: def $ax killed $ax killed $eax -; O1-NEXT: .Lpcsection150: +; O1-NEXT: .Lpcsection165: ; O1-NEXT: lock cmpxchgw %cx, (%rdi) -; O1-NEXT: .Lpcsection151: +; O1-NEXT: .Lpcsection166: ; O1-NEXT: # kill: def $ax killed $ax def $eax -; O1-NEXT: .Lpcsection152: +; O1-NEXT: .Lpcsection167: ; O1-NEXT: jne .LBB92_1 ; O1-NEXT: # %bb.2: # %atomicrmw.end ; O1-NEXT: movq $1, foo(%rip) @@ -4545,23 +4590,23 @@ ; O2-LABEL: atomic16_nand_seq_cst: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection146: +; O2-NEXT: .Lpcsection161: ; O2-NEXT: movzwl (%rdi), %eax ; O2-NEXT: .p2align 4, 0x90 ; O2-NEXT: .LBB92_1: # %atomicrmw.start ; O2-NEXT: # =>This Inner Loop Header: Depth=1 ; O2-NEXT: movl %eax, %ecx -; O2-NEXT: .Lpcsection147: +; O2-NEXT: .Lpcsection162: ; O2-NEXT: notl %ecx -; O2-NEXT: .Lpcsection148: +; O2-NEXT: .Lpcsection163: ; O2-NEXT: orl $65493, %ecx # imm = 0xFFD5 -; O2-NEXT: .Lpcsection149: +; O2-NEXT: .Lpcsection164: ; O2-NEXT: # kill: def $ax killed $ax killed $eax -; O2-NEXT: .Lpcsection150: +; O2-NEXT: .Lpcsection165: ; O2-NEXT: lock cmpxchgw %cx, (%rdi) -; O2-NEXT: .Lpcsection151: +; O2-NEXT: .Lpcsection166: ; O2-NEXT: # kill: def $ax killed $ax def $eax -; O2-NEXT: .Lpcsection152: +; O2-NEXT: .Lpcsection167: ; O2-NEXT: jne .LBB92_1 ; O2-NEXT: # %bb.2: # %atomicrmw.end ; O2-NEXT: movq $1, foo(%rip) @@ -4570,23 +4615,23 @@ ; O3-LABEL: atomic16_nand_seq_cst: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection146: +; O3-NEXT: .Lpcsection161: ; O3-NEXT: movzwl (%rdi), %eax ; O3-NEXT: .p2align 4, 0x90 ; O3-NEXT: .LBB92_1: # %atomicrmw.start ; O3-NEXT: # =>This Inner Loop Header: Depth=1 ; O3-NEXT: movl %eax, %ecx -; O3-NEXT: .Lpcsection147: +; O3-NEXT: .Lpcsection162: ; O3-NEXT: notl %ecx -; O3-NEXT: .Lpcsection148: +; O3-NEXT: .Lpcsection163: ; O3-NEXT: orl $65493, %ecx # imm = 0xFFD5 -; O3-NEXT: .Lpcsection149: +; O3-NEXT: .Lpcsection164: ; O3-NEXT: # kill: def $ax killed $ax killed $eax -; O3-NEXT: .Lpcsection150: +; O3-NEXT: .Lpcsection165: ; O3-NEXT: lock cmpxchgw %cx, (%rdi) -; O3-NEXT: .Lpcsection151: +; O3-NEXT: .Lpcsection166: ; O3-NEXT: # kill: def $ax killed $ax def $eax -; O3-NEXT: .Lpcsection152: +; O3-NEXT: .Lpcsection167: ; O3-NEXT: jne .LBB92_1 ; O3-NEXT: # %bb.2: # %atomicrmw.end ; O3-NEXT: movq $1, foo(%rip) @@ -4625,13 +4670,13 @@ ; O1-NEXT: movq foo(%rip), %rax ; O1-NEXT: movw $1, %cx ; O1-NEXT: movw $42, %ax -; O1-NEXT: .Lpcsection153: +; O1-NEXT: .Lpcsection168: ; O1-NEXT: lock cmpxchgw %cx, (%rdi) ; O1-NEXT: movw $42, %ax -; O1-NEXT: .Lpcsection154: +; O1-NEXT: .Lpcsection169: ; O1-NEXT: lock cmpxchgw %cx, (%rdi) ; O1-NEXT: movw $42, %ax -; O1-NEXT: .Lpcsection155: +; O1-NEXT: .Lpcsection170: ; O1-NEXT: lock cmpxchgw %cx, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -4641,13 +4686,13 @@ ; O2-NEXT: movq foo(%rip), %rax ; O2-NEXT: movw $1, %cx ; O2-NEXT: movw $42, %ax -; O2-NEXT: .Lpcsection153: +; O2-NEXT: .Lpcsection168: ; O2-NEXT: lock cmpxchgw %cx, (%rdi) ; O2-NEXT: movw $42, %ax -; O2-NEXT: .Lpcsection154: +; O2-NEXT: .Lpcsection169: ; O2-NEXT: lock cmpxchgw %cx, (%rdi) ; O2-NEXT: movw $42, %ax -; O2-NEXT: .Lpcsection155: +; O2-NEXT: .Lpcsection170: ; O2-NEXT: lock cmpxchgw %cx, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -4657,13 +4702,13 @@ ; O3-NEXT: movq foo(%rip), %rax ; O3-NEXT: movw $1, %cx ; O3-NEXT: movw $42, %ax -; O3-NEXT: .Lpcsection153: +; O3-NEXT: .Lpcsection168: ; O3-NEXT: lock cmpxchgw %cx, (%rdi) ; O3-NEXT: movw $42, %ax -; O3-NEXT: .Lpcsection154: +; O3-NEXT: .Lpcsection169: ; O3-NEXT: lock cmpxchgw %cx, (%rdi) ; O3-NEXT: movw $42, %ax -; O3-NEXT: .Lpcsection155: +; O3-NEXT: .Lpcsection170: ; O3-NEXT: lock cmpxchgw %cx, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -4703,13 +4748,13 @@ ; O1-NEXT: movq foo(%rip), %rax ; O1-NEXT: movw $1, %cx ; O1-NEXT: movw $42, %ax -; O1-NEXT: .Lpcsection156: +; O1-NEXT: .Lpcsection171: ; O1-NEXT: lock cmpxchgw %cx, (%rdi) ; O1-NEXT: movw $42, %ax -; O1-NEXT: .Lpcsection157: +; O1-NEXT: .Lpcsection172: ; O1-NEXT: lock cmpxchgw %cx, (%rdi) ; O1-NEXT: movw $42, %ax -; O1-NEXT: .Lpcsection158: +; O1-NEXT: .Lpcsection173: ; O1-NEXT: lock cmpxchgw %cx, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -4719,13 +4764,13 @@ ; O2-NEXT: movq foo(%rip), %rax ; O2-NEXT: movw $1, %cx ; O2-NEXT: movw $42, %ax -; O2-NEXT: .Lpcsection156: +; O2-NEXT: .Lpcsection171: ; O2-NEXT: lock cmpxchgw %cx, (%rdi) ; O2-NEXT: movw $42, %ax -; O2-NEXT: .Lpcsection157: +; O2-NEXT: .Lpcsection172: ; O2-NEXT: lock cmpxchgw %cx, (%rdi) ; O2-NEXT: movw $42, %ax -; O2-NEXT: .Lpcsection158: +; O2-NEXT: .Lpcsection173: ; O2-NEXT: lock cmpxchgw %cx, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -4735,13 +4780,13 @@ ; O3-NEXT: movq foo(%rip), %rax ; O3-NEXT: movw $1, %cx ; O3-NEXT: movw $42, %ax -; O3-NEXT: .Lpcsection156: +; O3-NEXT: .Lpcsection171: ; O3-NEXT: lock cmpxchgw %cx, (%rdi) ; O3-NEXT: movw $42, %ax -; O3-NEXT: .Lpcsection157: +; O3-NEXT: .Lpcsection172: ; O3-NEXT: lock cmpxchgw %cx, (%rdi) ; O3-NEXT: movw $42, %ax -; O3-NEXT: .Lpcsection158: +; O3-NEXT: .Lpcsection173: ; O3-NEXT: lock cmpxchgw %cx, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -4781,13 +4826,13 @@ ; O1-NEXT: movq foo(%rip), %rax ; O1-NEXT: movw $1, %cx ; O1-NEXT: movw $42, %ax -; O1-NEXT: .Lpcsection159: +; O1-NEXT: .Lpcsection174: ; O1-NEXT: lock cmpxchgw %cx, (%rdi) ; O1-NEXT: movw $42, %ax -; O1-NEXT: .Lpcsection160: +; O1-NEXT: .Lpcsection175: ; O1-NEXT: lock cmpxchgw %cx, (%rdi) ; O1-NEXT: movw $42, %ax -; O1-NEXT: .Lpcsection161: +; O1-NEXT: .Lpcsection176: ; O1-NEXT: lock cmpxchgw %cx, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -4797,13 +4842,13 @@ ; O2-NEXT: movq foo(%rip), %rax ; O2-NEXT: movw $1, %cx ; O2-NEXT: movw $42, %ax -; O2-NEXT: .Lpcsection159: +; O2-NEXT: .Lpcsection174: ; O2-NEXT: lock cmpxchgw %cx, (%rdi) ; O2-NEXT: movw $42, %ax -; O2-NEXT: .Lpcsection160: +; O2-NEXT: .Lpcsection175: ; O2-NEXT: lock cmpxchgw %cx, (%rdi) ; O2-NEXT: movw $42, %ax -; O2-NEXT: .Lpcsection161: +; O2-NEXT: .Lpcsection176: ; O2-NEXT: lock cmpxchgw %cx, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -4813,13 +4858,13 @@ ; O3-NEXT: movq foo(%rip), %rax ; O3-NEXT: movw $1, %cx ; O3-NEXT: movw $42, %ax -; O3-NEXT: .Lpcsection159: +; O3-NEXT: .Lpcsection174: ; O3-NEXT: lock cmpxchgw %cx, (%rdi) ; O3-NEXT: movw $42, %ax -; O3-NEXT: .Lpcsection160: +; O3-NEXT: .Lpcsection175: ; O3-NEXT: lock cmpxchgw %cx, (%rdi) ; O3-NEXT: movw $42, %ax -; O3-NEXT: .Lpcsection161: +; O3-NEXT: .Lpcsection176: ; O3-NEXT: lock cmpxchgw %cx, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -4859,13 +4904,13 @@ ; O1-NEXT: movq foo(%rip), %rax ; O1-NEXT: movw $1, %cx ; O1-NEXT: movw $42, %ax -; O1-NEXT: .Lpcsection162: +; O1-NEXT: .Lpcsection177: ; O1-NEXT: lock cmpxchgw %cx, (%rdi) ; O1-NEXT: movw $42, %ax -; O1-NEXT: .Lpcsection163: +; O1-NEXT: .Lpcsection178: ; O1-NEXT: lock cmpxchgw %cx, (%rdi) ; O1-NEXT: movw $42, %ax -; O1-NEXT: .Lpcsection164: +; O1-NEXT: .Lpcsection179: ; O1-NEXT: lock cmpxchgw %cx, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -4875,13 +4920,13 @@ ; O2-NEXT: movq foo(%rip), %rax ; O2-NEXT: movw $1, %cx ; O2-NEXT: movw $42, %ax -; O2-NEXT: .Lpcsection162: +; O2-NEXT: .Lpcsection177: ; O2-NEXT: lock cmpxchgw %cx, (%rdi) ; O2-NEXT: movw $42, %ax -; O2-NEXT: .Lpcsection163: +; O2-NEXT: .Lpcsection178: ; O2-NEXT: lock cmpxchgw %cx, (%rdi) ; O2-NEXT: movw $42, %ax -; O2-NEXT: .Lpcsection164: +; O2-NEXT: .Lpcsection179: ; O2-NEXT: lock cmpxchgw %cx, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -4891,13 +4936,13 @@ ; O3-NEXT: movq foo(%rip), %rax ; O3-NEXT: movw $1, %cx ; O3-NEXT: movw $42, %ax -; O3-NEXT: .Lpcsection162: +; O3-NEXT: .Lpcsection177: ; O3-NEXT: lock cmpxchgw %cx, (%rdi) ; O3-NEXT: movw $42, %ax -; O3-NEXT: .Lpcsection163: +; O3-NEXT: .Lpcsection178: ; O3-NEXT: lock cmpxchgw %cx, (%rdi) ; O3-NEXT: movw $42, %ax -; O3-NEXT: .Lpcsection164: +; O3-NEXT: .Lpcsection179: ; O3-NEXT: lock cmpxchgw %cx, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -4937,13 +4982,13 @@ ; O1-NEXT: movq foo(%rip), %rax ; O1-NEXT: movw $1, %cx ; O1-NEXT: movw $42, %ax -; O1-NEXT: .Lpcsection165: +; O1-NEXT: .Lpcsection180: ; O1-NEXT: lock cmpxchgw %cx, (%rdi) ; O1-NEXT: movw $42, %ax -; O1-NEXT: .Lpcsection166: +; O1-NEXT: .Lpcsection181: ; O1-NEXT: lock cmpxchgw %cx, (%rdi) ; O1-NEXT: movw $42, %ax -; O1-NEXT: .Lpcsection167: +; O1-NEXT: .Lpcsection182: ; O1-NEXT: lock cmpxchgw %cx, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -4953,13 +4998,13 @@ ; O2-NEXT: movq foo(%rip), %rax ; O2-NEXT: movw $1, %cx ; O2-NEXT: movw $42, %ax -; O2-NEXT: .Lpcsection165: +; O2-NEXT: .Lpcsection180: ; O2-NEXT: lock cmpxchgw %cx, (%rdi) ; O2-NEXT: movw $42, %ax -; O2-NEXT: .Lpcsection166: +; O2-NEXT: .Lpcsection181: ; O2-NEXT: lock cmpxchgw %cx, (%rdi) ; O2-NEXT: movw $42, %ax -; O2-NEXT: .Lpcsection167: +; O2-NEXT: .Lpcsection182: ; O2-NEXT: lock cmpxchgw %cx, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -4969,13 +5014,13 @@ ; O3-NEXT: movq foo(%rip), %rax ; O3-NEXT: movw $1, %cx ; O3-NEXT: movw $42, %ax -; O3-NEXT: .Lpcsection165: +; O3-NEXT: .Lpcsection180: ; O3-NEXT: lock cmpxchgw %cx, (%rdi) ; O3-NEXT: movw $42, %ax -; O3-NEXT: .Lpcsection166: +; O3-NEXT: .Lpcsection181: ; O3-NEXT: lock cmpxchgw %cx, (%rdi) ; O3-NEXT: movw $42, %ax -; O3-NEXT: .Lpcsection167: +; O3-NEXT: .Lpcsection182: ; O3-NEXT: lock cmpxchgw %cx, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -5000,7 +5045,7 @@ ; O1-LABEL: atomic32_load_unordered: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection168: +; O1-NEXT: .Lpcsection183: ; O1-NEXT: movl (%rdi), %eax ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -5008,7 +5053,7 @@ ; O2-LABEL: atomic32_load_unordered: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection168: +; O2-NEXT: .Lpcsection183: ; O2-NEXT: movl (%rdi), %eax ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -5016,7 +5061,7 @@ ; O3-LABEL: atomic32_load_unordered: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection168: +; O3-NEXT: .Lpcsection183: ; O3-NEXT: movl (%rdi), %eax ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -5039,7 +5084,7 @@ ; O1-LABEL: atomic32_load_monotonic: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection169: +; O1-NEXT: .Lpcsection184: ; O1-NEXT: movl (%rdi), %eax ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -5047,7 +5092,7 @@ ; O2-LABEL: atomic32_load_monotonic: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection169: +; O2-NEXT: .Lpcsection184: ; O2-NEXT: movl (%rdi), %eax ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -5055,7 +5100,7 @@ ; O3-LABEL: atomic32_load_monotonic: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection169: +; O3-NEXT: .Lpcsection184: ; O3-NEXT: movl (%rdi), %eax ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -5078,7 +5123,7 @@ ; O1-LABEL: atomic32_load_acquire: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection170: +; O1-NEXT: .Lpcsection185: ; O1-NEXT: movl (%rdi), %eax ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -5086,7 +5131,7 @@ ; O2-LABEL: atomic32_load_acquire: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection170: +; O2-NEXT: .Lpcsection185: ; O2-NEXT: movl (%rdi), %eax ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -5094,7 +5139,7 @@ ; O3-LABEL: atomic32_load_acquire: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection170: +; O3-NEXT: .Lpcsection185: ; O3-NEXT: movl (%rdi), %eax ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -5117,7 +5162,7 @@ ; O1-LABEL: atomic32_load_seq_cst: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection171: +; O1-NEXT: .Lpcsection186: ; O1-NEXT: movl (%rdi), %eax ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -5125,7 +5170,7 @@ ; O2-LABEL: atomic32_load_seq_cst: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection171: +; O2-NEXT: .Lpcsection186: ; O2-NEXT: movl (%rdi), %eax ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -5133,7 +5178,7 @@ ; O3-LABEL: atomic32_load_seq_cst: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection171: +; O3-NEXT: .Lpcsection186: ; O3-NEXT: movl (%rdi), %eax ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -5156,7 +5201,7 @@ ; O1-LABEL: atomic32_store_unordered: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection172: +; O1-NEXT: .Lpcsection187: ; O1-NEXT: movl $42, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -5164,7 +5209,7 @@ ; O2-LABEL: atomic32_store_unordered: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection172: +; O2-NEXT: .Lpcsection187: ; O2-NEXT: movl $42, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -5172,7 +5217,7 @@ ; O3-LABEL: atomic32_store_unordered: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection172: +; O3-NEXT: .Lpcsection187: ; O3-NEXT: movl $42, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -5195,7 +5240,7 @@ ; O1-LABEL: atomic32_store_monotonic: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection173: +; O1-NEXT: .Lpcsection188: ; O1-NEXT: movl $42, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -5203,7 +5248,7 @@ ; O2-LABEL: atomic32_store_monotonic: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection173: +; O2-NEXT: .Lpcsection188: ; O2-NEXT: movl $42, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -5211,7 +5256,7 @@ ; O3-LABEL: atomic32_store_monotonic: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection173: +; O3-NEXT: .Lpcsection188: ; O3-NEXT: movl $42, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -5234,7 +5279,7 @@ ; O1-LABEL: atomic32_store_release: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection174: +; O1-NEXT: .Lpcsection189: ; O1-NEXT: movl $42, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -5242,7 +5287,7 @@ ; O2-LABEL: atomic32_store_release: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection174: +; O2-NEXT: .Lpcsection189: ; O2-NEXT: movl $42, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -5250,7 +5295,7 @@ ; O3-LABEL: atomic32_store_release: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection174: +; O3-NEXT: .Lpcsection189: ; O3-NEXT: movl $42, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -5275,7 +5320,7 @@ ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax ; O1-NEXT: movl $42, %eax -; O1-NEXT: .Lpcsection175: +; O1-NEXT: .Lpcsection190: ; O1-NEXT: xchgl %eax, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -5284,7 +5329,7 @@ ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax ; O2-NEXT: movl $42, %eax -; O2-NEXT: .Lpcsection175: +; O2-NEXT: .Lpcsection190: ; O2-NEXT: xchgl %eax, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -5293,7 +5338,7 @@ ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax ; O3-NEXT: movl $42, %eax -; O3-NEXT: .Lpcsection175: +; O3-NEXT: .Lpcsection190: ; O3-NEXT: xchgl %eax, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -5318,7 +5363,7 @@ ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax ; O1-NEXT: movl $42, %eax -; O1-NEXT: .Lpcsection176: +; O1-NEXT: .Lpcsection191: ; O1-NEXT: xchgl %eax, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -5327,7 +5372,7 @@ ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax ; O2-NEXT: movl $42, %eax -; O2-NEXT: .Lpcsection176: +; O2-NEXT: .Lpcsection191: ; O2-NEXT: xchgl %eax, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -5336,7 +5381,7 @@ ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax ; O3-NEXT: movl $42, %eax -; O3-NEXT: .Lpcsection176: +; O3-NEXT: .Lpcsection191: ; O3-NEXT: xchgl %eax, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -5359,7 +5404,7 @@ ; O1-LABEL: atomic32_add_monotonic: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection177: +; O1-NEXT: .Lpcsection192: ; O1-NEXT: lock addl $42, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -5367,7 +5412,7 @@ ; O2-LABEL: atomic32_add_monotonic: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection177: +; O2-NEXT: .Lpcsection192: ; O2-NEXT: lock addl $42, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -5375,7 +5420,7 @@ ; O3-LABEL: atomic32_add_monotonic: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection177: +; O3-NEXT: .Lpcsection192: ; O3-NEXT: lock addl $42, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -5398,7 +5443,7 @@ ; O1-LABEL: atomic32_sub_monotonic: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection178: +; O1-NEXT: .Lpcsection193: ; O1-NEXT: lock subl $42, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -5406,7 +5451,7 @@ ; O2-LABEL: atomic32_sub_monotonic: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection178: +; O2-NEXT: .Lpcsection193: ; O2-NEXT: lock subl $42, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -5414,7 +5459,7 @@ ; O3-LABEL: atomic32_sub_monotonic: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection178: +; O3-NEXT: .Lpcsection193: ; O3-NEXT: lock subl $42, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -5437,7 +5482,7 @@ ; O1-LABEL: atomic32_and_monotonic: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection179: +; O1-NEXT: .Lpcsection194: ; O1-NEXT: lock andl $42, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -5445,7 +5490,7 @@ ; O2-LABEL: atomic32_and_monotonic: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection179: +; O2-NEXT: .Lpcsection194: ; O2-NEXT: lock andl $42, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -5453,7 +5498,7 @@ ; O3-LABEL: atomic32_and_monotonic: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection179: +; O3-NEXT: .Lpcsection194: ; O3-NEXT: lock andl $42, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -5476,7 +5521,7 @@ ; O1-LABEL: atomic32_or_monotonic: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection180: +; O1-NEXT: .Lpcsection195: ; O1-NEXT: lock orl $42, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -5484,7 +5529,7 @@ ; O2-LABEL: atomic32_or_monotonic: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection180: +; O2-NEXT: .Lpcsection195: ; O2-NEXT: lock orl $42, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -5492,7 +5537,7 @@ ; O3-LABEL: atomic32_or_monotonic: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection180: +; O3-NEXT: .Lpcsection195: ; O3-NEXT: lock orl $42, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -5515,7 +5560,7 @@ ; O1-LABEL: atomic32_xor_monotonic: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection181: +; O1-NEXT: .Lpcsection196: ; O1-NEXT: lock xorl $42, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -5523,7 +5568,7 @@ ; O2-LABEL: atomic32_xor_monotonic: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection181: +; O2-NEXT: .Lpcsection196: ; O2-NEXT: lock xorl $42, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -5531,7 +5576,7 @@ ; O3-LABEL: atomic32_xor_monotonic: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection181: +; O3-NEXT: .Lpcsection196: ; O3-NEXT: lock xorl $42, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -5576,19 +5621,19 @@ ; O1-LABEL: atomic32_nand_monotonic: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection182: +; O1-NEXT: .Lpcsection197: ; O1-NEXT: movl (%rdi), %eax ; O1-NEXT: .p2align 4, 0x90 ; O1-NEXT: .LBB112_1: # %atomicrmw.start ; O1-NEXT: # =>This Inner Loop Header: Depth=1 ; O1-NEXT: movl %eax, %ecx -; O1-NEXT: .Lpcsection183: +; O1-NEXT: .Lpcsection198: ; O1-NEXT: notl %ecx -; O1-NEXT: .Lpcsection184: +; O1-NEXT: .Lpcsection199: ; O1-NEXT: orl $-43, %ecx -; O1-NEXT: .Lpcsection185: +; O1-NEXT: .Lpcsection200: ; O1-NEXT: lock cmpxchgl %ecx, (%rdi) -; O1-NEXT: .Lpcsection186: +; O1-NEXT: .Lpcsection201: ; O1-NEXT: jne .LBB112_1 ; O1-NEXT: # %bb.2: # %atomicrmw.end ; O1-NEXT: movq $1, foo(%rip) @@ -5597,19 +5642,19 @@ ; O2-LABEL: atomic32_nand_monotonic: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection182: +; O2-NEXT: .Lpcsection197: ; O2-NEXT: movl (%rdi), %eax ; O2-NEXT: .p2align 4, 0x90 ; O2-NEXT: .LBB112_1: # %atomicrmw.start ; O2-NEXT: # =>This Inner Loop Header: Depth=1 ; O2-NEXT: movl %eax, %ecx -; O2-NEXT: .Lpcsection183: +; O2-NEXT: .Lpcsection198: ; O2-NEXT: notl %ecx -; O2-NEXT: .Lpcsection184: +; O2-NEXT: .Lpcsection199: ; O2-NEXT: orl $-43, %ecx -; O2-NEXT: .Lpcsection185: +; O2-NEXT: .Lpcsection200: ; O2-NEXT: lock cmpxchgl %ecx, (%rdi) -; O2-NEXT: .Lpcsection186: +; O2-NEXT: .Lpcsection201: ; O2-NEXT: jne .LBB112_1 ; O2-NEXT: # %bb.2: # %atomicrmw.end ; O2-NEXT: movq $1, foo(%rip) @@ -5618,19 +5663,19 @@ ; O3-LABEL: atomic32_nand_monotonic: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection182: +; O3-NEXT: .Lpcsection197: ; O3-NEXT: movl (%rdi), %eax ; O3-NEXT: .p2align 4, 0x90 ; O3-NEXT: .LBB112_1: # %atomicrmw.start ; O3-NEXT: # =>This Inner Loop Header: Depth=1 ; O3-NEXT: movl %eax, %ecx -; O3-NEXT: .Lpcsection183: +; O3-NEXT: .Lpcsection198: ; O3-NEXT: notl %ecx -; O3-NEXT: .Lpcsection184: +; O3-NEXT: .Lpcsection199: ; O3-NEXT: orl $-43, %ecx -; O3-NEXT: .Lpcsection185: +; O3-NEXT: .Lpcsection200: ; O3-NEXT: lock cmpxchgl %ecx, (%rdi) -; O3-NEXT: .Lpcsection186: +; O3-NEXT: .Lpcsection201: ; O3-NEXT: jne .LBB112_1 ; O3-NEXT: # %bb.2: # %atomicrmw.end ; O3-NEXT: movq $1, foo(%rip) @@ -5656,7 +5701,7 @@ ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax ; O1-NEXT: movl $42, %eax -; O1-NEXT: .Lpcsection187: +; O1-NEXT: .Lpcsection202: ; O1-NEXT: xchgl %eax, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -5665,7 +5710,7 @@ ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax ; O2-NEXT: movl $42, %eax -; O2-NEXT: .Lpcsection187: +; O2-NEXT: .Lpcsection202: ; O2-NEXT: xchgl %eax, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -5674,7 +5719,7 @@ ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax ; O3-NEXT: movl $42, %eax -; O3-NEXT: .Lpcsection187: +; O3-NEXT: .Lpcsection202: ; O3-NEXT: xchgl %eax, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -5697,7 +5742,7 @@ ; O1-LABEL: atomic32_add_acquire: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection188: +; O1-NEXT: .Lpcsection203: ; O1-NEXT: lock addl $42, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -5705,7 +5750,7 @@ ; O2-LABEL: atomic32_add_acquire: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection188: +; O2-NEXT: .Lpcsection203: ; O2-NEXT: lock addl $42, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -5713,7 +5758,7 @@ ; O3-LABEL: atomic32_add_acquire: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection188: +; O3-NEXT: .Lpcsection203: ; O3-NEXT: lock addl $42, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -5736,7 +5781,7 @@ ; O1-LABEL: atomic32_sub_acquire: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection189: +; O1-NEXT: .Lpcsection204: ; O1-NEXT: lock subl $42, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -5744,7 +5789,7 @@ ; O2-LABEL: atomic32_sub_acquire: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection189: +; O2-NEXT: .Lpcsection204: ; O2-NEXT: lock subl $42, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -5752,7 +5797,7 @@ ; O3-LABEL: atomic32_sub_acquire: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection189: +; O3-NEXT: .Lpcsection204: ; O3-NEXT: lock subl $42, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -5775,7 +5820,7 @@ ; O1-LABEL: atomic32_and_acquire: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection190: +; O1-NEXT: .Lpcsection205: ; O1-NEXT: lock andl $42, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -5783,7 +5828,7 @@ ; O2-LABEL: atomic32_and_acquire: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection190: +; O2-NEXT: .Lpcsection205: ; O2-NEXT: lock andl $42, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -5791,7 +5836,7 @@ ; O3-LABEL: atomic32_and_acquire: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection190: +; O3-NEXT: .Lpcsection205: ; O3-NEXT: lock andl $42, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -5814,7 +5859,7 @@ ; O1-LABEL: atomic32_or_acquire: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection191: +; O1-NEXT: .Lpcsection206: ; O1-NEXT: lock orl $42, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -5822,7 +5867,7 @@ ; O2-LABEL: atomic32_or_acquire: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection191: +; O2-NEXT: .Lpcsection206: ; O2-NEXT: lock orl $42, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -5830,7 +5875,7 @@ ; O3-LABEL: atomic32_or_acquire: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection191: +; O3-NEXT: .Lpcsection206: ; O3-NEXT: lock orl $42, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -5853,7 +5898,7 @@ ; O1-LABEL: atomic32_xor_acquire: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection192: +; O1-NEXT: .Lpcsection207: ; O1-NEXT: lock xorl $42, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -5861,7 +5906,7 @@ ; O2-LABEL: atomic32_xor_acquire: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection192: +; O2-NEXT: .Lpcsection207: ; O2-NEXT: lock xorl $42, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -5869,7 +5914,7 @@ ; O3-LABEL: atomic32_xor_acquire: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection192: +; O3-NEXT: .Lpcsection207: ; O3-NEXT: lock xorl $42, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -5914,19 +5959,19 @@ ; O1-LABEL: atomic32_nand_acquire: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection193: +; O1-NEXT: .Lpcsection208: ; O1-NEXT: movl (%rdi), %eax ; O1-NEXT: .p2align 4, 0x90 ; O1-NEXT: .LBB119_1: # %atomicrmw.start ; O1-NEXT: # =>This Inner Loop Header: Depth=1 ; O1-NEXT: movl %eax, %ecx -; O1-NEXT: .Lpcsection194: +; O1-NEXT: .Lpcsection209: ; O1-NEXT: notl %ecx -; O1-NEXT: .Lpcsection195: +; O1-NEXT: .Lpcsection210: ; O1-NEXT: orl $-43, %ecx -; O1-NEXT: .Lpcsection196: +; O1-NEXT: .Lpcsection211: ; O1-NEXT: lock cmpxchgl %ecx, (%rdi) -; O1-NEXT: .Lpcsection197: +; O1-NEXT: .Lpcsection212: ; O1-NEXT: jne .LBB119_1 ; O1-NEXT: # %bb.2: # %atomicrmw.end ; O1-NEXT: movq $1, foo(%rip) @@ -5935,19 +5980,19 @@ ; O2-LABEL: atomic32_nand_acquire: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection193: +; O2-NEXT: .Lpcsection208: ; O2-NEXT: movl (%rdi), %eax ; O2-NEXT: .p2align 4, 0x90 ; O2-NEXT: .LBB119_1: # %atomicrmw.start ; O2-NEXT: # =>This Inner Loop Header: Depth=1 ; O2-NEXT: movl %eax, %ecx -; O2-NEXT: .Lpcsection194: +; O2-NEXT: .Lpcsection209: ; O2-NEXT: notl %ecx -; O2-NEXT: .Lpcsection195: +; O2-NEXT: .Lpcsection210: ; O2-NEXT: orl $-43, %ecx -; O2-NEXT: .Lpcsection196: +; O2-NEXT: .Lpcsection211: ; O2-NEXT: lock cmpxchgl %ecx, (%rdi) -; O2-NEXT: .Lpcsection197: +; O2-NEXT: .Lpcsection212: ; O2-NEXT: jne .LBB119_1 ; O2-NEXT: # %bb.2: # %atomicrmw.end ; O2-NEXT: movq $1, foo(%rip) @@ -5956,19 +6001,19 @@ ; O3-LABEL: atomic32_nand_acquire: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection193: +; O3-NEXT: .Lpcsection208: ; O3-NEXT: movl (%rdi), %eax ; O3-NEXT: .p2align 4, 0x90 ; O3-NEXT: .LBB119_1: # %atomicrmw.start ; O3-NEXT: # =>This Inner Loop Header: Depth=1 ; O3-NEXT: movl %eax, %ecx -; O3-NEXT: .Lpcsection194: +; O3-NEXT: .Lpcsection209: ; O3-NEXT: notl %ecx -; O3-NEXT: .Lpcsection195: +; O3-NEXT: .Lpcsection210: ; O3-NEXT: orl $-43, %ecx -; O3-NEXT: .Lpcsection196: +; O3-NEXT: .Lpcsection211: ; O3-NEXT: lock cmpxchgl %ecx, (%rdi) -; O3-NEXT: .Lpcsection197: +; O3-NEXT: .Lpcsection212: ; O3-NEXT: jne .LBB119_1 ; O3-NEXT: # %bb.2: # %atomicrmw.end ; O3-NEXT: movq $1, foo(%rip) @@ -5994,7 +6039,7 @@ ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax ; O1-NEXT: movl $42, %eax -; O1-NEXT: .Lpcsection198: +; O1-NEXT: .Lpcsection213: ; O1-NEXT: xchgl %eax, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -6003,7 +6048,7 @@ ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax ; O2-NEXT: movl $42, %eax -; O2-NEXT: .Lpcsection198: +; O2-NEXT: .Lpcsection213: ; O2-NEXT: xchgl %eax, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -6012,7 +6057,7 @@ ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax ; O3-NEXT: movl $42, %eax -; O3-NEXT: .Lpcsection198: +; O3-NEXT: .Lpcsection213: ; O3-NEXT: xchgl %eax, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -6035,7 +6080,7 @@ ; O1-LABEL: atomic32_add_release: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection199: +; O1-NEXT: .Lpcsection214: ; O1-NEXT: lock addl $42, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -6043,7 +6088,7 @@ ; O2-LABEL: atomic32_add_release: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection199: +; O2-NEXT: .Lpcsection214: ; O2-NEXT: lock addl $42, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -6051,7 +6096,7 @@ ; O3-LABEL: atomic32_add_release: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection199: +; O3-NEXT: .Lpcsection214: ; O3-NEXT: lock addl $42, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -6074,7 +6119,7 @@ ; O1-LABEL: atomic32_sub_release: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection200: +; O1-NEXT: .Lpcsection215: ; O1-NEXT: lock subl $42, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -6082,7 +6127,7 @@ ; O2-LABEL: atomic32_sub_release: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection200: +; O2-NEXT: .Lpcsection215: ; O2-NEXT: lock subl $42, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -6090,7 +6135,7 @@ ; O3-LABEL: atomic32_sub_release: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection200: +; O3-NEXT: .Lpcsection215: ; O3-NEXT: lock subl $42, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -6113,7 +6158,7 @@ ; O1-LABEL: atomic32_and_release: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection201: +; O1-NEXT: .Lpcsection216: ; O1-NEXT: lock andl $42, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -6121,7 +6166,7 @@ ; O2-LABEL: atomic32_and_release: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection201: +; O2-NEXT: .Lpcsection216: ; O2-NEXT: lock andl $42, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -6129,7 +6174,7 @@ ; O3-LABEL: atomic32_and_release: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection201: +; O3-NEXT: .Lpcsection216: ; O3-NEXT: lock andl $42, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -6152,7 +6197,7 @@ ; O1-LABEL: atomic32_or_release: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection202: +; O1-NEXT: .Lpcsection217: ; O1-NEXT: lock orl $42, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -6160,7 +6205,7 @@ ; O2-LABEL: atomic32_or_release: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection202: +; O2-NEXT: .Lpcsection217: ; O2-NEXT: lock orl $42, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -6168,7 +6213,7 @@ ; O3-LABEL: atomic32_or_release: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection202: +; O3-NEXT: .Lpcsection217: ; O3-NEXT: lock orl $42, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -6191,7 +6236,7 @@ ; O1-LABEL: atomic32_xor_release: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection203: +; O1-NEXT: .Lpcsection218: ; O1-NEXT: lock xorl $42, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -6199,7 +6244,7 @@ ; O2-LABEL: atomic32_xor_release: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection203: +; O2-NEXT: .Lpcsection218: ; O2-NEXT: lock xorl $42, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -6207,7 +6252,7 @@ ; O3-LABEL: atomic32_xor_release: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection203: +; O3-NEXT: .Lpcsection218: ; O3-NEXT: lock xorl $42, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -6252,19 +6297,19 @@ ; O1-LABEL: atomic32_nand_release: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection204: +; O1-NEXT: .Lpcsection219: ; O1-NEXT: movl (%rdi), %eax ; O1-NEXT: .p2align 4, 0x90 ; O1-NEXT: .LBB126_1: # %atomicrmw.start ; O1-NEXT: # =>This Inner Loop Header: Depth=1 ; O1-NEXT: movl %eax, %ecx -; O1-NEXT: .Lpcsection205: +; O1-NEXT: .Lpcsection220: ; O1-NEXT: notl %ecx -; O1-NEXT: .Lpcsection206: +; O1-NEXT: .Lpcsection221: ; O1-NEXT: orl $-43, %ecx -; O1-NEXT: .Lpcsection207: +; O1-NEXT: .Lpcsection222: ; O1-NEXT: lock cmpxchgl %ecx, (%rdi) -; O1-NEXT: .Lpcsection208: +; O1-NEXT: .Lpcsection223: ; O1-NEXT: jne .LBB126_1 ; O1-NEXT: # %bb.2: # %atomicrmw.end ; O1-NEXT: movq $1, foo(%rip) @@ -6273,19 +6318,19 @@ ; O2-LABEL: atomic32_nand_release: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection204: +; O2-NEXT: .Lpcsection219: ; O2-NEXT: movl (%rdi), %eax ; O2-NEXT: .p2align 4, 0x90 ; O2-NEXT: .LBB126_1: # %atomicrmw.start ; O2-NEXT: # =>This Inner Loop Header: Depth=1 ; O2-NEXT: movl %eax, %ecx -; O2-NEXT: .Lpcsection205: +; O2-NEXT: .Lpcsection220: ; O2-NEXT: notl %ecx -; O2-NEXT: .Lpcsection206: +; O2-NEXT: .Lpcsection221: ; O2-NEXT: orl $-43, %ecx -; O2-NEXT: .Lpcsection207: +; O2-NEXT: .Lpcsection222: ; O2-NEXT: lock cmpxchgl %ecx, (%rdi) -; O2-NEXT: .Lpcsection208: +; O2-NEXT: .Lpcsection223: ; O2-NEXT: jne .LBB126_1 ; O2-NEXT: # %bb.2: # %atomicrmw.end ; O2-NEXT: movq $1, foo(%rip) @@ -6294,19 +6339,19 @@ ; O3-LABEL: atomic32_nand_release: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection204: +; O3-NEXT: .Lpcsection219: ; O3-NEXT: movl (%rdi), %eax ; O3-NEXT: .p2align 4, 0x90 ; O3-NEXT: .LBB126_1: # %atomicrmw.start ; O3-NEXT: # =>This Inner Loop Header: Depth=1 ; O3-NEXT: movl %eax, %ecx -; O3-NEXT: .Lpcsection205: +; O3-NEXT: .Lpcsection220: ; O3-NEXT: notl %ecx -; O3-NEXT: .Lpcsection206: +; O3-NEXT: .Lpcsection221: ; O3-NEXT: orl $-43, %ecx -; O3-NEXT: .Lpcsection207: +; O3-NEXT: .Lpcsection222: ; O3-NEXT: lock cmpxchgl %ecx, (%rdi) -; O3-NEXT: .Lpcsection208: +; O3-NEXT: .Lpcsection223: ; O3-NEXT: jne .LBB126_1 ; O3-NEXT: # %bb.2: # %atomicrmw.end ; O3-NEXT: movq $1, foo(%rip) @@ -6332,7 +6377,7 @@ ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax ; O1-NEXT: movl $42, %eax -; O1-NEXT: .Lpcsection209: +; O1-NEXT: .Lpcsection224: ; O1-NEXT: xchgl %eax, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -6341,7 +6386,7 @@ ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax ; O2-NEXT: movl $42, %eax -; O2-NEXT: .Lpcsection209: +; O2-NEXT: .Lpcsection224: ; O2-NEXT: xchgl %eax, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -6350,7 +6395,7 @@ ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax ; O3-NEXT: movl $42, %eax -; O3-NEXT: .Lpcsection209: +; O3-NEXT: .Lpcsection224: ; O3-NEXT: xchgl %eax, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -6373,7 +6418,7 @@ ; O1-LABEL: atomic32_add_acq_rel: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection210: +; O1-NEXT: .Lpcsection225: ; O1-NEXT: lock addl $42, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -6381,7 +6426,7 @@ ; O2-LABEL: atomic32_add_acq_rel: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection210: +; O2-NEXT: .Lpcsection225: ; O2-NEXT: lock addl $42, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -6389,7 +6434,7 @@ ; O3-LABEL: atomic32_add_acq_rel: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection210: +; O3-NEXT: .Lpcsection225: ; O3-NEXT: lock addl $42, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -6412,7 +6457,7 @@ ; O1-LABEL: atomic32_sub_acq_rel: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection211: +; O1-NEXT: .Lpcsection226: ; O1-NEXT: lock subl $42, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -6420,7 +6465,7 @@ ; O2-LABEL: atomic32_sub_acq_rel: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection211: +; O2-NEXT: .Lpcsection226: ; O2-NEXT: lock subl $42, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -6428,7 +6473,7 @@ ; O3-LABEL: atomic32_sub_acq_rel: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection211: +; O3-NEXT: .Lpcsection226: ; O3-NEXT: lock subl $42, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -6451,7 +6496,7 @@ ; O1-LABEL: atomic32_and_acq_rel: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection212: +; O1-NEXT: .Lpcsection227: ; O1-NEXT: lock andl $42, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -6459,7 +6504,7 @@ ; O2-LABEL: atomic32_and_acq_rel: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection212: +; O2-NEXT: .Lpcsection227: ; O2-NEXT: lock andl $42, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -6467,7 +6512,7 @@ ; O3-LABEL: atomic32_and_acq_rel: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection212: +; O3-NEXT: .Lpcsection227: ; O3-NEXT: lock andl $42, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -6490,7 +6535,7 @@ ; O1-LABEL: atomic32_or_acq_rel: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection213: +; O1-NEXT: .Lpcsection228: ; O1-NEXT: lock orl $42, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -6498,7 +6543,7 @@ ; O2-LABEL: atomic32_or_acq_rel: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection213: +; O2-NEXT: .Lpcsection228: ; O2-NEXT: lock orl $42, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -6506,7 +6551,7 @@ ; O3-LABEL: atomic32_or_acq_rel: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection213: +; O3-NEXT: .Lpcsection228: ; O3-NEXT: lock orl $42, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -6529,7 +6574,7 @@ ; O1-LABEL: atomic32_xor_acq_rel: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection214: +; O1-NEXT: .Lpcsection229: ; O1-NEXT: lock xorl $42, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -6537,7 +6582,7 @@ ; O2-LABEL: atomic32_xor_acq_rel: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection214: +; O2-NEXT: .Lpcsection229: ; O2-NEXT: lock xorl $42, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -6545,7 +6590,7 @@ ; O3-LABEL: atomic32_xor_acq_rel: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection214: +; O3-NEXT: .Lpcsection229: ; O3-NEXT: lock xorl $42, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -6590,19 +6635,19 @@ ; O1-LABEL: atomic32_nand_acq_rel: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection215: +; O1-NEXT: .Lpcsection230: ; O1-NEXT: movl (%rdi), %eax ; O1-NEXT: .p2align 4, 0x90 ; O1-NEXT: .LBB133_1: # %atomicrmw.start ; O1-NEXT: # =>This Inner Loop Header: Depth=1 ; O1-NEXT: movl %eax, %ecx -; O1-NEXT: .Lpcsection216: +; O1-NEXT: .Lpcsection231: ; O1-NEXT: notl %ecx -; O1-NEXT: .Lpcsection217: +; O1-NEXT: .Lpcsection232: ; O1-NEXT: orl $-43, %ecx -; O1-NEXT: .Lpcsection218: +; O1-NEXT: .Lpcsection233: ; O1-NEXT: lock cmpxchgl %ecx, (%rdi) -; O1-NEXT: .Lpcsection219: +; O1-NEXT: .Lpcsection234: ; O1-NEXT: jne .LBB133_1 ; O1-NEXT: # %bb.2: # %atomicrmw.end ; O1-NEXT: movq $1, foo(%rip) @@ -6611,19 +6656,19 @@ ; O2-LABEL: atomic32_nand_acq_rel: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection215: +; O2-NEXT: .Lpcsection230: ; O2-NEXT: movl (%rdi), %eax ; O2-NEXT: .p2align 4, 0x90 ; O2-NEXT: .LBB133_1: # %atomicrmw.start ; O2-NEXT: # =>This Inner Loop Header: Depth=1 ; O2-NEXT: movl %eax, %ecx -; O2-NEXT: .Lpcsection216: +; O2-NEXT: .Lpcsection231: ; O2-NEXT: notl %ecx -; O2-NEXT: .Lpcsection217: +; O2-NEXT: .Lpcsection232: ; O2-NEXT: orl $-43, %ecx -; O2-NEXT: .Lpcsection218: +; O2-NEXT: .Lpcsection233: ; O2-NEXT: lock cmpxchgl %ecx, (%rdi) -; O2-NEXT: .Lpcsection219: +; O2-NEXT: .Lpcsection234: ; O2-NEXT: jne .LBB133_1 ; O2-NEXT: # %bb.2: # %atomicrmw.end ; O2-NEXT: movq $1, foo(%rip) @@ -6632,19 +6677,19 @@ ; O3-LABEL: atomic32_nand_acq_rel: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection215: +; O3-NEXT: .Lpcsection230: ; O3-NEXT: movl (%rdi), %eax ; O3-NEXT: .p2align 4, 0x90 ; O3-NEXT: .LBB133_1: # %atomicrmw.start ; O3-NEXT: # =>This Inner Loop Header: Depth=1 ; O3-NEXT: movl %eax, %ecx -; O3-NEXT: .Lpcsection216: +; O3-NEXT: .Lpcsection231: ; O3-NEXT: notl %ecx -; O3-NEXT: .Lpcsection217: +; O3-NEXT: .Lpcsection232: ; O3-NEXT: orl $-43, %ecx -; O3-NEXT: .Lpcsection218: +; O3-NEXT: .Lpcsection233: ; O3-NEXT: lock cmpxchgl %ecx, (%rdi) -; O3-NEXT: .Lpcsection219: +; O3-NEXT: .Lpcsection234: ; O3-NEXT: jne .LBB133_1 ; O3-NEXT: # %bb.2: # %atomicrmw.end ; O3-NEXT: movq $1, foo(%rip) @@ -6670,7 +6715,7 @@ ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax ; O1-NEXT: movl $42, %eax -; O1-NEXT: .Lpcsection220: +; O1-NEXT: .Lpcsection235: ; O1-NEXT: xchgl %eax, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -6679,7 +6724,7 @@ ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax ; O2-NEXT: movl $42, %eax -; O2-NEXT: .Lpcsection220: +; O2-NEXT: .Lpcsection235: ; O2-NEXT: xchgl %eax, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -6688,7 +6733,7 @@ ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax ; O3-NEXT: movl $42, %eax -; O3-NEXT: .Lpcsection220: +; O3-NEXT: .Lpcsection235: ; O3-NEXT: xchgl %eax, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -6711,7 +6756,7 @@ ; O1-LABEL: atomic32_add_seq_cst: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection221: +; O1-NEXT: .Lpcsection236: ; O1-NEXT: lock addl $42, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -6719,7 +6764,7 @@ ; O2-LABEL: atomic32_add_seq_cst: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection221: +; O2-NEXT: .Lpcsection236: ; O2-NEXT: lock addl $42, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -6727,7 +6772,7 @@ ; O3-LABEL: atomic32_add_seq_cst: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection221: +; O3-NEXT: .Lpcsection236: ; O3-NEXT: lock addl $42, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -6750,7 +6795,7 @@ ; O1-LABEL: atomic32_sub_seq_cst: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection222: +; O1-NEXT: .Lpcsection237: ; O1-NEXT: lock subl $42, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -6758,7 +6803,7 @@ ; O2-LABEL: atomic32_sub_seq_cst: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection222: +; O2-NEXT: .Lpcsection237: ; O2-NEXT: lock subl $42, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -6766,7 +6811,7 @@ ; O3-LABEL: atomic32_sub_seq_cst: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection222: +; O3-NEXT: .Lpcsection237: ; O3-NEXT: lock subl $42, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -6789,7 +6834,7 @@ ; O1-LABEL: atomic32_and_seq_cst: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection223: +; O1-NEXT: .Lpcsection238: ; O1-NEXT: lock andl $42, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -6797,7 +6842,7 @@ ; O2-LABEL: atomic32_and_seq_cst: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection223: +; O2-NEXT: .Lpcsection238: ; O2-NEXT: lock andl $42, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -6805,7 +6850,7 @@ ; O3-LABEL: atomic32_and_seq_cst: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection223: +; O3-NEXT: .Lpcsection238: ; O3-NEXT: lock andl $42, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -6828,7 +6873,7 @@ ; O1-LABEL: atomic32_or_seq_cst: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection224: +; O1-NEXT: .Lpcsection239: ; O1-NEXT: lock orl $42, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -6836,7 +6881,7 @@ ; O2-LABEL: atomic32_or_seq_cst: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection224: +; O2-NEXT: .Lpcsection239: ; O2-NEXT: lock orl $42, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -6844,7 +6889,7 @@ ; O3-LABEL: atomic32_or_seq_cst: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection224: +; O3-NEXT: .Lpcsection239: ; O3-NEXT: lock orl $42, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -6867,7 +6912,7 @@ ; O1-LABEL: atomic32_xor_seq_cst: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection225: +; O1-NEXT: .Lpcsection240: ; O1-NEXT: lock xorl $42, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -6875,7 +6920,7 @@ ; O2-LABEL: atomic32_xor_seq_cst: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection225: +; O2-NEXT: .Lpcsection240: ; O2-NEXT: lock xorl $42, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -6883,7 +6928,7 @@ ; O3-LABEL: atomic32_xor_seq_cst: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection225: +; O3-NEXT: .Lpcsection240: ; O3-NEXT: lock xorl $42, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -6928,19 +6973,19 @@ ; O1-LABEL: atomic32_nand_seq_cst: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection226: +; O1-NEXT: .Lpcsection241: ; O1-NEXT: movl (%rdi), %eax ; O1-NEXT: .p2align 4, 0x90 ; O1-NEXT: .LBB140_1: # %atomicrmw.start ; O1-NEXT: # =>This Inner Loop Header: Depth=1 ; O1-NEXT: movl %eax, %ecx -; O1-NEXT: .Lpcsection227: +; O1-NEXT: .Lpcsection242: ; O1-NEXT: notl %ecx -; O1-NEXT: .Lpcsection228: +; O1-NEXT: .Lpcsection243: ; O1-NEXT: orl $-43, %ecx -; O1-NEXT: .Lpcsection229: +; O1-NEXT: .Lpcsection244: ; O1-NEXT: lock cmpxchgl %ecx, (%rdi) -; O1-NEXT: .Lpcsection230: +; O1-NEXT: .Lpcsection245: ; O1-NEXT: jne .LBB140_1 ; O1-NEXT: # %bb.2: # %atomicrmw.end ; O1-NEXT: movq $1, foo(%rip) @@ -6949,19 +6994,19 @@ ; O2-LABEL: atomic32_nand_seq_cst: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection226: +; O2-NEXT: .Lpcsection241: ; O2-NEXT: movl (%rdi), %eax ; O2-NEXT: .p2align 4, 0x90 ; O2-NEXT: .LBB140_1: # %atomicrmw.start ; O2-NEXT: # =>This Inner Loop Header: Depth=1 ; O2-NEXT: movl %eax, %ecx -; O2-NEXT: .Lpcsection227: +; O2-NEXT: .Lpcsection242: ; O2-NEXT: notl %ecx -; O2-NEXT: .Lpcsection228: +; O2-NEXT: .Lpcsection243: ; O2-NEXT: orl $-43, %ecx -; O2-NEXT: .Lpcsection229: +; O2-NEXT: .Lpcsection244: ; O2-NEXT: lock cmpxchgl %ecx, (%rdi) -; O2-NEXT: .Lpcsection230: +; O2-NEXT: .Lpcsection245: ; O2-NEXT: jne .LBB140_1 ; O2-NEXT: # %bb.2: # %atomicrmw.end ; O2-NEXT: movq $1, foo(%rip) @@ -6970,19 +7015,19 @@ ; O3-LABEL: atomic32_nand_seq_cst: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection226: +; O3-NEXT: .Lpcsection241: ; O3-NEXT: movl (%rdi), %eax ; O3-NEXT: .p2align 4, 0x90 ; O3-NEXT: .LBB140_1: # %atomicrmw.start ; O3-NEXT: # =>This Inner Loop Header: Depth=1 ; O3-NEXT: movl %eax, %ecx -; O3-NEXT: .Lpcsection227: +; O3-NEXT: .Lpcsection242: ; O3-NEXT: notl %ecx -; O3-NEXT: .Lpcsection228: +; O3-NEXT: .Lpcsection243: ; O3-NEXT: orl $-43, %ecx -; O3-NEXT: .Lpcsection229: +; O3-NEXT: .Lpcsection244: ; O3-NEXT: lock cmpxchgl %ecx, (%rdi) -; O3-NEXT: .Lpcsection230: +; O3-NEXT: .Lpcsection245: ; O3-NEXT: jne .LBB140_1 ; O3-NEXT: # %bb.2: # %atomicrmw.end ; O3-NEXT: movq $1, foo(%rip) @@ -7020,14 +7065,17 @@ ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax ; O1-NEXT: movl $1, %ecx +; O1-NEXT: .Lpcsection246: ; O1-NEXT: movl $42, %eax -; O1-NEXT: .Lpcsection231: +; O1-NEXT: .Lpcsection247: ; O1-NEXT: lock cmpxchgl %ecx, (%rdi) +; O1-NEXT: .Lpcsection248: ; O1-NEXT: movl $42, %eax -; O1-NEXT: .Lpcsection232: +; O1-NEXT: .Lpcsection249: ; O1-NEXT: lock cmpxchgl %ecx, (%rdi) +; O1-NEXT: .Lpcsection250: ; O1-NEXT: movl $42, %eax -; O1-NEXT: .Lpcsection233: +; O1-NEXT: .Lpcsection251: ; O1-NEXT: lock cmpxchgl %ecx, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -7036,14 +7084,17 @@ ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax ; O2-NEXT: movl $1, %ecx +; O2-NEXT: .Lpcsection246: ; O2-NEXT: movl $42, %eax -; O2-NEXT: .Lpcsection231: +; O2-NEXT: .Lpcsection247: ; O2-NEXT: lock cmpxchgl %ecx, (%rdi) +; O2-NEXT: .Lpcsection248: ; O2-NEXT: movl $42, %eax -; O2-NEXT: .Lpcsection232: +; O2-NEXT: .Lpcsection249: ; O2-NEXT: lock cmpxchgl %ecx, (%rdi) +; O2-NEXT: .Lpcsection250: ; O2-NEXT: movl $42, %eax -; O2-NEXT: .Lpcsection233: +; O2-NEXT: .Lpcsection251: ; O2-NEXT: lock cmpxchgl %ecx, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -7052,14 +7103,17 @@ ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax ; O3-NEXT: movl $1, %ecx +; O3-NEXT: .Lpcsection246: ; O3-NEXT: movl $42, %eax -; O3-NEXT: .Lpcsection231: +; O3-NEXT: .Lpcsection247: ; O3-NEXT: lock cmpxchgl %ecx, (%rdi) +; O3-NEXT: .Lpcsection248: ; O3-NEXT: movl $42, %eax -; O3-NEXT: .Lpcsection232: +; O3-NEXT: .Lpcsection249: ; O3-NEXT: lock cmpxchgl %ecx, (%rdi) +; O3-NEXT: .Lpcsection250: ; O3-NEXT: movl $42, %eax -; O3-NEXT: .Lpcsection233: +; O3-NEXT: .Lpcsection251: ; O3-NEXT: lock cmpxchgl %ecx, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -7098,14 +7152,17 @@ ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax ; O1-NEXT: movl $1, %ecx +; O1-NEXT: .Lpcsection252: ; O1-NEXT: movl $42, %eax -; O1-NEXT: .Lpcsection234: +; O1-NEXT: .Lpcsection253: ; O1-NEXT: lock cmpxchgl %ecx, (%rdi) +; O1-NEXT: .Lpcsection254: ; O1-NEXT: movl $42, %eax -; O1-NEXT: .Lpcsection235: +; O1-NEXT: .Lpcsection255: ; O1-NEXT: lock cmpxchgl %ecx, (%rdi) +; O1-NEXT: .Lpcsection256: ; O1-NEXT: movl $42, %eax -; O1-NEXT: .Lpcsection236: +; O1-NEXT: .Lpcsection257: ; O1-NEXT: lock cmpxchgl %ecx, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -7114,14 +7171,17 @@ ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax ; O2-NEXT: movl $1, %ecx +; O2-NEXT: .Lpcsection252: ; O2-NEXT: movl $42, %eax -; O2-NEXT: .Lpcsection234: +; O2-NEXT: .Lpcsection253: ; O2-NEXT: lock cmpxchgl %ecx, (%rdi) +; O2-NEXT: .Lpcsection254: ; O2-NEXT: movl $42, %eax -; O2-NEXT: .Lpcsection235: +; O2-NEXT: .Lpcsection255: ; O2-NEXT: lock cmpxchgl %ecx, (%rdi) +; O2-NEXT: .Lpcsection256: ; O2-NEXT: movl $42, %eax -; O2-NEXT: .Lpcsection236: +; O2-NEXT: .Lpcsection257: ; O2-NEXT: lock cmpxchgl %ecx, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -7130,14 +7190,17 @@ ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax ; O3-NEXT: movl $1, %ecx +; O3-NEXT: .Lpcsection252: ; O3-NEXT: movl $42, %eax -; O3-NEXT: .Lpcsection234: +; O3-NEXT: .Lpcsection253: ; O3-NEXT: lock cmpxchgl %ecx, (%rdi) +; O3-NEXT: .Lpcsection254: ; O3-NEXT: movl $42, %eax -; O3-NEXT: .Lpcsection235: +; O3-NEXT: .Lpcsection255: ; O3-NEXT: lock cmpxchgl %ecx, (%rdi) +; O3-NEXT: .Lpcsection256: ; O3-NEXT: movl $42, %eax -; O3-NEXT: .Lpcsection236: +; O3-NEXT: .Lpcsection257: ; O3-NEXT: lock cmpxchgl %ecx, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -7176,14 +7239,17 @@ ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax ; O1-NEXT: movl $1, %ecx +; O1-NEXT: .Lpcsection258: ; O1-NEXT: movl $42, %eax -; O1-NEXT: .Lpcsection237: +; O1-NEXT: .Lpcsection259: ; O1-NEXT: lock cmpxchgl %ecx, (%rdi) +; O1-NEXT: .Lpcsection260: ; O1-NEXT: movl $42, %eax -; O1-NEXT: .Lpcsection238: +; O1-NEXT: .Lpcsection261: ; O1-NEXT: lock cmpxchgl %ecx, (%rdi) +; O1-NEXT: .Lpcsection262: ; O1-NEXT: movl $42, %eax -; O1-NEXT: .Lpcsection239: +; O1-NEXT: .Lpcsection263: ; O1-NEXT: lock cmpxchgl %ecx, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -7192,14 +7258,17 @@ ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax ; O2-NEXT: movl $1, %ecx +; O2-NEXT: .Lpcsection258: ; O2-NEXT: movl $42, %eax -; O2-NEXT: .Lpcsection237: +; O2-NEXT: .Lpcsection259: ; O2-NEXT: lock cmpxchgl %ecx, (%rdi) +; O2-NEXT: .Lpcsection260: ; O2-NEXT: movl $42, %eax -; O2-NEXT: .Lpcsection238: +; O2-NEXT: .Lpcsection261: ; O2-NEXT: lock cmpxchgl %ecx, (%rdi) +; O2-NEXT: .Lpcsection262: ; O2-NEXT: movl $42, %eax -; O2-NEXT: .Lpcsection239: +; O2-NEXT: .Lpcsection263: ; O2-NEXT: lock cmpxchgl %ecx, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -7208,14 +7277,17 @@ ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax ; O3-NEXT: movl $1, %ecx +; O3-NEXT: .Lpcsection258: ; O3-NEXT: movl $42, %eax -; O3-NEXT: .Lpcsection237: +; O3-NEXT: .Lpcsection259: ; O3-NEXT: lock cmpxchgl %ecx, (%rdi) +; O3-NEXT: .Lpcsection260: ; O3-NEXT: movl $42, %eax -; O3-NEXT: .Lpcsection238: +; O3-NEXT: .Lpcsection261: ; O3-NEXT: lock cmpxchgl %ecx, (%rdi) +; O3-NEXT: .Lpcsection262: ; O3-NEXT: movl $42, %eax -; O3-NEXT: .Lpcsection239: +; O3-NEXT: .Lpcsection263: ; O3-NEXT: lock cmpxchgl %ecx, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -7254,14 +7326,17 @@ ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax ; O1-NEXT: movl $1, %ecx +; O1-NEXT: .Lpcsection264: ; O1-NEXT: movl $42, %eax -; O1-NEXT: .Lpcsection240: +; O1-NEXT: .Lpcsection265: ; O1-NEXT: lock cmpxchgl %ecx, (%rdi) +; O1-NEXT: .Lpcsection266: ; O1-NEXT: movl $42, %eax -; O1-NEXT: .Lpcsection241: +; O1-NEXT: .Lpcsection267: ; O1-NEXT: lock cmpxchgl %ecx, (%rdi) +; O1-NEXT: .Lpcsection268: ; O1-NEXT: movl $42, %eax -; O1-NEXT: .Lpcsection242: +; O1-NEXT: .Lpcsection269: ; O1-NEXT: lock cmpxchgl %ecx, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -7270,14 +7345,17 @@ ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax ; O2-NEXT: movl $1, %ecx +; O2-NEXT: .Lpcsection264: ; O2-NEXT: movl $42, %eax -; O2-NEXT: .Lpcsection240: +; O2-NEXT: .Lpcsection265: ; O2-NEXT: lock cmpxchgl %ecx, (%rdi) +; O2-NEXT: .Lpcsection266: ; O2-NEXT: movl $42, %eax -; O2-NEXT: .Lpcsection241: +; O2-NEXT: .Lpcsection267: ; O2-NEXT: lock cmpxchgl %ecx, (%rdi) +; O2-NEXT: .Lpcsection268: ; O2-NEXT: movl $42, %eax -; O2-NEXT: .Lpcsection242: +; O2-NEXT: .Lpcsection269: ; O2-NEXT: lock cmpxchgl %ecx, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -7286,14 +7364,17 @@ ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax ; O3-NEXT: movl $1, %ecx +; O3-NEXT: .Lpcsection264: ; O3-NEXT: movl $42, %eax -; O3-NEXT: .Lpcsection240: +; O3-NEXT: .Lpcsection265: ; O3-NEXT: lock cmpxchgl %ecx, (%rdi) +; O3-NEXT: .Lpcsection266: ; O3-NEXT: movl $42, %eax -; O3-NEXT: .Lpcsection241: +; O3-NEXT: .Lpcsection267: ; O3-NEXT: lock cmpxchgl %ecx, (%rdi) +; O3-NEXT: .Lpcsection268: ; O3-NEXT: movl $42, %eax -; O3-NEXT: .Lpcsection242: +; O3-NEXT: .Lpcsection269: ; O3-NEXT: lock cmpxchgl %ecx, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -7332,14 +7413,17 @@ ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax ; O1-NEXT: movl $1, %ecx +; O1-NEXT: .Lpcsection270: ; O1-NEXT: movl $42, %eax -; O1-NEXT: .Lpcsection243: +; O1-NEXT: .Lpcsection271: ; O1-NEXT: lock cmpxchgl %ecx, (%rdi) +; O1-NEXT: .Lpcsection272: ; O1-NEXT: movl $42, %eax -; O1-NEXT: .Lpcsection244: +; O1-NEXT: .Lpcsection273: ; O1-NEXT: lock cmpxchgl %ecx, (%rdi) +; O1-NEXT: .Lpcsection274: ; O1-NEXT: movl $42, %eax -; O1-NEXT: .Lpcsection245: +; O1-NEXT: .Lpcsection275: ; O1-NEXT: lock cmpxchgl %ecx, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -7348,14 +7432,17 @@ ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax ; O2-NEXT: movl $1, %ecx +; O2-NEXT: .Lpcsection270: ; O2-NEXT: movl $42, %eax -; O2-NEXT: .Lpcsection243: +; O2-NEXT: .Lpcsection271: ; O2-NEXT: lock cmpxchgl %ecx, (%rdi) +; O2-NEXT: .Lpcsection272: ; O2-NEXT: movl $42, %eax -; O2-NEXT: .Lpcsection244: +; O2-NEXT: .Lpcsection273: ; O2-NEXT: lock cmpxchgl %ecx, (%rdi) +; O2-NEXT: .Lpcsection274: ; O2-NEXT: movl $42, %eax -; O2-NEXT: .Lpcsection245: +; O2-NEXT: .Lpcsection275: ; O2-NEXT: lock cmpxchgl %ecx, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -7364,14 +7451,17 @@ ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax ; O3-NEXT: movl $1, %ecx +; O3-NEXT: .Lpcsection270: ; O3-NEXT: movl $42, %eax -; O3-NEXT: .Lpcsection243: +; O3-NEXT: .Lpcsection271: ; O3-NEXT: lock cmpxchgl %ecx, (%rdi) +; O3-NEXT: .Lpcsection272: ; O3-NEXT: movl $42, %eax -; O3-NEXT: .Lpcsection244: +; O3-NEXT: .Lpcsection273: ; O3-NEXT: lock cmpxchgl %ecx, (%rdi) +; O3-NEXT: .Lpcsection274: ; O3-NEXT: movl $42, %eax -; O3-NEXT: .Lpcsection245: +; O3-NEXT: .Lpcsection275: ; O3-NEXT: lock cmpxchgl %ecx, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -7396,7 +7486,7 @@ ; O1-LABEL: atomic64_load_unordered: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection246: +; O1-NEXT: .Lpcsection276: ; O1-NEXT: movq (%rdi), %rax ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -7404,7 +7494,7 @@ ; O2-LABEL: atomic64_load_unordered: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection246: +; O2-NEXT: .Lpcsection276: ; O2-NEXT: movq (%rdi), %rax ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -7412,7 +7502,7 @@ ; O3-LABEL: atomic64_load_unordered: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection246: +; O3-NEXT: .Lpcsection276: ; O3-NEXT: movq (%rdi), %rax ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -7435,7 +7525,7 @@ ; O1-LABEL: atomic64_load_monotonic: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection247: +; O1-NEXT: .Lpcsection277: ; O1-NEXT: movq (%rdi), %rax ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -7443,7 +7533,7 @@ ; O2-LABEL: atomic64_load_monotonic: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection247: +; O2-NEXT: .Lpcsection277: ; O2-NEXT: movq (%rdi), %rax ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -7451,7 +7541,7 @@ ; O3-LABEL: atomic64_load_monotonic: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection247: +; O3-NEXT: .Lpcsection277: ; O3-NEXT: movq (%rdi), %rax ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -7474,7 +7564,7 @@ ; O1-LABEL: atomic64_load_acquire: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection248: +; O1-NEXT: .Lpcsection278: ; O1-NEXT: movq (%rdi), %rax ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -7482,7 +7572,7 @@ ; O2-LABEL: atomic64_load_acquire: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection248: +; O2-NEXT: .Lpcsection278: ; O2-NEXT: movq (%rdi), %rax ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -7490,7 +7580,7 @@ ; O3-LABEL: atomic64_load_acquire: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection248: +; O3-NEXT: .Lpcsection278: ; O3-NEXT: movq (%rdi), %rax ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -7513,7 +7603,7 @@ ; O1-LABEL: atomic64_load_seq_cst: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection249: +; O1-NEXT: .Lpcsection279: ; O1-NEXT: movq (%rdi), %rax ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -7521,7 +7611,7 @@ ; O2-LABEL: atomic64_load_seq_cst: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection249: +; O2-NEXT: .Lpcsection279: ; O2-NEXT: movq (%rdi), %rax ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -7529,7 +7619,7 @@ ; O3-LABEL: atomic64_load_seq_cst: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection249: +; O3-NEXT: .Lpcsection279: ; O3-NEXT: movq (%rdi), %rax ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -7552,7 +7642,7 @@ ; O1-LABEL: atomic64_load_seq_cst_ptr_ty: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection250: +; O1-NEXT: .Lpcsection280: ; O1-NEXT: movq (%rdi), %rax ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -7560,7 +7650,7 @@ ; O2-LABEL: atomic64_load_seq_cst_ptr_ty: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection250: +; O2-NEXT: .Lpcsection280: ; O2-NEXT: movq (%rdi), %rax ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -7568,7 +7658,7 @@ ; O3-LABEL: atomic64_load_seq_cst_ptr_ty: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection250: +; O3-NEXT: .Lpcsection280: ; O3-NEXT: movq (%rdi), %rax ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -7591,7 +7681,7 @@ ; O1-LABEL: atomic64_store_unordered: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection251: +; O1-NEXT: .Lpcsection281: ; O1-NEXT: movq $42, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -7599,7 +7689,7 @@ ; O2-LABEL: atomic64_store_unordered: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection251: +; O2-NEXT: .Lpcsection281: ; O2-NEXT: movq $42, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -7607,7 +7697,7 @@ ; O3-LABEL: atomic64_store_unordered: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection251: +; O3-NEXT: .Lpcsection281: ; O3-NEXT: movq $42, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -7630,7 +7720,7 @@ ; O1-LABEL: atomic64_store_monotonic: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection252: +; O1-NEXT: .Lpcsection282: ; O1-NEXT: movq $42, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -7638,7 +7728,7 @@ ; O2-LABEL: atomic64_store_monotonic: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection252: +; O2-NEXT: .Lpcsection282: ; O2-NEXT: movq $42, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -7646,7 +7736,7 @@ ; O3-LABEL: atomic64_store_monotonic: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection252: +; O3-NEXT: .Lpcsection282: ; O3-NEXT: movq $42, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -7669,7 +7759,7 @@ ; O1-LABEL: atomic64_store_release: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection253: +; O1-NEXT: .Lpcsection283: ; O1-NEXT: movq $42, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -7677,7 +7767,7 @@ ; O2-LABEL: atomic64_store_release: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection253: +; O2-NEXT: .Lpcsection283: ; O2-NEXT: movq $42, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -7685,7 +7775,7 @@ ; O3-LABEL: atomic64_store_release: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection253: +; O3-NEXT: .Lpcsection283: ; O3-NEXT: movq $42, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -7710,7 +7800,7 @@ ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax ; O1-NEXT: movl $42, %eax -; O1-NEXT: .Lpcsection254: +; O1-NEXT: .Lpcsection284: ; O1-NEXT: xchgq %rax, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -7719,7 +7809,7 @@ ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax ; O2-NEXT: movl $42, %eax -; O2-NEXT: .Lpcsection254: +; O2-NEXT: .Lpcsection284: ; O2-NEXT: xchgq %rax, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -7728,7 +7818,7 @@ ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax ; O3-NEXT: movl $42, %eax -; O3-NEXT: .Lpcsection254: +; O3-NEXT: .Lpcsection284: ; O3-NEXT: xchgq %rax, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -7751,7 +7841,7 @@ ; O1-LABEL: atomic64_store_seq_cst_ptr_ty: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection255: +; O1-NEXT: .Lpcsection285: ; O1-NEXT: xchgq %rsi, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -7759,7 +7849,7 @@ ; O2-LABEL: atomic64_store_seq_cst_ptr_ty: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection255: +; O2-NEXT: .Lpcsection285: ; O2-NEXT: xchgq %rsi, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -7767,7 +7857,7 @@ ; O3-LABEL: atomic64_store_seq_cst_ptr_ty: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection255: +; O3-NEXT: .Lpcsection285: ; O3-NEXT: xchgq %rsi, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -7792,7 +7882,7 @@ ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax ; O1-NEXT: movl $42, %eax -; O1-NEXT: .Lpcsection256: +; O1-NEXT: .Lpcsection286: ; O1-NEXT: xchgq %rax, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -7801,7 +7891,7 @@ ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax ; O2-NEXT: movl $42, %eax -; O2-NEXT: .Lpcsection256: +; O2-NEXT: .Lpcsection286: ; O2-NEXT: xchgq %rax, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -7810,7 +7900,7 @@ ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax ; O3-NEXT: movl $42, %eax -; O3-NEXT: .Lpcsection256: +; O3-NEXT: .Lpcsection286: ; O3-NEXT: xchgq %rax, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -7833,7 +7923,7 @@ ; O1-LABEL: atomic64_add_monotonic: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection257: +; O1-NEXT: .Lpcsection287: ; O1-NEXT: lock addq $42, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -7841,7 +7931,7 @@ ; O2-LABEL: atomic64_add_monotonic: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection257: +; O2-NEXT: .Lpcsection287: ; O2-NEXT: lock addq $42, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -7849,7 +7939,7 @@ ; O3-LABEL: atomic64_add_monotonic: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection257: +; O3-NEXT: .Lpcsection287: ; O3-NEXT: lock addq $42, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -7872,7 +7962,7 @@ ; O1-LABEL: atomic64_sub_monotonic: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection258: +; O1-NEXT: .Lpcsection288: ; O1-NEXT: lock subq $42, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -7880,7 +7970,7 @@ ; O2-LABEL: atomic64_sub_monotonic: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection258: +; O2-NEXT: .Lpcsection288: ; O2-NEXT: lock subq $42, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -7888,7 +7978,7 @@ ; O3-LABEL: atomic64_sub_monotonic: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection258: +; O3-NEXT: .Lpcsection288: ; O3-NEXT: lock subq $42, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -7911,7 +8001,7 @@ ; O1-LABEL: atomic64_and_monotonic: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection259: +; O1-NEXT: .Lpcsection289: ; O1-NEXT: lock andq $42, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -7919,7 +8009,7 @@ ; O2-LABEL: atomic64_and_monotonic: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection259: +; O2-NEXT: .Lpcsection289: ; O2-NEXT: lock andq $42, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -7927,7 +8017,7 @@ ; O3-LABEL: atomic64_and_monotonic: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection259: +; O3-NEXT: .Lpcsection289: ; O3-NEXT: lock andq $42, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -7950,7 +8040,7 @@ ; O1-LABEL: atomic64_or_monotonic: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection260: +; O1-NEXT: .Lpcsection290: ; O1-NEXT: lock orq $42, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -7958,7 +8048,7 @@ ; O2-LABEL: atomic64_or_monotonic: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection260: +; O2-NEXT: .Lpcsection290: ; O2-NEXT: lock orq $42, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -7966,7 +8056,7 @@ ; O3-LABEL: atomic64_or_monotonic: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection260: +; O3-NEXT: .Lpcsection290: ; O3-NEXT: lock orq $42, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -7989,7 +8079,7 @@ ; O1-LABEL: atomic64_xor_monotonic: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection261: +; O1-NEXT: .Lpcsection291: ; O1-NEXT: lock xorq $42, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -7997,7 +8087,7 @@ ; O2-LABEL: atomic64_xor_monotonic: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection261: +; O2-NEXT: .Lpcsection291: ; O2-NEXT: lock xorq $42, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -8005,7 +8095,7 @@ ; O3-LABEL: atomic64_xor_monotonic: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection261: +; O3-NEXT: .Lpcsection291: ; O3-NEXT: lock xorq $42, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -8053,19 +8143,19 @@ ; O1-LABEL: atomic64_nand_monotonic: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection262: +; O1-NEXT: .Lpcsection292: ; O1-NEXT: movq (%rdi), %rax ; O1-NEXT: .p2align 4, 0x90 ; O1-NEXT: .LBB162_1: # %atomicrmw.start ; O1-NEXT: # =>This Inner Loop Header: Depth=1 ; O1-NEXT: movl %eax, %ecx -; O1-NEXT: .Lpcsection263: +; O1-NEXT: .Lpcsection293: ; O1-NEXT: notl %ecx -; O1-NEXT: .Lpcsection264: +; O1-NEXT: .Lpcsection294: ; O1-NEXT: orq $-43, %rcx -; O1-NEXT: .Lpcsection265: +; O1-NEXT: .Lpcsection295: ; O1-NEXT: lock cmpxchgq %rcx, (%rdi) -; O1-NEXT: .Lpcsection266: +; O1-NEXT: .Lpcsection296: ; O1-NEXT: jne .LBB162_1 ; O1-NEXT: # %bb.2: # %atomicrmw.end ; O1-NEXT: movq $1, foo(%rip) @@ -8074,19 +8164,19 @@ ; O2-LABEL: atomic64_nand_monotonic: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection262: +; O2-NEXT: .Lpcsection292: ; O2-NEXT: movq (%rdi), %rax ; O2-NEXT: .p2align 4, 0x90 ; O2-NEXT: .LBB162_1: # %atomicrmw.start ; O2-NEXT: # =>This Inner Loop Header: Depth=1 ; O2-NEXT: movl %eax, %ecx -; O2-NEXT: .Lpcsection263: +; O2-NEXT: .Lpcsection293: ; O2-NEXT: notl %ecx -; O2-NEXT: .Lpcsection264: +; O2-NEXT: .Lpcsection294: ; O2-NEXT: orq $-43, %rcx -; O2-NEXT: .Lpcsection265: +; O2-NEXT: .Lpcsection295: ; O2-NEXT: lock cmpxchgq %rcx, (%rdi) -; O2-NEXT: .Lpcsection266: +; O2-NEXT: .Lpcsection296: ; O2-NEXT: jne .LBB162_1 ; O2-NEXT: # %bb.2: # %atomicrmw.end ; O2-NEXT: movq $1, foo(%rip) @@ -8095,19 +8185,19 @@ ; O3-LABEL: atomic64_nand_monotonic: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection262: +; O3-NEXT: .Lpcsection292: ; O3-NEXT: movq (%rdi), %rax ; O3-NEXT: .p2align 4, 0x90 ; O3-NEXT: .LBB162_1: # %atomicrmw.start ; O3-NEXT: # =>This Inner Loop Header: Depth=1 ; O3-NEXT: movl %eax, %ecx -; O3-NEXT: .Lpcsection263: +; O3-NEXT: .Lpcsection293: ; O3-NEXT: notl %ecx -; O3-NEXT: .Lpcsection264: +; O3-NEXT: .Lpcsection294: ; O3-NEXT: orq $-43, %rcx -; O3-NEXT: .Lpcsection265: +; O3-NEXT: .Lpcsection295: ; O3-NEXT: lock cmpxchgq %rcx, (%rdi) -; O3-NEXT: .Lpcsection266: +; O3-NEXT: .Lpcsection296: ; O3-NEXT: jne .LBB162_1 ; O3-NEXT: # %bb.2: # %atomicrmw.end ; O3-NEXT: movq $1, foo(%rip) @@ -8133,7 +8223,7 @@ ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax ; O1-NEXT: movl $42, %eax -; O1-NEXT: .Lpcsection267: +; O1-NEXT: .Lpcsection297: ; O1-NEXT: xchgq %rax, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -8142,7 +8232,7 @@ ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax ; O2-NEXT: movl $42, %eax -; O2-NEXT: .Lpcsection267: +; O2-NEXT: .Lpcsection297: ; O2-NEXT: xchgq %rax, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -8151,7 +8241,7 @@ ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax ; O3-NEXT: movl $42, %eax -; O3-NEXT: .Lpcsection267: +; O3-NEXT: .Lpcsection297: ; O3-NEXT: xchgq %rax, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -8174,7 +8264,7 @@ ; O1-LABEL: atomic64_add_acquire: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection268: +; O1-NEXT: .Lpcsection298: ; O1-NEXT: lock addq $42, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -8182,7 +8272,7 @@ ; O2-LABEL: atomic64_add_acquire: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection268: +; O2-NEXT: .Lpcsection298: ; O2-NEXT: lock addq $42, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -8190,7 +8280,7 @@ ; O3-LABEL: atomic64_add_acquire: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection268: +; O3-NEXT: .Lpcsection298: ; O3-NEXT: lock addq $42, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -8213,7 +8303,7 @@ ; O1-LABEL: atomic64_sub_acquire: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection269: +; O1-NEXT: .Lpcsection299: ; O1-NEXT: lock subq $42, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -8221,7 +8311,7 @@ ; O2-LABEL: atomic64_sub_acquire: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection269: +; O2-NEXT: .Lpcsection299: ; O2-NEXT: lock subq $42, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -8229,7 +8319,7 @@ ; O3-LABEL: atomic64_sub_acquire: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection269: +; O3-NEXT: .Lpcsection299: ; O3-NEXT: lock subq $42, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -8252,7 +8342,7 @@ ; O1-LABEL: atomic64_and_acquire: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection270: +; O1-NEXT: .Lpcsection300: ; O1-NEXT: lock andq $42, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -8260,7 +8350,7 @@ ; O2-LABEL: atomic64_and_acquire: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection270: +; O2-NEXT: .Lpcsection300: ; O2-NEXT: lock andq $42, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -8268,7 +8358,7 @@ ; O3-LABEL: atomic64_and_acquire: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection270: +; O3-NEXT: .Lpcsection300: ; O3-NEXT: lock andq $42, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -8291,7 +8381,7 @@ ; O1-LABEL: atomic64_or_acquire: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection271: +; O1-NEXT: .Lpcsection301: ; O1-NEXT: lock orq $42, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -8299,7 +8389,7 @@ ; O2-LABEL: atomic64_or_acquire: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection271: +; O2-NEXT: .Lpcsection301: ; O2-NEXT: lock orq $42, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -8307,7 +8397,7 @@ ; O3-LABEL: atomic64_or_acquire: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection271: +; O3-NEXT: .Lpcsection301: ; O3-NEXT: lock orq $42, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -8330,7 +8420,7 @@ ; O1-LABEL: atomic64_xor_acquire: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection272: +; O1-NEXT: .Lpcsection302: ; O1-NEXT: lock xorq $42, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -8338,7 +8428,7 @@ ; O2-LABEL: atomic64_xor_acquire: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection272: +; O2-NEXT: .Lpcsection302: ; O2-NEXT: lock xorq $42, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -8346,7 +8436,7 @@ ; O3-LABEL: atomic64_xor_acquire: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection272: +; O3-NEXT: .Lpcsection302: ; O3-NEXT: lock xorq $42, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -8394,19 +8484,19 @@ ; O1-LABEL: atomic64_nand_acquire: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection273: +; O1-NEXT: .Lpcsection303: ; O1-NEXT: movq (%rdi), %rax ; O1-NEXT: .p2align 4, 0x90 ; O1-NEXT: .LBB169_1: # %atomicrmw.start ; O1-NEXT: # =>This Inner Loop Header: Depth=1 ; O1-NEXT: movl %eax, %ecx -; O1-NEXT: .Lpcsection274: +; O1-NEXT: .Lpcsection304: ; O1-NEXT: notl %ecx -; O1-NEXT: .Lpcsection275: +; O1-NEXT: .Lpcsection305: ; O1-NEXT: orq $-43, %rcx -; O1-NEXT: .Lpcsection276: +; O1-NEXT: .Lpcsection306: ; O1-NEXT: lock cmpxchgq %rcx, (%rdi) -; O1-NEXT: .Lpcsection277: +; O1-NEXT: .Lpcsection307: ; O1-NEXT: jne .LBB169_1 ; O1-NEXT: # %bb.2: # %atomicrmw.end ; O1-NEXT: movq $1, foo(%rip) @@ -8415,19 +8505,19 @@ ; O2-LABEL: atomic64_nand_acquire: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection273: +; O2-NEXT: .Lpcsection303: ; O2-NEXT: movq (%rdi), %rax ; O2-NEXT: .p2align 4, 0x90 ; O2-NEXT: .LBB169_1: # %atomicrmw.start ; O2-NEXT: # =>This Inner Loop Header: Depth=1 ; O2-NEXT: movl %eax, %ecx -; O2-NEXT: .Lpcsection274: +; O2-NEXT: .Lpcsection304: ; O2-NEXT: notl %ecx -; O2-NEXT: .Lpcsection275: +; O2-NEXT: .Lpcsection305: ; O2-NEXT: orq $-43, %rcx -; O2-NEXT: .Lpcsection276: +; O2-NEXT: .Lpcsection306: ; O2-NEXT: lock cmpxchgq %rcx, (%rdi) -; O2-NEXT: .Lpcsection277: +; O2-NEXT: .Lpcsection307: ; O2-NEXT: jne .LBB169_1 ; O2-NEXT: # %bb.2: # %atomicrmw.end ; O2-NEXT: movq $1, foo(%rip) @@ -8436,19 +8526,19 @@ ; O3-LABEL: atomic64_nand_acquire: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection273: +; O3-NEXT: .Lpcsection303: ; O3-NEXT: movq (%rdi), %rax ; O3-NEXT: .p2align 4, 0x90 ; O3-NEXT: .LBB169_1: # %atomicrmw.start ; O3-NEXT: # =>This Inner Loop Header: Depth=1 ; O3-NEXT: movl %eax, %ecx -; O3-NEXT: .Lpcsection274: +; O3-NEXT: .Lpcsection304: ; O3-NEXT: notl %ecx -; O3-NEXT: .Lpcsection275: +; O3-NEXT: .Lpcsection305: ; O3-NEXT: orq $-43, %rcx -; O3-NEXT: .Lpcsection276: +; O3-NEXT: .Lpcsection306: ; O3-NEXT: lock cmpxchgq %rcx, (%rdi) -; O3-NEXT: .Lpcsection277: +; O3-NEXT: .Lpcsection307: ; O3-NEXT: jne .LBB169_1 ; O3-NEXT: # %bb.2: # %atomicrmw.end ; O3-NEXT: movq $1, foo(%rip) @@ -8474,7 +8564,7 @@ ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax ; O1-NEXT: movl $42, %eax -; O1-NEXT: .Lpcsection278: +; O1-NEXT: .Lpcsection308: ; O1-NEXT: xchgq %rax, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -8483,7 +8573,7 @@ ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax ; O2-NEXT: movl $42, %eax -; O2-NEXT: .Lpcsection278: +; O2-NEXT: .Lpcsection308: ; O2-NEXT: xchgq %rax, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -8492,7 +8582,7 @@ ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax ; O3-NEXT: movl $42, %eax -; O3-NEXT: .Lpcsection278: +; O3-NEXT: .Lpcsection308: ; O3-NEXT: xchgq %rax, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -8515,7 +8605,7 @@ ; O1-LABEL: atomic64_add_release: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection279: +; O1-NEXT: .Lpcsection309: ; O1-NEXT: lock addq $42, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -8523,7 +8613,7 @@ ; O2-LABEL: atomic64_add_release: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection279: +; O2-NEXT: .Lpcsection309: ; O2-NEXT: lock addq $42, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -8531,7 +8621,7 @@ ; O3-LABEL: atomic64_add_release: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection279: +; O3-NEXT: .Lpcsection309: ; O3-NEXT: lock addq $42, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -8554,7 +8644,7 @@ ; O1-LABEL: atomic64_sub_release: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection280: +; O1-NEXT: .Lpcsection310: ; O1-NEXT: lock subq $42, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -8562,7 +8652,7 @@ ; O2-LABEL: atomic64_sub_release: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection280: +; O2-NEXT: .Lpcsection310: ; O2-NEXT: lock subq $42, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -8570,7 +8660,7 @@ ; O3-LABEL: atomic64_sub_release: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection280: +; O3-NEXT: .Lpcsection310: ; O3-NEXT: lock subq $42, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -8593,7 +8683,7 @@ ; O1-LABEL: atomic64_and_release: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection281: +; O1-NEXT: .Lpcsection311: ; O1-NEXT: lock andq $42, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -8601,7 +8691,7 @@ ; O2-LABEL: atomic64_and_release: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection281: +; O2-NEXT: .Lpcsection311: ; O2-NEXT: lock andq $42, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -8609,7 +8699,7 @@ ; O3-LABEL: atomic64_and_release: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection281: +; O3-NEXT: .Lpcsection311: ; O3-NEXT: lock andq $42, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -8632,7 +8722,7 @@ ; O1-LABEL: atomic64_or_release: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection282: +; O1-NEXT: .Lpcsection312: ; O1-NEXT: lock orq $42, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -8640,7 +8730,7 @@ ; O2-LABEL: atomic64_or_release: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection282: +; O2-NEXT: .Lpcsection312: ; O2-NEXT: lock orq $42, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -8648,7 +8738,7 @@ ; O3-LABEL: atomic64_or_release: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection282: +; O3-NEXT: .Lpcsection312: ; O3-NEXT: lock orq $42, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -8671,7 +8761,7 @@ ; O1-LABEL: atomic64_xor_release: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection283: +; O1-NEXT: .Lpcsection313: ; O1-NEXT: lock xorq $42, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -8679,7 +8769,7 @@ ; O2-LABEL: atomic64_xor_release: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection283: +; O2-NEXT: .Lpcsection313: ; O2-NEXT: lock xorq $42, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -8687,7 +8777,7 @@ ; O3-LABEL: atomic64_xor_release: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection283: +; O3-NEXT: .Lpcsection313: ; O3-NEXT: lock xorq $42, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -8735,19 +8825,19 @@ ; O1-LABEL: atomic64_nand_release: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection284: +; O1-NEXT: .Lpcsection314: ; O1-NEXT: movq (%rdi), %rax ; O1-NEXT: .p2align 4, 0x90 ; O1-NEXT: .LBB176_1: # %atomicrmw.start ; O1-NEXT: # =>This Inner Loop Header: Depth=1 ; O1-NEXT: movl %eax, %ecx -; O1-NEXT: .Lpcsection285: +; O1-NEXT: .Lpcsection315: ; O1-NEXT: notl %ecx -; O1-NEXT: .Lpcsection286: +; O1-NEXT: .Lpcsection316: ; O1-NEXT: orq $-43, %rcx -; O1-NEXT: .Lpcsection287: +; O1-NEXT: .Lpcsection317: ; O1-NEXT: lock cmpxchgq %rcx, (%rdi) -; O1-NEXT: .Lpcsection288: +; O1-NEXT: .Lpcsection318: ; O1-NEXT: jne .LBB176_1 ; O1-NEXT: # %bb.2: # %atomicrmw.end ; O1-NEXT: movq $1, foo(%rip) @@ -8756,19 +8846,19 @@ ; O2-LABEL: atomic64_nand_release: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection284: +; O2-NEXT: .Lpcsection314: ; O2-NEXT: movq (%rdi), %rax ; O2-NEXT: .p2align 4, 0x90 ; O2-NEXT: .LBB176_1: # %atomicrmw.start ; O2-NEXT: # =>This Inner Loop Header: Depth=1 ; O2-NEXT: movl %eax, %ecx -; O2-NEXT: .Lpcsection285: +; O2-NEXT: .Lpcsection315: ; O2-NEXT: notl %ecx -; O2-NEXT: .Lpcsection286: +; O2-NEXT: .Lpcsection316: ; O2-NEXT: orq $-43, %rcx -; O2-NEXT: .Lpcsection287: +; O2-NEXT: .Lpcsection317: ; O2-NEXT: lock cmpxchgq %rcx, (%rdi) -; O2-NEXT: .Lpcsection288: +; O2-NEXT: .Lpcsection318: ; O2-NEXT: jne .LBB176_1 ; O2-NEXT: # %bb.2: # %atomicrmw.end ; O2-NEXT: movq $1, foo(%rip) @@ -8777,19 +8867,19 @@ ; O3-LABEL: atomic64_nand_release: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection284: +; O3-NEXT: .Lpcsection314: ; O3-NEXT: movq (%rdi), %rax ; O3-NEXT: .p2align 4, 0x90 ; O3-NEXT: .LBB176_1: # %atomicrmw.start ; O3-NEXT: # =>This Inner Loop Header: Depth=1 ; O3-NEXT: movl %eax, %ecx -; O3-NEXT: .Lpcsection285: +; O3-NEXT: .Lpcsection315: ; O3-NEXT: notl %ecx -; O3-NEXT: .Lpcsection286: +; O3-NEXT: .Lpcsection316: ; O3-NEXT: orq $-43, %rcx -; O3-NEXT: .Lpcsection287: +; O3-NEXT: .Lpcsection317: ; O3-NEXT: lock cmpxchgq %rcx, (%rdi) -; O3-NEXT: .Lpcsection288: +; O3-NEXT: .Lpcsection318: ; O3-NEXT: jne .LBB176_1 ; O3-NEXT: # %bb.2: # %atomicrmw.end ; O3-NEXT: movq $1, foo(%rip) @@ -8815,7 +8905,7 @@ ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax ; O1-NEXT: movl $42, %eax -; O1-NEXT: .Lpcsection289: +; O1-NEXT: .Lpcsection319: ; O1-NEXT: xchgq %rax, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -8824,7 +8914,7 @@ ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax ; O2-NEXT: movl $42, %eax -; O2-NEXT: .Lpcsection289: +; O2-NEXT: .Lpcsection319: ; O2-NEXT: xchgq %rax, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -8833,7 +8923,7 @@ ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax ; O3-NEXT: movl $42, %eax -; O3-NEXT: .Lpcsection289: +; O3-NEXT: .Lpcsection319: ; O3-NEXT: xchgq %rax, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -8856,7 +8946,7 @@ ; O1-LABEL: atomic64_add_acq_rel: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection290: +; O1-NEXT: .Lpcsection320: ; O1-NEXT: lock addq $42, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -8864,7 +8954,7 @@ ; O2-LABEL: atomic64_add_acq_rel: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection290: +; O2-NEXT: .Lpcsection320: ; O2-NEXT: lock addq $42, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -8872,7 +8962,7 @@ ; O3-LABEL: atomic64_add_acq_rel: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection290: +; O3-NEXT: .Lpcsection320: ; O3-NEXT: lock addq $42, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -8895,7 +8985,7 @@ ; O1-LABEL: atomic64_sub_acq_rel: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection291: +; O1-NEXT: .Lpcsection321: ; O1-NEXT: lock subq $42, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -8903,7 +8993,7 @@ ; O2-LABEL: atomic64_sub_acq_rel: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection291: +; O2-NEXT: .Lpcsection321: ; O2-NEXT: lock subq $42, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -8911,7 +9001,7 @@ ; O3-LABEL: atomic64_sub_acq_rel: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection291: +; O3-NEXT: .Lpcsection321: ; O3-NEXT: lock subq $42, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -8934,7 +9024,7 @@ ; O1-LABEL: atomic64_and_acq_rel: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection292: +; O1-NEXT: .Lpcsection322: ; O1-NEXT: lock andq $42, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -8942,7 +9032,7 @@ ; O2-LABEL: atomic64_and_acq_rel: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection292: +; O2-NEXT: .Lpcsection322: ; O2-NEXT: lock andq $42, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -8950,7 +9040,7 @@ ; O3-LABEL: atomic64_and_acq_rel: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection292: +; O3-NEXT: .Lpcsection322: ; O3-NEXT: lock andq $42, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -8973,7 +9063,7 @@ ; O1-LABEL: atomic64_or_acq_rel: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection293: +; O1-NEXT: .Lpcsection323: ; O1-NEXT: lock orq $42, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -8981,7 +9071,7 @@ ; O2-LABEL: atomic64_or_acq_rel: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection293: +; O2-NEXT: .Lpcsection323: ; O2-NEXT: lock orq $42, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -8989,7 +9079,7 @@ ; O3-LABEL: atomic64_or_acq_rel: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection293: +; O3-NEXT: .Lpcsection323: ; O3-NEXT: lock orq $42, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -9012,7 +9102,7 @@ ; O1-LABEL: atomic64_xor_acq_rel: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection294: +; O1-NEXT: .Lpcsection324: ; O1-NEXT: lock xorq $42, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -9020,7 +9110,7 @@ ; O2-LABEL: atomic64_xor_acq_rel: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection294: +; O2-NEXT: .Lpcsection324: ; O2-NEXT: lock xorq $42, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -9028,7 +9118,7 @@ ; O3-LABEL: atomic64_xor_acq_rel: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection294: +; O3-NEXT: .Lpcsection324: ; O3-NEXT: lock xorq $42, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -9076,19 +9166,19 @@ ; O1-LABEL: atomic64_nand_acq_rel: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection295: +; O1-NEXT: .Lpcsection325: ; O1-NEXT: movq (%rdi), %rax ; O1-NEXT: .p2align 4, 0x90 ; O1-NEXT: .LBB183_1: # %atomicrmw.start ; O1-NEXT: # =>This Inner Loop Header: Depth=1 ; O1-NEXT: movl %eax, %ecx -; O1-NEXT: .Lpcsection296: +; O1-NEXT: .Lpcsection326: ; O1-NEXT: notl %ecx -; O1-NEXT: .Lpcsection297: +; O1-NEXT: .Lpcsection327: ; O1-NEXT: orq $-43, %rcx -; O1-NEXT: .Lpcsection298: +; O1-NEXT: .Lpcsection328: ; O1-NEXT: lock cmpxchgq %rcx, (%rdi) -; O1-NEXT: .Lpcsection299: +; O1-NEXT: .Lpcsection329: ; O1-NEXT: jne .LBB183_1 ; O1-NEXT: # %bb.2: # %atomicrmw.end ; O1-NEXT: movq $1, foo(%rip) @@ -9097,19 +9187,19 @@ ; O2-LABEL: atomic64_nand_acq_rel: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection295: +; O2-NEXT: .Lpcsection325: ; O2-NEXT: movq (%rdi), %rax ; O2-NEXT: .p2align 4, 0x90 ; O2-NEXT: .LBB183_1: # %atomicrmw.start ; O2-NEXT: # =>This Inner Loop Header: Depth=1 ; O2-NEXT: movl %eax, %ecx -; O2-NEXT: .Lpcsection296: +; O2-NEXT: .Lpcsection326: ; O2-NEXT: notl %ecx -; O2-NEXT: .Lpcsection297: +; O2-NEXT: .Lpcsection327: ; O2-NEXT: orq $-43, %rcx -; O2-NEXT: .Lpcsection298: +; O2-NEXT: .Lpcsection328: ; O2-NEXT: lock cmpxchgq %rcx, (%rdi) -; O2-NEXT: .Lpcsection299: +; O2-NEXT: .Lpcsection329: ; O2-NEXT: jne .LBB183_1 ; O2-NEXT: # %bb.2: # %atomicrmw.end ; O2-NEXT: movq $1, foo(%rip) @@ -9118,19 +9208,19 @@ ; O3-LABEL: atomic64_nand_acq_rel: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection295: +; O3-NEXT: .Lpcsection325: ; O3-NEXT: movq (%rdi), %rax ; O3-NEXT: .p2align 4, 0x90 ; O3-NEXT: .LBB183_1: # %atomicrmw.start ; O3-NEXT: # =>This Inner Loop Header: Depth=1 ; O3-NEXT: movl %eax, %ecx -; O3-NEXT: .Lpcsection296: +; O3-NEXT: .Lpcsection326: ; O3-NEXT: notl %ecx -; O3-NEXT: .Lpcsection297: +; O3-NEXT: .Lpcsection327: ; O3-NEXT: orq $-43, %rcx -; O3-NEXT: .Lpcsection298: +; O3-NEXT: .Lpcsection328: ; O3-NEXT: lock cmpxchgq %rcx, (%rdi) -; O3-NEXT: .Lpcsection299: +; O3-NEXT: .Lpcsection329: ; O3-NEXT: jne .LBB183_1 ; O3-NEXT: # %bb.2: # %atomicrmw.end ; O3-NEXT: movq $1, foo(%rip) @@ -9156,7 +9246,7 @@ ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax ; O1-NEXT: movl $42, %eax -; O1-NEXT: .Lpcsection300: +; O1-NEXT: .Lpcsection330: ; O1-NEXT: xchgq %rax, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -9165,7 +9255,7 @@ ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax ; O2-NEXT: movl $42, %eax -; O2-NEXT: .Lpcsection300: +; O2-NEXT: .Lpcsection330: ; O2-NEXT: xchgq %rax, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -9174,7 +9264,7 @@ ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax ; O3-NEXT: movl $42, %eax -; O3-NEXT: .Lpcsection300: +; O3-NEXT: .Lpcsection330: ; O3-NEXT: xchgq %rax, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -9197,7 +9287,7 @@ ; O1-LABEL: atomic64_add_seq_cst: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection301: +; O1-NEXT: .Lpcsection331: ; O1-NEXT: lock addq $42, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -9205,7 +9295,7 @@ ; O2-LABEL: atomic64_add_seq_cst: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection301: +; O2-NEXT: .Lpcsection331: ; O2-NEXT: lock addq $42, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -9213,7 +9303,7 @@ ; O3-LABEL: atomic64_add_seq_cst: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection301: +; O3-NEXT: .Lpcsection331: ; O3-NEXT: lock addq $42, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -9236,7 +9326,7 @@ ; O1-LABEL: atomic64_sub_seq_cst: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection302: +; O1-NEXT: .Lpcsection332: ; O1-NEXT: lock subq $42, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -9244,7 +9334,7 @@ ; O2-LABEL: atomic64_sub_seq_cst: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection302: +; O2-NEXT: .Lpcsection332: ; O2-NEXT: lock subq $42, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -9252,7 +9342,7 @@ ; O3-LABEL: atomic64_sub_seq_cst: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection302: +; O3-NEXT: .Lpcsection332: ; O3-NEXT: lock subq $42, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -9275,7 +9365,7 @@ ; O1-LABEL: atomic64_and_seq_cst: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection303: +; O1-NEXT: .Lpcsection333: ; O1-NEXT: lock andq $42, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -9283,7 +9373,7 @@ ; O2-LABEL: atomic64_and_seq_cst: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection303: +; O2-NEXT: .Lpcsection333: ; O2-NEXT: lock andq $42, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -9291,7 +9381,7 @@ ; O3-LABEL: atomic64_and_seq_cst: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection303: +; O3-NEXT: .Lpcsection333: ; O3-NEXT: lock andq $42, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -9314,7 +9404,7 @@ ; O1-LABEL: atomic64_or_seq_cst: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection304: +; O1-NEXT: .Lpcsection334: ; O1-NEXT: lock orq $42, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -9322,7 +9412,7 @@ ; O2-LABEL: atomic64_or_seq_cst: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection304: +; O2-NEXT: .Lpcsection334: ; O2-NEXT: lock orq $42, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -9330,7 +9420,7 @@ ; O3-LABEL: atomic64_or_seq_cst: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection304: +; O3-NEXT: .Lpcsection334: ; O3-NEXT: lock orq $42, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -9353,7 +9443,7 @@ ; O1-LABEL: atomic64_xor_seq_cst: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection305: +; O1-NEXT: .Lpcsection335: ; O1-NEXT: lock xorq $42, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -9361,7 +9451,7 @@ ; O2-LABEL: atomic64_xor_seq_cst: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection305: +; O2-NEXT: .Lpcsection335: ; O2-NEXT: lock xorq $42, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -9369,7 +9459,7 @@ ; O3-LABEL: atomic64_xor_seq_cst: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection305: +; O3-NEXT: .Lpcsection335: ; O3-NEXT: lock xorq $42, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -9417,19 +9507,19 @@ ; O1-LABEL: atomic64_nand_seq_cst: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection306: +; O1-NEXT: .Lpcsection336: ; O1-NEXT: movq (%rdi), %rax ; O1-NEXT: .p2align 4, 0x90 ; O1-NEXT: .LBB190_1: # %atomicrmw.start ; O1-NEXT: # =>This Inner Loop Header: Depth=1 ; O1-NEXT: movl %eax, %ecx -; O1-NEXT: .Lpcsection307: +; O1-NEXT: .Lpcsection337: ; O1-NEXT: notl %ecx -; O1-NEXT: .Lpcsection308: +; O1-NEXT: .Lpcsection338: ; O1-NEXT: orq $-43, %rcx -; O1-NEXT: .Lpcsection309: +; O1-NEXT: .Lpcsection339: ; O1-NEXT: lock cmpxchgq %rcx, (%rdi) -; O1-NEXT: .Lpcsection310: +; O1-NEXT: .Lpcsection340: ; O1-NEXT: jne .LBB190_1 ; O1-NEXT: # %bb.2: # %atomicrmw.end ; O1-NEXT: movq $1, foo(%rip) @@ -9438,19 +9528,19 @@ ; O2-LABEL: atomic64_nand_seq_cst: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection306: +; O2-NEXT: .Lpcsection336: ; O2-NEXT: movq (%rdi), %rax ; O2-NEXT: .p2align 4, 0x90 ; O2-NEXT: .LBB190_1: # %atomicrmw.start ; O2-NEXT: # =>This Inner Loop Header: Depth=1 ; O2-NEXT: movl %eax, %ecx -; O2-NEXT: .Lpcsection307: +; O2-NEXT: .Lpcsection337: ; O2-NEXT: notl %ecx -; O2-NEXT: .Lpcsection308: +; O2-NEXT: .Lpcsection338: ; O2-NEXT: orq $-43, %rcx -; O2-NEXT: .Lpcsection309: +; O2-NEXT: .Lpcsection339: ; O2-NEXT: lock cmpxchgq %rcx, (%rdi) -; O2-NEXT: .Lpcsection310: +; O2-NEXT: .Lpcsection340: ; O2-NEXT: jne .LBB190_1 ; O2-NEXT: # %bb.2: # %atomicrmw.end ; O2-NEXT: movq $1, foo(%rip) @@ -9459,19 +9549,19 @@ ; O3-LABEL: atomic64_nand_seq_cst: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection306: +; O3-NEXT: .Lpcsection336: ; O3-NEXT: movq (%rdi), %rax ; O3-NEXT: .p2align 4, 0x90 ; O3-NEXT: .LBB190_1: # %atomicrmw.start ; O3-NEXT: # =>This Inner Loop Header: Depth=1 ; O3-NEXT: movl %eax, %ecx -; O3-NEXT: .Lpcsection307: +; O3-NEXT: .Lpcsection337: ; O3-NEXT: notl %ecx -; O3-NEXT: .Lpcsection308: +; O3-NEXT: .Lpcsection338: ; O3-NEXT: orq $-43, %rcx -; O3-NEXT: .Lpcsection309: +; O3-NEXT: .Lpcsection339: ; O3-NEXT: lock cmpxchgq %rcx, (%rdi) -; O3-NEXT: .Lpcsection310: +; O3-NEXT: .Lpcsection340: ; O3-NEXT: jne .LBB190_1 ; O3-NEXT: # %bb.2: # %atomicrmw.end ; O3-NEXT: movq $1, foo(%rip) @@ -9510,13 +9600,13 @@ ; O1-NEXT: movq foo(%rip), %rax ; O1-NEXT: movl $1, %ecx ; O1-NEXT: movl $42, %eax -; O1-NEXT: .Lpcsection311: +; O1-NEXT: .Lpcsection341: ; O1-NEXT: lock cmpxchgq %rcx, (%rdi) ; O1-NEXT: movl $42, %eax -; O1-NEXT: .Lpcsection312: +; O1-NEXT: .Lpcsection342: ; O1-NEXT: lock cmpxchgq %rcx, (%rdi) ; O1-NEXT: movl $42, %eax -; O1-NEXT: .Lpcsection313: +; O1-NEXT: .Lpcsection343: ; O1-NEXT: lock cmpxchgq %rcx, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -9526,13 +9616,13 @@ ; O2-NEXT: movq foo(%rip), %rax ; O2-NEXT: movl $1, %ecx ; O2-NEXT: movl $42, %eax -; O2-NEXT: .Lpcsection311: +; O2-NEXT: .Lpcsection341: ; O2-NEXT: lock cmpxchgq %rcx, (%rdi) ; O2-NEXT: movl $42, %eax -; O2-NEXT: .Lpcsection312: +; O2-NEXT: .Lpcsection342: ; O2-NEXT: lock cmpxchgq %rcx, (%rdi) ; O2-NEXT: movl $42, %eax -; O2-NEXT: .Lpcsection313: +; O2-NEXT: .Lpcsection343: ; O2-NEXT: lock cmpxchgq %rcx, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -9542,13 +9632,13 @@ ; O3-NEXT: movq foo(%rip), %rax ; O3-NEXT: movl $1, %ecx ; O3-NEXT: movl $42, %eax -; O3-NEXT: .Lpcsection311: +; O3-NEXT: .Lpcsection341: ; O3-NEXT: lock cmpxchgq %rcx, (%rdi) ; O3-NEXT: movl $42, %eax -; O3-NEXT: .Lpcsection312: +; O3-NEXT: .Lpcsection342: ; O3-NEXT: lock cmpxchgq %rcx, (%rdi) ; O3-NEXT: movl $42, %eax -; O3-NEXT: .Lpcsection313: +; O3-NEXT: .Lpcsection343: ; O3-NEXT: lock cmpxchgq %rcx, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -9588,13 +9678,13 @@ ; O1-NEXT: movq foo(%rip), %rax ; O1-NEXT: movl $1, %ecx ; O1-NEXT: movl $42, %eax -; O1-NEXT: .Lpcsection314: +; O1-NEXT: .Lpcsection344: ; O1-NEXT: lock cmpxchgq %rcx, (%rdi) ; O1-NEXT: movl $42, %eax -; O1-NEXT: .Lpcsection315: +; O1-NEXT: .Lpcsection345: ; O1-NEXT: lock cmpxchgq %rcx, (%rdi) ; O1-NEXT: movl $42, %eax -; O1-NEXT: .Lpcsection316: +; O1-NEXT: .Lpcsection346: ; O1-NEXT: lock cmpxchgq %rcx, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -9604,13 +9694,13 @@ ; O2-NEXT: movq foo(%rip), %rax ; O2-NEXT: movl $1, %ecx ; O2-NEXT: movl $42, %eax -; O2-NEXT: .Lpcsection314: +; O2-NEXT: .Lpcsection344: ; O2-NEXT: lock cmpxchgq %rcx, (%rdi) ; O2-NEXT: movl $42, %eax -; O2-NEXT: .Lpcsection315: +; O2-NEXT: .Lpcsection345: ; O2-NEXT: lock cmpxchgq %rcx, (%rdi) ; O2-NEXT: movl $42, %eax -; O2-NEXT: .Lpcsection316: +; O2-NEXT: .Lpcsection346: ; O2-NEXT: lock cmpxchgq %rcx, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -9620,13 +9710,13 @@ ; O3-NEXT: movq foo(%rip), %rax ; O3-NEXT: movl $1, %ecx ; O3-NEXT: movl $42, %eax -; O3-NEXT: .Lpcsection314: +; O3-NEXT: .Lpcsection344: ; O3-NEXT: lock cmpxchgq %rcx, (%rdi) ; O3-NEXT: movl $42, %eax -; O3-NEXT: .Lpcsection315: +; O3-NEXT: .Lpcsection345: ; O3-NEXT: lock cmpxchgq %rcx, (%rdi) ; O3-NEXT: movl $42, %eax -; O3-NEXT: .Lpcsection316: +; O3-NEXT: .Lpcsection346: ; O3-NEXT: lock cmpxchgq %rcx, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -9666,13 +9756,13 @@ ; O1-NEXT: movq foo(%rip), %rax ; O1-NEXT: movl $1, %ecx ; O1-NEXT: movl $42, %eax -; O1-NEXT: .Lpcsection317: +; O1-NEXT: .Lpcsection347: ; O1-NEXT: lock cmpxchgq %rcx, (%rdi) ; O1-NEXT: movl $42, %eax -; O1-NEXT: .Lpcsection318: +; O1-NEXT: .Lpcsection348: ; O1-NEXT: lock cmpxchgq %rcx, (%rdi) ; O1-NEXT: movl $42, %eax -; O1-NEXT: .Lpcsection319: +; O1-NEXT: .Lpcsection349: ; O1-NEXT: lock cmpxchgq %rcx, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -9682,13 +9772,13 @@ ; O2-NEXT: movq foo(%rip), %rax ; O2-NEXT: movl $1, %ecx ; O2-NEXT: movl $42, %eax -; O2-NEXT: .Lpcsection317: +; O2-NEXT: .Lpcsection347: ; O2-NEXT: lock cmpxchgq %rcx, (%rdi) ; O2-NEXT: movl $42, %eax -; O2-NEXT: .Lpcsection318: +; O2-NEXT: .Lpcsection348: ; O2-NEXT: lock cmpxchgq %rcx, (%rdi) ; O2-NEXT: movl $42, %eax -; O2-NEXT: .Lpcsection319: +; O2-NEXT: .Lpcsection349: ; O2-NEXT: lock cmpxchgq %rcx, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -9698,13 +9788,13 @@ ; O3-NEXT: movq foo(%rip), %rax ; O3-NEXT: movl $1, %ecx ; O3-NEXT: movl $42, %eax -; O3-NEXT: .Lpcsection317: +; O3-NEXT: .Lpcsection347: ; O3-NEXT: lock cmpxchgq %rcx, (%rdi) ; O3-NEXT: movl $42, %eax -; O3-NEXT: .Lpcsection318: +; O3-NEXT: .Lpcsection348: ; O3-NEXT: lock cmpxchgq %rcx, (%rdi) ; O3-NEXT: movl $42, %eax -; O3-NEXT: .Lpcsection319: +; O3-NEXT: .Lpcsection349: ; O3-NEXT: lock cmpxchgq %rcx, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -9744,13 +9834,13 @@ ; O1-NEXT: movq foo(%rip), %rax ; O1-NEXT: movl $1, %ecx ; O1-NEXT: movl $42, %eax -; O1-NEXT: .Lpcsection320: +; O1-NEXT: .Lpcsection350: ; O1-NEXT: lock cmpxchgq %rcx, (%rdi) ; O1-NEXT: movl $42, %eax -; O1-NEXT: .Lpcsection321: +; O1-NEXT: .Lpcsection351: ; O1-NEXT: lock cmpxchgq %rcx, (%rdi) ; O1-NEXT: movl $42, %eax -; O1-NEXT: .Lpcsection322: +; O1-NEXT: .Lpcsection352: ; O1-NEXT: lock cmpxchgq %rcx, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -9760,13 +9850,13 @@ ; O2-NEXT: movq foo(%rip), %rax ; O2-NEXT: movl $1, %ecx ; O2-NEXT: movl $42, %eax -; O2-NEXT: .Lpcsection320: +; O2-NEXT: .Lpcsection350: ; O2-NEXT: lock cmpxchgq %rcx, (%rdi) ; O2-NEXT: movl $42, %eax -; O2-NEXT: .Lpcsection321: +; O2-NEXT: .Lpcsection351: ; O2-NEXT: lock cmpxchgq %rcx, (%rdi) ; O2-NEXT: movl $42, %eax -; O2-NEXT: .Lpcsection322: +; O2-NEXT: .Lpcsection352: ; O2-NEXT: lock cmpxchgq %rcx, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -9776,13 +9866,13 @@ ; O3-NEXT: movq foo(%rip), %rax ; O3-NEXT: movl $1, %ecx ; O3-NEXT: movl $42, %eax -; O3-NEXT: .Lpcsection320: +; O3-NEXT: .Lpcsection350: ; O3-NEXT: lock cmpxchgq %rcx, (%rdi) ; O3-NEXT: movl $42, %eax -; O3-NEXT: .Lpcsection321: +; O3-NEXT: .Lpcsection351: ; O3-NEXT: lock cmpxchgq %rcx, (%rdi) ; O3-NEXT: movl $42, %eax -; O3-NEXT: .Lpcsection322: +; O3-NEXT: .Lpcsection352: ; O3-NEXT: lock cmpxchgq %rcx, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -9822,13 +9912,13 @@ ; O1-NEXT: movq foo(%rip), %rax ; O1-NEXT: movl $1, %ecx ; O1-NEXT: movl $42, %eax -; O1-NEXT: .Lpcsection323: +; O1-NEXT: .Lpcsection353: ; O1-NEXT: lock cmpxchgq %rcx, (%rdi) ; O1-NEXT: movl $42, %eax -; O1-NEXT: .Lpcsection324: +; O1-NEXT: .Lpcsection354: ; O1-NEXT: lock cmpxchgq %rcx, (%rdi) ; O1-NEXT: movl $42, %eax -; O1-NEXT: .Lpcsection325: +; O1-NEXT: .Lpcsection355: ; O1-NEXT: lock cmpxchgq %rcx, (%rdi) ; O1-NEXT: movq $3, foo(%rip) ; O1-NEXT: retq @@ -9838,13 +9928,13 @@ ; O2-NEXT: movq foo(%rip), %rax ; O2-NEXT: movl $1, %ecx ; O2-NEXT: movl $42, %eax -; O2-NEXT: .Lpcsection323: +; O2-NEXT: .Lpcsection353: ; O2-NEXT: lock cmpxchgq %rcx, (%rdi) ; O2-NEXT: movl $42, %eax -; O2-NEXT: .Lpcsection324: +; O2-NEXT: .Lpcsection354: ; O2-NEXT: lock cmpxchgq %rcx, (%rdi) ; O2-NEXT: movl $42, %eax -; O2-NEXT: .Lpcsection325: +; O2-NEXT: .Lpcsection355: ; O2-NEXT: lock cmpxchgq %rcx, (%rdi) ; O2-NEXT: movq $3, foo(%rip) ; O2-NEXT: retq @@ -9854,13 +9944,13 @@ ; O3-NEXT: movq foo(%rip), %rax ; O3-NEXT: movl $1, %ecx ; O3-NEXT: movl $42, %eax -; O3-NEXT: .Lpcsection323: +; O3-NEXT: .Lpcsection353: ; O3-NEXT: lock cmpxchgq %rcx, (%rdi) ; O3-NEXT: movl $42, %eax -; O3-NEXT: .Lpcsection324: +; O3-NEXT: .Lpcsection354: ; O3-NEXT: lock cmpxchgq %rcx, (%rdi) ; O3-NEXT: movl $42, %eax -; O3-NEXT: .Lpcsection325: +; O3-NEXT: .Lpcsection355: ; O3-NEXT: lock cmpxchgq %rcx, (%rdi) ; O3-NEXT: movq $3, foo(%rip) ; O3-NEXT: retq @@ -9887,7 +9977,7 @@ ; O1: # %bb.0: # %entry ; O1-NEXT: movq %rsi, %rax ; O1-NEXT: movq foo(%rip), %rcx -; O1-NEXT: .Lpcsection326: +; O1-NEXT: .Lpcsection356: ; O1-NEXT: lock cmpxchgq %rdx, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -9896,7 +9986,7 @@ ; O2: # %bb.0: # %entry ; O2-NEXT: movq %rsi, %rax ; O2-NEXT: movq foo(%rip), %rcx -; O2-NEXT: .Lpcsection326: +; O2-NEXT: .Lpcsection356: ; O2-NEXT: lock cmpxchgq %rdx, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -9905,7 +9995,7 @@ ; O3: # %bb.0: # %entry ; O3-NEXT: movq %rsi, %rax ; O3-NEXT: movq foo(%rip), %rcx -; O3-NEXT: .Lpcsection326: +; O3-NEXT: .Lpcsection356: ; O3-NEXT: lock cmpxchgq %rdx, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -9934,7 +10024,7 @@ ; ; O1-LABEL: atomic_use_cond: ; O1: # %bb.0: # %entry -; O1-NEXT: .Lpcsection327: +; O1-NEXT: .Lpcsection357: ; O1-NEXT: lock decq (%rdi) ; O1-NEXT: jne .LBB197_2 ; O1-NEXT: # %bb.1: # %then @@ -9946,7 +10036,7 @@ ; ; O2-LABEL: atomic_use_cond: ; O2: # %bb.0: # %entry -; O2-NEXT: .Lpcsection327: +; O2-NEXT: .Lpcsection357: ; O2-NEXT: lock decq (%rdi) ; O2-NEXT: jne .LBB197_2 ; O2-NEXT: # %bb.1: # %then @@ -9958,7 +10048,7 @@ ; ; O3-LABEL: atomic_use_cond: ; O3: # %bb.0: # %entry -; O3-NEXT: .Lpcsection327: +; O3-NEXT: .Lpcsection357: ; O3-NEXT: lock decq (%rdi) ; O3-NEXT: jne .LBB197_2 ; O3-NEXT: # %bb.1: # %then @@ -10005,15 +10095,15 @@ ; O1-NEXT: .cfi_def_cfa_offset 16 ; O1-NEXT: .cfi_offset %rbx, -16 ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection328: +; O1-NEXT: .Lpcsection358: ; O1-NEXT: xorl %eax, %eax -; O1-NEXT: .Lpcsection329: +; O1-NEXT: .Lpcsection359: ; O1-NEXT: xorl %edx, %edx -; O1-NEXT: .Lpcsection330: +; O1-NEXT: .Lpcsection360: ; O1-NEXT: xorl %ecx, %ecx -; O1-NEXT: .Lpcsection331: +; O1-NEXT: .Lpcsection361: ; O1-NEXT: xorl %ebx, %ebx -; O1-NEXT: .Lpcsection332: +; O1-NEXT: .Lpcsection362: ; O1-NEXT: lock cmpxchg16b (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: popq %rbx @@ -10026,15 +10116,15 @@ ; O2-NEXT: .cfi_def_cfa_offset 16 ; O2-NEXT: .cfi_offset %rbx, -16 ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection328: +; O2-NEXT: .Lpcsection358: ; O2-NEXT: xorl %eax, %eax -; O2-NEXT: .Lpcsection329: +; O2-NEXT: .Lpcsection359: ; O2-NEXT: xorl %edx, %edx -; O2-NEXT: .Lpcsection330: +; O2-NEXT: .Lpcsection360: ; O2-NEXT: xorl %ecx, %ecx -; O2-NEXT: .Lpcsection331: +; O2-NEXT: .Lpcsection361: ; O2-NEXT: xorl %ebx, %ebx -; O2-NEXT: .Lpcsection332: +; O2-NEXT: .Lpcsection362: ; O2-NEXT: lock cmpxchg16b (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: popq %rbx @@ -10047,15 +10137,15 @@ ; O3-NEXT: .cfi_def_cfa_offset 16 ; O3-NEXT: .cfi_offset %rbx, -16 ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection328: +; O3-NEXT: .Lpcsection358: ; O3-NEXT: xorl %eax, %eax -; O3-NEXT: .Lpcsection329: +; O3-NEXT: .Lpcsection359: ; O3-NEXT: xorl %edx, %edx -; O3-NEXT: .Lpcsection330: +; O3-NEXT: .Lpcsection360: ; O3-NEXT: xorl %ecx, %ecx -; O3-NEXT: .Lpcsection331: +; O3-NEXT: .Lpcsection361: ; O3-NEXT: xorl %ebx, %ebx -; O3-NEXT: .Lpcsection332: +; O3-NEXT: .Lpcsection362: ; O3-NEXT: lock cmpxchg16b (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: popq %rbx @@ -10094,15 +10184,15 @@ ; O1-NEXT: .cfi_def_cfa_offset 16 ; O1-NEXT: .cfi_offset %rbx, -16 ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection333: +; O1-NEXT: .Lpcsection363: ; O1-NEXT: xorl %eax, %eax -; O1-NEXT: .Lpcsection334: +; O1-NEXT: .Lpcsection364: ; O1-NEXT: xorl %edx, %edx -; O1-NEXT: .Lpcsection335: +; O1-NEXT: .Lpcsection365: ; O1-NEXT: xorl %ecx, %ecx -; O1-NEXT: .Lpcsection336: +; O1-NEXT: .Lpcsection366: ; O1-NEXT: xorl %ebx, %ebx -; O1-NEXT: .Lpcsection337: +; O1-NEXT: .Lpcsection367: ; O1-NEXT: lock cmpxchg16b (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: popq %rbx @@ -10115,15 +10205,15 @@ ; O2-NEXT: .cfi_def_cfa_offset 16 ; O2-NEXT: .cfi_offset %rbx, -16 ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection333: +; O2-NEXT: .Lpcsection363: ; O2-NEXT: xorl %eax, %eax -; O2-NEXT: .Lpcsection334: +; O2-NEXT: .Lpcsection364: ; O2-NEXT: xorl %edx, %edx -; O2-NEXT: .Lpcsection335: +; O2-NEXT: .Lpcsection365: ; O2-NEXT: xorl %ecx, %ecx -; O2-NEXT: .Lpcsection336: +; O2-NEXT: .Lpcsection366: ; O2-NEXT: xorl %ebx, %ebx -; O2-NEXT: .Lpcsection337: +; O2-NEXT: .Lpcsection367: ; O2-NEXT: lock cmpxchg16b (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: popq %rbx @@ -10136,15 +10226,15 @@ ; O3-NEXT: .cfi_def_cfa_offset 16 ; O3-NEXT: .cfi_offset %rbx, -16 ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection333: +; O3-NEXT: .Lpcsection363: ; O3-NEXT: xorl %eax, %eax -; O3-NEXT: .Lpcsection334: +; O3-NEXT: .Lpcsection364: ; O3-NEXT: xorl %edx, %edx -; O3-NEXT: .Lpcsection335: +; O3-NEXT: .Lpcsection365: ; O3-NEXT: xorl %ecx, %ecx -; O3-NEXT: .Lpcsection336: +; O3-NEXT: .Lpcsection366: ; O3-NEXT: xorl %ebx, %ebx -; O3-NEXT: .Lpcsection337: +; O3-NEXT: .Lpcsection367: ; O3-NEXT: lock cmpxchg16b (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: popq %rbx @@ -10183,15 +10273,15 @@ ; O1-NEXT: .cfi_def_cfa_offset 16 ; O1-NEXT: .cfi_offset %rbx, -16 ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection338: +; O1-NEXT: .Lpcsection368: ; O1-NEXT: xorl %eax, %eax -; O1-NEXT: .Lpcsection339: +; O1-NEXT: .Lpcsection369: ; O1-NEXT: xorl %edx, %edx -; O1-NEXT: .Lpcsection340: +; O1-NEXT: .Lpcsection370: ; O1-NEXT: xorl %ecx, %ecx -; O1-NEXT: .Lpcsection341: +; O1-NEXT: .Lpcsection371: ; O1-NEXT: xorl %ebx, %ebx -; O1-NEXT: .Lpcsection342: +; O1-NEXT: .Lpcsection372: ; O1-NEXT: lock cmpxchg16b (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: popq %rbx @@ -10204,15 +10294,15 @@ ; O2-NEXT: .cfi_def_cfa_offset 16 ; O2-NEXT: .cfi_offset %rbx, -16 ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection338: +; O2-NEXT: .Lpcsection368: ; O2-NEXT: xorl %eax, %eax -; O2-NEXT: .Lpcsection339: +; O2-NEXT: .Lpcsection369: ; O2-NEXT: xorl %edx, %edx -; O2-NEXT: .Lpcsection340: +; O2-NEXT: .Lpcsection370: ; O2-NEXT: xorl %ecx, %ecx -; O2-NEXT: .Lpcsection341: +; O2-NEXT: .Lpcsection371: ; O2-NEXT: xorl %ebx, %ebx -; O2-NEXT: .Lpcsection342: +; O2-NEXT: .Lpcsection372: ; O2-NEXT: lock cmpxchg16b (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: popq %rbx @@ -10225,15 +10315,15 @@ ; O3-NEXT: .cfi_def_cfa_offset 16 ; O3-NEXT: .cfi_offset %rbx, -16 ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection338: +; O3-NEXT: .Lpcsection368: ; O3-NEXT: xorl %eax, %eax -; O3-NEXT: .Lpcsection339: +; O3-NEXT: .Lpcsection369: ; O3-NEXT: xorl %edx, %edx -; O3-NEXT: .Lpcsection340: +; O3-NEXT: .Lpcsection370: ; O3-NEXT: xorl %ecx, %ecx -; O3-NEXT: .Lpcsection341: +; O3-NEXT: .Lpcsection371: ; O3-NEXT: xorl %ebx, %ebx -; O3-NEXT: .Lpcsection342: +; O3-NEXT: .Lpcsection372: ; O3-NEXT: lock cmpxchg16b (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: popq %rbx @@ -10272,15 +10362,15 @@ ; O1-NEXT: .cfi_def_cfa_offset 16 ; O1-NEXT: .cfi_offset %rbx, -16 ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection343: +; O1-NEXT: .Lpcsection373: ; O1-NEXT: xorl %eax, %eax -; O1-NEXT: .Lpcsection344: +; O1-NEXT: .Lpcsection374: ; O1-NEXT: xorl %edx, %edx -; O1-NEXT: .Lpcsection345: +; O1-NEXT: .Lpcsection375: ; O1-NEXT: xorl %ecx, %ecx -; O1-NEXT: .Lpcsection346: +; O1-NEXT: .Lpcsection376: ; O1-NEXT: xorl %ebx, %ebx -; O1-NEXT: .Lpcsection347: +; O1-NEXT: .Lpcsection377: ; O1-NEXT: lock cmpxchg16b (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: popq %rbx @@ -10293,15 +10383,15 @@ ; O2-NEXT: .cfi_def_cfa_offset 16 ; O2-NEXT: .cfi_offset %rbx, -16 ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection343: +; O2-NEXT: .Lpcsection373: ; O2-NEXT: xorl %eax, %eax -; O2-NEXT: .Lpcsection344: +; O2-NEXT: .Lpcsection374: ; O2-NEXT: xorl %edx, %edx -; O2-NEXT: .Lpcsection345: +; O2-NEXT: .Lpcsection375: ; O2-NEXT: xorl %ecx, %ecx -; O2-NEXT: .Lpcsection346: +; O2-NEXT: .Lpcsection376: ; O2-NEXT: xorl %ebx, %ebx -; O2-NEXT: .Lpcsection347: +; O2-NEXT: .Lpcsection377: ; O2-NEXT: lock cmpxchg16b (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: popq %rbx @@ -10314,15 +10404,15 @@ ; O3-NEXT: .cfi_def_cfa_offset 16 ; O3-NEXT: .cfi_offset %rbx, -16 ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection343: +; O3-NEXT: .Lpcsection373: ; O3-NEXT: xorl %eax, %eax -; O3-NEXT: .Lpcsection344: +; O3-NEXT: .Lpcsection374: ; O3-NEXT: xorl %edx, %edx -; O3-NEXT: .Lpcsection345: +; O3-NEXT: .Lpcsection375: ; O3-NEXT: xorl %ecx, %ecx -; O3-NEXT: .Lpcsection346: +; O3-NEXT: .Lpcsection376: ; O3-NEXT: xorl %ebx, %ebx -; O3-NEXT: .Lpcsection347: +; O3-NEXT: .Lpcsection377: ; O3-NEXT: lock cmpxchg16b (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: popq %rbx @@ -10347,7 +10437,7 @@ ; O1-LABEL: atomic128_load_seq_cst_ptr_ty: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection348: +; O1-NEXT: .Lpcsection378: ; O1-NEXT: movq (%rdi), %rax ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -10355,7 +10445,7 @@ ; O2-LABEL: atomic128_load_seq_cst_ptr_ty: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection348: +; O2-NEXT: .Lpcsection378: ; O2-NEXT: movq (%rdi), %rax ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -10363,7 +10453,7 @@ ; O3-LABEL: atomic128_load_seq_cst_ptr_ty: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection348: +; O3-NEXT: .Lpcsection378: ; O3-NEXT: movq (%rdi), %rax ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -10420,20 +10510,20 @@ ; O1-NEXT: .cfi_def_cfa_offset 16 ; O1-NEXT: .cfi_offset %rbx, -16 ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection349: +; O1-NEXT: .Lpcsection379: ; O1-NEXT: movq (%rdi), %rax -; O1-NEXT: .Lpcsection350: +; O1-NEXT: .Lpcsection380: ; O1-NEXT: movq 8(%rdi), %rdx -; O1-NEXT: .Lpcsection351: +; O1-NEXT: .Lpcsection381: ; O1-NEXT: movl $42, %ebx ; O1-NEXT: .p2align 4, 0x90 ; O1-NEXT: .LBB203_1: # %atomicrmw.start ; O1-NEXT: # =>This Inner Loop Header: Depth=1 -; O1-NEXT: .Lpcsection352: +; O1-NEXT: .Lpcsection382: ; O1-NEXT: xorl %ecx, %ecx -; O1-NEXT: .Lpcsection353: +; O1-NEXT: .Lpcsection383: ; O1-NEXT: lock cmpxchg16b (%rdi) -; O1-NEXT: .Lpcsection354: +; O1-NEXT: .Lpcsection384: ; O1-NEXT: jne .LBB203_1 ; O1-NEXT: # %bb.2: # %atomicrmw.end ; O1-NEXT: movq $1, foo(%rip) @@ -10447,20 +10537,20 @@ ; O2-NEXT: .cfi_def_cfa_offset 16 ; O2-NEXT: .cfi_offset %rbx, -16 ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection349: +; O2-NEXT: .Lpcsection379: ; O2-NEXT: movq (%rdi), %rax -; O2-NEXT: .Lpcsection350: +; O2-NEXT: .Lpcsection380: ; O2-NEXT: movq 8(%rdi), %rdx -; O2-NEXT: .Lpcsection351: +; O2-NEXT: .Lpcsection381: ; O2-NEXT: movl $42, %ebx ; O2-NEXT: .p2align 4, 0x90 ; O2-NEXT: .LBB203_1: # %atomicrmw.start ; O2-NEXT: # =>This Inner Loop Header: Depth=1 -; O2-NEXT: .Lpcsection352: +; O2-NEXT: .Lpcsection382: ; O2-NEXT: xorl %ecx, %ecx -; O2-NEXT: .Lpcsection353: +; O2-NEXT: .Lpcsection383: ; O2-NEXT: lock cmpxchg16b (%rdi) -; O2-NEXT: .Lpcsection354: +; O2-NEXT: .Lpcsection384: ; O2-NEXT: jne .LBB203_1 ; O2-NEXT: # %bb.2: # %atomicrmw.end ; O2-NEXT: movq $1, foo(%rip) @@ -10474,20 +10564,20 @@ ; O3-NEXT: .cfi_def_cfa_offset 16 ; O3-NEXT: .cfi_offset %rbx, -16 ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection349: +; O3-NEXT: .Lpcsection379: ; O3-NEXT: movq (%rdi), %rax -; O3-NEXT: .Lpcsection350: +; O3-NEXT: .Lpcsection380: ; O3-NEXT: movq 8(%rdi), %rdx -; O3-NEXT: .Lpcsection351: +; O3-NEXT: .Lpcsection381: ; O3-NEXT: movl $42, %ebx ; O3-NEXT: .p2align 4, 0x90 ; O3-NEXT: .LBB203_1: # %atomicrmw.start ; O3-NEXT: # =>This Inner Loop Header: Depth=1 -; O3-NEXT: .Lpcsection352: +; O3-NEXT: .Lpcsection382: ; O3-NEXT: xorl %ecx, %ecx -; O3-NEXT: .Lpcsection353: +; O3-NEXT: .Lpcsection383: ; O3-NEXT: lock cmpxchg16b (%rdi) -; O3-NEXT: .Lpcsection354: +; O3-NEXT: .Lpcsection384: ; O3-NEXT: jne .LBB203_1 ; O3-NEXT: # %bb.2: # %atomicrmw.end ; O3-NEXT: movq $1, foo(%rip) @@ -10547,20 +10637,20 @@ ; O1-NEXT: .cfi_def_cfa_offset 16 ; O1-NEXT: .cfi_offset %rbx, -16 ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection355: +; O1-NEXT: .Lpcsection385: ; O1-NEXT: movq (%rdi), %rax -; O1-NEXT: .Lpcsection356: +; O1-NEXT: .Lpcsection386: ; O1-NEXT: movq 8(%rdi), %rdx -; O1-NEXT: .Lpcsection357: +; O1-NEXT: .Lpcsection387: ; O1-NEXT: movl $42, %ebx ; O1-NEXT: .p2align 4, 0x90 ; O1-NEXT: .LBB204_1: # %atomicrmw.start ; O1-NEXT: # =>This Inner Loop Header: Depth=1 -; O1-NEXT: .Lpcsection358: +; O1-NEXT: .Lpcsection388: ; O1-NEXT: xorl %ecx, %ecx -; O1-NEXT: .Lpcsection359: +; O1-NEXT: .Lpcsection389: ; O1-NEXT: lock cmpxchg16b (%rdi) -; O1-NEXT: .Lpcsection360: +; O1-NEXT: .Lpcsection390: ; O1-NEXT: jne .LBB204_1 ; O1-NEXT: # %bb.2: # %atomicrmw.end ; O1-NEXT: movq $1, foo(%rip) @@ -10574,20 +10664,20 @@ ; O2-NEXT: .cfi_def_cfa_offset 16 ; O2-NEXT: .cfi_offset %rbx, -16 ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection355: +; O2-NEXT: .Lpcsection385: ; O2-NEXT: movq (%rdi), %rax -; O2-NEXT: .Lpcsection356: +; O2-NEXT: .Lpcsection386: ; O2-NEXT: movq 8(%rdi), %rdx -; O2-NEXT: .Lpcsection357: +; O2-NEXT: .Lpcsection387: ; O2-NEXT: movl $42, %ebx ; O2-NEXT: .p2align 4, 0x90 ; O2-NEXT: .LBB204_1: # %atomicrmw.start ; O2-NEXT: # =>This Inner Loop Header: Depth=1 -; O2-NEXT: .Lpcsection358: +; O2-NEXT: .Lpcsection388: ; O2-NEXT: xorl %ecx, %ecx -; O2-NEXT: .Lpcsection359: +; O2-NEXT: .Lpcsection389: ; O2-NEXT: lock cmpxchg16b (%rdi) -; O2-NEXT: .Lpcsection360: +; O2-NEXT: .Lpcsection390: ; O2-NEXT: jne .LBB204_1 ; O2-NEXT: # %bb.2: # %atomicrmw.end ; O2-NEXT: movq $1, foo(%rip) @@ -10601,20 +10691,20 @@ ; O3-NEXT: .cfi_def_cfa_offset 16 ; O3-NEXT: .cfi_offset %rbx, -16 ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection355: +; O3-NEXT: .Lpcsection385: ; O3-NEXT: movq (%rdi), %rax -; O3-NEXT: .Lpcsection356: +; O3-NEXT: .Lpcsection386: ; O3-NEXT: movq 8(%rdi), %rdx -; O3-NEXT: .Lpcsection357: +; O3-NEXT: .Lpcsection387: ; O3-NEXT: movl $42, %ebx ; O3-NEXT: .p2align 4, 0x90 ; O3-NEXT: .LBB204_1: # %atomicrmw.start ; O3-NEXT: # =>This Inner Loop Header: Depth=1 -; O3-NEXT: .Lpcsection358: +; O3-NEXT: .Lpcsection388: ; O3-NEXT: xorl %ecx, %ecx -; O3-NEXT: .Lpcsection359: +; O3-NEXT: .Lpcsection389: ; O3-NEXT: lock cmpxchg16b (%rdi) -; O3-NEXT: .Lpcsection360: +; O3-NEXT: .Lpcsection390: ; O3-NEXT: jne .LBB204_1 ; O3-NEXT: # %bb.2: # %atomicrmw.end ; O3-NEXT: movq $1, foo(%rip) @@ -10674,20 +10764,20 @@ ; O1-NEXT: .cfi_def_cfa_offset 16 ; O1-NEXT: .cfi_offset %rbx, -16 ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection361: +; O1-NEXT: .Lpcsection391: ; O1-NEXT: movq (%rdi), %rax -; O1-NEXT: .Lpcsection362: +; O1-NEXT: .Lpcsection392: ; O1-NEXT: movq 8(%rdi), %rdx -; O1-NEXT: .Lpcsection363: +; O1-NEXT: .Lpcsection393: ; O1-NEXT: movl $42, %ebx ; O1-NEXT: .p2align 4, 0x90 ; O1-NEXT: .LBB205_1: # %atomicrmw.start ; O1-NEXT: # =>This Inner Loop Header: Depth=1 -; O1-NEXT: .Lpcsection364: +; O1-NEXT: .Lpcsection394: ; O1-NEXT: xorl %ecx, %ecx -; O1-NEXT: .Lpcsection365: +; O1-NEXT: .Lpcsection395: ; O1-NEXT: lock cmpxchg16b (%rdi) -; O1-NEXT: .Lpcsection366: +; O1-NEXT: .Lpcsection396: ; O1-NEXT: jne .LBB205_1 ; O1-NEXT: # %bb.2: # %atomicrmw.end ; O1-NEXT: movq $1, foo(%rip) @@ -10701,20 +10791,20 @@ ; O2-NEXT: .cfi_def_cfa_offset 16 ; O2-NEXT: .cfi_offset %rbx, -16 ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection361: +; O2-NEXT: .Lpcsection391: ; O2-NEXT: movq (%rdi), %rax -; O2-NEXT: .Lpcsection362: +; O2-NEXT: .Lpcsection392: ; O2-NEXT: movq 8(%rdi), %rdx -; O2-NEXT: .Lpcsection363: +; O2-NEXT: .Lpcsection393: ; O2-NEXT: movl $42, %ebx ; O2-NEXT: .p2align 4, 0x90 ; O2-NEXT: .LBB205_1: # %atomicrmw.start ; O2-NEXT: # =>This Inner Loop Header: Depth=1 -; O2-NEXT: .Lpcsection364: +; O2-NEXT: .Lpcsection394: ; O2-NEXT: xorl %ecx, %ecx -; O2-NEXT: .Lpcsection365: +; O2-NEXT: .Lpcsection395: ; O2-NEXT: lock cmpxchg16b (%rdi) -; O2-NEXT: .Lpcsection366: +; O2-NEXT: .Lpcsection396: ; O2-NEXT: jne .LBB205_1 ; O2-NEXT: # %bb.2: # %atomicrmw.end ; O2-NEXT: movq $1, foo(%rip) @@ -10728,20 +10818,20 @@ ; O3-NEXT: .cfi_def_cfa_offset 16 ; O3-NEXT: .cfi_offset %rbx, -16 ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection361: +; O3-NEXT: .Lpcsection391: ; O3-NEXT: movq (%rdi), %rax -; O3-NEXT: .Lpcsection362: +; O3-NEXT: .Lpcsection392: ; O3-NEXT: movq 8(%rdi), %rdx -; O3-NEXT: .Lpcsection363: +; O3-NEXT: .Lpcsection393: ; O3-NEXT: movl $42, %ebx ; O3-NEXT: .p2align 4, 0x90 ; O3-NEXT: .LBB205_1: # %atomicrmw.start ; O3-NEXT: # =>This Inner Loop Header: Depth=1 -; O3-NEXT: .Lpcsection364: +; O3-NEXT: .Lpcsection394: ; O3-NEXT: xorl %ecx, %ecx -; O3-NEXT: .Lpcsection365: +; O3-NEXT: .Lpcsection395: ; O3-NEXT: lock cmpxchg16b (%rdi) -; O3-NEXT: .Lpcsection366: +; O3-NEXT: .Lpcsection396: ; O3-NEXT: jne .LBB205_1 ; O3-NEXT: # %bb.2: # %atomicrmw.end ; O3-NEXT: movq $1, foo(%rip) @@ -10801,20 +10891,20 @@ ; O1-NEXT: .cfi_def_cfa_offset 16 ; O1-NEXT: .cfi_offset %rbx, -16 ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection367: +; O1-NEXT: .Lpcsection397: ; O1-NEXT: movq (%rdi), %rax -; O1-NEXT: .Lpcsection368: +; O1-NEXT: .Lpcsection398: ; O1-NEXT: movq 8(%rdi), %rdx -; O1-NEXT: .Lpcsection369: +; O1-NEXT: .Lpcsection399: ; O1-NEXT: movl $42, %ebx ; O1-NEXT: .p2align 4, 0x90 ; O1-NEXT: .LBB206_1: # %atomicrmw.start ; O1-NEXT: # =>This Inner Loop Header: Depth=1 -; O1-NEXT: .Lpcsection370: +; O1-NEXT: .Lpcsection400: ; O1-NEXT: xorl %ecx, %ecx -; O1-NEXT: .Lpcsection371: +; O1-NEXT: .Lpcsection401: ; O1-NEXT: lock cmpxchg16b (%rdi) -; O1-NEXT: .Lpcsection372: +; O1-NEXT: .Lpcsection402: ; O1-NEXT: jne .LBB206_1 ; O1-NEXT: # %bb.2: # %atomicrmw.end ; O1-NEXT: movq $1, foo(%rip) @@ -10828,20 +10918,20 @@ ; O2-NEXT: .cfi_def_cfa_offset 16 ; O2-NEXT: .cfi_offset %rbx, -16 ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection367: +; O2-NEXT: .Lpcsection397: ; O2-NEXT: movq (%rdi), %rax -; O2-NEXT: .Lpcsection368: +; O2-NEXT: .Lpcsection398: ; O2-NEXT: movq 8(%rdi), %rdx -; O2-NEXT: .Lpcsection369: +; O2-NEXT: .Lpcsection399: ; O2-NEXT: movl $42, %ebx ; O2-NEXT: .p2align 4, 0x90 ; O2-NEXT: .LBB206_1: # %atomicrmw.start ; O2-NEXT: # =>This Inner Loop Header: Depth=1 -; O2-NEXT: .Lpcsection370: +; O2-NEXT: .Lpcsection400: ; O2-NEXT: xorl %ecx, %ecx -; O2-NEXT: .Lpcsection371: +; O2-NEXT: .Lpcsection401: ; O2-NEXT: lock cmpxchg16b (%rdi) -; O2-NEXT: .Lpcsection372: +; O2-NEXT: .Lpcsection402: ; O2-NEXT: jne .LBB206_1 ; O2-NEXT: # %bb.2: # %atomicrmw.end ; O2-NEXT: movq $1, foo(%rip) @@ -10855,20 +10945,20 @@ ; O3-NEXT: .cfi_def_cfa_offset 16 ; O3-NEXT: .cfi_offset %rbx, -16 ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection367: +; O3-NEXT: .Lpcsection397: ; O3-NEXT: movq (%rdi), %rax -; O3-NEXT: .Lpcsection368: +; O3-NEXT: .Lpcsection398: ; O3-NEXT: movq 8(%rdi), %rdx -; O3-NEXT: .Lpcsection369: +; O3-NEXT: .Lpcsection399: ; O3-NEXT: movl $42, %ebx ; O3-NEXT: .p2align 4, 0x90 ; O3-NEXT: .LBB206_1: # %atomicrmw.start ; O3-NEXT: # =>This Inner Loop Header: Depth=1 -; O3-NEXT: .Lpcsection370: +; O3-NEXT: .Lpcsection400: ; O3-NEXT: xorl %ecx, %ecx -; O3-NEXT: .Lpcsection371: +; O3-NEXT: .Lpcsection401: ; O3-NEXT: lock cmpxchg16b (%rdi) -; O3-NEXT: .Lpcsection372: +; O3-NEXT: .Lpcsection402: ; O3-NEXT: jne .LBB206_1 ; O3-NEXT: # %bb.2: # %atomicrmw.end ; O3-NEXT: movq $1, foo(%rip) @@ -10894,7 +10984,7 @@ ; O1-LABEL: atomic128_store_seq_cst_ptr_ty: ; O1: # %bb.0: # %entry ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection373: +; O1-NEXT: .Lpcsection403: ; O1-NEXT: xchgq %rsi, (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: retq @@ -10902,7 +10992,7 @@ ; O2-LABEL: atomic128_store_seq_cst_ptr_ty: ; O2: # %bb.0: # %entry ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection373: +; O2-NEXT: .Lpcsection403: ; O2-NEXT: xchgq %rsi, (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: retq @@ -10910,7 +11000,7 @@ ; O3-LABEL: atomic128_store_seq_cst_ptr_ty: ; O3: # %bb.0: # %entry ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection373: +; O3-NEXT: .Lpcsection403: ; O3-NEXT: xchgq %rsi, (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: retq @@ -10967,20 +11057,20 @@ ; O1-NEXT: .cfi_def_cfa_offset 16 ; O1-NEXT: .cfi_offset %rbx, -16 ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection374: +; O1-NEXT: .Lpcsection404: ; O1-NEXT: movq (%rdi), %rax -; O1-NEXT: .Lpcsection375: +; O1-NEXT: .Lpcsection405: ; O1-NEXT: movq 8(%rdi), %rdx -; O1-NEXT: .Lpcsection376: +; O1-NEXT: .Lpcsection406: ; O1-NEXT: movl $42, %ebx ; O1-NEXT: .p2align 4, 0x90 ; O1-NEXT: .LBB208_1: # %atomicrmw.start ; O1-NEXT: # =>This Inner Loop Header: Depth=1 -; O1-NEXT: .Lpcsection377: +; O1-NEXT: .Lpcsection407: ; O1-NEXT: xorl %ecx, %ecx -; O1-NEXT: .Lpcsection378: +; O1-NEXT: .Lpcsection408: ; O1-NEXT: lock cmpxchg16b (%rdi) -; O1-NEXT: .Lpcsection379: +; O1-NEXT: .Lpcsection409: ; O1-NEXT: jne .LBB208_1 ; O1-NEXT: # %bb.2: # %atomicrmw.end ; O1-NEXT: movq $1, foo(%rip) @@ -10994,20 +11084,20 @@ ; O2-NEXT: .cfi_def_cfa_offset 16 ; O2-NEXT: .cfi_offset %rbx, -16 ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection374: +; O2-NEXT: .Lpcsection404: ; O2-NEXT: movq (%rdi), %rax -; O2-NEXT: .Lpcsection375: +; O2-NEXT: .Lpcsection405: ; O2-NEXT: movq 8(%rdi), %rdx -; O2-NEXT: .Lpcsection376: +; O2-NEXT: .Lpcsection406: ; O2-NEXT: movl $42, %ebx ; O2-NEXT: .p2align 4, 0x90 ; O2-NEXT: .LBB208_1: # %atomicrmw.start ; O2-NEXT: # =>This Inner Loop Header: Depth=1 -; O2-NEXT: .Lpcsection377: +; O2-NEXT: .Lpcsection407: ; O2-NEXT: xorl %ecx, %ecx -; O2-NEXT: .Lpcsection378: +; O2-NEXT: .Lpcsection408: ; O2-NEXT: lock cmpxchg16b (%rdi) -; O2-NEXT: .Lpcsection379: +; O2-NEXT: .Lpcsection409: ; O2-NEXT: jne .LBB208_1 ; O2-NEXT: # %bb.2: # %atomicrmw.end ; O2-NEXT: movq $1, foo(%rip) @@ -11021,20 +11111,20 @@ ; O3-NEXT: .cfi_def_cfa_offset 16 ; O3-NEXT: .cfi_offset %rbx, -16 ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection374: +; O3-NEXT: .Lpcsection404: ; O3-NEXT: movq (%rdi), %rax -; O3-NEXT: .Lpcsection375: +; O3-NEXT: .Lpcsection405: ; O3-NEXT: movq 8(%rdi), %rdx -; O3-NEXT: .Lpcsection376: +; O3-NEXT: .Lpcsection406: ; O3-NEXT: movl $42, %ebx ; O3-NEXT: .p2align 4, 0x90 ; O3-NEXT: .LBB208_1: # %atomicrmw.start ; O3-NEXT: # =>This Inner Loop Header: Depth=1 -; O3-NEXT: .Lpcsection377: +; O3-NEXT: .Lpcsection407: ; O3-NEXT: xorl %ecx, %ecx -; O3-NEXT: .Lpcsection378: +; O3-NEXT: .Lpcsection408: ; O3-NEXT: lock cmpxchg16b (%rdi) -; O3-NEXT: .Lpcsection379: +; O3-NEXT: .Lpcsection409: ; O3-NEXT: jne .LBB208_1 ; O3-NEXT: # %bb.2: # %atomicrmw.end ; O3-NEXT: movq $1, foo(%rip) @@ -11094,22 +11184,22 @@ ; O1-NEXT: .cfi_def_cfa_offset 16 ; O1-NEXT: .cfi_offset %rbx, -16 ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection380: +; O1-NEXT: .Lpcsection410: ; O1-NEXT: movq (%rdi), %rax -; O1-NEXT: .Lpcsection381: +; O1-NEXT: .Lpcsection411: ; O1-NEXT: movq 8(%rdi), %rdx ; O1-NEXT: .p2align 4, 0x90 ; O1-NEXT: .LBB209_1: # %atomicrmw.start ; O1-NEXT: # =>This Inner Loop Header: Depth=1 ; O1-NEXT: movq %rax, %rbx -; O1-NEXT: .Lpcsection382: +; O1-NEXT: .Lpcsection412: ; O1-NEXT: addq $42, %rbx ; O1-NEXT: movq %rdx, %rcx -; O1-NEXT: .Lpcsection383: +; O1-NEXT: .Lpcsection413: ; O1-NEXT: adcq $0, %rcx -; O1-NEXT: .Lpcsection384: +; O1-NEXT: .Lpcsection414: ; O1-NEXT: lock cmpxchg16b (%rdi) -; O1-NEXT: .Lpcsection385: +; O1-NEXT: .Lpcsection415: ; O1-NEXT: jne .LBB209_1 ; O1-NEXT: # %bb.2: # %atomicrmw.end ; O1-NEXT: movq $1, foo(%rip) @@ -11123,22 +11213,22 @@ ; O2-NEXT: .cfi_def_cfa_offset 16 ; O2-NEXT: .cfi_offset %rbx, -16 ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection380: +; O2-NEXT: .Lpcsection410: ; O2-NEXT: movq (%rdi), %rax -; O2-NEXT: .Lpcsection381: +; O2-NEXT: .Lpcsection411: ; O2-NEXT: movq 8(%rdi), %rdx ; O2-NEXT: .p2align 4, 0x90 ; O2-NEXT: .LBB209_1: # %atomicrmw.start ; O2-NEXT: # =>This Inner Loop Header: Depth=1 ; O2-NEXT: movq %rax, %rbx -; O2-NEXT: .Lpcsection382: +; O2-NEXT: .Lpcsection412: ; O2-NEXT: addq $42, %rbx ; O2-NEXT: movq %rdx, %rcx -; O2-NEXT: .Lpcsection383: +; O2-NEXT: .Lpcsection413: ; O2-NEXT: adcq $0, %rcx -; O2-NEXT: .Lpcsection384: +; O2-NEXT: .Lpcsection414: ; O2-NEXT: lock cmpxchg16b (%rdi) -; O2-NEXT: .Lpcsection385: +; O2-NEXT: .Lpcsection415: ; O2-NEXT: jne .LBB209_1 ; O2-NEXT: # %bb.2: # %atomicrmw.end ; O2-NEXT: movq $1, foo(%rip) @@ -11152,22 +11242,22 @@ ; O3-NEXT: .cfi_def_cfa_offset 16 ; O3-NEXT: .cfi_offset %rbx, -16 ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection380: +; O3-NEXT: .Lpcsection410: ; O3-NEXT: movq (%rdi), %rax -; O3-NEXT: .Lpcsection381: +; O3-NEXT: .Lpcsection411: ; O3-NEXT: movq 8(%rdi), %rdx ; O3-NEXT: .p2align 4, 0x90 ; O3-NEXT: .LBB209_1: # %atomicrmw.start ; O3-NEXT: # =>This Inner Loop Header: Depth=1 ; O3-NEXT: movq %rax, %rbx -; O3-NEXT: .Lpcsection382: +; O3-NEXT: .Lpcsection412: ; O3-NEXT: addq $42, %rbx ; O3-NEXT: movq %rdx, %rcx -; O3-NEXT: .Lpcsection383: +; O3-NEXT: .Lpcsection413: ; O3-NEXT: adcq $0, %rcx -; O3-NEXT: .Lpcsection384: +; O3-NEXT: .Lpcsection414: ; O3-NEXT: lock cmpxchg16b (%rdi) -; O3-NEXT: .Lpcsection385: +; O3-NEXT: .Lpcsection415: ; O3-NEXT: jne .LBB209_1 ; O3-NEXT: # %bb.2: # %atomicrmw.end ; O3-NEXT: movq $1, foo(%rip) @@ -11227,22 +11317,22 @@ ; O1-NEXT: .cfi_def_cfa_offset 16 ; O1-NEXT: .cfi_offset %rbx, -16 ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection386: +; O1-NEXT: .Lpcsection416: ; O1-NEXT: movq (%rdi), %rax -; O1-NEXT: .Lpcsection387: +; O1-NEXT: .Lpcsection417: ; O1-NEXT: movq 8(%rdi), %rdx ; O1-NEXT: .p2align 4, 0x90 ; O1-NEXT: .LBB210_1: # %atomicrmw.start ; O1-NEXT: # =>This Inner Loop Header: Depth=1 ; O1-NEXT: movq %rax, %rbx -; O1-NEXT: .Lpcsection388: +; O1-NEXT: .Lpcsection418: ; O1-NEXT: addq $-42, %rbx ; O1-NEXT: movq %rdx, %rcx -; O1-NEXT: .Lpcsection389: +; O1-NEXT: .Lpcsection419: ; O1-NEXT: adcq $-1, %rcx -; O1-NEXT: .Lpcsection390: +; O1-NEXT: .Lpcsection420: ; O1-NEXT: lock cmpxchg16b (%rdi) -; O1-NEXT: .Lpcsection391: +; O1-NEXT: .Lpcsection421: ; O1-NEXT: jne .LBB210_1 ; O1-NEXT: # %bb.2: # %atomicrmw.end ; O1-NEXT: movq $1, foo(%rip) @@ -11256,22 +11346,22 @@ ; O2-NEXT: .cfi_def_cfa_offset 16 ; O2-NEXT: .cfi_offset %rbx, -16 ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection386: +; O2-NEXT: .Lpcsection416: ; O2-NEXT: movq (%rdi), %rax -; O2-NEXT: .Lpcsection387: +; O2-NEXT: .Lpcsection417: ; O2-NEXT: movq 8(%rdi), %rdx ; O2-NEXT: .p2align 4, 0x90 ; O2-NEXT: .LBB210_1: # %atomicrmw.start ; O2-NEXT: # =>This Inner Loop Header: Depth=1 ; O2-NEXT: movq %rax, %rbx -; O2-NEXT: .Lpcsection388: +; O2-NEXT: .Lpcsection418: ; O2-NEXT: addq $-42, %rbx ; O2-NEXT: movq %rdx, %rcx -; O2-NEXT: .Lpcsection389: +; O2-NEXT: .Lpcsection419: ; O2-NEXT: adcq $-1, %rcx -; O2-NEXT: .Lpcsection390: +; O2-NEXT: .Lpcsection420: ; O2-NEXT: lock cmpxchg16b (%rdi) -; O2-NEXT: .Lpcsection391: +; O2-NEXT: .Lpcsection421: ; O2-NEXT: jne .LBB210_1 ; O2-NEXT: # %bb.2: # %atomicrmw.end ; O2-NEXT: movq $1, foo(%rip) @@ -11285,22 +11375,22 @@ ; O3-NEXT: .cfi_def_cfa_offset 16 ; O3-NEXT: .cfi_offset %rbx, -16 ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection386: +; O3-NEXT: .Lpcsection416: ; O3-NEXT: movq (%rdi), %rax -; O3-NEXT: .Lpcsection387: +; O3-NEXT: .Lpcsection417: ; O3-NEXT: movq 8(%rdi), %rdx ; O3-NEXT: .p2align 4, 0x90 ; O3-NEXT: .LBB210_1: # %atomicrmw.start ; O3-NEXT: # =>This Inner Loop Header: Depth=1 ; O3-NEXT: movq %rax, %rbx -; O3-NEXT: .Lpcsection388: +; O3-NEXT: .Lpcsection418: ; O3-NEXT: addq $-42, %rbx ; O3-NEXT: movq %rdx, %rcx -; O3-NEXT: .Lpcsection389: +; O3-NEXT: .Lpcsection419: ; O3-NEXT: adcq $-1, %rcx -; O3-NEXT: .Lpcsection390: +; O3-NEXT: .Lpcsection420: ; O3-NEXT: lock cmpxchg16b (%rdi) -; O3-NEXT: .Lpcsection391: +; O3-NEXT: .Lpcsection421: ; O3-NEXT: jne .LBB210_1 ; O3-NEXT: # %bb.2: # %atomicrmw.end ; O3-NEXT: movq $1, foo(%rip) @@ -11362,21 +11452,21 @@ ; O1-NEXT: .cfi_def_cfa_offset 16 ; O1-NEXT: .cfi_offset %rbx, -16 ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection392: +; O1-NEXT: .Lpcsection422: ; O1-NEXT: movq (%rdi), %rax -; O1-NEXT: .Lpcsection393: +; O1-NEXT: .Lpcsection423: ; O1-NEXT: movq 8(%rdi), %rdx ; O1-NEXT: .p2align 4, 0x90 ; O1-NEXT: .LBB211_1: # %atomicrmw.start ; O1-NEXT: # =>This Inner Loop Header: Depth=1 ; O1-NEXT: movl %eax, %ebx -; O1-NEXT: .Lpcsection394: +; O1-NEXT: .Lpcsection424: ; O1-NEXT: andl $42, %ebx -; O1-NEXT: .Lpcsection395: +; O1-NEXT: .Lpcsection425: ; O1-NEXT: xorl %ecx, %ecx -; O1-NEXT: .Lpcsection396: +; O1-NEXT: .Lpcsection426: ; O1-NEXT: lock cmpxchg16b (%rdi) -; O1-NEXT: .Lpcsection397: +; O1-NEXT: .Lpcsection427: ; O1-NEXT: jne .LBB211_1 ; O1-NEXT: # %bb.2: # %atomicrmw.end ; O1-NEXT: movq $1, foo(%rip) @@ -11390,21 +11480,21 @@ ; O2-NEXT: .cfi_def_cfa_offset 16 ; O2-NEXT: .cfi_offset %rbx, -16 ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection392: +; O2-NEXT: .Lpcsection422: ; O2-NEXT: movq (%rdi), %rax -; O2-NEXT: .Lpcsection393: +; O2-NEXT: .Lpcsection423: ; O2-NEXT: movq 8(%rdi), %rdx ; O2-NEXT: .p2align 4, 0x90 ; O2-NEXT: .LBB211_1: # %atomicrmw.start ; O2-NEXT: # =>This Inner Loop Header: Depth=1 ; O2-NEXT: movl %eax, %ebx -; O2-NEXT: .Lpcsection394: +; O2-NEXT: .Lpcsection424: ; O2-NEXT: andl $42, %ebx -; O2-NEXT: .Lpcsection395: +; O2-NEXT: .Lpcsection425: ; O2-NEXT: xorl %ecx, %ecx -; O2-NEXT: .Lpcsection396: +; O2-NEXT: .Lpcsection426: ; O2-NEXT: lock cmpxchg16b (%rdi) -; O2-NEXT: .Lpcsection397: +; O2-NEXT: .Lpcsection427: ; O2-NEXT: jne .LBB211_1 ; O2-NEXT: # %bb.2: # %atomicrmw.end ; O2-NEXT: movq $1, foo(%rip) @@ -11418,21 +11508,21 @@ ; O3-NEXT: .cfi_def_cfa_offset 16 ; O3-NEXT: .cfi_offset %rbx, -16 ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection392: +; O3-NEXT: .Lpcsection422: ; O3-NEXT: movq (%rdi), %rax -; O3-NEXT: .Lpcsection393: +; O3-NEXT: .Lpcsection423: ; O3-NEXT: movq 8(%rdi), %rdx ; O3-NEXT: .p2align 4, 0x90 ; O3-NEXT: .LBB211_1: # %atomicrmw.start ; O3-NEXT: # =>This Inner Loop Header: Depth=1 ; O3-NEXT: movl %eax, %ebx -; O3-NEXT: .Lpcsection394: +; O3-NEXT: .Lpcsection424: ; O3-NEXT: andl $42, %ebx -; O3-NEXT: .Lpcsection395: +; O3-NEXT: .Lpcsection425: ; O3-NEXT: xorl %ecx, %ecx -; O3-NEXT: .Lpcsection396: +; O3-NEXT: .Lpcsection426: ; O3-NEXT: lock cmpxchg16b (%rdi) -; O3-NEXT: .Lpcsection397: +; O3-NEXT: .Lpcsection427: ; O3-NEXT: jne .LBB211_1 ; O3-NEXT: # %bb.2: # %atomicrmw.end ; O3-NEXT: movq $1, foo(%rip) @@ -11490,20 +11580,20 @@ ; O1-NEXT: .cfi_def_cfa_offset 16 ; O1-NEXT: .cfi_offset %rbx, -16 ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection398: +; O1-NEXT: .Lpcsection428: ; O1-NEXT: movq (%rdi), %rax -; O1-NEXT: .Lpcsection399: +; O1-NEXT: .Lpcsection429: ; O1-NEXT: movq 8(%rdi), %rdx ; O1-NEXT: .p2align 4, 0x90 ; O1-NEXT: .LBB212_1: # %atomicrmw.start ; O1-NEXT: # =>This Inner Loop Header: Depth=1 ; O1-NEXT: movq %rax, %rbx -; O1-NEXT: .Lpcsection400: +; O1-NEXT: .Lpcsection430: ; O1-NEXT: orq $42, %rbx ; O1-NEXT: movq %rdx, %rcx -; O1-NEXT: .Lpcsection401: +; O1-NEXT: .Lpcsection431: ; O1-NEXT: lock cmpxchg16b (%rdi) -; O1-NEXT: .Lpcsection402: +; O1-NEXT: .Lpcsection432: ; O1-NEXT: jne .LBB212_1 ; O1-NEXT: # %bb.2: # %atomicrmw.end ; O1-NEXT: movq $1, foo(%rip) @@ -11517,20 +11607,20 @@ ; O2-NEXT: .cfi_def_cfa_offset 16 ; O2-NEXT: .cfi_offset %rbx, -16 ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection398: +; O2-NEXT: .Lpcsection428: ; O2-NEXT: movq (%rdi), %rax -; O2-NEXT: .Lpcsection399: +; O2-NEXT: .Lpcsection429: ; O2-NEXT: movq 8(%rdi), %rdx ; O2-NEXT: .p2align 4, 0x90 ; O2-NEXT: .LBB212_1: # %atomicrmw.start ; O2-NEXT: # =>This Inner Loop Header: Depth=1 ; O2-NEXT: movq %rax, %rbx -; O2-NEXT: .Lpcsection400: +; O2-NEXT: .Lpcsection430: ; O2-NEXT: orq $42, %rbx ; O2-NEXT: movq %rdx, %rcx -; O2-NEXT: .Lpcsection401: +; O2-NEXT: .Lpcsection431: ; O2-NEXT: lock cmpxchg16b (%rdi) -; O2-NEXT: .Lpcsection402: +; O2-NEXT: .Lpcsection432: ; O2-NEXT: jne .LBB212_1 ; O2-NEXT: # %bb.2: # %atomicrmw.end ; O2-NEXT: movq $1, foo(%rip) @@ -11544,20 +11634,20 @@ ; O3-NEXT: .cfi_def_cfa_offset 16 ; O3-NEXT: .cfi_offset %rbx, -16 ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection398: +; O3-NEXT: .Lpcsection428: ; O3-NEXT: movq (%rdi), %rax -; O3-NEXT: .Lpcsection399: +; O3-NEXT: .Lpcsection429: ; O3-NEXT: movq 8(%rdi), %rdx ; O3-NEXT: .p2align 4, 0x90 ; O3-NEXT: .LBB212_1: # %atomicrmw.start ; O3-NEXT: # =>This Inner Loop Header: Depth=1 ; O3-NEXT: movq %rax, %rbx -; O3-NEXT: .Lpcsection400: +; O3-NEXT: .Lpcsection430: ; O3-NEXT: orq $42, %rbx ; O3-NEXT: movq %rdx, %rcx -; O3-NEXT: .Lpcsection401: +; O3-NEXT: .Lpcsection431: ; O3-NEXT: lock cmpxchg16b (%rdi) -; O3-NEXT: .Lpcsection402: +; O3-NEXT: .Lpcsection432: ; O3-NEXT: jne .LBB212_1 ; O3-NEXT: # %bb.2: # %atomicrmw.end ; O3-NEXT: movq $1, foo(%rip) @@ -11615,20 +11705,20 @@ ; O1-NEXT: .cfi_def_cfa_offset 16 ; O1-NEXT: .cfi_offset %rbx, -16 ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection403: +; O1-NEXT: .Lpcsection433: ; O1-NEXT: movq (%rdi), %rax -; O1-NEXT: .Lpcsection404: +; O1-NEXT: .Lpcsection434: ; O1-NEXT: movq 8(%rdi), %rdx ; O1-NEXT: .p2align 4, 0x90 ; O1-NEXT: .LBB213_1: # %atomicrmw.start ; O1-NEXT: # =>This Inner Loop Header: Depth=1 ; O1-NEXT: movq %rax, %rbx -; O1-NEXT: .Lpcsection405: +; O1-NEXT: .Lpcsection435: ; O1-NEXT: xorq $42, %rbx ; O1-NEXT: movq %rdx, %rcx -; O1-NEXT: .Lpcsection406: +; O1-NEXT: .Lpcsection436: ; O1-NEXT: lock cmpxchg16b (%rdi) -; O1-NEXT: .Lpcsection407: +; O1-NEXT: .Lpcsection437: ; O1-NEXT: jne .LBB213_1 ; O1-NEXT: # %bb.2: # %atomicrmw.end ; O1-NEXT: movq $1, foo(%rip) @@ -11642,20 +11732,20 @@ ; O2-NEXT: .cfi_def_cfa_offset 16 ; O2-NEXT: .cfi_offset %rbx, -16 ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection403: +; O2-NEXT: .Lpcsection433: ; O2-NEXT: movq (%rdi), %rax -; O2-NEXT: .Lpcsection404: +; O2-NEXT: .Lpcsection434: ; O2-NEXT: movq 8(%rdi), %rdx ; O2-NEXT: .p2align 4, 0x90 ; O2-NEXT: .LBB213_1: # %atomicrmw.start ; O2-NEXT: # =>This Inner Loop Header: Depth=1 ; O2-NEXT: movq %rax, %rbx -; O2-NEXT: .Lpcsection405: +; O2-NEXT: .Lpcsection435: ; O2-NEXT: xorq $42, %rbx ; O2-NEXT: movq %rdx, %rcx -; O2-NEXT: .Lpcsection406: +; O2-NEXT: .Lpcsection436: ; O2-NEXT: lock cmpxchg16b (%rdi) -; O2-NEXT: .Lpcsection407: +; O2-NEXT: .Lpcsection437: ; O2-NEXT: jne .LBB213_1 ; O2-NEXT: # %bb.2: # %atomicrmw.end ; O2-NEXT: movq $1, foo(%rip) @@ -11669,20 +11759,20 @@ ; O3-NEXT: .cfi_def_cfa_offset 16 ; O3-NEXT: .cfi_offset %rbx, -16 ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection403: +; O3-NEXT: .Lpcsection433: ; O3-NEXT: movq (%rdi), %rax -; O3-NEXT: .Lpcsection404: +; O3-NEXT: .Lpcsection434: ; O3-NEXT: movq 8(%rdi), %rdx ; O3-NEXT: .p2align 4, 0x90 ; O3-NEXT: .LBB213_1: # %atomicrmw.start ; O3-NEXT: # =>This Inner Loop Header: Depth=1 ; O3-NEXT: movq %rax, %rbx -; O3-NEXT: .Lpcsection405: +; O3-NEXT: .Lpcsection435: ; O3-NEXT: xorq $42, %rbx ; O3-NEXT: movq %rdx, %rcx -; O3-NEXT: .Lpcsection406: +; O3-NEXT: .Lpcsection436: ; O3-NEXT: lock cmpxchg16b (%rdi) -; O3-NEXT: .Lpcsection407: +; O3-NEXT: .Lpcsection437: ; O3-NEXT: jne .LBB213_1 ; O3-NEXT: # %bb.2: # %atomicrmw.end ; O3-NEXT: movq $1, foo(%rip) @@ -11746,23 +11836,23 @@ ; O1-NEXT: .cfi_def_cfa_offset 16 ; O1-NEXT: .cfi_offset %rbx, -16 ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection408: +; O1-NEXT: .Lpcsection438: ; O1-NEXT: movq (%rdi), %rax -; O1-NEXT: .Lpcsection409: +; O1-NEXT: .Lpcsection439: ; O1-NEXT: movq 8(%rdi), %rdx -; O1-NEXT: .Lpcsection410: +; O1-NEXT: .Lpcsection440: ; O1-NEXT: movq $-1, %rcx ; O1-NEXT: .p2align 4, 0x90 ; O1-NEXT: .LBB214_1: # %atomicrmw.start ; O1-NEXT: # =>This Inner Loop Header: Depth=1 ; O1-NEXT: movl %eax, %ebx -; O1-NEXT: .Lpcsection411: +; O1-NEXT: .Lpcsection441: ; O1-NEXT: notl %ebx -; O1-NEXT: .Lpcsection412: +; O1-NEXT: .Lpcsection442: ; O1-NEXT: orq $-43, %rbx -; O1-NEXT: .Lpcsection413: +; O1-NEXT: .Lpcsection443: ; O1-NEXT: lock cmpxchg16b (%rdi) -; O1-NEXT: .Lpcsection414: +; O1-NEXT: .Lpcsection444: ; O1-NEXT: jne .LBB214_1 ; O1-NEXT: # %bb.2: # %atomicrmw.end ; O1-NEXT: movq $1, foo(%rip) @@ -11776,23 +11866,23 @@ ; O2-NEXT: .cfi_def_cfa_offset 16 ; O2-NEXT: .cfi_offset %rbx, -16 ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection408: +; O2-NEXT: .Lpcsection438: ; O2-NEXT: movq (%rdi), %rax -; O2-NEXT: .Lpcsection409: +; O2-NEXT: .Lpcsection439: ; O2-NEXT: movq 8(%rdi), %rdx -; O2-NEXT: .Lpcsection410: +; O2-NEXT: .Lpcsection440: ; O2-NEXT: movq $-1, %rcx ; O2-NEXT: .p2align 4, 0x90 ; O2-NEXT: .LBB214_1: # %atomicrmw.start ; O2-NEXT: # =>This Inner Loop Header: Depth=1 ; O2-NEXT: movl %eax, %ebx -; O2-NEXT: .Lpcsection411: +; O2-NEXT: .Lpcsection441: ; O2-NEXT: notl %ebx -; O2-NEXT: .Lpcsection412: +; O2-NEXT: .Lpcsection442: ; O2-NEXT: orq $-43, %rbx -; O2-NEXT: .Lpcsection413: +; O2-NEXT: .Lpcsection443: ; O2-NEXT: lock cmpxchg16b (%rdi) -; O2-NEXT: .Lpcsection414: +; O2-NEXT: .Lpcsection444: ; O2-NEXT: jne .LBB214_1 ; O2-NEXT: # %bb.2: # %atomicrmw.end ; O2-NEXT: movq $1, foo(%rip) @@ -11806,23 +11896,23 @@ ; O3-NEXT: .cfi_def_cfa_offset 16 ; O3-NEXT: .cfi_offset %rbx, -16 ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection408: +; O3-NEXT: .Lpcsection438: ; O3-NEXT: movq (%rdi), %rax -; O3-NEXT: .Lpcsection409: +; O3-NEXT: .Lpcsection439: ; O3-NEXT: movq 8(%rdi), %rdx -; O3-NEXT: .Lpcsection410: +; O3-NEXT: .Lpcsection440: ; O3-NEXT: movq $-1, %rcx ; O3-NEXT: .p2align 4, 0x90 ; O3-NEXT: .LBB214_1: # %atomicrmw.start ; O3-NEXT: # =>This Inner Loop Header: Depth=1 ; O3-NEXT: movl %eax, %ebx -; O3-NEXT: .Lpcsection411: +; O3-NEXT: .Lpcsection441: ; O3-NEXT: notl %ebx -; O3-NEXT: .Lpcsection412: +; O3-NEXT: .Lpcsection442: ; O3-NEXT: orq $-43, %rbx -; O3-NEXT: .Lpcsection413: +; O3-NEXT: .Lpcsection443: ; O3-NEXT: lock cmpxchg16b (%rdi) -; O3-NEXT: .Lpcsection414: +; O3-NEXT: .Lpcsection444: ; O3-NEXT: jne .LBB214_1 ; O3-NEXT: # %bb.2: # %atomicrmw.end ; O3-NEXT: movq $1, foo(%rip) @@ -11882,20 +11972,20 @@ ; O1-NEXT: .cfi_def_cfa_offset 16 ; O1-NEXT: .cfi_offset %rbx, -16 ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection415: +; O1-NEXT: .Lpcsection445: ; O1-NEXT: movq (%rdi), %rax -; O1-NEXT: .Lpcsection416: +; O1-NEXT: .Lpcsection446: ; O1-NEXT: movq 8(%rdi), %rdx -; O1-NEXT: .Lpcsection417: +; O1-NEXT: .Lpcsection447: ; O1-NEXT: movl $42, %ebx ; O1-NEXT: .p2align 4, 0x90 ; O1-NEXT: .LBB215_1: # %atomicrmw.start ; O1-NEXT: # =>This Inner Loop Header: Depth=1 -; O1-NEXT: .Lpcsection418: +; O1-NEXT: .Lpcsection448: ; O1-NEXT: xorl %ecx, %ecx -; O1-NEXT: .Lpcsection419: +; O1-NEXT: .Lpcsection449: ; O1-NEXT: lock cmpxchg16b (%rdi) -; O1-NEXT: .Lpcsection420: +; O1-NEXT: .Lpcsection450: ; O1-NEXT: jne .LBB215_1 ; O1-NEXT: # %bb.2: # %atomicrmw.end ; O1-NEXT: movq $1, foo(%rip) @@ -11909,20 +11999,20 @@ ; O2-NEXT: .cfi_def_cfa_offset 16 ; O2-NEXT: .cfi_offset %rbx, -16 ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection415: +; O2-NEXT: .Lpcsection445: ; O2-NEXT: movq (%rdi), %rax -; O2-NEXT: .Lpcsection416: +; O2-NEXT: .Lpcsection446: ; O2-NEXT: movq 8(%rdi), %rdx -; O2-NEXT: .Lpcsection417: +; O2-NEXT: .Lpcsection447: ; O2-NEXT: movl $42, %ebx ; O2-NEXT: .p2align 4, 0x90 ; O2-NEXT: .LBB215_1: # %atomicrmw.start ; O2-NEXT: # =>This Inner Loop Header: Depth=1 -; O2-NEXT: .Lpcsection418: +; O2-NEXT: .Lpcsection448: ; O2-NEXT: xorl %ecx, %ecx -; O2-NEXT: .Lpcsection419: +; O2-NEXT: .Lpcsection449: ; O2-NEXT: lock cmpxchg16b (%rdi) -; O2-NEXT: .Lpcsection420: +; O2-NEXT: .Lpcsection450: ; O2-NEXT: jne .LBB215_1 ; O2-NEXT: # %bb.2: # %atomicrmw.end ; O2-NEXT: movq $1, foo(%rip) @@ -11936,20 +12026,20 @@ ; O3-NEXT: .cfi_def_cfa_offset 16 ; O3-NEXT: .cfi_offset %rbx, -16 ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection415: +; O3-NEXT: .Lpcsection445: ; O3-NEXT: movq (%rdi), %rax -; O3-NEXT: .Lpcsection416: +; O3-NEXT: .Lpcsection446: ; O3-NEXT: movq 8(%rdi), %rdx -; O3-NEXT: .Lpcsection417: +; O3-NEXT: .Lpcsection447: ; O3-NEXT: movl $42, %ebx ; O3-NEXT: .p2align 4, 0x90 ; O3-NEXT: .LBB215_1: # %atomicrmw.start ; O3-NEXT: # =>This Inner Loop Header: Depth=1 -; O3-NEXT: .Lpcsection418: +; O3-NEXT: .Lpcsection448: ; O3-NEXT: xorl %ecx, %ecx -; O3-NEXT: .Lpcsection419: +; O3-NEXT: .Lpcsection449: ; O3-NEXT: lock cmpxchg16b (%rdi) -; O3-NEXT: .Lpcsection420: +; O3-NEXT: .Lpcsection450: ; O3-NEXT: jne .LBB215_1 ; O3-NEXT: # %bb.2: # %atomicrmw.end ; O3-NEXT: movq $1, foo(%rip) @@ -12009,22 +12099,22 @@ ; O1-NEXT: .cfi_def_cfa_offset 16 ; O1-NEXT: .cfi_offset %rbx, -16 ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection421: +; O1-NEXT: .Lpcsection451: ; O1-NEXT: movq (%rdi), %rax -; O1-NEXT: .Lpcsection422: +; O1-NEXT: .Lpcsection452: ; O1-NEXT: movq 8(%rdi), %rdx ; O1-NEXT: .p2align 4, 0x90 ; O1-NEXT: .LBB216_1: # %atomicrmw.start ; O1-NEXT: # =>This Inner Loop Header: Depth=1 ; O1-NEXT: movq %rax, %rbx -; O1-NEXT: .Lpcsection423: +; O1-NEXT: .Lpcsection453: ; O1-NEXT: addq $42, %rbx ; O1-NEXT: movq %rdx, %rcx -; O1-NEXT: .Lpcsection424: +; O1-NEXT: .Lpcsection454: ; O1-NEXT: adcq $0, %rcx -; O1-NEXT: .Lpcsection425: +; O1-NEXT: .Lpcsection455: ; O1-NEXT: lock cmpxchg16b (%rdi) -; O1-NEXT: .Lpcsection426: +; O1-NEXT: .Lpcsection456: ; O1-NEXT: jne .LBB216_1 ; O1-NEXT: # %bb.2: # %atomicrmw.end ; O1-NEXT: movq $1, foo(%rip) @@ -12038,22 +12128,22 @@ ; O2-NEXT: .cfi_def_cfa_offset 16 ; O2-NEXT: .cfi_offset %rbx, -16 ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection421: +; O2-NEXT: .Lpcsection451: ; O2-NEXT: movq (%rdi), %rax -; O2-NEXT: .Lpcsection422: +; O2-NEXT: .Lpcsection452: ; O2-NEXT: movq 8(%rdi), %rdx ; O2-NEXT: .p2align 4, 0x90 ; O2-NEXT: .LBB216_1: # %atomicrmw.start ; O2-NEXT: # =>This Inner Loop Header: Depth=1 ; O2-NEXT: movq %rax, %rbx -; O2-NEXT: .Lpcsection423: +; O2-NEXT: .Lpcsection453: ; O2-NEXT: addq $42, %rbx ; O2-NEXT: movq %rdx, %rcx -; O2-NEXT: .Lpcsection424: +; O2-NEXT: .Lpcsection454: ; O2-NEXT: adcq $0, %rcx -; O2-NEXT: .Lpcsection425: +; O2-NEXT: .Lpcsection455: ; O2-NEXT: lock cmpxchg16b (%rdi) -; O2-NEXT: .Lpcsection426: +; O2-NEXT: .Lpcsection456: ; O2-NEXT: jne .LBB216_1 ; O2-NEXT: # %bb.2: # %atomicrmw.end ; O2-NEXT: movq $1, foo(%rip) @@ -12067,22 +12157,22 @@ ; O3-NEXT: .cfi_def_cfa_offset 16 ; O3-NEXT: .cfi_offset %rbx, -16 ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection421: +; O3-NEXT: .Lpcsection451: ; O3-NEXT: movq (%rdi), %rax -; O3-NEXT: .Lpcsection422: +; O3-NEXT: .Lpcsection452: ; O3-NEXT: movq 8(%rdi), %rdx ; O3-NEXT: .p2align 4, 0x90 ; O3-NEXT: .LBB216_1: # %atomicrmw.start ; O3-NEXT: # =>This Inner Loop Header: Depth=1 ; O3-NEXT: movq %rax, %rbx -; O3-NEXT: .Lpcsection423: +; O3-NEXT: .Lpcsection453: ; O3-NEXT: addq $42, %rbx ; O3-NEXT: movq %rdx, %rcx -; O3-NEXT: .Lpcsection424: +; O3-NEXT: .Lpcsection454: ; O3-NEXT: adcq $0, %rcx -; O3-NEXT: .Lpcsection425: +; O3-NEXT: .Lpcsection455: ; O3-NEXT: lock cmpxchg16b (%rdi) -; O3-NEXT: .Lpcsection426: +; O3-NEXT: .Lpcsection456: ; O3-NEXT: jne .LBB216_1 ; O3-NEXT: # %bb.2: # %atomicrmw.end ; O3-NEXT: movq $1, foo(%rip) @@ -12142,22 +12232,22 @@ ; O1-NEXT: .cfi_def_cfa_offset 16 ; O1-NEXT: .cfi_offset %rbx, -16 ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection427: +; O1-NEXT: .Lpcsection457: ; O1-NEXT: movq (%rdi), %rax -; O1-NEXT: .Lpcsection428: +; O1-NEXT: .Lpcsection458: ; O1-NEXT: movq 8(%rdi), %rdx ; O1-NEXT: .p2align 4, 0x90 ; O1-NEXT: .LBB217_1: # %atomicrmw.start ; O1-NEXT: # =>This Inner Loop Header: Depth=1 ; O1-NEXT: movq %rax, %rbx -; O1-NEXT: .Lpcsection429: +; O1-NEXT: .Lpcsection459: ; O1-NEXT: addq $-42, %rbx ; O1-NEXT: movq %rdx, %rcx -; O1-NEXT: .Lpcsection430: +; O1-NEXT: .Lpcsection460: ; O1-NEXT: adcq $-1, %rcx -; O1-NEXT: .Lpcsection431: +; O1-NEXT: .Lpcsection461: ; O1-NEXT: lock cmpxchg16b (%rdi) -; O1-NEXT: .Lpcsection432: +; O1-NEXT: .Lpcsection462: ; O1-NEXT: jne .LBB217_1 ; O1-NEXT: # %bb.2: # %atomicrmw.end ; O1-NEXT: movq $1, foo(%rip) @@ -12171,22 +12261,22 @@ ; O2-NEXT: .cfi_def_cfa_offset 16 ; O2-NEXT: .cfi_offset %rbx, -16 ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection427: +; O2-NEXT: .Lpcsection457: ; O2-NEXT: movq (%rdi), %rax -; O2-NEXT: .Lpcsection428: +; O2-NEXT: .Lpcsection458: ; O2-NEXT: movq 8(%rdi), %rdx ; O2-NEXT: .p2align 4, 0x90 ; O2-NEXT: .LBB217_1: # %atomicrmw.start ; O2-NEXT: # =>This Inner Loop Header: Depth=1 ; O2-NEXT: movq %rax, %rbx -; O2-NEXT: .Lpcsection429: +; O2-NEXT: .Lpcsection459: ; O2-NEXT: addq $-42, %rbx ; O2-NEXT: movq %rdx, %rcx -; O2-NEXT: .Lpcsection430: +; O2-NEXT: .Lpcsection460: ; O2-NEXT: adcq $-1, %rcx -; O2-NEXT: .Lpcsection431: +; O2-NEXT: .Lpcsection461: ; O2-NEXT: lock cmpxchg16b (%rdi) -; O2-NEXT: .Lpcsection432: +; O2-NEXT: .Lpcsection462: ; O2-NEXT: jne .LBB217_1 ; O2-NEXT: # %bb.2: # %atomicrmw.end ; O2-NEXT: movq $1, foo(%rip) @@ -12200,22 +12290,22 @@ ; O3-NEXT: .cfi_def_cfa_offset 16 ; O3-NEXT: .cfi_offset %rbx, -16 ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection427: +; O3-NEXT: .Lpcsection457: ; O3-NEXT: movq (%rdi), %rax -; O3-NEXT: .Lpcsection428: +; O3-NEXT: .Lpcsection458: ; O3-NEXT: movq 8(%rdi), %rdx ; O3-NEXT: .p2align 4, 0x90 ; O3-NEXT: .LBB217_1: # %atomicrmw.start ; O3-NEXT: # =>This Inner Loop Header: Depth=1 ; O3-NEXT: movq %rax, %rbx -; O3-NEXT: .Lpcsection429: +; O3-NEXT: .Lpcsection459: ; O3-NEXT: addq $-42, %rbx ; O3-NEXT: movq %rdx, %rcx -; O3-NEXT: .Lpcsection430: +; O3-NEXT: .Lpcsection460: ; O3-NEXT: adcq $-1, %rcx -; O3-NEXT: .Lpcsection431: +; O3-NEXT: .Lpcsection461: ; O3-NEXT: lock cmpxchg16b (%rdi) -; O3-NEXT: .Lpcsection432: +; O3-NEXT: .Lpcsection462: ; O3-NEXT: jne .LBB217_1 ; O3-NEXT: # %bb.2: # %atomicrmw.end ; O3-NEXT: movq $1, foo(%rip) @@ -12277,21 +12367,21 @@ ; O1-NEXT: .cfi_def_cfa_offset 16 ; O1-NEXT: .cfi_offset %rbx, -16 ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection433: +; O1-NEXT: .Lpcsection463: ; O1-NEXT: movq (%rdi), %rax -; O1-NEXT: .Lpcsection434: +; O1-NEXT: .Lpcsection464: ; O1-NEXT: movq 8(%rdi), %rdx ; O1-NEXT: .p2align 4, 0x90 ; O1-NEXT: .LBB218_1: # %atomicrmw.start ; O1-NEXT: # =>This Inner Loop Header: Depth=1 ; O1-NEXT: movl %eax, %ebx -; O1-NEXT: .Lpcsection435: +; O1-NEXT: .Lpcsection465: ; O1-NEXT: andl $42, %ebx -; O1-NEXT: .Lpcsection436: +; O1-NEXT: .Lpcsection466: ; O1-NEXT: xorl %ecx, %ecx -; O1-NEXT: .Lpcsection437: +; O1-NEXT: .Lpcsection467: ; O1-NEXT: lock cmpxchg16b (%rdi) -; O1-NEXT: .Lpcsection438: +; O1-NEXT: .Lpcsection468: ; O1-NEXT: jne .LBB218_1 ; O1-NEXT: # %bb.2: # %atomicrmw.end ; O1-NEXT: movq $1, foo(%rip) @@ -12305,21 +12395,21 @@ ; O2-NEXT: .cfi_def_cfa_offset 16 ; O2-NEXT: .cfi_offset %rbx, -16 ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection433: +; O2-NEXT: .Lpcsection463: ; O2-NEXT: movq (%rdi), %rax -; O2-NEXT: .Lpcsection434: +; O2-NEXT: .Lpcsection464: ; O2-NEXT: movq 8(%rdi), %rdx ; O2-NEXT: .p2align 4, 0x90 ; O2-NEXT: .LBB218_1: # %atomicrmw.start ; O2-NEXT: # =>This Inner Loop Header: Depth=1 ; O2-NEXT: movl %eax, %ebx -; O2-NEXT: .Lpcsection435: +; O2-NEXT: .Lpcsection465: ; O2-NEXT: andl $42, %ebx -; O2-NEXT: .Lpcsection436: +; O2-NEXT: .Lpcsection466: ; O2-NEXT: xorl %ecx, %ecx -; O2-NEXT: .Lpcsection437: +; O2-NEXT: .Lpcsection467: ; O2-NEXT: lock cmpxchg16b (%rdi) -; O2-NEXT: .Lpcsection438: +; O2-NEXT: .Lpcsection468: ; O2-NEXT: jne .LBB218_1 ; O2-NEXT: # %bb.2: # %atomicrmw.end ; O2-NEXT: movq $1, foo(%rip) @@ -12333,21 +12423,21 @@ ; O3-NEXT: .cfi_def_cfa_offset 16 ; O3-NEXT: .cfi_offset %rbx, -16 ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection433: +; O3-NEXT: .Lpcsection463: ; O3-NEXT: movq (%rdi), %rax -; O3-NEXT: .Lpcsection434: +; O3-NEXT: .Lpcsection464: ; O3-NEXT: movq 8(%rdi), %rdx ; O3-NEXT: .p2align 4, 0x90 ; O3-NEXT: .LBB218_1: # %atomicrmw.start ; O3-NEXT: # =>This Inner Loop Header: Depth=1 ; O3-NEXT: movl %eax, %ebx -; O3-NEXT: .Lpcsection435: +; O3-NEXT: .Lpcsection465: ; O3-NEXT: andl $42, %ebx -; O3-NEXT: .Lpcsection436: +; O3-NEXT: .Lpcsection466: ; O3-NEXT: xorl %ecx, %ecx -; O3-NEXT: .Lpcsection437: +; O3-NEXT: .Lpcsection467: ; O3-NEXT: lock cmpxchg16b (%rdi) -; O3-NEXT: .Lpcsection438: +; O3-NEXT: .Lpcsection468: ; O3-NEXT: jne .LBB218_1 ; O3-NEXT: # %bb.2: # %atomicrmw.end ; O3-NEXT: movq $1, foo(%rip) @@ -12405,20 +12495,20 @@ ; O1-NEXT: .cfi_def_cfa_offset 16 ; O1-NEXT: .cfi_offset %rbx, -16 ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection439: +; O1-NEXT: .Lpcsection469: ; O1-NEXT: movq (%rdi), %rax -; O1-NEXT: .Lpcsection440: +; O1-NEXT: .Lpcsection470: ; O1-NEXT: movq 8(%rdi), %rdx ; O1-NEXT: .p2align 4, 0x90 ; O1-NEXT: .LBB219_1: # %atomicrmw.start ; O1-NEXT: # =>This Inner Loop Header: Depth=1 ; O1-NEXT: movq %rax, %rbx -; O1-NEXT: .Lpcsection441: +; O1-NEXT: .Lpcsection471: ; O1-NEXT: orq $42, %rbx ; O1-NEXT: movq %rdx, %rcx -; O1-NEXT: .Lpcsection442: +; O1-NEXT: .Lpcsection472: ; O1-NEXT: lock cmpxchg16b (%rdi) -; O1-NEXT: .Lpcsection443: +; O1-NEXT: .Lpcsection473: ; O1-NEXT: jne .LBB219_1 ; O1-NEXT: # %bb.2: # %atomicrmw.end ; O1-NEXT: movq $1, foo(%rip) @@ -12432,20 +12522,20 @@ ; O2-NEXT: .cfi_def_cfa_offset 16 ; O2-NEXT: .cfi_offset %rbx, -16 ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection439: +; O2-NEXT: .Lpcsection469: ; O2-NEXT: movq (%rdi), %rax -; O2-NEXT: .Lpcsection440: +; O2-NEXT: .Lpcsection470: ; O2-NEXT: movq 8(%rdi), %rdx ; O2-NEXT: .p2align 4, 0x90 ; O2-NEXT: .LBB219_1: # %atomicrmw.start ; O2-NEXT: # =>This Inner Loop Header: Depth=1 ; O2-NEXT: movq %rax, %rbx -; O2-NEXT: .Lpcsection441: +; O2-NEXT: .Lpcsection471: ; O2-NEXT: orq $42, %rbx ; O2-NEXT: movq %rdx, %rcx -; O2-NEXT: .Lpcsection442: +; O2-NEXT: .Lpcsection472: ; O2-NEXT: lock cmpxchg16b (%rdi) -; O2-NEXT: .Lpcsection443: +; O2-NEXT: .Lpcsection473: ; O2-NEXT: jne .LBB219_1 ; O2-NEXT: # %bb.2: # %atomicrmw.end ; O2-NEXT: movq $1, foo(%rip) @@ -12459,20 +12549,20 @@ ; O3-NEXT: .cfi_def_cfa_offset 16 ; O3-NEXT: .cfi_offset %rbx, -16 ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection439: +; O3-NEXT: .Lpcsection469: ; O3-NEXT: movq (%rdi), %rax -; O3-NEXT: .Lpcsection440: +; O3-NEXT: .Lpcsection470: ; O3-NEXT: movq 8(%rdi), %rdx ; O3-NEXT: .p2align 4, 0x90 ; O3-NEXT: .LBB219_1: # %atomicrmw.start ; O3-NEXT: # =>This Inner Loop Header: Depth=1 ; O3-NEXT: movq %rax, %rbx -; O3-NEXT: .Lpcsection441: +; O3-NEXT: .Lpcsection471: ; O3-NEXT: orq $42, %rbx ; O3-NEXT: movq %rdx, %rcx -; O3-NEXT: .Lpcsection442: +; O3-NEXT: .Lpcsection472: ; O3-NEXT: lock cmpxchg16b (%rdi) -; O3-NEXT: .Lpcsection443: +; O3-NEXT: .Lpcsection473: ; O3-NEXT: jne .LBB219_1 ; O3-NEXT: # %bb.2: # %atomicrmw.end ; O3-NEXT: movq $1, foo(%rip) @@ -12530,20 +12620,20 @@ ; O1-NEXT: .cfi_def_cfa_offset 16 ; O1-NEXT: .cfi_offset %rbx, -16 ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection444: +; O1-NEXT: .Lpcsection474: ; O1-NEXT: movq (%rdi), %rax -; O1-NEXT: .Lpcsection445: +; O1-NEXT: .Lpcsection475: ; O1-NEXT: movq 8(%rdi), %rdx ; O1-NEXT: .p2align 4, 0x90 ; O1-NEXT: .LBB220_1: # %atomicrmw.start ; O1-NEXT: # =>This Inner Loop Header: Depth=1 ; O1-NEXT: movq %rax, %rbx -; O1-NEXT: .Lpcsection446: +; O1-NEXT: .Lpcsection476: ; O1-NEXT: xorq $42, %rbx ; O1-NEXT: movq %rdx, %rcx -; O1-NEXT: .Lpcsection447: +; O1-NEXT: .Lpcsection477: ; O1-NEXT: lock cmpxchg16b (%rdi) -; O1-NEXT: .Lpcsection448: +; O1-NEXT: .Lpcsection478: ; O1-NEXT: jne .LBB220_1 ; O1-NEXT: # %bb.2: # %atomicrmw.end ; O1-NEXT: movq $1, foo(%rip) @@ -12557,20 +12647,20 @@ ; O2-NEXT: .cfi_def_cfa_offset 16 ; O2-NEXT: .cfi_offset %rbx, -16 ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection444: +; O2-NEXT: .Lpcsection474: ; O2-NEXT: movq (%rdi), %rax -; O2-NEXT: .Lpcsection445: +; O2-NEXT: .Lpcsection475: ; O2-NEXT: movq 8(%rdi), %rdx ; O2-NEXT: .p2align 4, 0x90 ; O2-NEXT: .LBB220_1: # %atomicrmw.start ; O2-NEXT: # =>This Inner Loop Header: Depth=1 ; O2-NEXT: movq %rax, %rbx -; O2-NEXT: .Lpcsection446: +; O2-NEXT: .Lpcsection476: ; O2-NEXT: xorq $42, %rbx ; O2-NEXT: movq %rdx, %rcx -; O2-NEXT: .Lpcsection447: +; O2-NEXT: .Lpcsection477: ; O2-NEXT: lock cmpxchg16b (%rdi) -; O2-NEXT: .Lpcsection448: +; O2-NEXT: .Lpcsection478: ; O2-NEXT: jne .LBB220_1 ; O2-NEXT: # %bb.2: # %atomicrmw.end ; O2-NEXT: movq $1, foo(%rip) @@ -12584,20 +12674,20 @@ ; O3-NEXT: .cfi_def_cfa_offset 16 ; O3-NEXT: .cfi_offset %rbx, -16 ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection444: +; O3-NEXT: .Lpcsection474: ; O3-NEXT: movq (%rdi), %rax -; O3-NEXT: .Lpcsection445: +; O3-NEXT: .Lpcsection475: ; O3-NEXT: movq 8(%rdi), %rdx ; O3-NEXT: .p2align 4, 0x90 ; O3-NEXT: .LBB220_1: # %atomicrmw.start ; O3-NEXT: # =>This Inner Loop Header: Depth=1 ; O3-NEXT: movq %rax, %rbx -; O3-NEXT: .Lpcsection446: +; O3-NEXT: .Lpcsection476: ; O3-NEXT: xorq $42, %rbx ; O3-NEXT: movq %rdx, %rcx -; O3-NEXT: .Lpcsection447: +; O3-NEXT: .Lpcsection477: ; O3-NEXT: lock cmpxchg16b (%rdi) -; O3-NEXT: .Lpcsection448: +; O3-NEXT: .Lpcsection478: ; O3-NEXT: jne .LBB220_1 ; O3-NEXT: # %bb.2: # %atomicrmw.end ; O3-NEXT: movq $1, foo(%rip) @@ -12661,23 +12751,23 @@ ; O1-NEXT: .cfi_def_cfa_offset 16 ; O1-NEXT: .cfi_offset %rbx, -16 ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection449: +; O1-NEXT: .Lpcsection479: ; O1-NEXT: movq (%rdi), %rax -; O1-NEXT: .Lpcsection450: +; O1-NEXT: .Lpcsection480: ; O1-NEXT: movq 8(%rdi), %rdx -; O1-NEXT: .Lpcsection451: +; O1-NEXT: .Lpcsection481: ; O1-NEXT: movq $-1, %rcx ; O1-NEXT: .p2align 4, 0x90 ; O1-NEXT: .LBB221_1: # %atomicrmw.start ; O1-NEXT: # =>This Inner Loop Header: Depth=1 ; O1-NEXT: movl %eax, %ebx -; O1-NEXT: .Lpcsection452: +; O1-NEXT: .Lpcsection482: ; O1-NEXT: notl %ebx -; O1-NEXT: .Lpcsection453: +; O1-NEXT: .Lpcsection483: ; O1-NEXT: orq $-43, %rbx -; O1-NEXT: .Lpcsection454: +; O1-NEXT: .Lpcsection484: ; O1-NEXT: lock cmpxchg16b (%rdi) -; O1-NEXT: .Lpcsection455: +; O1-NEXT: .Lpcsection485: ; O1-NEXT: jne .LBB221_1 ; O1-NEXT: # %bb.2: # %atomicrmw.end ; O1-NEXT: movq $1, foo(%rip) @@ -12691,23 +12781,23 @@ ; O2-NEXT: .cfi_def_cfa_offset 16 ; O2-NEXT: .cfi_offset %rbx, -16 ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection449: +; O2-NEXT: .Lpcsection479: ; O2-NEXT: movq (%rdi), %rax -; O2-NEXT: .Lpcsection450: +; O2-NEXT: .Lpcsection480: ; O2-NEXT: movq 8(%rdi), %rdx -; O2-NEXT: .Lpcsection451: +; O2-NEXT: .Lpcsection481: ; O2-NEXT: movq $-1, %rcx ; O2-NEXT: .p2align 4, 0x90 ; O2-NEXT: .LBB221_1: # %atomicrmw.start ; O2-NEXT: # =>This Inner Loop Header: Depth=1 ; O2-NEXT: movl %eax, %ebx -; O2-NEXT: .Lpcsection452: +; O2-NEXT: .Lpcsection482: ; O2-NEXT: notl %ebx -; O2-NEXT: .Lpcsection453: +; O2-NEXT: .Lpcsection483: ; O2-NEXT: orq $-43, %rbx -; O2-NEXT: .Lpcsection454: +; O2-NEXT: .Lpcsection484: ; O2-NEXT: lock cmpxchg16b (%rdi) -; O2-NEXT: .Lpcsection455: +; O2-NEXT: .Lpcsection485: ; O2-NEXT: jne .LBB221_1 ; O2-NEXT: # %bb.2: # %atomicrmw.end ; O2-NEXT: movq $1, foo(%rip) @@ -12721,23 +12811,23 @@ ; O3-NEXT: .cfi_def_cfa_offset 16 ; O3-NEXT: .cfi_offset %rbx, -16 ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection449: +; O3-NEXT: .Lpcsection479: ; O3-NEXT: movq (%rdi), %rax -; O3-NEXT: .Lpcsection450: +; O3-NEXT: .Lpcsection480: ; O3-NEXT: movq 8(%rdi), %rdx -; O3-NEXT: .Lpcsection451: +; O3-NEXT: .Lpcsection481: ; O3-NEXT: movq $-1, %rcx ; O3-NEXT: .p2align 4, 0x90 ; O3-NEXT: .LBB221_1: # %atomicrmw.start ; O3-NEXT: # =>This Inner Loop Header: Depth=1 ; O3-NEXT: movl %eax, %ebx -; O3-NEXT: .Lpcsection452: +; O3-NEXT: .Lpcsection482: ; O3-NEXT: notl %ebx -; O3-NEXT: .Lpcsection453: +; O3-NEXT: .Lpcsection483: ; O3-NEXT: orq $-43, %rbx -; O3-NEXT: .Lpcsection454: +; O3-NEXT: .Lpcsection484: ; O3-NEXT: lock cmpxchg16b (%rdi) -; O3-NEXT: .Lpcsection455: +; O3-NEXT: .Lpcsection485: ; O3-NEXT: jne .LBB221_1 ; O3-NEXT: # %bb.2: # %atomicrmw.end ; O3-NEXT: movq $1, foo(%rip) @@ -12797,20 +12887,20 @@ ; O1-NEXT: .cfi_def_cfa_offset 16 ; O1-NEXT: .cfi_offset %rbx, -16 ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection456: +; O1-NEXT: .Lpcsection486: ; O1-NEXT: movq (%rdi), %rax -; O1-NEXT: .Lpcsection457: +; O1-NEXT: .Lpcsection487: ; O1-NEXT: movq 8(%rdi), %rdx -; O1-NEXT: .Lpcsection458: +; O1-NEXT: .Lpcsection488: ; O1-NEXT: movl $42, %ebx ; O1-NEXT: .p2align 4, 0x90 ; O1-NEXT: .LBB222_1: # %atomicrmw.start ; O1-NEXT: # =>This Inner Loop Header: Depth=1 -; O1-NEXT: .Lpcsection459: +; O1-NEXT: .Lpcsection489: ; O1-NEXT: xorl %ecx, %ecx -; O1-NEXT: .Lpcsection460: +; O1-NEXT: .Lpcsection490: ; O1-NEXT: lock cmpxchg16b (%rdi) -; O1-NEXT: .Lpcsection461: +; O1-NEXT: .Lpcsection491: ; O1-NEXT: jne .LBB222_1 ; O1-NEXT: # %bb.2: # %atomicrmw.end ; O1-NEXT: movq $1, foo(%rip) @@ -12824,20 +12914,20 @@ ; O2-NEXT: .cfi_def_cfa_offset 16 ; O2-NEXT: .cfi_offset %rbx, -16 ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection456: +; O2-NEXT: .Lpcsection486: ; O2-NEXT: movq (%rdi), %rax -; O2-NEXT: .Lpcsection457: +; O2-NEXT: .Lpcsection487: ; O2-NEXT: movq 8(%rdi), %rdx -; O2-NEXT: .Lpcsection458: +; O2-NEXT: .Lpcsection488: ; O2-NEXT: movl $42, %ebx ; O2-NEXT: .p2align 4, 0x90 ; O2-NEXT: .LBB222_1: # %atomicrmw.start ; O2-NEXT: # =>This Inner Loop Header: Depth=1 -; O2-NEXT: .Lpcsection459: +; O2-NEXT: .Lpcsection489: ; O2-NEXT: xorl %ecx, %ecx -; O2-NEXT: .Lpcsection460: +; O2-NEXT: .Lpcsection490: ; O2-NEXT: lock cmpxchg16b (%rdi) -; O2-NEXT: .Lpcsection461: +; O2-NEXT: .Lpcsection491: ; O2-NEXT: jne .LBB222_1 ; O2-NEXT: # %bb.2: # %atomicrmw.end ; O2-NEXT: movq $1, foo(%rip) @@ -12851,20 +12941,20 @@ ; O3-NEXT: .cfi_def_cfa_offset 16 ; O3-NEXT: .cfi_offset %rbx, -16 ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection456: +; O3-NEXT: .Lpcsection486: ; O3-NEXT: movq (%rdi), %rax -; O3-NEXT: .Lpcsection457: +; O3-NEXT: .Lpcsection487: ; O3-NEXT: movq 8(%rdi), %rdx -; O3-NEXT: .Lpcsection458: +; O3-NEXT: .Lpcsection488: ; O3-NEXT: movl $42, %ebx ; O3-NEXT: .p2align 4, 0x90 ; O3-NEXT: .LBB222_1: # %atomicrmw.start ; O3-NEXT: # =>This Inner Loop Header: Depth=1 -; O3-NEXT: .Lpcsection459: +; O3-NEXT: .Lpcsection489: ; O3-NEXT: xorl %ecx, %ecx -; O3-NEXT: .Lpcsection460: +; O3-NEXT: .Lpcsection490: ; O3-NEXT: lock cmpxchg16b (%rdi) -; O3-NEXT: .Lpcsection461: +; O3-NEXT: .Lpcsection491: ; O3-NEXT: jne .LBB222_1 ; O3-NEXT: # %bb.2: # %atomicrmw.end ; O3-NEXT: movq $1, foo(%rip) @@ -12923,22 +13013,22 @@ ; O1-NEXT: .cfi_def_cfa_offset 16 ; O1-NEXT: .cfi_offset %rbx, -16 ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection462: +; O1-NEXT: .Lpcsection492: ; O1-NEXT: movq (%rdi), %rax -; O1-NEXT: .Lpcsection463: +; O1-NEXT: .Lpcsection493: ; O1-NEXT: movq 8(%rdi), %rdx ; O1-NEXT: .p2align 4, 0x90 ; O1-NEXT: .LBB223_1: # %atomicrmw.start ; O1-NEXT: # =>This Inner Loop Header: Depth=1 ; O1-NEXT: movq %rax, %rbx -; O1-NEXT: .Lpcsection464: +; O1-NEXT: .Lpcsection494: ; O1-NEXT: addq $42, %rbx ; O1-NEXT: movq %rdx, %rcx -; O1-NEXT: .Lpcsection465: +; O1-NEXT: .Lpcsection495: ; O1-NEXT: adcq $0, %rcx -; O1-NEXT: .Lpcsection466: +; O1-NEXT: .Lpcsection496: ; O1-NEXT: lock cmpxchg16b (%rdi) -; O1-NEXT: .Lpcsection467: +; O1-NEXT: .Lpcsection497: ; O1-NEXT: jne .LBB223_1 ; O1-NEXT: # %bb.2: # %atomicrmw.end ; O1-NEXT: movq $1, foo(%rip) @@ -12952,22 +13042,22 @@ ; O2-NEXT: .cfi_def_cfa_offset 16 ; O2-NEXT: .cfi_offset %rbx, -16 ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection462: +; O2-NEXT: .Lpcsection492: ; O2-NEXT: movq (%rdi), %rax -; O2-NEXT: .Lpcsection463: +; O2-NEXT: .Lpcsection493: ; O2-NEXT: movq 8(%rdi), %rdx ; O2-NEXT: .p2align 4, 0x90 ; O2-NEXT: .LBB223_1: # %atomicrmw.start ; O2-NEXT: # =>This Inner Loop Header: Depth=1 ; O2-NEXT: movq %rax, %rbx -; O2-NEXT: .Lpcsection464: +; O2-NEXT: .Lpcsection494: ; O2-NEXT: addq $42, %rbx ; O2-NEXT: movq %rdx, %rcx -; O2-NEXT: .Lpcsection465: +; O2-NEXT: .Lpcsection495: ; O2-NEXT: adcq $0, %rcx -; O2-NEXT: .Lpcsection466: +; O2-NEXT: .Lpcsection496: ; O2-NEXT: lock cmpxchg16b (%rdi) -; O2-NEXT: .Lpcsection467: +; O2-NEXT: .Lpcsection497: ; O2-NEXT: jne .LBB223_1 ; O2-NEXT: # %bb.2: # %atomicrmw.end ; O2-NEXT: movq $1, foo(%rip) @@ -12981,22 +13071,22 @@ ; O3-NEXT: .cfi_def_cfa_offset 16 ; O3-NEXT: .cfi_offset %rbx, -16 ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection462: +; O3-NEXT: .Lpcsection492: ; O3-NEXT: movq (%rdi), %rax -; O3-NEXT: .Lpcsection463: +; O3-NEXT: .Lpcsection493: ; O3-NEXT: movq 8(%rdi), %rdx ; O3-NEXT: .p2align 4, 0x90 ; O3-NEXT: .LBB223_1: # %atomicrmw.start ; O3-NEXT: # =>This Inner Loop Header: Depth=1 ; O3-NEXT: movq %rax, %rbx -; O3-NEXT: .Lpcsection464: +; O3-NEXT: .Lpcsection494: ; O3-NEXT: addq $42, %rbx ; O3-NEXT: movq %rdx, %rcx -; O3-NEXT: .Lpcsection465: +; O3-NEXT: .Lpcsection495: ; O3-NEXT: adcq $0, %rcx -; O3-NEXT: .Lpcsection466: +; O3-NEXT: .Lpcsection496: ; O3-NEXT: lock cmpxchg16b (%rdi) -; O3-NEXT: .Lpcsection467: +; O3-NEXT: .Lpcsection497: ; O3-NEXT: jne .LBB223_1 ; O3-NEXT: # %bb.2: # %atomicrmw.end ; O3-NEXT: movq $1, foo(%rip) @@ -13056,22 +13146,22 @@ ; O1-NEXT: .cfi_def_cfa_offset 16 ; O1-NEXT: .cfi_offset %rbx, -16 ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection468: +; O1-NEXT: .Lpcsection498: ; O1-NEXT: movq (%rdi), %rax -; O1-NEXT: .Lpcsection469: +; O1-NEXT: .Lpcsection499: ; O1-NEXT: movq 8(%rdi), %rdx ; O1-NEXT: .p2align 4, 0x90 ; O1-NEXT: .LBB224_1: # %atomicrmw.start ; O1-NEXT: # =>This Inner Loop Header: Depth=1 ; O1-NEXT: movq %rax, %rbx -; O1-NEXT: .Lpcsection470: +; O1-NEXT: .Lpcsection500: ; O1-NEXT: addq $-42, %rbx ; O1-NEXT: movq %rdx, %rcx -; O1-NEXT: .Lpcsection471: +; O1-NEXT: .Lpcsection501: ; O1-NEXT: adcq $-1, %rcx -; O1-NEXT: .Lpcsection472: +; O1-NEXT: .Lpcsection502: ; O1-NEXT: lock cmpxchg16b (%rdi) -; O1-NEXT: .Lpcsection473: +; O1-NEXT: .Lpcsection503: ; O1-NEXT: jne .LBB224_1 ; O1-NEXT: # %bb.2: # %atomicrmw.end ; O1-NEXT: movq $1, foo(%rip) @@ -13085,22 +13175,22 @@ ; O2-NEXT: .cfi_def_cfa_offset 16 ; O2-NEXT: .cfi_offset %rbx, -16 ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection468: +; O2-NEXT: .Lpcsection498: ; O2-NEXT: movq (%rdi), %rax -; O2-NEXT: .Lpcsection469: +; O2-NEXT: .Lpcsection499: ; O2-NEXT: movq 8(%rdi), %rdx ; O2-NEXT: .p2align 4, 0x90 ; O2-NEXT: .LBB224_1: # %atomicrmw.start ; O2-NEXT: # =>This Inner Loop Header: Depth=1 ; O2-NEXT: movq %rax, %rbx -; O2-NEXT: .Lpcsection470: +; O2-NEXT: .Lpcsection500: ; O2-NEXT: addq $-42, %rbx ; O2-NEXT: movq %rdx, %rcx -; O2-NEXT: .Lpcsection471: +; O2-NEXT: .Lpcsection501: ; O2-NEXT: adcq $-1, %rcx -; O2-NEXT: .Lpcsection472: +; O2-NEXT: .Lpcsection502: ; O2-NEXT: lock cmpxchg16b (%rdi) -; O2-NEXT: .Lpcsection473: +; O2-NEXT: .Lpcsection503: ; O2-NEXT: jne .LBB224_1 ; O2-NEXT: # %bb.2: # %atomicrmw.end ; O2-NEXT: movq $1, foo(%rip) @@ -13114,22 +13204,22 @@ ; O3-NEXT: .cfi_def_cfa_offset 16 ; O3-NEXT: .cfi_offset %rbx, -16 ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection468: +; O3-NEXT: .Lpcsection498: ; O3-NEXT: movq (%rdi), %rax -; O3-NEXT: .Lpcsection469: +; O3-NEXT: .Lpcsection499: ; O3-NEXT: movq 8(%rdi), %rdx ; O3-NEXT: .p2align 4, 0x90 ; O3-NEXT: .LBB224_1: # %atomicrmw.start ; O3-NEXT: # =>This Inner Loop Header: Depth=1 ; O3-NEXT: movq %rax, %rbx -; O3-NEXT: .Lpcsection470: +; O3-NEXT: .Lpcsection500: ; O3-NEXT: addq $-42, %rbx ; O3-NEXT: movq %rdx, %rcx -; O3-NEXT: .Lpcsection471: +; O3-NEXT: .Lpcsection501: ; O3-NEXT: adcq $-1, %rcx -; O3-NEXT: .Lpcsection472: +; O3-NEXT: .Lpcsection502: ; O3-NEXT: lock cmpxchg16b (%rdi) -; O3-NEXT: .Lpcsection473: +; O3-NEXT: .Lpcsection503: ; O3-NEXT: jne .LBB224_1 ; O3-NEXT: # %bb.2: # %atomicrmw.end ; O3-NEXT: movq $1, foo(%rip) @@ -13191,21 +13281,21 @@ ; O1-NEXT: .cfi_def_cfa_offset 16 ; O1-NEXT: .cfi_offset %rbx, -16 ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection474: +; O1-NEXT: .Lpcsection504: ; O1-NEXT: movq (%rdi), %rax -; O1-NEXT: .Lpcsection475: +; O1-NEXT: .Lpcsection505: ; O1-NEXT: movq 8(%rdi), %rdx ; O1-NEXT: .p2align 4, 0x90 ; O1-NEXT: .LBB225_1: # %atomicrmw.start ; O1-NEXT: # =>This Inner Loop Header: Depth=1 ; O1-NEXT: movl %eax, %ebx -; O1-NEXT: .Lpcsection476: +; O1-NEXT: .Lpcsection506: ; O1-NEXT: andl $42, %ebx -; O1-NEXT: .Lpcsection477: +; O1-NEXT: .Lpcsection507: ; O1-NEXT: xorl %ecx, %ecx -; O1-NEXT: .Lpcsection478: +; O1-NEXT: .Lpcsection508: ; O1-NEXT: lock cmpxchg16b (%rdi) -; O1-NEXT: .Lpcsection479: +; O1-NEXT: .Lpcsection509: ; O1-NEXT: jne .LBB225_1 ; O1-NEXT: # %bb.2: # %atomicrmw.end ; O1-NEXT: movq $1, foo(%rip) @@ -13219,21 +13309,21 @@ ; O2-NEXT: .cfi_def_cfa_offset 16 ; O2-NEXT: .cfi_offset %rbx, -16 ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection474: +; O2-NEXT: .Lpcsection504: ; O2-NEXT: movq (%rdi), %rax -; O2-NEXT: .Lpcsection475: +; O2-NEXT: .Lpcsection505: ; O2-NEXT: movq 8(%rdi), %rdx ; O2-NEXT: .p2align 4, 0x90 ; O2-NEXT: .LBB225_1: # %atomicrmw.start ; O2-NEXT: # =>This Inner Loop Header: Depth=1 ; O2-NEXT: movl %eax, %ebx -; O2-NEXT: .Lpcsection476: +; O2-NEXT: .Lpcsection506: ; O2-NEXT: andl $42, %ebx -; O2-NEXT: .Lpcsection477: +; O2-NEXT: .Lpcsection507: ; O2-NEXT: xorl %ecx, %ecx -; O2-NEXT: .Lpcsection478: +; O2-NEXT: .Lpcsection508: ; O2-NEXT: lock cmpxchg16b (%rdi) -; O2-NEXT: .Lpcsection479: +; O2-NEXT: .Lpcsection509: ; O2-NEXT: jne .LBB225_1 ; O2-NEXT: # %bb.2: # %atomicrmw.end ; O2-NEXT: movq $1, foo(%rip) @@ -13247,21 +13337,21 @@ ; O3-NEXT: .cfi_def_cfa_offset 16 ; O3-NEXT: .cfi_offset %rbx, -16 ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection474: +; O3-NEXT: .Lpcsection504: ; O3-NEXT: movq (%rdi), %rax -; O3-NEXT: .Lpcsection475: +; O3-NEXT: .Lpcsection505: ; O3-NEXT: movq 8(%rdi), %rdx ; O3-NEXT: .p2align 4, 0x90 ; O3-NEXT: .LBB225_1: # %atomicrmw.start ; O3-NEXT: # =>This Inner Loop Header: Depth=1 ; O3-NEXT: movl %eax, %ebx -; O3-NEXT: .Lpcsection476: +; O3-NEXT: .Lpcsection506: ; O3-NEXT: andl $42, %ebx -; O3-NEXT: .Lpcsection477: +; O3-NEXT: .Lpcsection507: ; O3-NEXT: xorl %ecx, %ecx -; O3-NEXT: .Lpcsection478: +; O3-NEXT: .Lpcsection508: ; O3-NEXT: lock cmpxchg16b (%rdi) -; O3-NEXT: .Lpcsection479: +; O3-NEXT: .Lpcsection509: ; O3-NEXT: jne .LBB225_1 ; O3-NEXT: # %bb.2: # %atomicrmw.end ; O3-NEXT: movq $1, foo(%rip) @@ -13319,20 +13409,20 @@ ; O1-NEXT: .cfi_def_cfa_offset 16 ; O1-NEXT: .cfi_offset %rbx, -16 ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection480: +; O1-NEXT: .Lpcsection510: ; O1-NEXT: movq (%rdi), %rax -; O1-NEXT: .Lpcsection481: +; O1-NEXT: .Lpcsection511: ; O1-NEXT: movq 8(%rdi), %rdx ; O1-NEXT: .p2align 4, 0x90 ; O1-NEXT: .LBB226_1: # %atomicrmw.start ; O1-NEXT: # =>This Inner Loop Header: Depth=1 ; O1-NEXT: movq %rax, %rbx -; O1-NEXT: .Lpcsection482: +; O1-NEXT: .Lpcsection512: ; O1-NEXT: orq $42, %rbx ; O1-NEXT: movq %rdx, %rcx -; O1-NEXT: .Lpcsection483: +; O1-NEXT: .Lpcsection513: ; O1-NEXT: lock cmpxchg16b (%rdi) -; O1-NEXT: .Lpcsection484: +; O1-NEXT: .Lpcsection514: ; O1-NEXT: jne .LBB226_1 ; O1-NEXT: # %bb.2: # %atomicrmw.end ; O1-NEXT: movq $1, foo(%rip) @@ -13346,20 +13436,20 @@ ; O2-NEXT: .cfi_def_cfa_offset 16 ; O2-NEXT: .cfi_offset %rbx, -16 ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection480: +; O2-NEXT: .Lpcsection510: ; O2-NEXT: movq (%rdi), %rax -; O2-NEXT: .Lpcsection481: +; O2-NEXT: .Lpcsection511: ; O2-NEXT: movq 8(%rdi), %rdx ; O2-NEXT: .p2align 4, 0x90 ; O2-NEXT: .LBB226_1: # %atomicrmw.start ; O2-NEXT: # =>This Inner Loop Header: Depth=1 ; O2-NEXT: movq %rax, %rbx -; O2-NEXT: .Lpcsection482: +; O2-NEXT: .Lpcsection512: ; O2-NEXT: orq $42, %rbx ; O2-NEXT: movq %rdx, %rcx -; O2-NEXT: .Lpcsection483: +; O2-NEXT: .Lpcsection513: ; O2-NEXT: lock cmpxchg16b (%rdi) -; O2-NEXT: .Lpcsection484: +; O2-NEXT: .Lpcsection514: ; O2-NEXT: jne .LBB226_1 ; O2-NEXT: # %bb.2: # %atomicrmw.end ; O2-NEXT: movq $1, foo(%rip) @@ -13373,20 +13463,20 @@ ; O3-NEXT: .cfi_def_cfa_offset 16 ; O3-NEXT: .cfi_offset %rbx, -16 ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection480: +; O3-NEXT: .Lpcsection510: ; O3-NEXT: movq (%rdi), %rax -; O3-NEXT: .Lpcsection481: +; O3-NEXT: .Lpcsection511: ; O3-NEXT: movq 8(%rdi), %rdx ; O3-NEXT: .p2align 4, 0x90 ; O3-NEXT: .LBB226_1: # %atomicrmw.start ; O3-NEXT: # =>This Inner Loop Header: Depth=1 ; O3-NEXT: movq %rax, %rbx -; O3-NEXT: .Lpcsection482: +; O3-NEXT: .Lpcsection512: ; O3-NEXT: orq $42, %rbx ; O3-NEXT: movq %rdx, %rcx -; O3-NEXT: .Lpcsection483: +; O3-NEXT: .Lpcsection513: ; O3-NEXT: lock cmpxchg16b (%rdi) -; O3-NEXT: .Lpcsection484: +; O3-NEXT: .Lpcsection514: ; O3-NEXT: jne .LBB226_1 ; O3-NEXT: # %bb.2: # %atomicrmw.end ; O3-NEXT: movq $1, foo(%rip) @@ -13444,20 +13534,20 @@ ; O1-NEXT: .cfi_def_cfa_offset 16 ; O1-NEXT: .cfi_offset %rbx, -16 ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection485: +; O1-NEXT: .Lpcsection515: ; O1-NEXT: movq (%rdi), %rax -; O1-NEXT: .Lpcsection486: +; O1-NEXT: .Lpcsection516: ; O1-NEXT: movq 8(%rdi), %rdx ; O1-NEXT: .p2align 4, 0x90 ; O1-NEXT: .LBB227_1: # %atomicrmw.start ; O1-NEXT: # =>This Inner Loop Header: Depth=1 ; O1-NEXT: movq %rax, %rbx -; O1-NEXT: .Lpcsection487: +; O1-NEXT: .Lpcsection517: ; O1-NEXT: xorq $42, %rbx ; O1-NEXT: movq %rdx, %rcx -; O1-NEXT: .Lpcsection488: +; O1-NEXT: .Lpcsection518: ; O1-NEXT: lock cmpxchg16b (%rdi) -; O1-NEXT: .Lpcsection489: +; O1-NEXT: .Lpcsection519: ; O1-NEXT: jne .LBB227_1 ; O1-NEXT: # %bb.2: # %atomicrmw.end ; O1-NEXT: movq $1, foo(%rip) @@ -13471,20 +13561,20 @@ ; O2-NEXT: .cfi_def_cfa_offset 16 ; O2-NEXT: .cfi_offset %rbx, -16 ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection485: +; O2-NEXT: .Lpcsection515: ; O2-NEXT: movq (%rdi), %rax -; O2-NEXT: .Lpcsection486: +; O2-NEXT: .Lpcsection516: ; O2-NEXT: movq 8(%rdi), %rdx ; O2-NEXT: .p2align 4, 0x90 ; O2-NEXT: .LBB227_1: # %atomicrmw.start ; O2-NEXT: # =>This Inner Loop Header: Depth=1 ; O2-NEXT: movq %rax, %rbx -; O2-NEXT: .Lpcsection487: +; O2-NEXT: .Lpcsection517: ; O2-NEXT: xorq $42, %rbx ; O2-NEXT: movq %rdx, %rcx -; O2-NEXT: .Lpcsection488: +; O2-NEXT: .Lpcsection518: ; O2-NEXT: lock cmpxchg16b (%rdi) -; O2-NEXT: .Lpcsection489: +; O2-NEXT: .Lpcsection519: ; O2-NEXT: jne .LBB227_1 ; O2-NEXT: # %bb.2: # %atomicrmw.end ; O2-NEXT: movq $1, foo(%rip) @@ -13498,20 +13588,20 @@ ; O3-NEXT: .cfi_def_cfa_offset 16 ; O3-NEXT: .cfi_offset %rbx, -16 ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection485: +; O3-NEXT: .Lpcsection515: ; O3-NEXT: movq (%rdi), %rax -; O3-NEXT: .Lpcsection486: +; O3-NEXT: .Lpcsection516: ; O3-NEXT: movq 8(%rdi), %rdx ; O3-NEXT: .p2align 4, 0x90 ; O3-NEXT: .LBB227_1: # %atomicrmw.start ; O3-NEXT: # =>This Inner Loop Header: Depth=1 ; O3-NEXT: movq %rax, %rbx -; O3-NEXT: .Lpcsection487: +; O3-NEXT: .Lpcsection517: ; O3-NEXT: xorq $42, %rbx ; O3-NEXT: movq %rdx, %rcx -; O3-NEXT: .Lpcsection488: +; O3-NEXT: .Lpcsection518: ; O3-NEXT: lock cmpxchg16b (%rdi) -; O3-NEXT: .Lpcsection489: +; O3-NEXT: .Lpcsection519: ; O3-NEXT: jne .LBB227_1 ; O3-NEXT: # %bb.2: # %atomicrmw.end ; O3-NEXT: movq $1, foo(%rip) @@ -13575,23 +13665,23 @@ ; O1-NEXT: .cfi_def_cfa_offset 16 ; O1-NEXT: .cfi_offset %rbx, -16 ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection490: +; O1-NEXT: .Lpcsection520: ; O1-NEXT: movq (%rdi), %rax -; O1-NEXT: .Lpcsection491: +; O1-NEXT: .Lpcsection521: ; O1-NEXT: movq 8(%rdi), %rdx -; O1-NEXT: .Lpcsection492: +; O1-NEXT: .Lpcsection522: ; O1-NEXT: movq $-1, %rcx ; O1-NEXT: .p2align 4, 0x90 ; O1-NEXT: .LBB228_1: # %atomicrmw.start ; O1-NEXT: # =>This Inner Loop Header: Depth=1 ; O1-NEXT: movl %eax, %ebx -; O1-NEXT: .Lpcsection493: +; O1-NEXT: .Lpcsection523: ; O1-NEXT: notl %ebx -; O1-NEXT: .Lpcsection494: +; O1-NEXT: .Lpcsection524: ; O1-NEXT: orq $-43, %rbx -; O1-NEXT: .Lpcsection495: +; O1-NEXT: .Lpcsection525: ; O1-NEXT: lock cmpxchg16b (%rdi) -; O1-NEXT: .Lpcsection496: +; O1-NEXT: .Lpcsection526: ; O1-NEXT: jne .LBB228_1 ; O1-NEXT: # %bb.2: # %atomicrmw.end ; O1-NEXT: movq $1, foo(%rip) @@ -13605,23 +13695,23 @@ ; O2-NEXT: .cfi_def_cfa_offset 16 ; O2-NEXT: .cfi_offset %rbx, -16 ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection490: +; O2-NEXT: .Lpcsection520: ; O2-NEXT: movq (%rdi), %rax -; O2-NEXT: .Lpcsection491: +; O2-NEXT: .Lpcsection521: ; O2-NEXT: movq 8(%rdi), %rdx -; O2-NEXT: .Lpcsection492: +; O2-NEXT: .Lpcsection522: ; O2-NEXT: movq $-1, %rcx ; O2-NEXT: .p2align 4, 0x90 ; O2-NEXT: .LBB228_1: # %atomicrmw.start ; O2-NEXT: # =>This Inner Loop Header: Depth=1 ; O2-NEXT: movl %eax, %ebx -; O2-NEXT: .Lpcsection493: +; O2-NEXT: .Lpcsection523: ; O2-NEXT: notl %ebx -; O2-NEXT: .Lpcsection494: +; O2-NEXT: .Lpcsection524: ; O2-NEXT: orq $-43, %rbx -; O2-NEXT: .Lpcsection495: +; O2-NEXT: .Lpcsection525: ; O2-NEXT: lock cmpxchg16b (%rdi) -; O2-NEXT: .Lpcsection496: +; O2-NEXT: .Lpcsection526: ; O2-NEXT: jne .LBB228_1 ; O2-NEXT: # %bb.2: # %atomicrmw.end ; O2-NEXT: movq $1, foo(%rip) @@ -13635,23 +13725,23 @@ ; O3-NEXT: .cfi_def_cfa_offset 16 ; O3-NEXT: .cfi_offset %rbx, -16 ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection490: +; O3-NEXT: .Lpcsection520: ; O3-NEXT: movq (%rdi), %rax -; O3-NEXT: .Lpcsection491: +; O3-NEXT: .Lpcsection521: ; O3-NEXT: movq 8(%rdi), %rdx -; O3-NEXT: .Lpcsection492: +; O3-NEXT: .Lpcsection522: ; O3-NEXT: movq $-1, %rcx ; O3-NEXT: .p2align 4, 0x90 ; O3-NEXT: .LBB228_1: # %atomicrmw.start ; O3-NEXT: # =>This Inner Loop Header: Depth=1 ; O3-NEXT: movl %eax, %ebx -; O3-NEXT: .Lpcsection493: +; O3-NEXT: .Lpcsection523: ; O3-NEXT: notl %ebx -; O3-NEXT: .Lpcsection494: +; O3-NEXT: .Lpcsection524: ; O3-NEXT: orq $-43, %rbx -; O3-NEXT: .Lpcsection495: +; O3-NEXT: .Lpcsection525: ; O3-NEXT: lock cmpxchg16b (%rdi) -; O3-NEXT: .Lpcsection496: +; O3-NEXT: .Lpcsection526: ; O3-NEXT: jne .LBB228_1 ; O3-NEXT: # %bb.2: # %atomicrmw.end ; O3-NEXT: movq $1, foo(%rip) @@ -13711,20 +13801,20 @@ ; O1-NEXT: .cfi_def_cfa_offset 16 ; O1-NEXT: .cfi_offset %rbx, -16 ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection497: +; O1-NEXT: .Lpcsection527: ; O1-NEXT: movq (%rdi), %rax -; O1-NEXT: .Lpcsection498: +; O1-NEXT: .Lpcsection528: ; O1-NEXT: movq 8(%rdi), %rdx -; O1-NEXT: .Lpcsection499: +; O1-NEXT: .Lpcsection529: ; O1-NEXT: movl $42, %ebx ; O1-NEXT: .p2align 4, 0x90 ; O1-NEXT: .LBB229_1: # %atomicrmw.start ; O1-NEXT: # =>This Inner Loop Header: Depth=1 -; O1-NEXT: .Lpcsection500: +; O1-NEXT: .Lpcsection530: ; O1-NEXT: xorl %ecx, %ecx -; O1-NEXT: .Lpcsection501: +; O1-NEXT: .Lpcsection531: ; O1-NEXT: lock cmpxchg16b (%rdi) -; O1-NEXT: .Lpcsection502: +; O1-NEXT: .Lpcsection532: ; O1-NEXT: jne .LBB229_1 ; O1-NEXT: # %bb.2: # %atomicrmw.end ; O1-NEXT: movq $1, foo(%rip) @@ -13738,20 +13828,20 @@ ; O2-NEXT: .cfi_def_cfa_offset 16 ; O2-NEXT: .cfi_offset %rbx, -16 ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection497: +; O2-NEXT: .Lpcsection527: ; O2-NEXT: movq (%rdi), %rax -; O2-NEXT: .Lpcsection498: +; O2-NEXT: .Lpcsection528: ; O2-NEXT: movq 8(%rdi), %rdx -; O2-NEXT: .Lpcsection499: +; O2-NEXT: .Lpcsection529: ; O2-NEXT: movl $42, %ebx ; O2-NEXT: .p2align 4, 0x90 ; O2-NEXT: .LBB229_1: # %atomicrmw.start ; O2-NEXT: # =>This Inner Loop Header: Depth=1 -; O2-NEXT: .Lpcsection500: +; O2-NEXT: .Lpcsection530: ; O2-NEXT: xorl %ecx, %ecx -; O2-NEXT: .Lpcsection501: +; O2-NEXT: .Lpcsection531: ; O2-NEXT: lock cmpxchg16b (%rdi) -; O2-NEXT: .Lpcsection502: +; O2-NEXT: .Lpcsection532: ; O2-NEXT: jne .LBB229_1 ; O2-NEXT: # %bb.2: # %atomicrmw.end ; O2-NEXT: movq $1, foo(%rip) @@ -13765,20 +13855,20 @@ ; O3-NEXT: .cfi_def_cfa_offset 16 ; O3-NEXT: .cfi_offset %rbx, -16 ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection497: +; O3-NEXT: .Lpcsection527: ; O3-NEXT: movq (%rdi), %rax -; O3-NEXT: .Lpcsection498: +; O3-NEXT: .Lpcsection528: ; O3-NEXT: movq 8(%rdi), %rdx -; O3-NEXT: .Lpcsection499: +; O3-NEXT: .Lpcsection529: ; O3-NEXT: movl $42, %ebx ; O3-NEXT: .p2align 4, 0x90 ; O3-NEXT: .LBB229_1: # %atomicrmw.start ; O3-NEXT: # =>This Inner Loop Header: Depth=1 -; O3-NEXT: .Lpcsection500: +; O3-NEXT: .Lpcsection530: ; O3-NEXT: xorl %ecx, %ecx -; O3-NEXT: .Lpcsection501: +; O3-NEXT: .Lpcsection531: ; O3-NEXT: lock cmpxchg16b (%rdi) -; O3-NEXT: .Lpcsection502: +; O3-NEXT: .Lpcsection532: ; O3-NEXT: jne .LBB229_1 ; O3-NEXT: # %bb.2: # %atomicrmw.end ; O3-NEXT: movq $1, foo(%rip) @@ -13838,22 +13928,22 @@ ; O1-NEXT: .cfi_def_cfa_offset 16 ; O1-NEXT: .cfi_offset %rbx, -16 ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection503: +; O1-NEXT: .Lpcsection533: ; O1-NEXT: movq (%rdi), %rax -; O1-NEXT: .Lpcsection504: +; O1-NEXT: .Lpcsection534: ; O1-NEXT: movq 8(%rdi), %rdx ; O1-NEXT: .p2align 4, 0x90 ; O1-NEXT: .LBB230_1: # %atomicrmw.start ; O1-NEXT: # =>This Inner Loop Header: Depth=1 ; O1-NEXT: movq %rax, %rbx -; O1-NEXT: .Lpcsection505: +; O1-NEXT: .Lpcsection535: ; O1-NEXT: addq $42, %rbx ; O1-NEXT: movq %rdx, %rcx -; O1-NEXT: .Lpcsection506: +; O1-NEXT: .Lpcsection536: ; O1-NEXT: adcq $0, %rcx -; O1-NEXT: .Lpcsection507: +; O1-NEXT: .Lpcsection537: ; O1-NEXT: lock cmpxchg16b (%rdi) -; O1-NEXT: .Lpcsection508: +; O1-NEXT: .Lpcsection538: ; O1-NEXT: jne .LBB230_1 ; O1-NEXT: # %bb.2: # %atomicrmw.end ; O1-NEXT: movq $1, foo(%rip) @@ -13867,22 +13957,22 @@ ; O2-NEXT: .cfi_def_cfa_offset 16 ; O2-NEXT: .cfi_offset %rbx, -16 ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection503: +; O2-NEXT: .Lpcsection533: ; O2-NEXT: movq (%rdi), %rax -; O2-NEXT: .Lpcsection504: +; O2-NEXT: .Lpcsection534: ; O2-NEXT: movq 8(%rdi), %rdx ; O2-NEXT: .p2align 4, 0x90 ; O2-NEXT: .LBB230_1: # %atomicrmw.start ; O2-NEXT: # =>This Inner Loop Header: Depth=1 ; O2-NEXT: movq %rax, %rbx -; O2-NEXT: .Lpcsection505: +; O2-NEXT: .Lpcsection535: ; O2-NEXT: addq $42, %rbx ; O2-NEXT: movq %rdx, %rcx -; O2-NEXT: .Lpcsection506: +; O2-NEXT: .Lpcsection536: ; O2-NEXT: adcq $0, %rcx -; O2-NEXT: .Lpcsection507: +; O2-NEXT: .Lpcsection537: ; O2-NEXT: lock cmpxchg16b (%rdi) -; O2-NEXT: .Lpcsection508: +; O2-NEXT: .Lpcsection538: ; O2-NEXT: jne .LBB230_1 ; O2-NEXT: # %bb.2: # %atomicrmw.end ; O2-NEXT: movq $1, foo(%rip) @@ -13896,22 +13986,22 @@ ; O3-NEXT: .cfi_def_cfa_offset 16 ; O3-NEXT: .cfi_offset %rbx, -16 ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection503: +; O3-NEXT: .Lpcsection533: ; O3-NEXT: movq (%rdi), %rax -; O3-NEXT: .Lpcsection504: +; O3-NEXT: .Lpcsection534: ; O3-NEXT: movq 8(%rdi), %rdx ; O3-NEXT: .p2align 4, 0x90 ; O3-NEXT: .LBB230_1: # %atomicrmw.start ; O3-NEXT: # =>This Inner Loop Header: Depth=1 ; O3-NEXT: movq %rax, %rbx -; O3-NEXT: .Lpcsection505: +; O3-NEXT: .Lpcsection535: ; O3-NEXT: addq $42, %rbx ; O3-NEXT: movq %rdx, %rcx -; O3-NEXT: .Lpcsection506: +; O3-NEXT: .Lpcsection536: ; O3-NEXT: adcq $0, %rcx -; O3-NEXT: .Lpcsection507: +; O3-NEXT: .Lpcsection537: ; O3-NEXT: lock cmpxchg16b (%rdi) -; O3-NEXT: .Lpcsection508: +; O3-NEXT: .Lpcsection538: ; O3-NEXT: jne .LBB230_1 ; O3-NEXT: # %bb.2: # %atomicrmw.end ; O3-NEXT: movq $1, foo(%rip) @@ -13971,22 +14061,22 @@ ; O1-NEXT: .cfi_def_cfa_offset 16 ; O1-NEXT: .cfi_offset %rbx, -16 ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection509: +; O1-NEXT: .Lpcsection539: ; O1-NEXT: movq (%rdi), %rax -; O1-NEXT: .Lpcsection510: +; O1-NEXT: .Lpcsection540: ; O1-NEXT: movq 8(%rdi), %rdx ; O1-NEXT: .p2align 4, 0x90 ; O1-NEXT: .LBB231_1: # %atomicrmw.start ; O1-NEXT: # =>This Inner Loop Header: Depth=1 ; O1-NEXT: movq %rax, %rbx -; O1-NEXT: .Lpcsection511: +; O1-NEXT: .Lpcsection541: ; O1-NEXT: addq $-42, %rbx ; O1-NEXT: movq %rdx, %rcx -; O1-NEXT: .Lpcsection512: +; O1-NEXT: .Lpcsection542: ; O1-NEXT: adcq $-1, %rcx -; O1-NEXT: .Lpcsection513: +; O1-NEXT: .Lpcsection543: ; O1-NEXT: lock cmpxchg16b (%rdi) -; O1-NEXT: .Lpcsection514: +; O1-NEXT: .Lpcsection544: ; O1-NEXT: jne .LBB231_1 ; O1-NEXT: # %bb.2: # %atomicrmw.end ; O1-NEXT: movq $1, foo(%rip) @@ -14000,22 +14090,22 @@ ; O2-NEXT: .cfi_def_cfa_offset 16 ; O2-NEXT: .cfi_offset %rbx, -16 ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection509: +; O2-NEXT: .Lpcsection539: ; O2-NEXT: movq (%rdi), %rax -; O2-NEXT: .Lpcsection510: +; O2-NEXT: .Lpcsection540: ; O2-NEXT: movq 8(%rdi), %rdx ; O2-NEXT: .p2align 4, 0x90 ; O2-NEXT: .LBB231_1: # %atomicrmw.start ; O2-NEXT: # =>This Inner Loop Header: Depth=1 ; O2-NEXT: movq %rax, %rbx -; O2-NEXT: .Lpcsection511: +; O2-NEXT: .Lpcsection541: ; O2-NEXT: addq $-42, %rbx ; O2-NEXT: movq %rdx, %rcx -; O2-NEXT: .Lpcsection512: +; O2-NEXT: .Lpcsection542: ; O2-NEXT: adcq $-1, %rcx -; O2-NEXT: .Lpcsection513: +; O2-NEXT: .Lpcsection543: ; O2-NEXT: lock cmpxchg16b (%rdi) -; O2-NEXT: .Lpcsection514: +; O2-NEXT: .Lpcsection544: ; O2-NEXT: jne .LBB231_1 ; O2-NEXT: # %bb.2: # %atomicrmw.end ; O2-NEXT: movq $1, foo(%rip) @@ -14029,22 +14119,22 @@ ; O3-NEXT: .cfi_def_cfa_offset 16 ; O3-NEXT: .cfi_offset %rbx, -16 ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection509: +; O3-NEXT: .Lpcsection539: ; O3-NEXT: movq (%rdi), %rax -; O3-NEXT: .Lpcsection510: +; O3-NEXT: .Lpcsection540: ; O3-NEXT: movq 8(%rdi), %rdx ; O3-NEXT: .p2align 4, 0x90 ; O3-NEXT: .LBB231_1: # %atomicrmw.start ; O3-NEXT: # =>This Inner Loop Header: Depth=1 ; O3-NEXT: movq %rax, %rbx -; O3-NEXT: .Lpcsection511: +; O3-NEXT: .Lpcsection541: ; O3-NEXT: addq $-42, %rbx ; O3-NEXT: movq %rdx, %rcx -; O3-NEXT: .Lpcsection512: +; O3-NEXT: .Lpcsection542: ; O3-NEXT: adcq $-1, %rcx -; O3-NEXT: .Lpcsection513: +; O3-NEXT: .Lpcsection543: ; O3-NEXT: lock cmpxchg16b (%rdi) -; O3-NEXT: .Lpcsection514: +; O3-NEXT: .Lpcsection544: ; O3-NEXT: jne .LBB231_1 ; O3-NEXT: # %bb.2: # %atomicrmw.end ; O3-NEXT: movq $1, foo(%rip) @@ -14106,21 +14196,21 @@ ; O1-NEXT: .cfi_def_cfa_offset 16 ; O1-NEXT: .cfi_offset %rbx, -16 ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection515: +; O1-NEXT: .Lpcsection545: ; O1-NEXT: movq (%rdi), %rax -; O1-NEXT: .Lpcsection516: +; O1-NEXT: .Lpcsection546: ; O1-NEXT: movq 8(%rdi), %rdx ; O1-NEXT: .p2align 4, 0x90 ; O1-NEXT: .LBB232_1: # %atomicrmw.start ; O1-NEXT: # =>This Inner Loop Header: Depth=1 ; O1-NEXT: movl %eax, %ebx -; O1-NEXT: .Lpcsection517: +; O1-NEXT: .Lpcsection547: ; O1-NEXT: andl $42, %ebx -; O1-NEXT: .Lpcsection518: +; O1-NEXT: .Lpcsection548: ; O1-NEXT: xorl %ecx, %ecx -; O1-NEXT: .Lpcsection519: +; O1-NEXT: .Lpcsection549: ; O1-NEXT: lock cmpxchg16b (%rdi) -; O1-NEXT: .Lpcsection520: +; O1-NEXT: .Lpcsection550: ; O1-NEXT: jne .LBB232_1 ; O1-NEXT: # %bb.2: # %atomicrmw.end ; O1-NEXT: movq $1, foo(%rip) @@ -14134,21 +14224,21 @@ ; O2-NEXT: .cfi_def_cfa_offset 16 ; O2-NEXT: .cfi_offset %rbx, -16 ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection515: +; O2-NEXT: .Lpcsection545: ; O2-NEXT: movq (%rdi), %rax -; O2-NEXT: .Lpcsection516: +; O2-NEXT: .Lpcsection546: ; O2-NEXT: movq 8(%rdi), %rdx ; O2-NEXT: .p2align 4, 0x90 ; O2-NEXT: .LBB232_1: # %atomicrmw.start ; O2-NEXT: # =>This Inner Loop Header: Depth=1 ; O2-NEXT: movl %eax, %ebx -; O2-NEXT: .Lpcsection517: +; O2-NEXT: .Lpcsection547: ; O2-NEXT: andl $42, %ebx -; O2-NEXT: .Lpcsection518: +; O2-NEXT: .Lpcsection548: ; O2-NEXT: xorl %ecx, %ecx -; O2-NEXT: .Lpcsection519: +; O2-NEXT: .Lpcsection549: ; O2-NEXT: lock cmpxchg16b (%rdi) -; O2-NEXT: .Lpcsection520: +; O2-NEXT: .Lpcsection550: ; O2-NEXT: jne .LBB232_1 ; O2-NEXT: # %bb.2: # %atomicrmw.end ; O2-NEXT: movq $1, foo(%rip) @@ -14162,21 +14252,21 @@ ; O3-NEXT: .cfi_def_cfa_offset 16 ; O3-NEXT: .cfi_offset %rbx, -16 ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection515: +; O3-NEXT: .Lpcsection545: ; O3-NEXT: movq (%rdi), %rax -; O3-NEXT: .Lpcsection516: +; O3-NEXT: .Lpcsection546: ; O3-NEXT: movq 8(%rdi), %rdx ; O3-NEXT: .p2align 4, 0x90 ; O3-NEXT: .LBB232_1: # %atomicrmw.start ; O3-NEXT: # =>This Inner Loop Header: Depth=1 ; O3-NEXT: movl %eax, %ebx -; O3-NEXT: .Lpcsection517: +; O3-NEXT: .Lpcsection547: ; O3-NEXT: andl $42, %ebx -; O3-NEXT: .Lpcsection518: +; O3-NEXT: .Lpcsection548: ; O3-NEXT: xorl %ecx, %ecx -; O3-NEXT: .Lpcsection519: +; O3-NEXT: .Lpcsection549: ; O3-NEXT: lock cmpxchg16b (%rdi) -; O3-NEXT: .Lpcsection520: +; O3-NEXT: .Lpcsection550: ; O3-NEXT: jne .LBB232_1 ; O3-NEXT: # %bb.2: # %atomicrmw.end ; O3-NEXT: movq $1, foo(%rip) @@ -14234,20 +14324,20 @@ ; O1-NEXT: .cfi_def_cfa_offset 16 ; O1-NEXT: .cfi_offset %rbx, -16 ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection521: +; O1-NEXT: .Lpcsection551: ; O1-NEXT: movq (%rdi), %rax -; O1-NEXT: .Lpcsection522: +; O1-NEXT: .Lpcsection552: ; O1-NEXT: movq 8(%rdi), %rdx ; O1-NEXT: .p2align 4, 0x90 ; O1-NEXT: .LBB233_1: # %atomicrmw.start ; O1-NEXT: # =>This Inner Loop Header: Depth=1 ; O1-NEXT: movq %rax, %rbx -; O1-NEXT: .Lpcsection523: +; O1-NEXT: .Lpcsection553: ; O1-NEXT: orq $42, %rbx ; O1-NEXT: movq %rdx, %rcx -; O1-NEXT: .Lpcsection524: +; O1-NEXT: .Lpcsection554: ; O1-NEXT: lock cmpxchg16b (%rdi) -; O1-NEXT: .Lpcsection525: +; O1-NEXT: .Lpcsection555: ; O1-NEXT: jne .LBB233_1 ; O1-NEXT: # %bb.2: # %atomicrmw.end ; O1-NEXT: movq $1, foo(%rip) @@ -14261,20 +14351,20 @@ ; O2-NEXT: .cfi_def_cfa_offset 16 ; O2-NEXT: .cfi_offset %rbx, -16 ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection521: +; O2-NEXT: .Lpcsection551: ; O2-NEXT: movq (%rdi), %rax -; O2-NEXT: .Lpcsection522: +; O2-NEXT: .Lpcsection552: ; O2-NEXT: movq 8(%rdi), %rdx ; O2-NEXT: .p2align 4, 0x90 ; O2-NEXT: .LBB233_1: # %atomicrmw.start ; O2-NEXT: # =>This Inner Loop Header: Depth=1 ; O2-NEXT: movq %rax, %rbx -; O2-NEXT: .Lpcsection523: +; O2-NEXT: .Lpcsection553: ; O2-NEXT: orq $42, %rbx ; O2-NEXT: movq %rdx, %rcx -; O2-NEXT: .Lpcsection524: +; O2-NEXT: .Lpcsection554: ; O2-NEXT: lock cmpxchg16b (%rdi) -; O2-NEXT: .Lpcsection525: +; O2-NEXT: .Lpcsection555: ; O2-NEXT: jne .LBB233_1 ; O2-NEXT: # %bb.2: # %atomicrmw.end ; O2-NEXT: movq $1, foo(%rip) @@ -14288,20 +14378,20 @@ ; O3-NEXT: .cfi_def_cfa_offset 16 ; O3-NEXT: .cfi_offset %rbx, -16 ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection521: +; O3-NEXT: .Lpcsection551: ; O3-NEXT: movq (%rdi), %rax -; O3-NEXT: .Lpcsection522: +; O3-NEXT: .Lpcsection552: ; O3-NEXT: movq 8(%rdi), %rdx ; O3-NEXT: .p2align 4, 0x90 ; O3-NEXT: .LBB233_1: # %atomicrmw.start ; O3-NEXT: # =>This Inner Loop Header: Depth=1 ; O3-NEXT: movq %rax, %rbx -; O3-NEXT: .Lpcsection523: +; O3-NEXT: .Lpcsection553: ; O3-NEXT: orq $42, %rbx ; O3-NEXT: movq %rdx, %rcx -; O3-NEXT: .Lpcsection524: +; O3-NEXT: .Lpcsection554: ; O3-NEXT: lock cmpxchg16b (%rdi) -; O3-NEXT: .Lpcsection525: +; O3-NEXT: .Lpcsection555: ; O3-NEXT: jne .LBB233_1 ; O3-NEXT: # %bb.2: # %atomicrmw.end ; O3-NEXT: movq $1, foo(%rip) @@ -14359,20 +14449,20 @@ ; O1-NEXT: .cfi_def_cfa_offset 16 ; O1-NEXT: .cfi_offset %rbx, -16 ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection526: +; O1-NEXT: .Lpcsection556: ; O1-NEXT: movq (%rdi), %rax -; O1-NEXT: .Lpcsection527: +; O1-NEXT: .Lpcsection557: ; O1-NEXT: movq 8(%rdi), %rdx ; O1-NEXT: .p2align 4, 0x90 ; O1-NEXT: .LBB234_1: # %atomicrmw.start ; O1-NEXT: # =>This Inner Loop Header: Depth=1 ; O1-NEXT: movq %rax, %rbx -; O1-NEXT: .Lpcsection528: +; O1-NEXT: .Lpcsection558: ; O1-NEXT: xorq $42, %rbx ; O1-NEXT: movq %rdx, %rcx -; O1-NEXT: .Lpcsection529: +; O1-NEXT: .Lpcsection559: ; O1-NEXT: lock cmpxchg16b (%rdi) -; O1-NEXT: .Lpcsection530: +; O1-NEXT: .Lpcsection560: ; O1-NEXT: jne .LBB234_1 ; O1-NEXT: # %bb.2: # %atomicrmw.end ; O1-NEXT: movq $1, foo(%rip) @@ -14386,20 +14476,20 @@ ; O2-NEXT: .cfi_def_cfa_offset 16 ; O2-NEXT: .cfi_offset %rbx, -16 ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection526: +; O2-NEXT: .Lpcsection556: ; O2-NEXT: movq (%rdi), %rax -; O2-NEXT: .Lpcsection527: +; O2-NEXT: .Lpcsection557: ; O2-NEXT: movq 8(%rdi), %rdx ; O2-NEXT: .p2align 4, 0x90 ; O2-NEXT: .LBB234_1: # %atomicrmw.start ; O2-NEXT: # =>This Inner Loop Header: Depth=1 ; O2-NEXT: movq %rax, %rbx -; O2-NEXT: .Lpcsection528: +; O2-NEXT: .Lpcsection558: ; O2-NEXT: xorq $42, %rbx ; O2-NEXT: movq %rdx, %rcx -; O2-NEXT: .Lpcsection529: +; O2-NEXT: .Lpcsection559: ; O2-NEXT: lock cmpxchg16b (%rdi) -; O2-NEXT: .Lpcsection530: +; O2-NEXT: .Lpcsection560: ; O2-NEXT: jne .LBB234_1 ; O2-NEXT: # %bb.2: # %atomicrmw.end ; O2-NEXT: movq $1, foo(%rip) @@ -14413,20 +14503,20 @@ ; O3-NEXT: .cfi_def_cfa_offset 16 ; O3-NEXT: .cfi_offset %rbx, -16 ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection526: +; O3-NEXT: .Lpcsection556: ; O3-NEXT: movq (%rdi), %rax -; O3-NEXT: .Lpcsection527: +; O3-NEXT: .Lpcsection557: ; O3-NEXT: movq 8(%rdi), %rdx ; O3-NEXT: .p2align 4, 0x90 ; O3-NEXT: .LBB234_1: # %atomicrmw.start ; O3-NEXT: # =>This Inner Loop Header: Depth=1 ; O3-NEXT: movq %rax, %rbx -; O3-NEXT: .Lpcsection528: +; O3-NEXT: .Lpcsection558: ; O3-NEXT: xorq $42, %rbx ; O3-NEXT: movq %rdx, %rcx -; O3-NEXT: .Lpcsection529: +; O3-NEXT: .Lpcsection559: ; O3-NEXT: lock cmpxchg16b (%rdi) -; O3-NEXT: .Lpcsection530: +; O3-NEXT: .Lpcsection560: ; O3-NEXT: jne .LBB234_1 ; O3-NEXT: # %bb.2: # %atomicrmw.end ; O3-NEXT: movq $1, foo(%rip) @@ -14490,23 +14580,23 @@ ; O1-NEXT: .cfi_def_cfa_offset 16 ; O1-NEXT: .cfi_offset %rbx, -16 ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection531: +; O1-NEXT: .Lpcsection561: ; O1-NEXT: movq (%rdi), %rax -; O1-NEXT: .Lpcsection532: +; O1-NEXT: .Lpcsection562: ; O1-NEXT: movq 8(%rdi), %rdx -; O1-NEXT: .Lpcsection533: +; O1-NEXT: .Lpcsection563: ; O1-NEXT: movq $-1, %rcx ; O1-NEXT: .p2align 4, 0x90 ; O1-NEXT: .LBB235_1: # %atomicrmw.start ; O1-NEXT: # =>This Inner Loop Header: Depth=1 ; O1-NEXT: movl %eax, %ebx -; O1-NEXT: .Lpcsection534: +; O1-NEXT: .Lpcsection564: ; O1-NEXT: notl %ebx -; O1-NEXT: .Lpcsection535: +; O1-NEXT: .Lpcsection565: ; O1-NEXT: orq $-43, %rbx -; O1-NEXT: .Lpcsection536: +; O1-NEXT: .Lpcsection566: ; O1-NEXT: lock cmpxchg16b (%rdi) -; O1-NEXT: .Lpcsection537: +; O1-NEXT: .Lpcsection567: ; O1-NEXT: jne .LBB235_1 ; O1-NEXT: # %bb.2: # %atomicrmw.end ; O1-NEXT: movq $1, foo(%rip) @@ -14520,23 +14610,23 @@ ; O2-NEXT: .cfi_def_cfa_offset 16 ; O2-NEXT: .cfi_offset %rbx, -16 ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection531: +; O2-NEXT: .Lpcsection561: ; O2-NEXT: movq (%rdi), %rax -; O2-NEXT: .Lpcsection532: +; O2-NEXT: .Lpcsection562: ; O2-NEXT: movq 8(%rdi), %rdx -; O2-NEXT: .Lpcsection533: +; O2-NEXT: .Lpcsection563: ; O2-NEXT: movq $-1, %rcx ; O2-NEXT: .p2align 4, 0x90 ; O2-NEXT: .LBB235_1: # %atomicrmw.start ; O2-NEXT: # =>This Inner Loop Header: Depth=1 ; O2-NEXT: movl %eax, %ebx -; O2-NEXT: .Lpcsection534: +; O2-NEXT: .Lpcsection564: ; O2-NEXT: notl %ebx -; O2-NEXT: .Lpcsection535: +; O2-NEXT: .Lpcsection565: ; O2-NEXT: orq $-43, %rbx -; O2-NEXT: .Lpcsection536: +; O2-NEXT: .Lpcsection566: ; O2-NEXT: lock cmpxchg16b (%rdi) -; O2-NEXT: .Lpcsection537: +; O2-NEXT: .Lpcsection567: ; O2-NEXT: jne .LBB235_1 ; O2-NEXT: # %bb.2: # %atomicrmw.end ; O2-NEXT: movq $1, foo(%rip) @@ -14550,23 +14640,23 @@ ; O3-NEXT: .cfi_def_cfa_offset 16 ; O3-NEXT: .cfi_offset %rbx, -16 ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection531: +; O3-NEXT: .Lpcsection561: ; O3-NEXT: movq (%rdi), %rax -; O3-NEXT: .Lpcsection532: +; O3-NEXT: .Lpcsection562: ; O3-NEXT: movq 8(%rdi), %rdx -; O3-NEXT: .Lpcsection533: +; O3-NEXT: .Lpcsection563: ; O3-NEXT: movq $-1, %rcx ; O3-NEXT: .p2align 4, 0x90 ; O3-NEXT: .LBB235_1: # %atomicrmw.start ; O3-NEXT: # =>This Inner Loop Header: Depth=1 ; O3-NEXT: movl %eax, %ebx -; O3-NEXT: .Lpcsection534: +; O3-NEXT: .Lpcsection564: ; O3-NEXT: notl %ebx -; O3-NEXT: .Lpcsection535: +; O3-NEXT: .Lpcsection565: ; O3-NEXT: orq $-43, %rbx -; O3-NEXT: .Lpcsection536: +; O3-NEXT: .Lpcsection566: ; O3-NEXT: lock cmpxchg16b (%rdi) -; O3-NEXT: .Lpcsection537: +; O3-NEXT: .Lpcsection567: ; O3-NEXT: jne .LBB235_1 ; O3-NEXT: # %bb.2: # %atomicrmw.end ; O3-NEXT: movq $1, foo(%rip) @@ -14626,20 +14716,20 @@ ; O1-NEXT: .cfi_def_cfa_offset 16 ; O1-NEXT: .cfi_offset %rbx, -16 ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection538: +; O1-NEXT: .Lpcsection568: ; O1-NEXT: movq (%rdi), %rax -; O1-NEXT: .Lpcsection539: +; O1-NEXT: .Lpcsection569: ; O1-NEXT: movq 8(%rdi), %rdx -; O1-NEXT: .Lpcsection540: +; O1-NEXT: .Lpcsection570: ; O1-NEXT: movl $42, %ebx ; O1-NEXT: .p2align 4, 0x90 ; O1-NEXT: .LBB236_1: # %atomicrmw.start ; O1-NEXT: # =>This Inner Loop Header: Depth=1 -; O1-NEXT: .Lpcsection541: +; O1-NEXT: .Lpcsection571: ; O1-NEXT: xorl %ecx, %ecx -; O1-NEXT: .Lpcsection542: +; O1-NEXT: .Lpcsection572: ; O1-NEXT: lock cmpxchg16b (%rdi) -; O1-NEXT: .Lpcsection543: +; O1-NEXT: .Lpcsection573: ; O1-NEXT: jne .LBB236_1 ; O1-NEXT: # %bb.2: # %atomicrmw.end ; O1-NEXT: movq $1, foo(%rip) @@ -14653,20 +14743,20 @@ ; O2-NEXT: .cfi_def_cfa_offset 16 ; O2-NEXT: .cfi_offset %rbx, -16 ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection538: +; O2-NEXT: .Lpcsection568: ; O2-NEXT: movq (%rdi), %rax -; O2-NEXT: .Lpcsection539: +; O2-NEXT: .Lpcsection569: ; O2-NEXT: movq 8(%rdi), %rdx -; O2-NEXT: .Lpcsection540: +; O2-NEXT: .Lpcsection570: ; O2-NEXT: movl $42, %ebx ; O2-NEXT: .p2align 4, 0x90 ; O2-NEXT: .LBB236_1: # %atomicrmw.start ; O2-NEXT: # =>This Inner Loop Header: Depth=1 -; O2-NEXT: .Lpcsection541: +; O2-NEXT: .Lpcsection571: ; O2-NEXT: xorl %ecx, %ecx -; O2-NEXT: .Lpcsection542: +; O2-NEXT: .Lpcsection572: ; O2-NEXT: lock cmpxchg16b (%rdi) -; O2-NEXT: .Lpcsection543: +; O2-NEXT: .Lpcsection573: ; O2-NEXT: jne .LBB236_1 ; O2-NEXT: # %bb.2: # %atomicrmw.end ; O2-NEXT: movq $1, foo(%rip) @@ -14680,20 +14770,20 @@ ; O3-NEXT: .cfi_def_cfa_offset 16 ; O3-NEXT: .cfi_offset %rbx, -16 ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection538: +; O3-NEXT: .Lpcsection568: ; O3-NEXT: movq (%rdi), %rax -; O3-NEXT: .Lpcsection539: +; O3-NEXT: .Lpcsection569: ; O3-NEXT: movq 8(%rdi), %rdx -; O3-NEXT: .Lpcsection540: +; O3-NEXT: .Lpcsection570: ; O3-NEXT: movl $42, %ebx ; O3-NEXT: .p2align 4, 0x90 ; O3-NEXT: .LBB236_1: # %atomicrmw.start ; O3-NEXT: # =>This Inner Loop Header: Depth=1 -; O3-NEXT: .Lpcsection541: +; O3-NEXT: .Lpcsection571: ; O3-NEXT: xorl %ecx, %ecx -; O3-NEXT: .Lpcsection542: +; O3-NEXT: .Lpcsection572: ; O3-NEXT: lock cmpxchg16b (%rdi) -; O3-NEXT: .Lpcsection543: +; O3-NEXT: .Lpcsection573: ; O3-NEXT: jne .LBB236_1 ; O3-NEXT: # %bb.2: # %atomicrmw.end ; O3-NEXT: movq $1, foo(%rip) @@ -14753,22 +14843,22 @@ ; O1-NEXT: .cfi_def_cfa_offset 16 ; O1-NEXT: .cfi_offset %rbx, -16 ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection544: +; O1-NEXT: .Lpcsection574: ; O1-NEXT: movq (%rdi), %rax -; O1-NEXT: .Lpcsection545: +; O1-NEXT: .Lpcsection575: ; O1-NEXT: movq 8(%rdi), %rdx ; O1-NEXT: .p2align 4, 0x90 ; O1-NEXT: .LBB237_1: # %atomicrmw.start ; O1-NEXT: # =>This Inner Loop Header: Depth=1 ; O1-NEXT: movq %rax, %rbx -; O1-NEXT: .Lpcsection546: +; O1-NEXT: .Lpcsection576: ; O1-NEXT: addq $42, %rbx ; O1-NEXT: movq %rdx, %rcx -; O1-NEXT: .Lpcsection547: +; O1-NEXT: .Lpcsection577: ; O1-NEXT: adcq $0, %rcx -; O1-NEXT: .Lpcsection548: +; O1-NEXT: .Lpcsection578: ; O1-NEXT: lock cmpxchg16b (%rdi) -; O1-NEXT: .Lpcsection549: +; O1-NEXT: .Lpcsection579: ; O1-NEXT: jne .LBB237_1 ; O1-NEXT: # %bb.2: # %atomicrmw.end ; O1-NEXT: movq $1, foo(%rip) @@ -14782,22 +14872,22 @@ ; O2-NEXT: .cfi_def_cfa_offset 16 ; O2-NEXT: .cfi_offset %rbx, -16 ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection544: +; O2-NEXT: .Lpcsection574: ; O2-NEXT: movq (%rdi), %rax -; O2-NEXT: .Lpcsection545: +; O2-NEXT: .Lpcsection575: ; O2-NEXT: movq 8(%rdi), %rdx ; O2-NEXT: .p2align 4, 0x90 ; O2-NEXT: .LBB237_1: # %atomicrmw.start ; O2-NEXT: # =>This Inner Loop Header: Depth=1 ; O2-NEXT: movq %rax, %rbx -; O2-NEXT: .Lpcsection546: +; O2-NEXT: .Lpcsection576: ; O2-NEXT: addq $42, %rbx ; O2-NEXT: movq %rdx, %rcx -; O2-NEXT: .Lpcsection547: +; O2-NEXT: .Lpcsection577: ; O2-NEXT: adcq $0, %rcx -; O2-NEXT: .Lpcsection548: +; O2-NEXT: .Lpcsection578: ; O2-NEXT: lock cmpxchg16b (%rdi) -; O2-NEXT: .Lpcsection549: +; O2-NEXT: .Lpcsection579: ; O2-NEXT: jne .LBB237_1 ; O2-NEXT: # %bb.2: # %atomicrmw.end ; O2-NEXT: movq $1, foo(%rip) @@ -14811,22 +14901,22 @@ ; O3-NEXT: .cfi_def_cfa_offset 16 ; O3-NEXT: .cfi_offset %rbx, -16 ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection544: +; O3-NEXT: .Lpcsection574: ; O3-NEXT: movq (%rdi), %rax -; O3-NEXT: .Lpcsection545: +; O3-NEXT: .Lpcsection575: ; O3-NEXT: movq 8(%rdi), %rdx ; O3-NEXT: .p2align 4, 0x90 ; O3-NEXT: .LBB237_1: # %atomicrmw.start ; O3-NEXT: # =>This Inner Loop Header: Depth=1 ; O3-NEXT: movq %rax, %rbx -; O3-NEXT: .Lpcsection546: +; O3-NEXT: .Lpcsection576: ; O3-NEXT: addq $42, %rbx ; O3-NEXT: movq %rdx, %rcx -; O3-NEXT: .Lpcsection547: +; O3-NEXT: .Lpcsection577: ; O3-NEXT: adcq $0, %rcx -; O3-NEXT: .Lpcsection548: +; O3-NEXT: .Lpcsection578: ; O3-NEXT: lock cmpxchg16b (%rdi) -; O3-NEXT: .Lpcsection549: +; O3-NEXT: .Lpcsection579: ; O3-NEXT: jne .LBB237_1 ; O3-NEXT: # %bb.2: # %atomicrmw.end ; O3-NEXT: movq $1, foo(%rip) @@ -14886,22 +14976,22 @@ ; O1-NEXT: .cfi_def_cfa_offset 16 ; O1-NEXT: .cfi_offset %rbx, -16 ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection550: +; O1-NEXT: .Lpcsection580: ; O1-NEXT: movq (%rdi), %rax -; O1-NEXT: .Lpcsection551: +; O1-NEXT: .Lpcsection581: ; O1-NEXT: movq 8(%rdi), %rdx ; O1-NEXT: .p2align 4, 0x90 ; O1-NEXT: .LBB238_1: # %atomicrmw.start ; O1-NEXT: # =>This Inner Loop Header: Depth=1 ; O1-NEXT: movq %rax, %rbx -; O1-NEXT: .Lpcsection552: +; O1-NEXT: .Lpcsection582: ; O1-NEXT: addq $-42, %rbx ; O1-NEXT: movq %rdx, %rcx -; O1-NEXT: .Lpcsection553: +; O1-NEXT: .Lpcsection583: ; O1-NEXT: adcq $-1, %rcx -; O1-NEXT: .Lpcsection554: +; O1-NEXT: .Lpcsection584: ; O1-NEXT: lock cmpxchg16b (%rdi) -; O1-NEXT: .Lpcsection555: +; O1-NEXT: .Lpcsection585: ; O1-NEXT: jne .LBB238_1 ; O1-NEXT: # %bb.2: # %atomicrmw.end ; O1-NEXT: movq $1, foo(%rip) @@ -14915,22 +15005,22 @@ ; O2-NEXT: .cfi_def_cfa_offset 16 ; O2-NEXT: .cfi_offset %rbx, -16 ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection550: +; O2-NEXT: .Lpcsection580: ; O2-NEXT: movq (%rdi), %rax -; O2-NEXT: .Lpcsection551: +; O2-NEXT: .Lpcsection581: ; O2-NEXT: movq 8(%rdi), %rdx ; O2-NEXT: .p2align 4, 0x90 ; O2-NEXT: .LBB238_1: # %atomicrmw.start ; O2-NEXT: # =>This Inner Loop Header: Depth=1 ; O2-NEXT: movq %rax, %rbx -; O2-NEXT: .Lpcsection552: +; O2-NEXT: .Lpcsection582: ; O2-NEXT: addq $-42, %rbx ; O2-NEXT: movq %rdx, %rcx -; O2-NEXT: .Lpcsection553: +; O2-NEXT: .Lpcsection583: ; O2-NEXT: adcq $-1, %rcx -; O2-NEXT: .Lpcsection554: +; O2-NEXT: .Lpcsection584: ; O2-NEXT: lock cmpxchg16b (%rdi) -; O2-NEXT: .Lpcsection555: +; O2-NEXT: .Lpcsection585: ; O2-NEXT: jne .LBB238_1 ; O2-NEXT: # %bb.2: # %atomicrmw.end ; O2-NEXT: movq $1, foo(%rip) @@ -14944,22 +15034,22 @@ ; O3-NEXT: .cfi_def_cfa_offset 16 ; O3-NEXT: .cfi_offset %rbx, -16 ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection550: +; O3-NEXT: .Lpcsection580: ; O3-NEXT: movq (%rdi), %rax -; O3-NEXT: .Lpcsection551: +; O3-NEXT: .Lpcsection581: ; O3-NEXT: movq 8(%rdi), %rdx ; O3-NEXT: .p2align 4, 0x90 ; O3-NEXT: .LBB238_1: # %atomicrmw.start ; O3-NEXT: # =>This Inner Loop Header: Depth=1 ; O3-NEXT: movq %rax, %rbx -; O3-NEXT: .Lpcsection552: +; O3-NEXT: .Lpcsection582: ; O3-NEXT: addq $-42, %rbx ; O3-NEXT: movq %rdx, %rcx -; O3-NEXT: .Lpcsection553: +; O3-NEXT: .Lpcsection583: ; O3-NEXT: adcq $-1, %rcx -; O3-NEXT: .Lpcsection554: +; O3-NEXT: .Lpcsection584: ; O3-NEXT: lock cmpxchg16b (%rdi) -; O3-NEXT: .Lpcsection555: +; O3-NEXT: .Lpcsection585: ; O3-NEXT: jne .LBB238_1 ; O3-NEXT: # %bb.2: # %atomicrmw.end ; O3-NEXT: movq $1, foo(%rip) @@ -15021,21 +15111,21 @@ ; O1-NEXT: .cfi_def_cfa_offset 16 ; O1-NEXT: .cfi_offset %rbx, -16 ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection556: +; O1-NEXT: .Lpcsection586: ; O1-NEXT: movq (%rdi), %rax -; O1-NEXT: .Lpcsection557: +; O1-NEXT: .Lpcsection587: ; O1-NEXT: movq 8(%rdi), %rdx ; O1-NEXT: .p2align 4, 0x90 ; O1-NEXT: .LBB239_1: # %atomicrmw.start ; O1-NEXT: # =>This Inner Loop Header: Depth=1 ; O1-NEXT: movl %eax, %ebx -; O1-NEXT: .Lpcsection558: +; O1-NEXT: .Lpcsection588: ; O1-NEXT: andl $42, %ebx -; O1-NEXT: .Lpcsection559: +; O1-NEXT: .Lpcsection589: ; O1-NEXT: xorl %ecx, %ecx -; O1-NEXT: .Lpcsection560: +; O1-NEXT: .Lpcsection590: ; O1-NEXT: lock cmpxchg16b (%rdi) -; O1-NEXT: .Lpcsection561: +; O1-NEXT: .Lpcsection591: ; O1-NEXT: jne .LBB239_1 ; O1-NEXT: # %bb.2: # %atomicrmw.end ; O1-NEXT: movq $1, foo(%rip) @@ -15049,21 +15139,21 @@ ; O2-NEXT: .cfi_def_cfa_offset 16 ; O2-NEXT: .cfi_offset %rbx, -16 ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection556: +; O2-NEXT: .Lpcsection586: ; O2-NEXT: movq (%rdi), %rax -; O2-NEXT: .Lpcsection557: +; O2-NEXT: .Lpcsection587: ; O2-NEXT: movq 8(%rdi), %rdx ; O2-NEXT: .p2align 4, 0x90 ; O2-NEXT: .LBB239_1: # %atomicrmw.start ; O2-NEXT: # =>This Inner Loop Header: Depth=1 ; O2-NEXT: movl %eax, %ebx -; O2-NEXT: .Lpcsection558: +; O2-NEXT: .Lpcsection588: ; O2-NEXT: andl $42, %ebx -; O2-NEXT: .Lpcsection559: +; O2-NEXT: .Lpcsection589: ; O2-NEXT: xorl %ecx, %ecx -; O2-NEXT: .Lpcsection560: +; O2-NEXT: .Lpcsection590: ; O2-NEXT: lock cmpxchg16b (%rdi) -; O2-NEXT: .Lpcsection561: +; O2-NEXT: .Lpcsection591: ; O2-NEXT: jne .LBB239_1 ; O2-NEXT: # %bb.2: # %atomicrmw.end ; O2-NEXT: movq $1, foo(%rip) @@ -15077,21 +15167,21 @@ ; O3-NEXT: .cfi_def_cfa_offset 16 ; O3-NEXT: .cfi_offset %rbx, -16 ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection556: +; O3-NEXT: .Lpcsection586: ; O3-NEXT: movq (%rdi), %rax -; O3-NEXT: .Lpcsection557: +; O3-NEXT: .Lpcsection587: ; O3-NEXT: movq 8(%rdi), %rdx ; O3-NEXT: .p2align 4, 0x90 ; O3-NEXT: .LBB239_1: # %atomicrmw.start ; O3-NEXT: # =>This Inner Loop Header: Depth=1 ; O3-NEXT: movl %eax, %ebx -; O3-NEXT: .Lpcsection558: +; O3-NEXT: .Lpcsection588: ; O3-NEXT: andl $42, %ebx -; O3-NEXT: .Lpcsection559: +; O3-NEXT: .Lpcsection589: ; O3-NEXT: xorl %ecx, %ecx -; O3-NEXT: .Lpcsection560: +; O3-NEXT: .Lpcsection590: ; O3-NEXT: lock cmpxchg16b (%rdi) -; O3-NEXT: .Lpcsection561: +; O3-NEXT: .Lpcsection591: ; O3-NEXT: jne .LBB239_1 ; O3-NEXT: # %bb.2: # %atomicrmw.end ; O3-NEXT: movq $1, foo(%rip) @@ -15149,20 +15239,20 @@ ; O1-NEXT: .cfi_def_cfa_offset 16 ; O1-NEXT: .cfi_offset %rbx, -16 ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection562: +; O1-NEXT: .Lpcsection592: ; O1-NEXT: movq (%rdi), %rax -; O1-NEXT: .Lpcsection563: +; O1-NEXT: .Lpcsection593: ; O1-NEXT: movq 8(%rdi), %rdx ; O1-NEXT: .p2align 4, 0x90 ; O1-NEXT: .LBB240_1: # %atomicrmw.start ; O1-NEXT: # =>This Inner Loop Header: Depth=1 ; O1-NEXT: movq %rax, %rbx -; O1-NEXT: .Lpcsection564: +; O1-NEXT: .Lpcsection594: ; O1-NEXT: orq $42, %rbx ; O1-NEXT: movq %rdx, %rcx -; O1-NEXT: .Lpcsection565: +; O1-NEXT: .Lpcsection595: ; O1-NEXT: lock cmpxchg16b (%rdi) -; O1-NEXT: .Lpcsection566: +; O1-NEXT: .Lpcsection596: ; O1-NEXT: jne .LBB240_1 ; O1-NEXT: # %bb.2: # %atomicrmw.end ; O1-NEXT: movq $1, foo(%rip) @@ -15176,20 +15266,20 @@ ; O2-NEXT: .cfi_def_cfa_offset 16 ; O2-NEXT: .cfi_offset %rbx, -16 ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection562: +; O2-NEXT: .Lpcsection592: ; O2-NEXT: movq (%rdi), %rax -; O2-NEXT: .Lpcsection563: +; O2-NEXT: .Lpcsection593: ; O2-NEXT: movq 8(%rdi), %rdx ; O2-NEXT: .p2align 4, 0x90 ; O2-NEXT: .LBB240_1: # %atomicrmw.start ; O2-NEXT: # =>This Inner Loop Header: Depth=1 ; O2-NEXT: movq %rax, %rbx -; O2-NEXT: .Lpcsection564: +; O2-NEXT: .Lpcsection594: ; O2-NEXT: orq $42, %rbx ; O2-NEXT: movq %rdx, %rcx -; O2-NEXT: .Lpcsection565: +; O2-NEXT: .Lpcsection595: ; O2-NEXT: lock cmpxchg16b (%rdi) -; O2-NEXT: .Lpcsection566: +; O2-NEXT: .Lpcsection596: ; O2-NEXT: jne .LBB240_1 ; O2-NEXT: # %bb.2: # %atomicrmw.end ; O2-NEXT: movq $1, foo(%rip) @@ -15203,20 +15293,20 @@ ; O3-NEXT: .cfi_def_cfa_offset 16 ; O3-NEXT: .cfi_offset %rbx, -16 ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection562: +; O3-NEXT: .Lpcsection592: ; O3-NEXT: movq (%rdi), %rax -; O3-NEXT: .Lpcsection563: +; O3-NEXT: .Lpcsection593: ; O3-NEXT: movq 8(%rdi), %rdx ; O3-NEXT: .p2align 4, 0x90 ; O3-NEXT: .LBB240_1: # %atomicrmw.start ; O3-NEXT: # =>This Inner Loop Header: Depth=1 ; O3-NEXT: movq %rax, %rbx -; O3-NEXT: .Lpcsection564: +; O3-NEXT: .Lpcsection594: ; O3-NEXT: orq $42, %rbx ; O3-NEXT: movq %rdx, %rcx -; O3-NEXT: .Lpcsection565: +; O3-NEXT: .Lpcsection595: ; O3-NEXT: lock cmpxchg16b (%rdi) -; O3-NEXT: .Lpcsection566: +; O3-NEXT: .Lpcsection596: ; O3-NEXT: jne .LBB240_1 ; O3-NEXT: # %bb.2: # %atomicrmw.end ; O3-NEXT: movq $1, foo(%rip) @@ -15274,20 +15364,20 @@ ; O1-NEXT: .cfi_def_cfa_offset 16 ; O1-NEXT: .cfi_offset %rbx, -16 ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection567: +; O1-NEXT: .Lpcsection597: ; O1-NEXT: movq (%rdi), %rax -; O1-NEXT: .Lpcsection568: +; O1-NEXT: .Lpcsection598: ; O1-NEXT: movq 8(%rdi), %rdx ; O1-NEXT: .p2align 4, 0x90 ; O1-NEXT: .LBB241_1: # %atomicrmw.start ; O1-NEXT: # =>This Inner Loop Header: Depth=1 ; O1-NEXT: movq %rax, %rbx -; O1-NEXT: .Lpcsection569: +; O1-NEXT: .Lpcsection599: ; O1-NEXT: xorq $42, %rbx ; O1-NEXT: movq %rdx, %rcx -; O1-NEXT: .Lpcsection570: +; O1-NEXT: .Lpcsection600: ; O1-NEXT: lock cmpxchg16b (%rdi) -; O1-NEXT: .Lpcsection571: +; O1-NEXT: .Lpcsection601: ; O1-NEXT: jne .LBB241_1 ; O1-NEXT: # %bb.2: # %atomicrmw.end ; O1-NEXT: movq $1, foo(%rip) @@ -15301,20 +15391,20 @@ ; O2-NEXT: .cfi_def_cfa_offset 16 ; O2-NEXT: .cfi_offset %rbx, -16 ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection567: +; O2-NEXT: .Lpcsection597: ; O2-NEXT: movq (%rdi), %rax -; O2-NEXT: .Lpcsection568: +; O2-NEXT: .Lpcsection598: ; O2-NEXT: movq 8(%rdi), %rdx ; O2-NEXT: .p2align 4, 0x90 ; O2-NEXT: .LBB241_1: # %atomicrmw.start ; O2-NEXT: # =>This Inner Loop Header: Depth=1 ; O2-NEXT: movq %rax, %rbx -; O2-NEXT: .Lpcsection569: +; O2-NEXT: .Lpcsection599: ; O2-NEXT: xorq $42, %rbx ; O2-NEXT: movq %rdx, %rcx -; O2-NEXT: .Lpcsection570: +; O2-NEXT: .Lpcsection600: ; O2-NEXT: lock cmpxchg16b (%rdi) -; O2-NEXT: .Lpcsection571: +; O2-NEXT: .Lpcsection601: ; O2-NEXT: jne .LBB241_1 ; O2-NEXT: # %bb.2: # %atomicrmw.end ; O2-NEXT: movq $1, foo(%rip) @@ -15328,20 +15418,20 @@ ; O3-NEXT: .cfi_def_cfa_offset 16 ; O3-NEXT: .cfi_offset %rbx, -16 ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection567: +; O3-NEXT: .Lpcsection597: ; O3-NEXT: movq (%rdi), %rax -; O3-NEXT: .Lpcsection568: +; O3-NEXT: .Lpcsection598: ; O3-NEXT: movq 8(%rdi), %rdx ; O3-NEXT: .p2align 4, 0x90 ; O3-NEXT: .LBB241_1: # %atomicrmw.start ; O3-NEXT: # =>This Inner Loop Header: Depth=1 ; O3-NEXT: movq %rax, %rbx -; O3-NEXT: .Lpcsection569: +; O3-NEXT: .Lpcsection599: ; O3-NEXT: xorq $42, %rbx ; O3-NEXT: movq %rdx, %rcx -; O3-NEXT: .Lpcsection570: +; O3-NEXT: .Lpcsection600: ; O3-NEXT: lock cmpxchg16b (%rdi) -; O3-NEXT: .Lpcsection571: +; O3-NEXT: .Lpcsection601: ; O3-NEXT: jne .LBB241_1 ; O3-NEXT: # %bb.2: # %atomicrmw.end ; O3-NEXT: movq $1, foo(%rip) @@ -15405,23 +15495,23 @@ ; O1-NEXT: .cfi_def_cfa_offset 16 ; O1-NEXT: .cfi_offset %rbx, -16 ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection572: +; O1-NEXT: .Lpcsection602: ; O1-NEXT: movq (%rdi), %rax -; O1-NEXT: .Lpcsection573: +; O1-NEXT: .Lpcsection603: ; O1-NEXT: movq 8(%rdi), %rdx -; O1-NEXT: .Lpcsection574: +; O1-NEXT: .Lpcsection604: ; O1-NEXT: movq $-1, %rcx ; O1-NEXT: .p2align 4, 0x90 ; O1-NEXT: .LBB242_1: # %atomicrmw.start ; O1-NEXT: # =>This Inner Loop Header: Depth=1 ; O1-NEXT: movl %eax, %ebx -; O1-NEXT: .Lpcsection575: +; O1-NEXT: .Lpcsection605: ; O1-NEXT: notl %ebx -; O1-NEXT: .Lpcsection576: +; O1-NEXT: .Lpcsection606: ; O1-NEXT: orq $-43, %rbx -; O1-NEXT: .Lpcsection577: +; O1-NEXT: .Lpcsection607: ; O1-NEXT: lock cmpxchg16b (%rdi) -; O1-NEXT: .Lpcsection578: +; O1-NEXT: .Lpcsection608: ; O1-NEXT: jne .LBB242_1 ; O1-NEXT: # %bb.2: # %atomicrmw.end ; O1-NEXT: movq $1, foo(%rip) @@ -15435,23 +15525,23 @@ ; O2-NEXT: .cfi_def_cfa_offset 16 ; O2-NEXT: .cfi_offset %rbx, -16 ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection572: +; O2-NEXT: .Lpcsection602: ; O2-NEXT: movq (%rdi), %rax -; O2-NEXT: .Lpcsection573: +; O2-NEXT: .Lpcsection603: ; O2-NEXT: movq 8(%rdi), %rdx -; O2-NEXT: .Lpcsection574: +; O2-NEXT: .Lpcsection604: ; O2-NEXT: movq $-1, %rcx ; O2-NEXT: .p2align 4, 0x90 ; O2-NEXT: .LBB242_1: # %atomicrmw.start ; O2-NEXT: # =>This Inner Loop Header: Depth=1 ; O2-NEXT: movl %eax, %ebx -; O2-NEXT: .Lpcsection575: +; O2-NEXT: .Lpcsection605: ; O2-NEXT: notl %ebx -; O2-NEXT: .Lpcsection576: +; O2-NEXT: .Lpcsection606: ; O2-NEXT: orq $-43, %rbx -; O2-NEXT: .Lpcsection577: +; O2-NEXT: .Lpcsection607: ; O2-NEXT: lock cmpxchg16b (%rdi) -; O2-NEXT: .Lpcsection578: +; O2-NEXT: .Lpcsection608: ; O2-NEXT: jne .LBB242_1 ; O2-NEXT: # %bb.2: # %atomicrmw.end ; O2-NEXT: movq $1, foo(%rip) @@ -15465,23 +15555,23 @@ ; O3-NEXT: .cfi_def_cfa_offset 16 ; O3-NEXT: .cfi_offset %rbx, -16 ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection572: +; O3-NEXT: .Lpcsection602: ; O3-NEXT: movq (%rdi), %rax -; O3-NEXT: .Lpcsection573: +; O3-NEXT: .Lpcsection603: ; O3-NEXT: movq 8(%rdi), %rdx -; O3-NEXT: .Lpcsection574: +; O3-NEXT: .Lpcsection604: ; O3-NEXT: movq $-1, %rcx ; O3-NEXT: .p2align 4, 0x90 ; O3-NEXT: .LBB242_1: # %atomicrmw.start ; O3-NEXT: # =>This Inner Loop Header: Depth=1 ; O3-NEXT: movl %eax, %ebx -; O3-NEXT: .Lpcsection575: +; O3-NEXT: .Lpcsection605: ; O3-NEXT: notl %ebx -; O3-NEXT: .Lpcsection576: +; O3-NEXT: .Lpcsection606: ; O3-NEXT: orq $-43, %rbx -; O3-NEXT: .Lpcsection577: +; O3-NEXT: .Lpcsection607: ; O3-NEXT: lock cmpxchg16b (%rdi) -; O3-NEXT: .Lpcsection578: +; O3-NEXT: .Lpcsection608: ; O3-NEXT: jne .LBB242_1 ; O3-NEXT: # %bb.2: # %atomicrmw.end ; O3-NEXT: movq $1, foo(%rip) @@ -15542,31 +15632,31 @@ ; O1-NEXT: .cfi_def_cfa_offset 16 ; O1-NEXT: .cfi_offset %rbx, -16 ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection579: +; O1-NEXT: .Lpcsection609: ; O1-NEXT: movl $42, %eax -; O1-NEXT: .Lpcsection580: +; O1-NEXT: .Lpcsection610: ; O1-NEXT: movl $1, %ebx -; O1-NEXT: .Lpcsection581: +; O1-NEXT: .Lpcsection611: ; O1-NEXT: xorl %edx, %edx -; O1-NEXT: .Lpcsection582: +; O1-NEXT: .Lpcsection612: ; O1-NEXT: xorl %ecx, %ecx -; O1-NEXT: .Lpcsection583: +; O1-NEXT: .Lpcsection613: ; O1-NEXT: lock cmpxchg16b (%rdi) -; O1-NEXT: .Lpcsection584: +; O1-NEXT: .Lpcsection614: ; O1-NEXT: movl $42, %eax -; O1-NEXT: .Lpcsection585: +; O1-NEXT: .Lpcsection615: ; O1-NEXT: xorl %edx, %edx -; O1-NEXT: .Lpcsection586: +; O1-NEXT: .Lpcsection616: ; O1-NEXT: xorl %ecx, %ecx -; O1-NEXT: .Lpcsection587: +; O1-NEXT: .Lpcsection617: ; O1-NEXT: lock cmpxchg16b (%rdi) -; O1-NEXT: .Lpcsection588: +; O1-NEXT: .Lpcsection618: ; O1-NEXT: movl $42, %eax -; O1-NEXT: .Lpcsection589: +; O1-NEXT: .Lpcsection619: ; O1-NEXT: xorl %edx, %edx -; O1-NEXT: .Lpcsection590: +; O1-NEXT: .Lpcsection620: ; O1-NEXT: xorl %ecx, %ecx -; O1-NEXT: .Lpcsection591: +; O1-NEXT: .Lpcsection621: ; O1-NEXT: lock cmpxchg16b (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: popq %rbx @@ -15579,31 +15669,31 @@ ; O2-NEXT: .cfi_def_cfa_offset 16 ; O2-NEXT: .cfi_offset %rbx, -16 ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection579: +; O2-NEXT: .Lpcsection609: ; O2-NEXT: movl $42, %eax -; O2-NEXT: .Lpcsection580: +; O2-NEXT: .Lpcsection610: ; O2-NEXT: movl $1, %ebx -; O2-NEXT: .Lpcsection581: +; O2-NEXT: .Lpcsection611: ; O2-NEXT: xorl %edx, %edx -; O2-NEXT: .Lpcsection582: +; O2-NEXT: .Lpcsection612: ; O2-NEXT: xorl %ecx, %ecx -; O2-NEXT: .Lpcsection583: +; O2-NEXT: .Lpcsection613: ; O2-NEXT: lock cmpxchg16b (%rdi) -; O2-NEXT: .Lpcsection584: +; O2-NEXT: .Lpcsection614: ; O2-NEXT: movl $42, %eax -; O2-NEXT: .Lpcsection585: +; O2-NEXT: .Lpcsection615: ; O2-NEXT: xorl %edx, %edx -; O2-NEXT: .Lpcsection586: +; O2-NEXT: .Lpcsection616: ; O2-NEXT: xorl %ecx, %ecx -; O2-NEXT: .Lpcsection587: +; O2-NEXT: .Lpcsection617: ; O2-NEXT: lock cmpxchg16b (%rdi) -; O2-NEXT: .Lpcsection588: +; O2-NEXT: .Lpcsection618: ; O2-NEXT: movl $42, %eax -; O2-NEXT: .Lpcsection589: +; O2-NEXT: .Lpcsection619: ; O2-NEXT: xorl %edx, %edx -; O2-NEXT: .Lpcsection590: +; O2-NEXT: .Lpcsection620: ; O2-NEXT: xorl %ecx, %ecx -; O2-NEXT: .Lpcsection591: +; O2-NEXT: .Lpcsection621: ; O2-NEXT: lock cmpxchg16b (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: popq %rbx @@ -15616,31 +15706,31 @@ ; O3-NEXT: .cfi_def_cfa_offset 16 ; O3-NEXT: .cfi_offset %rbx, -16 ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection579: +; O3-NEXT: .Lpcsection609: ; O3-NEXT: movl $42, %eax -; O3-NEXT: .Lpcsection580: +; O3-NEXT: .Lpcsection610: ; O3-NEXT: movl $1, %ebx -; O3-NEXT: .Lpcsection581: +; O3-NEXT: .Lpcsection611: ; O3-NEXT: xorl %edx, %edx -; O3-NEXT: .Lpcsection582: +; O3-NEXT: .Lpcsection612: ; O3-NEXT: xorl %ecx, %ecx -; O3-NEXT: .Lpcsection583: +; O3-NEXT: .Lpcsection613: ; O3-NEXT: lock cmpxchg16b (%rdi) -; O3-NEXT: .Lpcsection584: +; O3-NEXT: .Lpcsection614: ; O3-NEXT: movl $42, %eax -; O3-NEXT: .Lpcsection585: +; O3-NEXT: .Lpcsection615: ; O3-NEXT: xorl %edx, %edx -; O3-NEXT: .Lpcsection586: +; O3-NEXT: .Lpcsection616: ; O3-NEXT: xorl %ecx, %ecx -; O3-NEXT: .Lpcsection587: +; O3-NEXT: .Lpcsection617: ; O3-NEXT: lock cmpxchg16b (%rdi) -; O3-NEXT: .Lpcsection588: +; O3-NEXT: .Lpcsection618: ; O3-NEXT: movl $42, %eax -; O3-NEXT: .Lpcsection589: +; O3-NEXT: .Lpcsection619: ; O3-NEXT: xorl %edx, %edx -; O3-NEXT: .Lpcsection590: +; O3-NEXT: .Lpcsection620: ; O3-NEXT: xorl %ecx, %ecx -; O3-NEXT: .Lpcsection591: +; O3-NEXT: .Lpcsection621: ; O3-NEXT: lock cmpxchg16b (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: popq %rbx @@ -15702,31 +15792,31 @@ ; O1-NEXT: .cfi_def_cfa_offset 16 ; O1-NEXT: .cfi_offset %rbx, -16 ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection592: +; O1-NEXT: .Lpcsection622: ; O1-NEXT: movl $42, %eax -; O1-NEXT: .Lpcsection593: +; O1-NEXT: .Lpcsection623: ; O1-NEXT: movl $1, %ebx -; O1-NEXT: .Lpcsection594: +; O1-NEXT: .Lpcsection624: ; O1-NEXT: xorl %edx, %edx -; O1-NEXT: .Lpcsection595: +; O1-NEXT: .Lpcsection625: ; O1-NEXT: xorl %ecx, %ecx -; O1-NEXT: .Lpcsection596: +; O1-NEXT: .Lpcsection626: ; O1-NEXT: lock cmpxchg16b (%rdi) -; O1-NEXT: .Lpcsection597: +; O1-NEXT: .Lpcsection627: ; O1-NEXT: movl $42, %eax -; O1-NEXT: .Lpcsection598: +; O1-NEXT: .Lpcsection628: ; O1-NEXT: xorl %edx, %edx -; O1-NEXT: .Lpcsection599: +; O1-NEXT: .Lpcsection629: ; O1-NEXT: xorl %ecx, %ecx -; O1-NEXT: .Lpcsection600: +; O1-NEXT: .Lpcsection630: ; O1-NEXT: lock cmpxchg16b (%rdi) -; O1-NEXT: .Lpcsection601: +; O1-NEXT: .Lpcsection631: ; O1-NEXT: movl $42, %eax -; O1-NEXT: .Lpcsection602: +; O1-NEXT: .Lpcsection632: ; O1-NEXT: xorl %edx, %edx -; O1-NEXT: .Lpcsection603: +; O1-NEXT: .Lpcsection633: ; O1-NEXT: xorl %ecx, %ecx -; O1-NEXT: .Lpcsection604: +; O1-NEXT: .Lpcsection634: ; O1-NEXT: lock cmpxchg16b (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: popq %rbx @@ -15739,31 +15829,31 @@ ; O2-NEXT: .cfi_def_cfa_offset 16 ; O2-NEXT: .cfi_offset %rbx, -16 ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection592: +; O2-NEXT: .Lpcsection622: ; O2-NEXT: movl $42, %eax -; O2-NEXT: .Lpcsection593: +; O2-NEXT: .Lpcsection623: ; O2-NEXT: movl $1, %ebx -; O2-NEXT: .Lpcsection594: +; O2-NEXT: .Lpcsection624: ; O2-NEXT: xorl %edx, %edx -; O2-NEXT: .Lpcsection595: +; O2-NEXT: .Lpcsection625: ; O2-NEXT: xorl %ecx, %ecx -; O2-NEXT: .Lpcsection596: +; O2-NEXT: .Lpcsection626: ; O2-NEXT: lock cmpxchg16b (%rdi) -; O2-NEXT: .Lpcsection597: +; O2-NEXT: .Lpcsection627: ; O2-NEXT: movl $42, %eax -; O2-NEXT: .Lpcsection598: +; O2-NEXT: .Lpcsection628: ; O2-NEXT: xorl %edx, %edx -; O2-NEXT: .Lpcsection599: +; O2-NEXT: .Lpcsection629: ; O2-NEXT: xorl %ecx, %ecx -; O2-NEXT: .Lpcsection600: +; O2-NEXT: .Lpcsection630: ; O2-NEXT: lock cmpxchg16b (%rdi) -; O2-NEXT: .Lpcsection601: +; O2-NEXT: .Lpcsection631: ; O2-NEXT: movl $42, %eax -; O2-NEXT: .Lpcsection602: +; O2-NEXT: .Lpcsection632: ; O2-NEXT: xorl %edx, %edx -; O2-NEXT: .Lpcsection603: +; O2-NEXT: .Lpcsection633: ; O2-NEXT: xorl %ecx, %ecx -; O2-NEXT: .Lpcsection604: +; O2-NEXT: .Lpcsection634: ; O2-NEXT: lock cmpxchg16b (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: popq %rbx @@ -15776,31 +15866,31 @@ ; O3-NEXT: .cfi_def_cfa_offset 16 ; O3-NEXT: .cfi_offset %rbx, -16 ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection592: +; O3-NEXT: .Lpcsection622: ; O3-NEXT: movl $42, %eax -; O3-NEXT: .Lpcsection593: +; O3-NEXT: .Lpcsection623: ; O3-NEXT: movl $1, %ebx -; O3-NEXT: .Lpcsection594: +; O3-NEXT: .Lpcsection624: ; O3-NEXT: xorl %edx, %edx -; O3-NEXT: .Lpcsection595: +; O3-NEXT: .Lpcsection625: ; O3-NEXT: xorl %ecx, %ecx -; O3-NEXT: .Lpcsection596: +; O3-NEXT: .Lpcsection626: ; O3-NEXT: lock cmpxchg16b (%rdi) -; O3-NEXT: .Lpcsection597: +; O3-NEXT: .Lpcsection627: ; O3-NEXT: movl $42, %eax -; O3-NEXT: .Lpcsection598: +; O3-NEXT: .Lpcsection628: ; O3-NEXT: xorl %edx, %edx -; O3-NEXT: .Lpcsection599: +; O3-NEXT: .Lpcsection629: ; O3-NEXT: xorl %ecx, %ecx -; O3-NEXT: .Lpcsection600: +; O3-NEXT: .Lpcsection630: ; O3-NEXT: lock cmpxchg16b (%rdi) -; O3-NEXT: .Lpcsection601: +; O3-NEXT: .Lpcsection631: ; O3-NEXT: movl $42, %eax -; O3-NEXT: .Lpcsection602: +; O3-NEXT: .Lpcsection632: ; O3-NEXT: xorl %edx, %edx -; O3-NEXT: .Lpcsection603: +; O3-NEXT: .Lpcsection633: ; O3-NEXT: xorl %ecx, %ecx -; O3-NEXT: .Lpcsection604: +; O3-NEXT: .Lpcsection634: ; O3-NEXT: lock cmpxchg16b (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: popq %rbx @@ -15862,31 +15952,31 @@ ; O1-NEXT: .cfi_def_cfa_offset 16 ; O1-NEXT: .cfi_offset %rbx, -16 ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection605: +; O1-NEXT: .Lpcsection635: ; O1-NEXT: movl $42, %eax -; O1-NEXT: .Lpcsection606: +; O1-NEXT: .Lpcsection636: ; O1-NEXT: movl $1, %ebx -; O1-NEXT: .Lpcsection607: +; O1-NEXT: .Lpcsection637: ; O1-NEXT: xorl %edx, %edx -; O1-NEXT: .Lpcsection608: +; O1-NEXT: .Lpcsection638: ; O1-NEXT: xorl %ecx, %ecx -; O1-NEXT: .Lpcsection609: +; O1-NEXT: .Lpcsection639: ; O1-NEXT: lock cmpxchg16b (%rdi) -; O1-NEXT: .Lpcsection610: +; O1-NEXT: .Lpcsection640: ; O1-NEXT: movl $42, %eax -; O1-NEXT: .Lpcsection611: +; O1-NEXT: .Lpcsection641: ; O1-NEXT: xorl %edx, %edx -; O1-NEXT: .Lpcsection612: +; O1-NEXT: .Lpcsection642: ; O1-NEXT: xorl %ecx, %ecx -; O1-NEXT: .Lpcsection613: +; O1-NEXT: .Lpcsection643: ; O1-NEXT: lock cmpxchg16b (%rdi) -; O1-NEXT: .Lpcsection614: +; O1-NEXT: .Lpcsection644: ; O1-NEXT: movl $42, %eax -; O1-NEXT: .Lpcsection615: +; O1-NEXT: .Lpcsection645: ; O1-NEXT: xorl %edx, %edx -; O1-NEXT: .Lpcsection616: +; O1-NEXT: .Lpcsection646: ; O1-NEXT: xorl %ecx, %ecx -; O1-NEXT: .Lpcsection617: +; O1-NEXT: .Lpcsection647: ; O1-NEXT: lock cmpxchg16b (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: popq %rbx @@ -15899,31 +15989,31 @@ ; O2-NEXT: .cfi_def_cfa_offset 16 ; O2-NEXT: .cfi_offset %rbx, -16 ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection605: +; O2-NEXT: .Lpcsection635: ; O2-NEXT: movl $42, %eax -; O2-NEXT: .Lpcsection606: +; O2-NEXT: .Lpcsection636: ; O2-NEXT: movl $1, %ebx -; O2-NEXT: .Lpcsection607: +; O2-NEXT: .Lpcsection637: ; O2-NEXT: xorl %edx, %edx -; O2-NEXT: .Lpcsection608: +; O2-NEXT: .Lpcsection638: ; O2-NEXT: xorl %ecx, %ecx -; O2-NEXT: .Lpcsection609: +; O2-NEXT: .Lpcsection639: ; O2-NEXT: lock cmpxchg16b (%rdi) -; O2-NEXT: .Lpcsection610: +; O2-NEXT: .Lpcsection640: ; O2-NEXT: movl $42, %eax -; O2-NEXT: .Lpcsection611: +; O2-NEXT: .Lpcsection641: ; O2-NEXT: xorl %edx, %edx -; O2-NEXT: .Lpcsection612: +; O2-NEXT: .Lpcsection642: ; O2-NEXT: xorl %ecx, %ecx -; O2-NEXT: .Lpcsection613: +; O2-NEXT: .Lpcsection643: ; O2-NEXT: lock cmpxchg16b (%rdi) -; O2-NEXT: .Lpcsection614: +; O2-NEXT: .Lpcsection644: ; O2-NEXT: movl $42, %eax -; O2-NEXT: .Lpcsection615: +; O2-NEXT: .Lpcsection645: ; O2-NEXT: xorl %edx, %edx -; O2-NEXT: .Lpcsection616: +; O2-NEXT: .Lpcsection646: ; O2-NEXT: xorl %ecx, %ecx -; O2-NEXT: .Lpcsection617: +; O2-NEXT: .Lpcsection647: ; O2-NEXT: lock cmpxchg16b (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: popq %rbx @@ -15936,31 +16026,31 @@ ; O3-NEXT: .cfi_def_cfa_offset 16 ; O3-NEXT: .cfi_offset %rbx, -16 ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection605: +; O3-NEXT: .Lpcsection635: ; O3-NEXT: movl $42, %eax -; O3-NEXT: .Lpcsection606: +; O3-NEXT: .Lpcsection636: ; O3-NEXT: movl $1, %ebx -; O3-NEXT: .Lpcsection607: +; O3-NEXT: .Lpcsection637: ; O3-NEXT: xorl %edx, %edx -; O3-NEXT: .Lpcsection608: +; O3-NEXT: .Lpcsection638: ; O3-NEXT: xorl %ecx, %ecx -; O3-NEXT: .Lpcsection609: +; O3-NEXT: .Lpcsection639: ; O3-NEXT: lock cmpxchg16b (%rdi) -; O3-NEXT: .Lpcsection610: +; O3-NEXT: .Lpcsection640: ; O3-NEXT: movl $42, %eax -; O3-NEXT: .Lpcsection611: +; O3-NEXT: .Lpcsection641: ; O3-NEXT: xorl %edx, %edx -; O3-NEXT: .Lpcsection612: +; O3-NEXT: .Lpcsection642: ; O3-NEXT: xorl %ecx, %ecx -; O3-NEXT: .Lpcsection613: +; O3-NEXT: .Lpcsection643: ; O3-NEXT: lock cmpxchg16b (%rdi) -; O3-NEXT: .Lpcsection614: +; O3-NEXT: .Lpcsection644: ; O3-NEXT: movl $42, %eax -; O3-NEXT: .Lpcsection615: +; O3-NEXT: .Lpcsection645: ; O3-NEXT: xorl %edx, %edx -; O3-NEXT: .Lpcsection616: +; O3-NEXT: .Lpcsection646: ; O3-NEXT: xorl %ecx, %ecx -; O3-NEXT: .Lpcsection617: +; O3-NEXT: .Lpcsection647: ; O3-NEXT: lock cmpxchg16b (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: popq %rbx @@ -16022,31 +16112,31 @@ ; O1-NEXT: .cfi_def_cfa_offset 16 ; O1-NEXT: .cfi_offset %rbx, -16 ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection618: +; O1-NEXT: .Lpcsection648: ; O1-NEXT: movl $42, %eax -; O1-NEXT: .Lpcsection619: +; O1-NEXT: .Lpcsection649: ; O1-NEXT: movl $1, %ebx -; O1-NEXT: .Lpcsection620: +; O1-NEXT: .Lpcsection650: ; O1-NEXT: xorl %edx, %edx -; O1-NEXT: .Lpcsection621: +; O1-NEXT: .Lpcsection651: ; O1-NEXT: xorl %ecx, %ecx -; O1-NEXT: .Lpcsection622: +; O1-NEXT: .Lpcsection652: ; O1-NEXT: lock cmpxchg16b (%rdi) -; O1-NEXT: .Lpcsection623: +; O1-NEXT: .Lpcsection653: ; O1-NEXT: movl $42, %eax -; O1-NEXT: .Lpcsection624: +; O1-NEXT: .Lpcsection654: ; O1-NEXT: xorl %edx, %edx -; O1-NEXT: .Lpcsection625: +; O1-NEXT: .Lpcsection655: ; O1-NEXT: xorl %ecx, %ecx -; O1-NEXT: .Lpcsection626: +; O1-NEXT: .Lpcsection656: ; O1-NEXT: lock cmpxchg16b (%rdi) -; O1-NEXT: .Lpcsection627: +; O1-NEXT: .Lpcsection657: ; O1-NEXT: movl $42, %eax -; O1-NEXT: .Lpcsection628: +; O1-NEXT: .Lpcsection658: ; O1-NEXT: xorl %edx, %edx -; O1-NEXT: .Lpcsection629: +; O1-NEXT: .Lpcsection659: ; O1-NEXT: xorl %ecx, %ecx -; O1-NEXT: .Lpcsection630: +; O1-NEXT: .Lpcsection660: ; O1-NEXT: lock cmpxchg16b (%rdi) ; O1-NEXT: movq $1, foo(%rip) ; O1-NEXT: popq %rbx @@ -16059,31 +16149,31 @@ ; O2-NEXT: .cfi_def_cfa_offset 16 ; O2-NEXT: .cfi_offset %rbx, -16 ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection618: +; O2-NEXT: .Lpcsection648: ; O2-NEXT: movl $42, %eax -; O2-NEXT: .Lpcsection619: +; O2-NEXT: .Lpcsection649: ; O2-NEXT: movl $1, %ebx -; O2-NEXT: .Lpcsection620: +; O2-NEXT: .Lpcsection650: ; O2-NEXT: xorl %edx, %edx -; O2-NEXT: .Lpcsection621: +; O2-NEXT: .Lpcsection651: ; O2-NEXT: xorl %ecx, %ecx -; O2-NEXT: .Lpcsection622: +; O2-NEXT: .Lpcsection652: ; O2-NEXT: lock cmpxchg16b (%rdi) -; O2-NEXT: .Lpcsection623: +; O2-NEXT: .Lpcsection653: ; O2-NEXT: movl $42, %eax -; O2-NEXT: .Lpcsection624: +; O2-NEXT: .Lpcsection654: ; O2-NEXT: xorl %edx, %edx -; O2-NEXT: .Lpcsection625: +; O2-NEXT: .Lpcsection655: ; O2-NEXT: xorl %ecx, %ecx -; O2-NEXT: .Lpcsection626: +; O2-NEXT: .Lpcsection656: ; O2-NEXT: lock cmpxchg16b (%rdi) -; O2-NEXT: .Lpcsection627: +; O2-NEXT: .Lpcsection657: ; O2-NEXT: movl $42, %eax -; O2-NEXT: .Lpcsection628: +; O2-NEXT: .Lpcsection658: ; O2-NEXT: xorl %edx, %edx -; O2-NEXT: .Lpcsection629: +; O2-NEXT: .Lpcsection659: ; O2-NEXT: xorl %ecx, %ecx -; O2-NEXT: .Lpcsection630: +; O2-NEXT: .Lpcsection660: ; O2-NEXT: lock cmpxchg16b (%rdi) ; O2-NEXT: movq $1, foo(%rip) ; O2-NEXT: popq %rbx @@ -16096,31 +16186,31 @@ ; O3-NEXT: .cfi_def_cfa_offset 16 ; O3-NEXT: .cfi_offset %rbx, -16 ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection618: +; O3-NEXT: .Lpcsection648: ; O3-NEXT: movl $42, %eax -; O3-NEXT: .Lpcsection619: +; O3-NEXT: .Lpcsection649: ; O3-NEXT: movl $1, %ebx -; O3-NEXT: .Lpcsection620: +; O3-NEXT: .Lpcsection650: ; O3-NEXT: xorl %edx, %edx -; O3-NEXT: .Lpcsection621: +; O3-NEXT: .Lpcsection651: ; O3-NEXT: xorl %ecx, %ecx -; O3-NEXT: .Lpcsection622: +; O3-NEXT: .Lpcsection652: ; O3-NEXT: lock cmpxchg16b (%rdi) -; O3-NEXT: .Lpcsection623: +; O3-NEXT: .Lpcsection653: ; O3-NEXT: movl $42, %eax -; O3-NEXT: .Lpcsection624: +; O3-NEXT: .Lpcsection654: ; O3-NEXT: xorl %edx, %edx -; O3-NEXT: .Lpcsection625: +; O3-NEXT: .Lpcsection655: ; O3-NEXT: xorl %ecx, %ecx -; O3-NEXT: .Lpcsection626: +; O3-NEXT: .Lpcsection656: ; O3-NEXT: lock cmpxchg16b (%rdi) -; O3-NEXT: .Lpcsection627: +; O3-NEXT: .Lpcsection657: ; O3-NEXT: movl $42, %eax -; O3-NEXT: .Lpcsection628: +; O3-NEXT: .Lpcsection658: ; O3-NEXT: xorl %edx, %edx -; O3-NEXT: .Lpcsection629: +; O3-NEXT: .Lpcsection659: ; O3-NEXT: xorl %ecx, %ecx -; O3-NEXT: .Lpcsection630: +; O3-NEXT: .Lpcsection660: ; O3-NEXT: lock cmpxchg16b (%rdi) ; O3-NEXT: movq $1, foo(%rip) ; O3-NEXT: popq %rbx @@ -16182,31 +16272,31 @@ ; O1-NEXT: .cfi_def_cfa_offset 16 ; O1-NEXT: .cfi_offset %rbx, -16 ; O1-NEXT: movq foo(%rip), %rax -; O1-NEXT: .Lpcsection631: +; O1-NEXT: .Lpcsection661: ; O1-NEXT: movl $42, %eax -; O1-NEXT: .Lpcsection632: +; O1-NEXT: .Lpcsection662: ; O1-NEXT: movl $1, %ebx -; O1-NEXT: .Lpcsection633: +; O1-NEXT: .Lpcsection663: ; O1-NEXT: xorl %edx, %edx -; O1-NEXT: .Lpcsection634: +; O1-NEXT: .Lpcsection664: ; O1-NEXT: xorl %ecx, %ecx -; O1-NEXT: .Lpcsection635: +; O1-NEXT: .Lpcsection665: ; O1-NEXT: lock cmpxchg16b (%rdi) -; O1-NEXT: .Lpcsection636: +; O1-NEXT: .Lpcsection666: ; O1-NEXT: movl $42, %eax -; O1-NEXT: .Lpcsection637: +; O1-NEXT: .Lpcsection667: ; O1-NEXT: xorl %edx, %edx -; O1-NEXT: .Lpcsection638: +; O1-NEXT: .Lpcsection668: ; O1-NEXT: xorl %ecx, %ecx -; O1-NEXT: .Lpcsection639: +; O1-NEXT: .Lpcsection669: ; O1-NEXT: lock cmpxchg16b (%rdi) -; O1-NEXT: .Lpcsection640: +; O1-NEXT: .Lpcsection670: ; O1-NEXT: movl $42, %eax -; O1-NEXT: .Lpcsection641: +; O1-NEXT: .Lpcsection671: ; O1-NEXT: xorl %edx, %edx -; O1-NEXT: .Lpcsection642: +; O1-NEXT: .Lpcsection672: ; O1-NEXT: xorl %ecx, %ecx -; O1-NEXT: .Lpcsection643: +; O1-NEXT: .Lpcsection673: ; O1-NEXT: lock cmpxchg16b (%rdi) ; O1-NEXT: movq $3, foo(%rip) ; O1-NEXT: popq %rbx @@ -16219,31 +16309,31 @@ ; O2-NEXT: .cfi_def_cfa_offset 16 ; O2-NEXT: .cfi_offset %rbx, -16 ; O2-NEXT: movq foo(%rip), %rax -; O2-NEXT: .Lpcsection631: +; O2-NEXT: .Lpcsection661: ; O2-NEXT: movl $42, %eax -; O2-NEXT: .Lpcsection632: +; O2-NEXT: .Lpcsection662: ; O2-NEXT: movl $1, %ebx -; O2-NEXT: .Lpcsection633: +; O2-NEXT: .Lpcsection663: ; O2-NEXT: xorl %edx, %edx -; O2-NEXT: .Lpcsection634: +; O2-NEXT: .Lpcsection664: ; O2-NEXT: xorl %ecx, %ecx -; O2-NEXT: .Lpcsection635: +; O2-NEXT: .Lpcsection665: ; O2-NEXT: lock cmpxchg16b (%rdi) -; O2-NEXT: .Lpcsection636: +; O2-NEXT: .Lpcsection666: ; O2-NEXT: movl $42, %eax -; O2-NEXT: .Lpcsection637: +; O2-NEXT: .Lpcsection667: ; O2-NEXT: xorl %edx, %edx -; O2-NEXT: .Lpcsection638: +; O2-NEXT: .Lpcsection668: ; O2-NEXT: xorl %ecx, %ecx -; O2-NEXT: .Lpcsection639: +; O2-NEXT: .Lpcsection669: ; O2-NEXT: lock cmpxchg16b (%rdi) -; O2-NEXT: .Lpcsection640: +; O2-NEXT: .Lpcsection670: ; O2-NEXT: movl $42, %eax -; O2-NEXT: .Lpcsection641: +; O2-NEXT: .Lpcsection671: ; O2-NEXT: xorl %edx, %edx -; O2-NEXT: .Lpcsection642: +; O2-NEXT: .Lpcsection672: ; O2-NEXT: xorl %ecx, %ecx -; O2-NEXT: .Lpcsection643: +; O2-NEXT: .Lpcsection673: ; O2-NEXT: lock cmpxchg16b (%rdi) ; O2-NEXT: movq $3, foo(%rip) ; O2-NEXT: popq %rbx @@ -16256,31 +16346,31 @@ ; O3-NEXT: .cfi_def_cfa_offset 16 ; O3-NEXT: .cfi_offset %rbx, -16 ; O3-NEXT: movq foo(%rip), %rax -; O3-NEXT: .Lpcsection631: +; O3-NEXT: .Lpcsection661: ; O3-NEXT: movl $42, %eax -; O3-NEXT: .Lpcsection632: +; O3-NEXT: .Lpcsection662: ; O3-NEXT: movl $1, %ebx -; O3-NEXT: .Lpcsection633: +; O3-NEXT: .Lpcsection663: ; O3-NEXT: xorl %edx, %edx -; O3-NEXT: .Lpcsection634: +; O3-NEXT: .Lpcsection664: ; O3-NEXT: xorl %ecx, %ecx -; O3-NEXT: .Lpcsection635: +; O3-NEXT: .Lpcsection665: ; O3-NEXT: lock cmpxchg16b (%rdi) -; O3-NEXT: .Lpcsection636: +; O3-NEXT: .Lpcsection666: ; O3-NEXT: movl $42, %eax -; O3-NEXT: .Lpcsection637: +; O3-NEXT: .Lpcsection667: ; O3-NEXT: xorl %edx, %edx -; O3-NEXT: .Lpcsection638: +; O3-NEXT: .Lpcsection668: ; O3-NEXT: xorl %ecx, %ecx -; O3-NEXT: .Lpcsection639: +; O3-NEXT: .Lpcsection669: ; O3-NEXT: lock cmpxchg16b (%rdi) -; O3-NEXT: .Lpcsection640: +; O3-NEXT: .Lpcsection670: ; O3-NEXT: movl $42, %eax -; O3-NEXT: .Lpcsection641: +; O3-NEXT: .Lpcsection671: ; O3-NEXT: xorl %edx, %edx -; O3-NEXT: .Lpcsection642: +; O3-NEXT: .Lpcsection672: ; O3-NEXT: xorl %ecx, %ecx -; O3-NEXT: .Lpcsection643: +; O3-NEXT: .Lpcsection673: ; O3-NEXT: lock cmpxchg16b (%rdi) ; O3-NEXT: movq $3, foo(%rip) ; O3-NEXT: popq %rbx Index: llvm/test/CodeGen/X86/physreg-pairs.ll =================================================================== --- llvm/test/CodeGen/X86/physreg-pairs.ll +++ llvm/test/CodeGen/X86/physreg-pairs.ll @@ -145,8 +145,8 @@ ; CHECK-LABEL: test_ebp: ; CHECK: # %bb.0: # %entry ; CHECK-NEXT: pushl %ebp -; CHECK-NEXT: movl $19088743, %esp # imm = 0x1234567 ; CHECK-NEXT: movl $-1985229329, %ebp # imm = 0x89ABCDEF +; CHECK-NEXT: movl $19088743, %esp # imm = 0x1234567 ; CHECK-NEXT: #APP ; CHECK-NEXT: movl %ebp, %eax ; CHECK-NEXT: #NO_APP Index: llvm/test/CodeGen/X86/popcnt.ll =================================================================== --- llvm/test/CodeGen/X86/popcnt.ll +++ llvm/test/CodeGen/X86/popcnt.ll @@ -615,12 +615,11 @@ ; X86-NEXT: shrl %ecx ; X86-NEXT: andl $1431655765, %ecx # imm = 0x55555555 ; X86-NEXT: subl %ecx, %eax -; X86-NEXT: movl $858993459, %ecx # imm = 0x33333333 -; X86-NEXT: movl %eax, %edx -; X86-NEXT: andl %ecx, %edx +; X86-NEXT: movl %eax, %ecx +; X86-NEXT: andl $858993459, %ecx # imm = 0x33333333 ; X86-NEXT: shrl $2, %eax -; X86-NEXT: andl %ecx, %eax -; X86-NEXT: addl %edx, %eax +; X86-NEXT: andl $858993459, %eax # imm = 0x33333333 +; X86-NEXT: addl %ecx, %eax ; X86-NEXT: movl %eax, %ecx ; X86-NEXT: shrl $4, %ecx ; X86-NEXT: addl %eax, %ecx @@ -635,12 +634,11 @@ ; X64-NEXT: shrl %eax ; X64-NEXT: andl $1431655765, %eax # imm = 0x55555555 ; X64-NEXT: subl %eax, %edi -; X64-NEXT: movl $858993459, %eax # imm = 0x33333333 -; X64-NEXT: movl %edi, %ecx -; X64-NEXT: andl %eax, %ecx +; X64-NEXT: movl %edi, %eax +; X64-NEXT: andl $858993459, %eax # imm = 0x33333333 ; X64-NEXT: shrl $2, %edi -; X64-NEXT: andl %eax, %edi -; X64-NEXT: addl %ecx, %edi +; X64-NEXT: andl $858993459, %edi # imm = 0x33333333 +; X64-NEXT: addl %eax, %edi ; X64-NEXT: movl %edi, %eax ; X64-NEXT: shrl $4, %eax ; X64-NEXT: addl %edi, %eax @@ -665,49 +663,40 @@ define i64 @cnt64_optsize(i64 %x) nounwind readnone optsize { ; X86-NOSSE-LABEL: cnt64_optsize: ; X86-NOSSE: # %bb.0: -; X86-NOSSE-NEXT: pushl %ebx -; X86-NOSSE-NEXT: pushl %edi -; X86-NOSSE-NEXT: pushl %esi ; X86-NOSSE-NEXT: movl {{[0-9]+}}(%esp), %eax -; X86-NOSSE-NEXT: movl {{[0-9]+}}(%esp), %esi -; X86-NOSSE-NEXT: movl %esi, %ecx -; X86-NOSSE-NEXT: shrl %ecx -; X86-NOSSE-NEXT: movl $1431655765, %edx # imm = 0x55555555 -; X86-NOSSE-NEXT: andl %edx, %ecx -; X86-NOSSE-NEXT: subl %ecx, %esi -; X86-NOSSE-NEXT: movl $858993459, %ecx # imm = 0x33333333 -; X86-NOSSE-NEXT: movl %esi, %edi -; X86-NOSSE-NEXT: andl %ecx, %edi -; X86-NOSSE-NEXT: shrl $2, %esi -; X86-NOSSE-NEXT: andl %ecx, %esi -; X86-NOSSE-NEXT: addl %edi, %esi -; X86-NOSSE-NEXT: movl %esi, %ebx -; X86-NOSSE-NEXT: shrl $4, %ebx -; X86-NOSSE-NEXT: addl %esi, %ebx -; X86-NOSSE-NEXT: movl $252645135, %edi # imm = 0xF0F0F0F -; X86-NOSSE-NEXT: andl %edi, %ebx -; X86-NOSSE-NEXT: imull $16843009, %ebx, %esi # imm = 0x1010101 -; X86-NOSSE-NEXT: shrl $24, %esi -; X86-NOSSE-NEXT: movl %eax, %ebx -; X86-NOSSE-NEXT: shrl %ebx -; X86-NOSSE-NEXT: andl %edx, %ebx -; X86-NOSSE-NEXT: subl %ebx, %eax +; X86-NOSSE-NEXT: movl {{[0-9]+}}(%esp), %ecx +; X86-NOSSE-NEXT: movl %ecx, %edx +; X86-NOSSE-NEXT: shrl %edx +; X86-NOSSE-NEXT: andl $1431655765, %edx # imm = 0x55555555 +; X86-NOSSE-NEXT: subl %edx, %ecx +; X86-NOSSE-NEXT: movl %ecx, %edx +; X86-NOSSE-NEXT: andl $858993459, %edx # imm = 0x33333333 +; X86-NOSSE-NEXT: shrl $2, %ecx +; X86-NOSSE-NEXT: andl $858993459, %ecx # imm = 0x33333333 +; X86-NOSSE-NEXT: addl %edx, %ecx +; X86-NOSSE-NEXT: movl %ecx, %edx +; X86-NOSSE-NEXT: shrl $4, %edx +; X86-NOSSE-NEXT: addl %ecx, %edx +; X86-NOSSE-NEXT: andl $252645135, %edx # imm = 0xF0F0F0F +; X86-NOSSE-NEXT: imull $16843009, %edx, %ecx # imm = 0x1010101 +; X86-NOSSE-NEXT: shrl $24, %ecx +; X86-NOSSE-NEXT: movl %eax, %edx +; X86-NOSSE-NEXT: shrl %edx +; X86-NOSSE-NEXT: andl $1431655765, %edx # imm = 0x55555555 +; X86-NOSSE-NEXT: subl %edx, %eax ; X86-NOSSE-NEXT: movl %eax, %edx -; X86-NOSSE-NEXT: andl %ecx, %edx +; X86-NOSSE-NEXT: andl $858993459, %edx # imm = 0x33333333 ; X86-NOSSE-NEXT: shrl $2, %eax -; X86-NOSSE-NEXT: andl %ecx, %eax +; X86-NOSSE-NEXT: andl $858993459, %eax # imm = 0x33333333 ; X86-NOSSE-NEXT: addl %edx, %eax -; X86-NOSSE-NEXT: movl %eax, %ecx -; X86-NOSSE-NEXT: shrl $4, %ecx -; X86-NOSSE-NEXT: addl %eax, %ecx -; X86-NOSSE-NEXT: andl %edi, %ecx -; X86-NOSSE-NEXT: imull $16843009, %ecx, %eax # imm = 0x1010101 +; X86-NOSSE-NEXT: movl %eax, %edx +; X86-NOSSE-NEXT: shrl $4, %edx +; X86-NOSSE-NEXT: addl %eax, %edx +; X86-NOSSE-NEXT: andl $252645135, %edx # imm = 0xF0F0F0F +; X86-NOSSE-NEXT: imull $16843009, %edx, %eax # imm = 0x1010101 ; X86-NOSSE-NEXT: shrl $24, %eax -; X86-NOSSE-NEXT: addl %esi, %eax +; X86-NOSSE-NEXT: addl %ecx, %eax ; X86-NOSSE-NEXT: xorl %edx, %edx -; X86-NOSSE-NEXT: popl %esi -; X86-NOSSE-NEXT: popl %edi -; X86-NOSSE-NEXT: popl %ebx ; X86-NOSSE-NEXT: retl ; ; X64-LABEL: cnt64_optsize: @@ -794,92 +783,85 @@ define i128 @cnt128_optsize(i128 %x) nounwind readnone optsize { ; X86-NOSSE-LABEL: cnt128_optsize: ; X86-NOSSE: # %bb.0: -; X86-NOSSE-NEXT: pushl %ebp ; X86-NOSSE-NEXT: pushl %ebx ; X86-NOSSE-NEXT: pushl %edi ; X86-NOSSE-NEXT: pushl %esi +; X86-NOSSE-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NOSSE-NEXT: movl {{[0-9]+}}(%esp), %ecx ; X86-NOSSE-NEXT: movl {{[0-9]+}}(%esp), %edx ; X86-NOSSE-NEXT: movl {{[0-9]+}}(%esp), %esi -; X86-NOSSE-NEXT: movl {{[0-9]+}}(%esp), %eax -; X86-NOSSE-NEXT: movl {{[0-9]+}}(%esp), %ebx -; X86-NOSSE-NEXT: movl %ebx, %ecx -; X86-NOSSE-NEXT: shrl %ecx -; X86-NOSSE-NEXT: movl $1431655765, %edi # imm = 0x55555555 -; X86-NOSSE-NEXT: andl %edi, %ecx -; X86-NOSSE-NEXT: subl %ecx, %ebx -; X86-NOSSE-NEXT: movl $858993459, %ecx # imm = 0x33333333 -; X86-NOSSE-NEXT: movl %ebx, %ebp -; X86-NOSSE-NEXT: andl %ecx, %ebp -; X86-NOSSE-NEXT: shrl $2, %ebx -; X86-NOSSE-NEXT: andl %ecx, %ebx -; X86-NOSSE-NEXT: addl %ebp, %ebx -; X86-NOSSE-NEXT: movl %ebx, %ebp -; X86-NOSSE-NEXT: shrl $4, %ebp -; X86-NOSSE-NEXT: addl %ebx, %ebp -; X86-NOSSE-NEXT: movl %eax, %ebx +; X86-NOSSE-NEXT: movl {{[0-9]+}}(%esp), %edi +; X86-NOSSE-NEXT: movl %edi, %ebx ; X86-NOSSE-NEXT: shrl %ebx -; X86-NOSSE-NEXT: andl %edi, %ebx -; X86-NOSSE-NEXT: subl %ebx, %eax -; X86-NOSSE-NEXT: movl %eax, %ebx -; X86-NOSSE-NEXT: andl %ecx, %ebx -; X86-NOSSE-NEXT: shrl $2, %eax -; X86-NOSSE-NEXT: andl %ecx, %eax -; X86-NOSSE-NEXT: addl %ebx, %eax -; X86-NOSSE-NEXT: movl %eax, %edi -; X86-NOSSE-NEXT: shrl $4, %edi -; X86-NOSSE-NEXT: addl %eax, %edi -; X86-NOSSE-NEXT: movl $252645135, %ebx # imm = 0xF0F0F0F -; X86-NOSSE-NEXT: andl %ebx, %ebp -; X86-NOSSE-NEXT: imull $16843009, %ebp, %eax # imm = 0x1010101 -; X86-NOSSE-NEXT: shrl $24, %eax -; X86-NOSSE-NEXT: andl %ebx, %edi -; X86-NOSSE-NEXT: imull $16843009, %edi, %edi # imm = 0x1010101 +; X86-NOSSE-NEXT: andl $1431655765, %ebx # imm = 0x55555555 +; X86-NOSSE-NEXT: subl %ebx, %edi +; X86-NOSSE-NEXT: movl %edi, %ebx +; X86-NOSSE-NEXT: andl $858993459, %ebx # imm = 0x33333333 +; X86-NOSSE-NEXT: shrl $2, %edi +; X86-NOSSE-NEXT: andl $858993459, %edi # imm = 0x33333333 +; X86-NOSSE-NEXT: addl %ebx, %edi +; X86-NOSSE-NEXT: movl %edi, %ebx +; X86-NOSSE-NEXT: shrl $4, %ebx +; X86-NOSSE-NEXT: addl %edi, %ebx +; X86-NOSSE-NEXT: andl $252645135, %ebx # imm = 0xF0F0F0F +; X86-NOSSE-NEXT: imull $16843009, %ebx, %edi # imm = 0x1010101 ; X86-NOSSE-NEXT: shrl $24, %edi -; X86-NOSSE-NEXT: addl %eax, %edi -; X86-NOSSE-NEXT: movl %esi, %eax -; X86-NOSSE-NEXT: shrl %eax -; X86-NOSSE-NEXT: movl $1431655765, %ebp # imm = 0x55555555 -; X86-NOSSE-NEXT: andl %ebp, %eax -; X86-NOSSE-NEXT: subl %eax, %esi -; X86-NOSSE-NEXT: movl %esi, %eax -; X86-NOSSE-NEXT: andl %ecx, %eax +; X86-NOSSE-NEXT: movl %esi, %ebx +; X86-NOSSE-NEXT: shrl %ebx +; X86-NOSSE-NEXT: andl $1431655765, %ebx # imm = 0x55555555 +; X86-NOSSE-NEXT: subl %ebx, %esi +; X86-NOSSE-NEXT: movl %esi, %ebx +; X86-NOSSE-NEXT: andl $858993459, %ebx # imm = 0x33333333 ; X86-NOSSE-NEXT: shrl $2, %esi -; X86-NOSSE-NEXT: andl %ecx, %esi -; X86-NOSSE-NEXT: addl %eax, %esi -; X86-NOSSE-NEXT: movl %esi, %ebp -; X86-NOSSE-NEXT: shrl $4, %ebp -; X86-NOSSE-NEXT: addl %esi, %ebp -; X86-NOSSE-NEXT: movl %edx, %eax -; X86-NOSSE-NEXT: shrl %eax -; X86-NOSSE-NEXT: movl $1431655765, %esi # imm = 0x55555555 -; X86-NOSSE-NEXT: andl %esi, %eax -; X86-NOSSE-NEXT: subl %eax, %edx -; X86-NOSSE-NEXT: movl %edx, %eax -; X86-NOSSE-NEXT: andl %ecx, %eax +; X86-NOSSE-NEXT: andl $858993459, %esi # imm = 0x33333333 +; X86-NOSSE-NEXT: addl %ebx, %esi +; X86-NOSSE-NEXT: movl %esi, %ebx +; X86-NOSSE-NEXT: shrl $4, %ebx +; X86-NOSSE-NEXT: addl %esi, %ebx +; X86-NOSSE-NEXT: andl $252645135, %ebx # imm = 0xF0F0F0F +; X86-NOSSE-NEXT: imull $16843009, %ebx, %esi # imm = 0x1010101 +; X86-NOSSE-NEXT: shrl $24, %esi +; X86-NOSSE-NEXT: addl %edi, %esi +; X86-NOSSE-NEXT: movl %edx, %edi +; X86-NOSSE-NEXT: shrl %edi +; X86-NOSSE-NEXT: andl $1431655765, %edi # imm = 0x55555555 +; X86-NOSSE-NEXT: subl %edi, %edx +; X86-NOSSE-NEXT: movl %edx, %edi +; X86-NOSSE-NEXT: andl $858993459, %edi # imm = 0x33333333 ; X86-NOSSE-NEXT: shrl $2, %edx -; X86-NOSSE-NEXT: andl %ecx, %edx -; X86-NOSSE-NEXT: addl %eax, %edx -; X86-NOSSE-NEXT: movl %edx, %eax -; X86-NOSSE-NEXT: shrl $4, %eax -; X86-NOSSE-NEXT: addl %edx, %eax -; X86-NOSSE-NEXT: andl %ebx, %ebp -; X86-NOSSE-NEXT: andl %ebx, %eax -; X86-NOSSE-NEXT: imull $16843009, %ebp, %ecx # imm = 0x1010101 -; X86-NOSSE-NEXT: shrl $24, %ecx -; X86-NOSSE-NEXT: imull $16843009, %eax, %edx # imm = 0x1010101 -; X86-NOSSE-NEXT: shrl $24, %edx -; X86-NOSSE-NEXT: addl %ecx, %edx -; X86-NOSSE-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NOSSE-NEXT: andl $858993459, %edx # imm = 0x33333333 ; X86-NOSSE-NEXT: addl %edi, %edx -; X86-NOSSE-NEXT: xorl %ecx, %ecx -; X86-NOSSE-NEXT: movl %ecx, 12(%eax) -; X86-NOSSE-NEXT: movl %ecx, 8(%eax) -; X86-NOSSE-NEXT: movl %ecx, 4(%eax) -; X86-NOSSE-NEXT: movl %edx, (%eax) +; X86-NOSSE-NEXT: movl %edx, %edi +; X86-NOSSE-NEXT: shrl $4, %edi +; X86-NOSSE-NEXT: addl %edx, %edi +; X86-NOSSE-NEXT: andl $252645135, %edi # imm = 0xF0F0F0F +; X86-NOSSE-NEXT: imull $16843009, %edi, %edx # imm = 0x1010101 +; X86-NOSSE-NEXT: shrl $24, %edx +; X86-NOSSE-NEXT: movl %ecx, %edi +; X86-NOSSE-NEXT: shrl %edi +; X86-NOSSE-NEXT: andl $1431655765, %edi # imm = 0x55555555 +; X86-NOSSE-NEXT: subl %edi, %ecx +; X86-NOSSE-NEXT: movl %ecx, %edi +; X86-NOSSE-NEXT: andl $858993459, %edi # imm = 0x33333333 +; X86-NOSSE-NEXT: shrl $2, %ecx +; X86-NOSSE-NEXT: andl $858993459, %ecx # imm = 0x33333333 +; X86-NOSSE-NEXT: addl %edi, %ecx +; X86-NOSSE-NEXT: movl %ecx, %edi +; X86-NOSSE-NEXT: shrl $4, %edi +; X86-NOSSE-NEXT: addl %ecx, %edi +; X86-NOSSE-NEXT: andl $252645135, %edi # imm = 0xF0F0F0F +; X86-NOSSE-NEXT: imull $16843009, %edi, %ecx # imm = 0x1010101 +; X86-NOSSE-NEXT: shrl $24, %ecx +; X86-NOSSE-NEXT: addl %edx, %ecx +; X86-NOSSE-NEXT: addl %esi, %ecx +; X86-NOSSE-NEXT: xorl %edx, %edx +; X86-NOSSE-NEXT: movl %edx, 12(%eax) +; X86-NOSSE-NEXT: movl %edx, 8(%eax) +; X86-NOSSE-NEXT: movl %edx, 4(%eax) +; X86-NOSSE-NEXT: movl %ecx, (%eax) ; X86-NOSSE-NEXT: popl %esi ; X86-NOSSE-NEXT: popl %edi ; X86-NOSSE-NEXT: popl %ebx -; X86-NOSSE-NEXT: popl %ebp ; X86-NOSSE-NEXT: retl $4 ; ; X64-LABEL: cnt128_optsize: @@ -1044,12 +1026,11 @@ ; X86-NEXT: shrl %ecx ; X86-NEXT: andl $1431655765, %ecx # imm = 0x55555555 ; X86-NEXT: subl %ecx, %eax -; X86-NEXT: movl $858993459, %ecx # imm = 0x33333333 -; X86-NEXT: movl %eax, %edx -; X86-NEXT: andl %ecx, %edx +; X86-NEXT: movl %eax, %ecx +; X86-NEXT: andl $858993459, %ecx # imm = 0x33333333 ; X86-NEXT: shrl $2, %eax -; X86-NEXT: andl %ecx, %eax -; X86-NEXT: addl %edx, %eax +; X86-NEXT: andl $858993459, %eax # imm = 0x33333333 +; X86-NEXT: addl %ecx, %eax ; X86-NEXT: movl %eax, %ecx ; X86-NEXT: shrl $4, %ecx ; X86-NEXT: addl %eax, %ecx @@ -1064,12 +1045,11 @@ ; X64-NEXT: shrl %eax ; X64-NEXT: andl $1431655765, %eax # imm = 0x55555555 ; X64-NEXT: subl %eax, %edi -; X64-NEXT: movl $858993459, %eax # imm = 0x33333333 -; X64-NEXT: movl %edi, %ecx -; X64-NEXT: andl %eax, %ecx +; X64-NEXT: movl %edi, %eax +; X64-NEXT: andl $858993459, %eax # imm = 0x33333333 ; X64-NEXT: shrl $2, %edi -; X64-NEXT: andl %eax, %edi -; X64-NEXT: addl %ecx, %edi +; X64-NEXT: andl $858993459, %edi # imm = 0x33333333 +; X64-NEXT: addl %eax, %edi ; X64-NEXT: movl %edi, %eax ; X64-NEXT: shrl $4, %eax ; X64-NEXT: addl %edi, %eax @@ -1094,49 +1074,40 @@ define i64 @cnt64_pgso(i64 %x) nounwind readnone !prof !14 { ; X86-NOSSE-LABEL: cnt64_pgso: ; X86-NOSSE: # %bb.0: -; X86-NOSSE-NEXT: pushl %ebx -; X86-NOSSE-NEXT: pushl %edi -; X86-NOSSE-NEXT: pushl %esi ; X86-NOSSE-NEXT: movl {{[0-9]+}}(%esp), %eax -; X86-NOSSE-NEXT: movl {{[0-9]+}}(%esp), %esi -; X86-NOSSE-NEXT: movl %esi, %ecx -; X86-NOSSE-NEXT: shrl %ecx -; X86-NOSSE-NEXT: movl $1431655765, %edx # imm = 0x55555555 -; X86-NOSSE-NEXT: andl %edx, %ecx -; X86-NOSSE-NEXT: subl %ecx, %esi -; X86-NOSSE-NEXT: movl $858993459, %ecx # imm = 0x33333333 -; X86-NOSSE-NEXT: movl %esi, %edi -; X86-NOSSE-NEXT: andl %ecx, %edi -; X86-NOSSE-NEXT: shrl $2, %esi -; X86-NOSSE-NEXT: andl %ecx, %esi -; X86-NOSSE-NEXT: addl %edi, %esi -; X86-NOSSE-NEXT: movl %esi, %ebx -; X86-NOSSE-NEXT: shrl $4, %ebx -; X86-NOSSE-NEXT: addl %esi, %ebx -; X86-NOSSE-NEXT: movl $252645135, %edi # imm = 0xF0F0F0F -; X86-NOSSE-NEXT: andl %edi, %ebx -; X86-NOSSE-NEXT: imull $16843009, %ebx, %esi # imm = 0x1010101 -; X86-NOSSE-NEXT: shrl $24, %esi -; X86-NOSSE-NEXT: movl %eax, %ebx -; X86-NOSSE-NEXT: shrl %ebx -; X86-NOSSE-NEXT: andl %edx, %ebx -; X86-NOSSE-NEXT: subl %ebx, %eax +; X86-NOSSE-NEXT: movl {{[0-9]+}}(%esp), %ecx +; X86-NOSSE-NEXT: movl %ecx, %edx +; X86-NOSSE-NEXT: shrl %edx +; X86-NOSSE-NEXT: andl $1431655765, %edx # imm = 0x55555555 +; X86-NOSSE-NEXT: subl %edx, %ecx +; X86-NOSSE-NEXT: movl %ecx, %edx +; X86-NOSSE-NEXT: andl $858993459, %edx # imm = 0x33333333 +; X86-NOSSE-NEXT: shrl $2, %ecx +; X86-NOSSE-NEXT: andl $858993459, %ecx # imm = 0x33333333 +; X86-NOSSE-NEXT: addl %edx, %ecx +; X86-NOSSE-NEXT: movl %ecx, %edx +; X86-NOSSE-NEXT: shrl $4, %edx +; X86-NOSSE-NEXT: addl %ecx, %edx +; X86-NOSSE-NEXT: andl $252645135, %edx # imm = 0xF0F0F0F +; X86-NOSSE-NEXT: imull $16843009, %edx, %ecx # imm = 0x1010101 +; X86-NOSSE-NEXT: shrl $24, %ecx +; X86-NOSSE-NEXT: movl %eax, %edx +; X86-NOSSE-NEXT: shrl %edx +; X86-NOSSE-NEXT: andl $1431655765, %edx # imm = 0x55555555 +; X86-NOSSE-NEXT: subl %edx, %eax ; X86-NOSSE-NEXT: movl %eax, %edx -; X86-NOSSE-NEXT: andl %ecx, %edx +; X86-NOSSE-NEXT: andl $858993459, %edx # imm = 0x33333333 ; X86-NOSSE-NEXT: shrl $2, %eax -; X86-NOSSE-NEXT: andl %ecx, %eax +; X86-NOSSE-NEXT: andl $858993459, %eax # imm = 0x33333333 ; X86-NOSSE-NEXT: addl %edx, %eax -; X86-NOSSE-NEXT: movl %eax, %ecx -; X86-NOSSE-NEXT: shrl $4, %ecx -; X86-NOSSE-NEXT: addl %eax, %ecx -; X86-NOSSE-NEXT: andl %edi, %ecx -; X86-NOSSE-NEXT: imull $16843009, %ecx, %eax # imm = 0x1010101 +; X86-NOSSE-NEXT: movl %eax, %edx +; X86-NOSSE-NEXT: shrl $4, %edx +; X86-NOSSE-NEXT: addl %eax, %edx +; X86-NOSSE-NEXT: andl $252645135, %edx # imm = 0xF0F0F0F +; X86-NOSSE-NEXT: imull $16843009, %edx, %eax # imm = 0x1010101 ; X86-NOSSE-NEXT: shrl $24, %eax -; X86-NOSSE-NEXT: addl %esi, %eax +; X86-NOSSE-NEXT: addl %ecx, %eax ; X86-NOSSE-NEXT: xorl %edx, %edx -; X86-NOSSE-NEXT: popl %esi -; X86-NOSSE-NEXT: popl %edi -; X86-NOSSE-NEXT: popl %ebx ; X86-NOSSE-NEXT: retl ; ; X64-LABEL: cnt64_pgso: @@ -1223,92 +1194,85 @@ define i128 @cnt128_pgso(i128 %x) nounwind readnone !prof !14 { ; X86-NOSSE-LABEL: cnt128_pgso: ; X86-NOSSE: # %bb.0: -; X86-NOSSE-NEXT: pushl %ebp ; X86-NOSSE-NEXT: pushl %ebx ; X86-NOSSE-NEXT: pushl %edi ; X86-NOSSE-NEXT: pushl %esi +; X86-NOSSE-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NOSSE-NEXT: movl {{[0-9]+}}(%esp), %ecx ; X86-NOSSE-NEXT: movl {{[0-9]+}}(%esp), %edx ; X86-NOSSE-NEXT: movl {{[0-9]+}}(%esp), %esi -; X86-NOSSE-NEXT: movl {{[0-9]+}}(%esp), %eax -; X86-NOSSE-NEXT: movl {{[0-9]+}}(%esp), %ebx -; X86-NOSSE-NEXT: movl %ebx, %ecx -; X86-NOSSE-NEXT: shrl %ecx -; X86-NOSSE-NEXT: movl $1431655765, %edi # imm = 0x55555555 -; X86-NOSSE-NEXT: andl %edi, %ecx -; X86-NOSSE-NEXT: subl %ecx, %ebx -; X86-NOSSE-NEXT: movl $858993459, %ecx # imm = 0x33333333 -; X86-NOSSE-NEXT: movl %ebx, %ebp -; X86-NOSSE-NEXT: andl %ecx, %ebp -; X86-NOSSE-NEXT: shrl $2, %ebx -; X86-NOSSE-NEXT: andl %ecx, %ebx -; X86-NOSSE-NEXT: addl %ebp, %ebx -; X86-NOSSE-NEXT: movl %ebx, %ebp -; X86-NOSSE-NEXT: shrl $4, %ebp -; X86-NOSSE-NEXT: addl %ebx, %ebp -; X86-NOSSE-NEXT: movl %eax, %ebx +; X86-NOSSE-NEXT: movl {{[0-9]+}}(%esp), %edi +; X86-NOSSE-NEXT: movl %edi, %ebx ; X86-NOSSE-NEXT: shrl %ebx -; X86-NOSSE-NEXT: andl %edi, %ebx -; X86-NOSSE-NEXT: subl %ebx, %eax -; X86-NOSSE-NEXT: movl %eax, %ebx -; X86-NOSSE-NEXT: andl %ecx, %ebx -; X86-NOSSE-NEXT: shrl $2, %eax -; X86-NOSSE-NEXT: andl %ecx, %eax -; X86-NOSSE-NEXT: addl %ebx, %eax -; X86-NOSSE-NEXT: movl %eax, %edi -; X86-NOSSE-NEXT: shrl $4, %edi -; X86-NOSSE-NEXT: addl %eax, %edi -; X86-NOSSE-NEXT: movl $252645135, %ebx # imm = 0xF0F0F0F -; X86-NOSSE-NEXT: andl %ebx, %ebp -; X86-NOSSE-NEXT: imull $16843009, %ebp, %eax # imm = 0x1010101 -; X86-NOSSE-NEXT: shrl $24, %eax -; X86-NOSSE-NEXT: andl %ebx, %edi -; X86-NOSSE-NEXT: imull $16843009, %edi, %edi # imm = 0x1010101 +; X86-NOSSE-NEXT: andl $1431655765, %ebx # imm = 0x55555555 +; X86-NOSSE-NEXT: subl %ebx, %edi +; X86-NOSSE-NEXT: movl %edi, %ebx +; X86-NOSSE-NEXT: andl $858993459, %ebx # imm = 0x33333333 +; X86-NOSSE-NEXT: shrl $2, %edi +; X86-NOSSE-NEXT: andl $858993459, %edi # imm = 0x33333333 +; X86-NOSSE-NEXT: addl %ebx, %edi +; X86-NOSSE-NEXT: movl %edi, %ebx +; X86-NOSSE-NEXT: shrl $4, %ebx +; X86-NOSSE-NEXT: addl %edi, %ebx +; X86-NOSSE-NEXT: andl $252645135, %ebx # imm = 0xF0F0F0F +; X86-NOSSE-NEXT: imull $16843009, %ebx, %edi # imm = 0x1010101 ; X86-NOSSE-NEXT: shrl $24, %edi -; X86-NOSSE-NEXT: addl %eax, %edi -; X86-NOSSE-NEXT: movl %esi, %eax -; X86-NOSSE-NEXT: shrl %eax -; X86-NOSSE-NEXT: movl $1431655765, %ebp # imm = 0x55555555 -; X86-NOSSE-NEXT: andl %ebp, %eax -; X86-NOSSE-NEXT: subl %eax, %esi -; X86-NOSSE-NEXT: movl %esi, %eax -; X86-NOSSE-NEXT: andl %ecx, %eax +; X86-NOSSE-NEXT: movl %esi, %ebx +; X86-NOSSE-NEXT: shrl %ebx +; X86-NOSSE-NEXT: andl $1431655765, %ebx # imm = 0x55555555 +; X86-NOSSE-NEXT: subl %ebx, %esi +; X86-NOSSE-NEXT: movl %esi, %ebx +; X86-NOSSE-NEXT: andl $858993459, %ebx # imm = 0x33333333 ; X86-NOSSE-NEXT: shrl $2, %esi -; X86-NOSSE-NEXT: andl %ecx, %esi -; X86-NOSSE-NEXT: addl %eax, %esi -; X86-NOSSE-NEXT: movl %esi, %ebp -; X86-NOSSE-NEXT: shrl $4, %ebp -; X86-NOSSE-NEXT: addl %esi, %ebp -; X86-NOSSE-NEXT: movl %edx, %eax -; X86-NOSSE-NEXT: shrl %eax -; X86-NOSSE-NEXT: movl $1431655765, %esi # imm = 0x55555555 -; X86-NOSSE-NEXT: andl %esi, %eax -; X86-NOSSE-NEXT: subl %eax, %edx -; X86-NOSSE-NEXT: movl %edx, %eax -; X86-NOSSE-NEXT: andl %ecx, %eax +; X86-NOSSE-NEXT: andl $858993459, %esi # imm = 0x33333333 +; X86-NOSSE-NEXT: addl %ebx, %esi +; X86-NOSSE-NEXT: movl %esi, %ebx +; X86-NOSSE-NEXT: shrl $4, %ebx +; X86-NOSSE-NEXT: addl %esi, %ebx +; X86-NOSSE-NEXT: andl $252645135, %ebx # imm = 0xF0F0F0F +; X86-NOSSE-NEXT: imull $16843009, %ebx, %esi # imm = 0x1010101 +; X86-NOSSE-NEXT: shrl $24, %esi +; X86-NOSSE-NEXT: addl %edi, %esi +; X86-NOSSE-NEXT: movl %edx, %edi +; X86-NOSSE-NEXT: shrl %edi +; X86-NOSSE-NEXT: andl $1431655765, %edi # imm = 0x55555555 +; X86-NOSSE-NEXT: subl %edi, %edx +; X86-NOSSE-NEXT: movl %edx, %edi +; X86-NOSSE-NEXT: andl $858993459, %edi # imm = 0x33333333 ; X86-NOSSE-NEXT: shrl $2, %edx -; X86-NOSSE-NEXT: andl %ecx, %edx -; X86-NOSSE-NEXT: addl %eax, %edx -; X86-NOSSE-NEXT: movl %edx, %eax -; X86-NOSSE-NEXT: shrl $4, %eax -; X86-NOSSE-NEXT: addl %edx, %eax -; X86-NOSSE-NEXT: andl %ebx, %ebp -; X86-NOSSE-NEXT: andl %ebx, %eax -; X86-NOSSE-NEXT: imull $16843009, %ebp, %ecx # imm = 0x1010101 -; X86-NOSSE-NEXT: shrl $24, %ecx -; X86-NOSSE-NEXT: imull $16843009, %eax, %edx # imm = 0x1010101 -; X86-NOSSE-NEXT: shrl $24, %edx -; X86-NOSSE-NEXT: addl %ecx, %edx -; X86-NOSSE-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NOSSE-NEXT: andl $858993459, %edx # imm = 0x33333333 ; X86-NOSSE-NEXT: addl %edi, %edx -; X86-NOSSE-NEXT: xorl %ecx, %ecx -; X86-NOSSE-NEXT: movl %ecx, 12(%eax) -; X86-NOSSE-NEXT: movl %ecx, 8(%eax) -; X86-NOSSE-NEXT: movl %ecx, 4(%eax) -; X86-NOSSE-NEXT: movl %edx, (%eax) +; X86-NOSSE-NEXT: movl %edx, %edi +; X86-NOSSE-NEXT: shrl $4, %edi +; X86-NOSSE-NEXT: addl %edx, %edi +; X86-NOSSE-NEXT: andl $252645135, %edi # imm = 0xF0F0F0F +; X86-NOSSE-NEXT: imull $16843009, %edi, %edx # imm = 0x1010101 +; X86-NOSSE-NEXT: shrl $24, %edx +; X86-NOSSE-NEXT: movl %ecx, %edi +; X86-NOSSE-NEXT: shrl %edi +; X86-NOSSE-NEXT: andl $1431655765, %edi # imm = 0x55555555 +; X86-NOSSE-NEXT: subl %edi, %ecx +; X86-NOSSE-NEXT: movl %ecx, %edi +; X86-NOSSE-NEXT: andl $858993459, %edi # imm = 0x33333333 +; X86-NOSSE-NEXT: shrl $2, %ecx +; X86-NOSSE-NEXT: andl $858993459, %ecx # imm = 0x33333333 +; X86-NOSSE-NEXT: addl %edi, %ecx +; X86-NOSSE-NEXT: movl %ecx, %edi +; X86-NOSSE-NEXT: shrl $4, %edi +; X86-NOSSE-NEXT: addl %ecx, %edi +; X86-NOSSE-NEXT: andl $252645135, %edi # imm = 0xF0F0F0F +; X86-NOSSE-NEXT: imull $16843009, %edi, %ecx # imm = 0x1010101 +; X86-NOSSE-NEXT: shrl $24, %ecx +; X86-NOSSE-NEXT: addl %edx, %ecx +; X86-NOSSE-NEXT: addl %esi, %ecx +; X86-NOSSE-NEXT: xorl %edx, %edx +; X86-NOSSE-NEXT: movl %edx, 12(%eax) +; X86-NOSSE-NEXT: movl %edx, 8(%eax) +; X86-NOSSE-NEXT: movl %edx, 4(%eax) +; X86-NOSSE-NEXT: movl %ecx, (%eax) ; X86-NOSSE-NEXT: popl %esi ; X86-NOSSE-NEXT: popl %edi ; X86-NOSSE-NEXT: popl %ebx -; X86-NOSSE-NEXT: popl %ebp ; X86-NOSSE-NEXT: retl $4 ; ; X64-LABEL: cnt128_pgso: Index: llvm/test/CodeGen/X86/ragreedy-hoist-spill.ll =================================================================== --- llvm/test/CodeGen/X86/ragreedy-hoist-spill.ll +++ llvm/test/CodeGen/X86/ragreedy-hoist-spill.ll @@ -46,11 +46,11 @@ ; CHECK-NEXT: ## %bb.2: ## %if.then4 ; CHECK-NEXT: xorl %eax, %eax ; CHECK-NEXT: testb %al, %al -; CHECK-NEXT: je LBB0_54 +; CHECK-NEXT: je LBB0_55 ; CHECK-NEXT: ## %bb.3: ## %SyTime.exit ; CHECK-NEXT: xorl %eax, %eax ; CHECK-NEXT: testb %al, %al -; CHECK-NEXT: je LBB0_54 +; CHECK-NEXT: je LBB0_55 ; CHECK-NEXT: LBB0_4: ## %cleanup ; CHECK-NEXT: addq $552, %rsp ## imm = 0x228 ; CHECK-NEXT: popq %rbx @@ -63,10 +63,10 @@ ; CHECK-NEXT: LBB0_5: ## %if.end25 ; CHECK-NEXT: xorl %eax, %eax ; CHECK-NEXT: testb %al, %al -; CHECK-NEXT: je LBB0_54 +; CHECK-NEXT: je LBB0_55 ; CHECK-NEXT: ## %bb.6: ## %SyTime.exit2720 ; CHECK-NEXT: movq %rdx, %r14 -; CHECK-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) ## 8-byte Spill +; CHECK-NEXT: movq %rdi, %rbx ; CHECK-NEXT: leaq {{[0-9]+}}(%rsp), %rax ; CHECK-NEXT: leaq {{[0-9]+}}(%rsp), %rcx ; CHECK-NEXT: cmpq %rax, %rcx @@ -80,8 +80,8 @@ ; CHECK-NEXT: movq _syBuf@GOTPCREL(%rip), %rcx ; CHECK-NEXT: leaq 8(%rcx,%rax), %rax ; CHECK-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) ## 8-byte Spill -; CHECK-NEXT: movl $1, %r15d ; CHECK-NEXT: movq _syCTRO@GOTPCREL(%rip), %rax +; CHECK-NEXT: movl $1, %r15d ; CHECK-NEXT: movb $1, %cl ; CHECK-NEXT: .p2align 4, 0x90 ; CHECK-NEXT: LBB0_9: ## %do.body @@ -90,223 +90,221 @@ ; CHECK-NEXT: testb %cl, %cl ; CHECK-NEXT: jne LBB0_9 ; CHECK-NEXT: ## %bb.10: ## %do.end -; CHECK-NEXT: xorl %ebx, %ebx -; CHECK-NEXT: testb %bl, %bl -; CHECK-NEXT: jne LBB0_11 -; CHECK-NEXT: ## %bb.12: ## %while.body200.preheader +; CHECK-NEXT: movq %rbx, {{[-0-9]+}}(%r{{[sb]}}p) ## 8-byte Spill +; CHECK-NEXT: xorl %r14d, %r14d +; CHECK-NEXT: testb %r14b, %r14b +; CHECK-NEXT: jne LBB0_41 +; CHECK-NEXT: ## %bb.11: ## %while.body200.preheader ; CHECK-NEXT: xorl %r13d, %r13d ; CHECK-NEXT: leaq LJTI0_0(%rip), %rdx -; CHECK-NEXT: leaq LJTI0_1(%rip), %r14 +; CHECK-NEXT: leaq LJTI0_1(%rip), %rbx ; CHECK-NEXT: movl $0, {{[-0-9]+}}(%r{{[sb]}}p) ## 4-byte Folded Spill ; CHECK-NEXT: xorl %r12d, %r12d -; CHECK-NEXT: jmp LBB0_13 -; CHECK-NEXT: LBB0_43: ## %while.cond1037.preheader -; CHECK-NEXT: ## in Loop: Header=BB0_13 Depth=1 +; CHECK-NEXT: jmp LBB0_14 +; CHECK-NEXT: LBB0_12: ## %while.cond1037.preheader +; CHECK-NEXT: ## in Loop: Header=BB0_14 Depth=1 ; CHECK-NEXT: testb %r13b, %r13b -; CHECK-NEXT: je LBB0_54 +; CHECK-NEXT: je LBB0_55 ; CHECK-NEXT: .p2align 4, 0x90 -; CHECK-NEXT: LBB0_20: ## %while.cond197.backedge -; CHECK-NEXT: ## in Loop: Header=BB0_13 Depth=1 +; CHECK-NEXT: LBB0_13: ## %while.cond197.backedge +; CHECK-NEXT: ## in Loop: Header=BB0_14 Depth=1 ; CHECK-NEXT: decl %r15d ; CHECK-NEXT: testl %r15d, %r15d -; CHECK-NEXT: movl %ebx, %r12d -; CHECK-NEXT: jle LBB0_21 -; CHECK-NEXT: LBB0_13: ## %while.body200 +; CHECK-NEXT: movl %r14d, %r12d +; CHECK-NEXT: jle LBB0_42 +; CHECK-NEXT: LBB0_14: ## %while.body200 ; CHECK-NEXT: ## =>This Loop Header: Depth=1 -; CHECK-NEXT: ## Child Loop BB0_28 Depth 2 -; CHECK-NEXT: ## Child Loop BB0_37 Depth 2 -; CHECK-NEXT: leal -268(%rbx), %eax +; CHECK-NEXT: ## Child Loop BB0_20 Depth 2 +; CHECK-NEXT: ## Child Loop BB0_36 Depth 2 +; CHECK-NEXT: leal -268(%r14), %eax ; CHECK-NEXT: cmpl $105, %eax -; CHECK-NEXT: ja LBB0_14 -; CHECK-NEXT: ## %bb.55: ## %while.body200 -; CHECK-NEXT: ## in Loop: Header=BB0_13 Depth=1 -; CHECK-NEXT: movslq (%r14,%rax,4), %rax -; CHECK-NEXT: addq %r14, %rax +; CHECK-NEXT: ja LBB0_23 +; CHECK-NEXT: ## %bb.15: ## %while.body200 +; CHECK-NEXT: ## in Loop: Header=BB0_14 Depth=1 +; CHECK-NEXT: movslq (%rbx,%rax,4), %rax +; CHECK-NEXT: addq %rbx, %rax ; CHECK-NEXT: jmpq *%rax -; CHECK-NEXT: LBB0_25: ## %sw.bb474 -; CHECK-NEXT: ## in Loop: Header=BB0_13 Depth=1 +; CHECK-NEXT: LBB0_16: ## %sw.bb474 +; CHECK-NEXT: ## in Loop: Header=BB0_14 Depth=1 ; CHECK-NEXT: testb %r13b, %r13b ; CHECK-NEXT: ## implicit-def: $rbp -; CHECK-NEXT: jne LBB0_33 -; CHECK-NEXT: ## %bb.26: ## %do.body479.preheader -; CHECK-NEXT: ## in Loop: Header=BB0_13 Depth=1 +; CHECK-NEXT: jne LBB0_31 +; CHECK-NEXT: ## %bb.17: ## %do.body479.preheader +; CHECK-NEXT: ## in Loop: Header=BB0_14 Depth=1 ; CHECK-NEXT: testb %r13b, %r13b ; CHECK-NEXT: ## implicit-def: $rbp -; CHECK-NEXT: jne LBB0_33 -; CHECK-NEXT: ## %bb.27: ## %land.rhs485.preheader -; CHECK-NEXT: ## in Loop: Header=BB0_13 Depth=1 +; CHECK-NEXT: jne LBB0_31 +; CHECK-NEXT: ## %bb.18: ## %land.rhs485.preheader +; CHECK-NEXT: ## in Loop: Header=BB0_14 Depth=1 ; CHECK-NEXT: ## implicit-def: $rax -; CHECK-NEXT: jmp LBB0_28 +; CHECK-NEXT: jmp LBB0_20 ; CHECK-NEXT: .p2align 4, 0x90 -; CHECK-NEXT: LBB0_31: ## %do.body479.backedge -; CHECK-NEXT: ## in Loop: Header=BB0_28 Depth=2 +; CHECK-NEXT: LBB0_19: ## %do.body479.backedge +; CHECK-NEXT: ## in Loop: Header=BB0_20 Depth=2 ; CHECK-NEXT: leaq 1(%rbp), %rax ; CHECK-NEXT: testb %r13b, %r13b -; CHECK-NEXT: je LBB0_32 -; CHECK-NEXT: LBB0_28: ## %land.rhs485 -; CHECK-NEXT: ## Parent Loop BB0_13 Depth=1 +; CHECK-NEXT: je LBB0_30 +; CHECK-NEXT: LBB0_20: ## %land.rhs485 +; CHECK-NEXT: ## Parent Loop BB0_14 Depth=1 ; CHECK-NEXT: ## => This Inner Loop Header: Depth=2 ; CHECK-NEXT: testb %al, %al -; CHECK-NEXT: js LBB0_54 -; CHECK-NEXT: ## %bb.29: ## %cond.true.i.i2780 -; CHECK-NEXT: ## in Loop: Header=BB0_28 Depth=2 +; CHECK-NEXT: js LBB0_55 +; CHECK-NEXT: ## %bb.21: ## %cond.true.i.i2780 +; CHECK-NEXT: ## in Loop: Header=BB0_20 Depth=2 ; CHECK-NEXT: movq %rax, %rbp ; CHECK-NEXT: testb %r13b, %r13b -; CHECK-NEXT: jne LBB0_31 -; CHECK-NEXT: ## %bb.30: ## %lor.rhs500 -; CHECK-NEXT: ## in Loop: Header=BB0_28 Depth=2 +; CHECK-NEXT: jne LBB0_19 +; CHECK-NEXT: ## %bb.22: ## %lor.rhs500 +; CHECK-NEXT: ## in Loop: Header=BB0_20 Depth=2 ; CHECK-NEXT: movl $256, %esi ## imm = 0x100 ; CHECK-NEXT: callq ___maskrune ; CHECK-NEXT: testb %r13b, %r13b -; CHECK-NEXT: jne LBB0_31 -; CHECK-NEXT: jmp LBB0_33 +; CHECK-NEXT: jne LBB0_19 +; CHECK-NEXT: jmp LBB0_31 ; CHECK-NEXT: .p2align 4, 0x90 -; CHECK-NEXT: LBB0_14: ## %while.body200 -; CHECK-NEXT: ## in Loop: Header=BB0_13 Depth=1 -; CHECK-NEXT: leal 1(%rbx), %eax +; CHECK-NEXT: LBB0_23: ## %while.body200 +; CHECK-NEXT: ## in Loop: Header=BB0_14 Depth=1 +; CHECK-NEXT: leal 1(%r14), %eax ; CHECK-NEXT: cmpl $21, %eax -; CHECK-NEXT: ja LBB0_20 -; CHECK-NEXT: ## %bb.15: ## %while.body200 -; CHECK-NEXT: ## in Loop: Header=BB0_13 Depth=1 +; CHECK-NEXT: ja LBB0_13 +; CHECK-NEXT: ## %bb.24: ## %while.body200 +; CHECK-NEXT: ## in Loop: Header=BB0_14 Depth=1 ; CHECK-NEXT: movslq (%rdx,%rax,4), %rax ; CHECK-NEXT: addq %rdx, %rax ; CHECK-NEXT: jmpq *%rax -; CHECK-NEXT: LBB0_18: ## %while.cond201.preheader -; CHECK-NEXT: ## in Loop: Header=BB0_13 Depth=1 -; CHECK-NEXT: movl $1, %ebx -; CHECK-NEXT: jmp LBB0_20 -; CHECK-NEXT: LBB0_44: ## %sw.bb1134 -; CHECK-NEXT: ## in Loop: Header=BB0_13 Depth=1 +; CHECK-NEXT: LBB0_25: ## %while.cond201.preheader +; CHECK-NEXT: ## in Loop: Header=BB0_14 Depth=1 +; CHECK-NEXT: movl $1, %r14d +; CHECK-NEXT: jmp LBB0_13 +; CHECK-NEXT: LBB0_26: ## %sw.bb1134 +; CHECK-NEXT: ## in Loop: Header=BB0_14 Depth=1 ; CHECK-NEXT: leaq {{[0-9]+}}(%rsp), %rax ; CHECK-NEXT: leaq {{[0-9]+}}(%rsp), %rcx ; CHECK-NEXT: cmpq %rax, %rcx -; CHECK-NEXT: jb LBB0_54 -; CHECK-NEXT: ## %bb.45: ## in Loop: Header=BB0_13 Depth=1 +; CHECK-NEXT: jb LBB0_55 +; CHECK-NEXT: ## %bb.27: ## in Loop: Header=BB0_14 Depth=1 ; CHECK-NEXT: movl $0, {{[-0-9]+}}(%r{{[sb]}}p) ## 4-byte Folded Spill -; CHECK-NEXT: movl $268, %ebx ## imm = 0x10C -; CHECK-NEXT: jmp LBB0_20 -; CHECK-NEXT: LBB0_39: ## %sw.bb566 -; CHECK-NEXT: ## in Loop: Header=BB0_13 Depth=1 -; CHECK-NEXT: movl $20, %ebx -; CHECK-NEXT: jmp LBB0_20 -; CHECK-NEXT: LBB0_19: ## %sw.bb243 -; CHECK-NEXT: ## in Loop: Header=BB0_13 Depth=1 -; CHECK-NEXT: movl $2, %ebx -; CHECK-NEXT: jmp LBB0_20 -; CHECK-NEXT: LBB0_32: ## %if.end517.loopexitsplit -; CHECK-NEXT: ## in Loop: Header=BB0_13 Depth=1 +; CHECK-NEXT: movl $268, %r14d ## imm = 0x10C +; CHECK-NEXT: jmp LBB0_13 +; CHECK-NEXT: LBB0_28: ## %sw.bb566 +; CHECK-NEXT: ## in Loop: Header=BB0_14 Depth=1 +; CHECK-NEXT: movl $20, %r14d +; CHECK-NEXT: jmp LBB0_13 +; CHECK-NEXT: LBB0_29: ## %sw.bb243 +; CHECK-NEXT: ## in Loop: Header=BB0_14 Depth=1 +; CHECK-NEXT: movl $2, %r14d +; CHECK-NEXT: jmp LBB0_13 +; CHECK-NEXT: LBB0_30: ## %if.end517.loopexitsplit +; CHECK-NEXT: ## in Loop: Header=BB0_14 Depth=1 ; CHECK-NEXT: incq %rbp -; CHECK-NEXT: LBB0_33: ## %if.end517 -; CHECK-NEXT: ## in Loop: Header=BB0_13 Depth=1 +; CHECK-NEXT: LBB0_31: ## %if.end517 +; CHECK-NEXT: ## in Loop: Header=BB0_14 Depth=1 ; CHECK-NEXT: leal -324(%r12), %eax ; CHECK-NEXT: cmpl $59, %eax -; CHECK-NEXT: ja LBB0_34 -; CHECK-NEXT: ## %bb.56: ## %if.end517 -; CHECK-NEXT: ## in Loop: Header=BB0_13 Depth=1 +; CHECK-NEXT: ja LBB0_33 +; CHECK-NEXT: ## %bb.32: ## %if.end517 +; CHECK-NEXT: ## in Loop: Header=BB0_14 Depth=1 ; CHECK-NEXT: movabsq $576460756598390785, %rcx ## imm = 0x800000100000001 ; CHECK-NEXT: btq %rax, %rcx -; CHECK-NEXT: jb LBB0_37 -; CHECK-NEXT: LBB0_34: ## %if.end517 -; CHECK-NEXT: ## in Loop: Header=BB0_13 Depth=1 +; CHECK-NEXT: jb LBB0_36 +; CHECK-NEXT: LBB0_33: ## %if.end517 +; CHECK-NEXT: ## in Loop: Header=BB0_14 Depth=1 ; CHECK-NEXT: cmpl $11, %r12d -; CHECK-NEXT: je LBB0_37 -; CHECK-NEXT: ## %bb.35: ## %if.end517 -; CHECK-NEXT: ## in Loop: Header=BB0_13 Depth=1 +; CHECK-NEXT: je LBB0_36 +; CHECK-NEXT: ## %bb.34: ## %if.end517 +; CHECK-NEXT: ## in Loop: Header=BB0_14 Depth=1 ; CHECK-NEXT: cmpl $24, %r12d -; CHECK-NEXT: je LBB0_37 -; CHECK-NEXT: ## %bb.36: ## %if.then532 -; CHECK-NEXT: ## in Loop: Header=BB0_13 Depth=1 +; CHECK-NEXT: je LBB0_36 +; CHECK-NEXT: ## %bb.35: ## %if.then532 +; CHECK-NEXT: ## in Loop: Header=BB0_14 Depth=1 ; CHECK-NEXT: movq _SyFgets.yank@GOTPCREL(%rip), %rax ; CHECK-NEXT: movb $0, (%rax) ; CHECK-NEXT: .p2align 4, 0x90 -; CHECK-NEXT: LBB0_37: ## %for.cond534 -; CHECK-NEXT: ## Parent Loop BB0_13 Depth=1 +; CHECK-NEXT: LBB0_36: ## %for.cond534 +; CHECK-NEXT: ## Parent Loop BB0_14 Depth=1 ; CHECK-NEXT: ## => This Inner Loop Header: Depth=2 ; CHECK-NEXT: testb %r13b, %r13b -; CHECK-NEXT: jne LBB0_37 -; CHECK-NEXT: ## %bb.38: ## %for.cond542.preheader -; CHECK-NEXT: ## in Loop: Header=BB0_13 Depth=1 +; CHECK-NEXT: jne LBB0_36 +; CHECK-NEXT: ## %bb.37: ## %for.cond542.preheader +; CHECK-NEXT: ## in Loop: Header=BB0_14 Depth=1 ; CHECK-NEXT: testb %r13b, %r13b ; CHECK-NEXT: movb $0, (%rbp) ; CHECK-NEXT: leaq LJTI0_0(%rip), %rdx -; CHECK-NEXT: jmp LBB0_20 +; CHECK-NEXT: jmp LBB0_13 ; CHECK-NEXT: .p2align 4, 0x90 -; CHECK-NEXT: LBB0_41: ## %while.cond864 +; CHECK-NEXT: LBB0_38: ## %while.cond864 ; CHECK-NEXT: ## =>This Inner Loop Header: Depth=1 -; CHECK-NEXT: jmp LBB0_41 +; CHECK-NEXT: jmp LBB0_38 ; CHECK-NEXT: .p2align 4, 0x90 -; CHECK-NEXT: LBB0_42: ## %while.cond962 +; CHECK-NEXT: LBB0_39: ## %while.cond962 ; CHECK-NEXT: ## =>This Inner Loop Header: Depth=1 -; CHECK-NEXT: jmp LBB0_42 +; CHECK-NEXT: jmp LBB0_39 ; CHECK-NEXT: .p2align 4, 0x90 -; CHECK-NEXT: LBB0_24: ## %for.cond357 +; CHECK-NEXT: LBB0_40: ## %for.cond357 ; CHECK-NEXT: ## =>This Inner Loop Header: Depth=1 -; CHECK-NEXT: jmp LBB0_24 -; CHECK-NEXT: LBB0_11: +; CHECK-NEXT: jmp LBB0_40 +; CHECK-NEXT: LBB0_41: ; CHECK-NEXT: movl $0, {{[-0-9]+}}(%r{{[sb]}}p) ## 4-byte Folded Spill -; CHECK-NEXT: LBB0_21: ## %while.end1465 -; CHECK-NEXT: incl %ebx -; CHECK-NEXT: cmpl $16, %ebx -; CHECK-NEXT: ja LBB0_49 -; CHECK-NEXT: ## %bb.22: ## %while.end1465 +; CHECK-NEXT: LBB0_42: ## %while.end1465 +; CHECK-NEXT: incl %r14d +; CHECK-NEXT: cmpl $16, %r14d +; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rbx ## 8-byte Reload +; CHECK-NEXT: ja LBB0_50 +; CHECK-NEXT: ## %bb.43: ## %while.end1465 ; CHECK-NEXT: movl $83969, %eax ## imm = 0x14801 -; CHECK-NEXT: btl %ebx, %eax -; CHECK-NEXT: jae LBB0_49 -; CHECK-NEXT: ## %bb.23: -; CHECK-NEXT: xorl %ebx, %ebx -; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r14 ## 8-byte Reload -; CHECK-NEXT: LBB0_47: ## %if.then1477 +; CHECK-NEXT: btl %r14d, %eax +; CHECK-NEXT: jae LBB0_50 +; CHECK-NEXT: LBB0_44: ## %if.then1477 ; CHECK-NEXT: movl $1, %edx ; CHECK-NEXT: callq _write -; CHECK-NEXT: subq %rbx, %r14 ; CHECK-NEXT: movq _syHistory@GOTPCREL(%rip), %rax -; CHECK-NEXT: leaq 8189(%r14,%rax), %rax +; CHECK-NEXT: leaq 8189(%rbx,%rax), %rax ; CHECK-NEXT: .p2align 4, 0x90 -; CHECK-NEXT: LBB0_48: ## %for.body1723 +; CHECK-NEXT: LBB0_45: ## %for.body1723 ; CHECK-NEXT: ## =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: decq %rax -; CHECK-NEXT: jmp LBB0_48 +; CHECK-NEXT: jmp LBB0_45 ; CHECK-NEXT: LBB0_46: ## %if.then1477.loopexit -; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r14 ## 8-byte Reload -; CHECK-NEXT: movq %r14, %rbx -; CHECK-NEXT: jmp LBB0_47 -; CHECK-NEXT: LBB0_16: ## %while.cond635.preheader +; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rbx ## 8-byte Reload +; CHECK-NEXT: subq %rbx, %rbx +; CHECK-NEXT: jmp LBB0_44 +; CHECK-NEXT: LBB0_47: ## %while.cond635.preheader ; CHECK-NEXT: xorl %eax, %eax ; CHECK-NEXT: testb %al, %al -; CHECK-NEXT: je LBB0_40 +; CHECK-NEXT: je LBB0_49 ; CHECK-NEXT: .p2align 4, 0x90 -; CHECK-NEXT: LBB0_17: ## %for.body643.us +; CHECK-NEXT: LBB0_48: ## %for.body643.us ; CHECK-NEXT: ## =>This Inner Loop Header: Depth=1 -; CHECK-NEXT: jmp LBB0_17 +; CHECK-NEXT: jmp LBB0_48 ; CHECK-NEXT: .p2align 4, 0x90 -; CHECK-NEXT: LBB0_40: ## %while.cond661 +; CHECK-NEXT: LBB0_49: ## %while.cond661 ; CHECK-NEXT: ## =>This Inner Loop Header: Depth=1 -; CHECK-NEXT: jmp LBB0_40 -; CHECK-NEXT: LBB0_49: ## %for.cond1480.preheader +; CHECK-NEXT: jmp LBB0_49 +; CHECK-NEXT: LBB0_50: ## %for.cond1480.preheader ; CHECK-NEXT: movl $512, %eax ## imm = 0x200 ; CHECK-NEXT: cmpq %rax, %rax -; CHECK-NEXT: jae LBB0_54 -; CHECK-NEXT: ## %bb.50: ## %for.body1664.lr.ph +; CHECK-NEXT: jae LBB0_55 +; CHECK-NEXT: ## %bb.51: ## %for.body1664.lr.ph ; CHECK-NEXT: xorl %eax, %eax ; CHECK-NEXT: testb %al, %al ; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rbx ## 8-byte Reload ; CHECK-NEXT: movl {{[-0-9]+}}(%r{{[sb]}}p), %ebp ## 4-byte Reload -; CHECK-NEXT: jne LBB0_53 -; CHECK-NEXT: ## %bb.51: ## %while.body1679.preheader +; CHECK-NEXT: jne LBB0_54 +; CHECK-NEXT: ## %bb.52: ## %while.body1679.preheader ; CHECK-NEXT: incl %ebp ; CHECK-NEXT: .p2align 4, 0x90 -; CHECK-NEXT: LBB0_52: ## %while.body1679 +; CHECK-NEXT: LBB0_53: ## %while.body1679 ; CHECK-NEXT: ## =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: movq (%rbx), %rdi ; CHECK-NEXT: callq _fileno ; CHECK-NEXT: movslq %ebp, %rax ; CHECK-NEXT: leal 1(%rax), %ebp ; CHECK-NEXT: cmpq %rax, %rax -; CHECK-NEXT: jl LBB0_52 -; CHECK-NEXT: LBB0_53: ## %while.cond1683.preheader +; CHECK-NEXT: jl LBB0_53 +; CHECK-NEXT: LBB0_54: ## %while.cond1683.preheader ; CHECK-NEXT: xorl %eax, %eax ; CHECK-NEXT: testb %al, %al -; CHECK-NEXT: LBB0_54: ## %if.then.i +; CHECK-NEXT: LBB0_55: ## %if.then.i ; CHECK-NEXT: ud2 entry: %sub.ptr.rhs.cast646 = ptrtoint ptr %line to i64 Index: llvm/test/CodeGen/X86/remat-phys-dead.ll =================================================================== --- llvm/test/CodeGen/X86/remat-phys-dead.ll +++ llvm/test/CodeGen/X86/remat-phys-dead.ll @@ -18,6 +18,5 @@ define i32 @test_remat32() { ret i32 0 ; CHECK: REGISTER COALESCER -; CHECK: Remat: $eax = MOV32r0 implicit-def dead $eflags } Index: llvm/test/CodeGen/X86/speculative-load-hardening-call-and-ret.ll =================================================================== --- llvm/test/CodeGen/X86/speculative-load-hardening-call-and-ret.ll +++ llvm/test/CodeGen/X86/speculative-load-hardening-call-and-ret.ll @@ -283,15 +283,14 @@ ; X64-NOPIC-NEXT: pushq %rbp ; X64-NOPIC-NEXT: pushq %r15 ; X64-NOPIC-NEXT: pushq %r14 -; X64-NOPIC-NEXT: pushq %r13 ; X64-NOPIC-NEXT: pushq %r12 ; X64-NOPIC-NEXT: pushq %rbx -; X64-NOPIC-NEXT: subq $24, %rsp +; X64-NOPIC-NEXT: subq $16, %rsp ; X64-NOPIC-NEXT: movq %rsp, %rax ; X64-NOPIC-NEXT: movq %rdi, %rbx ; X64-NOPIC-NEXT: movq $-1, %r15 ; X64-NOPIC-NEXT: sarq $63, %rax -; X64-NOPIC-NEXT: leaq {{[0-9]+}}(%rsp), %r14 +; X64-NOPIC-NEXT: movq %rsp, %r14 ; X64-NOPIC-NEXT: shlq $47, %rax ; X64-NOPIC-NEXT: movq %r14, %rdi ; X64-NOPIC-NEXT: orq %rax, %rsp @@ -302,24 +301,23 @@ ; X64-NOPIC-NEXT: sarq $63, %rax ; X64-NOPIC-NEXT: cmpq $.Lslh_ret_addr4, %r12 ; X64-NOPIC-NEXT: cmovneq %r15, %rax -; X64-NOPIC-NEXT: movl (%rbx), %r12d -; X64-NOPIC-NEXT: movl $42, %ebp +; X64-NOPIC-NEXT: movl (%rbx), %ebp ; X64-NOPIC-NEXT: shlq $47, %rax ; X64-NOPIC-NEXT: movq %r14, %rdi -; X64-NOPIC-NEXT: movl %ebp, %esi +; X64-NOPIC-NEXT: movl $42, %esi ; X64-NOPIC-NEXT: orq %rax, %rsp -; X64-NOPIC-NEXT: movq $.Lslh_ret_addr5, %r13 +; X64-NOPIC-NEXT: movq $.Lslh_ret_addr5, %r12 ; X64-NOPIC-NEXT: callq sigsetjmp@PLT ; X64-NOPIC-NEXT: .Lslh_ret_addr5: ; X64-NOPIC-NEXT: movq %rsp, %rax ; X64-NOPIC-NEXT: sarq $63, %rax -; X64-NOPIC-NEXT: cmpq $.Lslh_ret_addr5, %r13 +; X64-NOPIC-NEXT: cmpq $.Lslh_ret_addr5, %r12 ; X64-NOPIC-NEXT: cmovneq %r15, %rax -; X64-NOPIC-NEXT: addl (%rbx), %r12d +; X64-NOPIC-NEXT: addl (%rbx), %ebp ; X64-NOPIC-NEXT: shlq $47, %rax ; X64-NOPIC-NEXT: movq %r14, %rdi ; X64-NOPIC-NEXT: movq %r14, %rsi -; X64-NOPIC-NEXT: movl %ebp, %edx +; X64-NOPIC-NEXT: movl $42, %edx ; X64-NOPIC-NEXT: orq %rax, %rsp ; X64-NOPIC-NEXT: movq $.Lslh_ret_addr6, %r14 ; X64-NOPIC-NEXT: callq __sigsetjmp@PLT @@ -329,15 +327,14 @@ ; X64-NOPIC-NEXT: cmpq $.Lslh_ret_addr6, %r14 ; X64-NOPIC-NEXT: movq %rax, %rcx ; X64-NOPIC-NEXT: cmovneq %r15, %rcx -; X64-NOPIC-NEXT: addl (%rbx), %r12d -; X64-NOPIC-NEXT: movl %r12d, %eax +; X64-NOPIC-NEXT: addl (%rbx), %ebp +; X64-NOPIC-NEXT: movl %ebp, %eax ; X64-NOPIC-NEXT: orl %ecx, %eax ; X64-NOPIC-NEXT: shlq $47, %rcx ; X64-NOPIC-NEXT: orq %rcx, %rsp -; X64-NOPIC-NEXT: addq $24, %rsp +; X64-NOPIC-NEXT: addq $16, %rsp ; X64-NOPIC-NEXT: popq %rbx ; X64-NOPIC-NEXT: popq %r12 -; X64-NOPIC-NEXT: popq %r13 ; X64-NOPIC-NEXT: popq %r14 ; X64-NOPIC-NEXT: popq %r15 ; X64-NOPIC-NEXT: popq %rbp @@ -348,15 +345,14 @@ ; X64-NOPIC-MCM-NEXT: pushq %rbp ; X64-NOPIC-MCM-NEXT: pushq %r15 ; X64-NOPIC-MCM-NEXT: pushq %r14 -; X64-NOPIC-MCM-NEXT: pushq %r13 ; X64-NOPIC-MCM-NEXT: pushq %r12 ; X64-NOPIC-MCM-NEXT: pushq %rbx -; X64-NOPIC-MCM-NEXT: subq $24, %rsp +; X64-NOPIC-MCM-NEXT: subq $16, %rsp ; X64-NOPIC-MCM-NEXT: movq %rsp, %rax ; X64-NOPIC-MCM-NEXT: movq %rdi, %rbx ; X64-NOPIC-MCM-NEXT: movq $-1, %r15 ; X64-NOPIC-MCM-NEXT: sarq $63, %rax -; X64-NOPIC-MCM-NEXT: leaq {{[0-9]+}}(%rsp), %r14 +; X64-NOPIC-MCM-NEXT: movq %rsp, %r14 ; X64-NOPIC-MCM-NEXT: shlq $47, %rax ; X64-NOPIC-MCM-NEXT: movq %r14, %rdi ; X64-NOPIC-MCM-NEXT: orq %rax, %rsp @@ -368,25 +364,24 @@ ; X64-NOPIC-MCM-NEXT: leaq .Lslh_ret_addr4(%rip), %rcx ; X64-NOPIC-MCM-NEXT: cmpq %rcx, %r12 ; X64-NOPIC-MCM-NEXT: cmovneq %r15, %rax -; X64-NOPIC-MCM-NEXT: movl (%rbx), %r12d -; X64-NOPIC-MCM-NEXT: movl $42, %ebp +; X64-NOPIC-MCM-NEXT: movl (%rbx), %ebp ; X64-NOPIC-MCM-NEXT: shlq $47, %rax ; X64-NOPIC-MCM-NEXT: movq %r14, %rdi -; X64-NOPIC-MCM-NEXT: movl %ebp, %esi +; X64-NOPIC-MCM-NEXT: movl $42, %esi ; X64-NOPIC-MCM-NEXT: orq %rax, %rsp -; X64-NOPIC-MCM-NEXT: leaq .Lslh_ret_addr5(%rip), %r13 +; X64-NOPIC-MCM-NEXT: leaq .Lslh_ret_addr5(%rip), %r12 ; X64-NOPIC-MCM-NEXT: callq sigsetjmp@PLT ; X64-NOPIC-MCM-NEXT: .Lslh_ret_addr5: ; X64-NOPIC-MCM-NEXT: movq %rsp, %rax ; X64-NOPIC-MCM-NEXT: sarq $63, %rax ; X64-NOPIC-MCM-NEXT: leaq .Lslh_ret_addr5(%rip), %rcx -; X64-NOPIC-MCM-NEXT: cmpq %rcx, %r13 +; X64-NOPIC-MCM-NEXT: cmpq %rcx, %r12 ; X64-NOPIC-MCM-NEXT: cmovneq %r15, %rax -; X64-NOPIC-MCM-NEXT: addl (%rbx), %r12d +; X64-NOPIC-MCM-NEXT: addl (%rbx), %ebp ; X64-NOPIC-MCM-NEXT: shlq $47, %rax ; X64-NOPIC-MCM-NEXT: movq %r14, %rdi ; X64-NOPIC-MCM-NEXT: movq %r14, %rsi -; X64-NOPIC-MCM-NEXT: movl %ebp, %edx +; X64-NOPIC-MCM-NEXT: movl $42, %edx ; X64-NOPIC-MCM-NEXT: orq %rax, %rsp ; X64-NOPIC-MCM-NEXT: leaq .Lslh_ret_addr6(%rip), %r14 ; X64-NOPIC-MCM-NEXT: callq __sigsetjmp@PLT @@ -397,15 +392,14 @@ ; X64-NOPIC-MCM-NEXT: cmpq %rcx, %r14 ; X64-NOPIC-MCM-NEXT: movq %rax, %rcx ; X64-NOPIC-MCM-NEXT: cmovneq %r15, %rcx -; X64-NOPIC-MCM-NEXT: addl (%rbx), %r12d -; X64-NOPIC-MCM-NEXT: movl %r12d, %eax +; X64-NOPIC-MCM-NEXT: addl (%rbx), %ebp +; X64-NOPIC-MCM-NEXT: movl %ebp, %eax ; X64-NOPIC-MCM-NEXT: orl %ecx, %eax ; X64-NOPIC-MCM-NEXT: shlq $47, %rcx ; X64-NOPIC-MCM-NEXT: orq %rcx, %rsp -; X64-NOPIC-MCM-NEXT: addq $24, %rsp +; X64-NOPIC-MCM-NEXT: addq $16, %rsp ; X64-NOPIC-MCM-NEXT: popq %rbx ; X64-NOPIC-MCM-NEXT: popq %r12 -; X64-NOPIC-MCM-NEXT: popq %r13 ; X64-NOPIC-MCM-NEXT: popq %r14 ; X64-NOPIC-MCM-NEXT: popq %r15 ; X64-NOPIC-MCM-NEXT: popq %rbp @@ -416,15 +410,14 @@ ; X64-PIC-NEXT: pushq %rbp ; X64-PIC-NEXT: pushq %r15 ; X64-PIC-NEXT: pushq %r14 -; X64-PIC-NEXT: pushq %r13 ; X64-PIC-NEXT: pushq %r12 ; X64-PIC-NEXT: pushq %rbx -; X64-PIC-NEXT: subq $24, %rsp +; X64-PIC-NEXT: subq $16, %rsp ; X64-PIC-NEXT: movq %rsp, %rax ; X64-PIC-NEXT: movq %rdi, %rbx ; X64-PIC-NEXT: movq $-1, %r15 ; X64-PIC-NEXT: sarq $63, %rax -; X64-PIC-NEXT: leaq {{[0-9]+}}(%rsp), %r14 +; X64-PIC-NEXT: movq %rsp, %r14 ; X64-PIC-NEXT: shlq $47, %rax ; X64-PIC-NEXT: movq %r14, %rdi ; X64-PIC-NEXT: orq %rax, %rsp @@ -436,25 +429,24 @@ ; X64-PIC-NEXT: leaq .Lslh_ret_addr4(%rip), %rcx ; X64-PIC-NEXT: cmpq %rcx, %r12 ; X64-PIC-NEXT: cmovneq %r15, %rax -; X64-PIC-NEXT: movl (%rbx), %r12d -; X64-PIC-NEXT: movl $42, %ebp +; X64-PIC-NEXT: movl (%rbx), %ebp ; X64-PIC-NEXT: shlq $47, %rax ; X64-PIC-NEXT: movq %r14, %rdi -; X64-PIC-NEXT: movl %ebp, %esi +; X64-PIC-NEXT: movl $42, %esi ; X64-PIC-NEXT: orq %rax, %rsp -; X64-PIC-NEXT: leaq .Lslh_ret_addr5(%rip), %r13 +; X64-PIC-NEXT: leaq .Lslh_ret_addr5(%rip), %r12 ; X64-PIC-NEXT: callq sigsetjmp@PLT ; X64-PIC-NEXT: .Lslh_ret_addr5: ; X64-PIC-NEXT: movq %rsp, %rax ; X64-PIC-NEXT: sarq $63, %rax ; X64-PIC-NEXT: leaq .Lslh_ret_addr5(%rip), %rcx -; X64-PIC-NEXT: cmpq %rcx, %r13 +; X64-PIC-NEXT: cmpq %rcx, %r12 ; X64-PIC-NEXT: cmovneq %r15, %rax -; X64-PIC-NEXT: addl (%rbx), %r12d +; X64-PIC-NEXT: addl (%rbx), %ebp ; X64-PIC-NEXT: shlq $47, %rax ; X64-PIC-NEXT: movq %r14, %rdi ; X64-PIC-NEXT: movq %r14, %rsi -; X64-PIC-NEXT: movl %ebp, %edx +; X64-PIC-NEXT: movl $42, %edx ; X64-PIC-NEXT: orq %rax, %rsp ; X64-PIC-NEXT: leaq .Lslh_ret_addr6(%rip), %r14 ; X64-PIC-NEXT: callq __sigsetjmp@PLT @@ -465,15 +457,14 @@ ; X64-PIC-NEXT: cmpq %rcx, %r14 ; X64-PIC-NEXT: movq %rax, %rcx ; X64-PIC-NEXT: cmovneq %r15, %rcx -; X64-PIC-NEXT: addl (%rbx), %r12d -; X64-PIC-NEXT: movl %r12d, %eax +; X64-PIC-NEXT: addl (%rbx), %ebp +; X64-PIC-NEXT: movl %ebp, %eax ; X64-PIC-NEXT: orl %ecx, %eax ; X64-PIC-NEXT: shlq $47, %rcx ; X64-PIC-NEXT: orq %rcx, %rsp -; X64-PIC-NEXT: addq $24, %rsp +; X64-PIC-NEXT: addq $16, %rsp ; X64-PIC-NEXT: popq %rbx ; X64-PIC-NEXT: popq %r12 -; X64-PIC-NEXT: popq %r13 ; X64-PIC-NEXT: popq %r14 ; X64-PIC-NEXT: popq %r15 ; X64-PIC-NEXT: popq %rbp Index: llvm/test/CodeGen/X86/vector-shuffle-combining-avx512bwvl.ll =================================================================== --- llvm/test/CodeGen/X86/vector-shuffle-combining-avx512bwvl.ll +++ llvm/test/CodeGen/X86/vector-shuffle-combining-avx512bwvl.ll @@ -173,13 +173,14 @@ define i64 @PR55050() { ; X86-LABEL: PR55050: ; X86: # %bb.0: # %entry +; X86-NEXT: xorl %edx, %edx ; X86-NEXT: xorl %eax, %eax -; X86-NEXT: testb %al, %al +; X86-NEXT: testb %dl, %dl ; X86-NEXT: jne .LBB10_2 ; X86-NEXT: # %bb.1: # %if ; X86-NEXT: xorl %eax, %eax +; X86-NEXT: xorl %edx, %edx ; X86-NEXT: .LBB10_2: # %exit -; X86-NEXT: movl %eax, %edx ; X86-NEXT: retl ; ; X64-LABEL: PR55050: