Allow unordered loads to be considered invariant in CodeGen

[reames]

Summary

The actual code change is fairly straight forward, but exercising it isn't. First, it turned out we weren't adding the appropriate flags in SelectionDAG. Second, it turned out that we've got some optimization gaps, so obvious test cases don't work.

My first attempt (in atomic-unordered.ll) points out a deficiency in our peephole-opt folding logic which I plan to fix separately. Instead, I'm exercising this through MachineLICM.

[jfb]

This part isn't something I've very familiar with, better if someone else reviews.

lib/CodeGen/MachineInstr.cpp

1319 ↗

(On Diff #190682)

Do you ever have volatile unordered?

[jlebar]

LGTM modulo jfb's outstanding comment.

lib/CodeGen/MachineInstr.cpp

1318 ↗

(On Diff #190682)

"need to be updated"?

[reames]

lib/CodeGen/MachineInstr.cpp

1319 ↗

(On Diff #190682)

isUnordered returns false for volatiles.

[reames]

lib/CodeGen/MachineInstr.cpp

1318 ↗

(On Diff #190682)

I left the wording here because this is a possible change with possible downsides design wise. I don't what to give the impression it's a straight forward change.

llvm/trunk/lib/CodeGen/MachineInstr.cpp

bool MachineInstr::isDereferenceableInvariantLoad(AliasAnalysis *AA) const {
   // If the instruction has lost its memoperands, conservatively assume that
   // it may not be an invariant load.
   if (memoperands_empty())
     return false;
 
   const MachineFrameInfo &MFI = getParent()->getParent()->getFrameInfo();
 
   for (MachineMemOperand *MMO : memoperands()) {
-    if (MMO->isVolatile()) return false;
-    // TODO: Figure out whether isAtomic is really necessary (see D57601).
-    if (MMO->isAtomic()) return false;
+    if (!MMO->isUnordered())
+      // If the memory operand has ordering side effects, we can't move the
+      // instruction.  Such an instruction is technically an invariant load,
+      // but the caller code would need updated to expect that.
+      return false;
     if (MMO->isStore()) return false;
     if (MMO->isInvariant() && MMO->isDereferenceable())
       continue;
 
     // A load from a constant PseudoSourceValue is invariant.
     if (const PseudoSourceValue *PSV = MMO->getPseudoValue())
       if (PSV->isConstant(&MFI))
         continue;

llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp

void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) {
 
   if (!TLI.supportsUnalignedAtomics() &&
       I.getAlignment() < VT.getStoreSize())
     report_fatal_error("Cannot generate unaligned atomic load");
 
   auto Flags = MachineMemOperand::MOLoad;
   if (I.isVolatile())
     Flags |= MachineMemOperand::MOVolatile;
+  if (I.getMetadata(LLVMContext::MD_invariant_load) != nullptr)
+    Flags |= MachineMemOperand::MOInvariant;
+  if (isDereferenceablePointer(I.getPointerOperand(), DAG.getDataLayout()))
+    Flags |= MachineMemOperand::MODereferenceable;
+
   Flags |= TLI.getMMOFlags(I);
 
   MachineMemOperand *MMO =
       DAG.getMachineFunction().
       getMachineMemOperand(MachinePointerInfo(I.getPointerOperand()),
                            Flags, VT.getStoreSize(),
                            I.getAlignment() ? I.getAlignment() :
                                               DAG.getEVTAlignment(VT),

llvm/trunk/test/CodeGen/X86/atomic-unordered.ll

 ; CHECK-O3-NEXT:    addq %rdi, %rax
 ; CHECK-O3-NEXT:    retq
   %v = load atomic i64, i64* @Constant unordered, align 8
   fence seq_cst
   %ret = add i64 %v, %arg
   ret i64 %ret
 }
 
-define i64 @fold_invariant_clobber(i64* %p, i64 %arg) {
+define i64 @fold_invariant_clobber(i64* dereferenceable(8) %p, i64 %arg) {
 ; CHECK-O0-LABEL: fold_invariant_clobber:
 ; CHECK-O0:       # %bb.0:
 ; CHECK-O0-NEXT:    movq (%rdi), %rax
 ; CHECK-O0-NEXT:    movq $5, (%rdi)
 ; CHECK-O0-NEXT:    addq %rsi, %rax
 ; CHECK-O0-NEXT:    retq
 ;
 ; CHECK-O3-LABEL: fold_invariant_clobber:
 ; CHECK-O3:       # %bb.0:
 ; CHECK-O3-NEXT:    movq (%rdi), %rax
 ; CHECK-O3-NEXT:    movq $5, (%rdi)
 ; CHECK-O3-NEXT:    addq %rsi, %rax
 ; CHECK-O3-NEXT:    retq
   %v = load atomic i64, i64* %p unordered, align 8, !invariant.load !{}
   store i64 5, i64* %p
   %ret = add i64 %v, %arg
   ret i64 %ret
 }
 
 
-define i64 @fold_invariant_fence(i64* %p, i64 %arg) {
+define i64 @fold_invariant_fence(i64* dereferenceable(8) %p, i64 %arg) {
 ; CHECK-O0-LABEL: fold_invariant_fence:
 ; CHECK-O0:       # %bb.0:
 ; CHECK-O0-NEXT:    movq (%rdi), %rdi
 ; CHECK-O0-NEXT:    mfence
 ; CHECK-O0-NEXT:    addq %rsi, %rdi
 ; CHECK-O0-NEXT:    movq %rdi, %rax
 ; CHECK-O0-NEXT:    retq
 ;

llvm/trunk/test/CodeGen/X86/hoist-invariant-load.ll

for.end:                                          ; preds = %for.body
   ret void
 }
 
 define void @test_unordered(i8* %x) uwtable ssp {
 ; CHECK-LABEL: test_unordered:
 ; CHECK:       ## %bb.0: ## %entry
 ; CHECK-NEXT:    pushq %rbp
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    pushq %r15
 ; CHECK-NEXT:    .cfi_def_cfa_offset 24
-; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    pushq %r14
 ; CHECK-NEXT:    .cfi_def_cfa_offset 32
-; CHECK-NEXT:    .cfi_offset %rbx, -32
-; CHECK-NEXT:    .cfi_offset %r14, -24
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    .cfi_offset %rbx, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
 ; CHECK-NEXT:    .cfi_offset %rbp, -16
 ; CHECK-NEXT:    movq %rdi, %rbx
 ; CHECK-NEXT:    movl $10000, %ebp ## imm = 0x2710
-; CHECK-NEXT:    movq _objc_msgSend@{{.*}}(%rip), %r14
+; CHECK-NEXT:    movq {{.*}}(%rip), %r14
+; CHECK-NEXT:    movq _objc_msgSend@{{.*}}(%rip), %r15
 ; CHECK-NEXT:    .p2align 4, 0x90
 ; CHECK-NEXT:  LBB1_1: ## %for.body
 ; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    movq {{.*}}(%rip), %rsi
 ; CHECK-NEXT:    movq %rbx, %rdi
-; CHECK-NEXT:    callq *%r14
+; CHECK-NEXT:    movq %r14, %rsi
+; CHECK-NEXT:    callq *%r15
 ; CHECK-NEXT:    decl %ebp
 ; CHECK-NEXT:    jne LBB1_1
 ; CHECK-NEXT:  ## %bb.2: ## %for.end
+; CHECK-NEXT:    addq $8, %rsp
 ; CHECK-NEXT:    popq %rbx
 ; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    popq %r15
 ; CHECK-NEXT:    popq %rbp
 ; CHECK-NEXT:    retq
 entry:
   br label %for.body
 
 for.body:                                         ; preds = %for.body, %entry
   %i.01 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
   %0 = load atomic i8*, i8** @"\01L_OBJC_SELECTOR_REFERENCES_" unordered, align 8, !invariant.load !0