[llvm][SelectionDAGBuilder] use getRegistersForValue to generate legal copies

[nickdesaulniers]

Summary

Fixes crashes observed when building the ARCH=i386 and ARCH=powerpc
Linux kernel.

Rewrite bits of SelectionDAGBuilder::visitCallBrLandingPad to re-use
machinery similar to SelectionDAGBuilder::visitInlineAsm. This is a bit
more robust for Flag Output Operands, outputs with a ConstraintVT that's
illegal for a given target, and outputs that require more than one
register.

[nickdesaulniers]

• git clang-format HEAD~

llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp

11658

unused

[nathanchance]

I applied this change along with the others from https://discourse.llvm.org/t/rfc-syncing-asm-goto-with-outputs-with-gcc/65453/11 and built and boot tested a variety of configurations on the architectures that the Linux kernel cares about and saw no new issues.

[nickdesaulniers]

• remove unnecessary Flag variable that was being shadowed

[efriedma]

LGTM

[nickdesaulniers]

Squashed into https://reviews.llvm.org/D140160.

llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp

if (MI->getOpcode() == TargetOpcode::COPY) {
     MI = MRI.def_begin(Reg)->getParent();
   }
   // The start of the chain must be an INLINEASM_BR.
   assert(MI->getOpcode() == TargetOpcode::INLINEASM_BR &&
          "end of copy chain MUST be INLINEASM_BR");
   return Reg;
 }
 
-static size_t PopulateResultValues(const CallBrInst &CBR, SelectionDAG &DAG,
-                                   SmallVectorImpl<EVT> &ResultVTs,
-                                   SmallVectorImpl<SDValue> &ResultValues) {
-  Type *CBRType = CBR.getType();
-  assert(!CBRType->isVoidTy() &&
-         "callbr landing pad for callbr without returns");
-  ArrayRef<Type *> ResultTypes = CBRType->isStructTy()
-                                     ? cast<StructType>(CBRType)->elements()
-                                     : ArrayRef(CBRType);
-  size_t NumReturnValues = ResultTypes.size();
-  ResultVTs.reserve(NumReturnValues);
-  ResultValues.reserve(NumReturnValues);
-  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
-  for (Type *T : ResultTypes)
-    ResultVTs.emplace_back(TLI.getValueType(DAG.getDataLayout(), T));
-  return NumReturnValues;
-}
-
 // We must do this walk rather than the simpler
 //   setValue(&I, getCopyFromRegs(CBR, CBR->getType()));
 // otherwise we will end up with copies of virtregs only valid along direct
 // edges.
 void SelectionDAGBuilder::visitCallBrLandingPad(const CallInst &I) {
   SmallVector<EVT, 8> ResultVTs;
   SmallVector<SDValue, 8> ResultValues;
   const auto *CBR =
       cast<CallBrInst>(I.getParent()->getUniquePredecessor()->getTerminator());
-  const size_t NumReturnValues =
-      PopulateResultValues(*CBR, DAG, ResultVTs, ResultValues);
+
+  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
+  const TargetRegisterInfo *TRI = DAG.getSubtarget().getRegisterInfo();
   MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
-  unsigned Reg = FuncInfo.ValueMap[CBR];
+
+  unsigned InitialDef = FuncInfo.ValueMap[CBR];
   SDValue Chain = DAG.getRoot();
 
-  for (size_t i = 0; i != NumReturnValues; ++i, ++Reg) {
-    Register Output = FollowCopyChain(MRI, Reg);
+  // Re-parse the asm constraints string.
+  TargetLowering::AsmOperandInfoVector TargetConstraints =
+      TLI.ParseConstraints(DAG.getDataLayout(), TRI, *CBR);
+  for (auto &T : TargetConstraints) {
+    SDISelAsmOperandInfo OpInfo(T);
+    if (OpInfo.Type != InlineAsm::isOutput)
+      continue;
 
-    if (Register::isPhysicalRegister(Output))
-      FuncInfo.MBB->addLiveIn(Output);
+    // Pencil in OpInfo.ConstraintType and OpInfo.ConstraintVT based on the
+    // individual constraint.
+    TLI.ComputeConstraintToUse(OpInfo, OpInfo.CallOperand, &DAG);
+
+    switch (OpInfo.ConstraintType) {
+    case TargetLowering::C_Register:
+    case TargetLowering::C_RegisterClass: {
+      // Fill in OpInfo.AssignedRegs.Regs.
+      getRegistersForValue(DAG, getCurSDLoc(), OpInfo, OpInfo);
+
+      // getRegistersForValue may produce 1 to many registers based on whether
+      // the OpInfo.ConstraintVT is legal on the target or not.
+      for (size_t i = 0, e = OpInfo.AssignedRegs.Regs.size(); i != e; ++i) {
+        Register OriginalDef = FollowCopyChain(MRI, InitialDef++);
+        if (Register::isPhysicalRegister(OriginalDef))
+          FuncInfo.MBB->addLiveIn(OriginalDef);
+        // Update the assigned registers to use the original defs.
+        OpInfo.AssignedRegs.Regs[i] = OriginalDef;
+      }
 
-    SDValue V = DAG.getCopyFromReg(Chain, getCurSDLoc(), Output, ResultVTs[i]);
+      SDValue V = OpInfo.AssignedRegs.getCopyFromRegs(
+          DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, CBR);
+      ResultValues.push_back(V);
+      ResultVTs.push_back(OpInfo.ConstraintVT);
+      break;
+    }
+    case TargetLowering::C_Other: {
+      SDValue Flag;
+      SDValue V = TLI.LowerAsmOutputForConstraint(Chain, Flag, getCurSDLoc(),
+                                                  OpInfo, DAG);
+      ++InitialDef;
       ResultValues.push_back(V);
+      ResultVTs.push_back(OpInfo.ConstraintVT);
+      break;
+    }
+    default:
+      break;
+    }
   }
   SDValue V = DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
                           DAG.getVTList(ResultVTs), ResultValues);
   setValue(&I, V);
 }

llvm/test/CodeGen/AArch64/callbr-asm-outputs-indirect-isel.ll

define i32 @test0() {
   ; CHECK-NEXT:   successors: %bb.5(0x80000000)
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:gpr32all = COPY %5
   ; CHECK-NEXT:   B %bb.5
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.2.direct:
   ; CHECK-NEXT:   successors: %bb.4(0x80000000), %bb.3(0x00000000)
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   INLINEASM_BR &"# $0", 0 /* attdialect */, 1507338 /* regdef:GPR32common */, def %6, 13 /* imm */, %bb.3
-  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:gpr32all = COPY %6
+  ; CHECK-NEXT:   INLINEASM_BR &"# $0", 0 /* attdialect */, 1507338 /* regdef:GPR32common */, def %7, 13 /* imm */, %bb.3
+  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:gpr32all = COPY %7
   ; CHECK-NEXT:   B %bb.4
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.3.direct.indirect_crit_edge (machine-block-address-taken, inlineasm-br-indirect-target):
   ; CHECK-NEXT:   successors: %bb.5(0x80000000)
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   [[COPY3:%[0-9]+]]:gpr32all = COPY %6
+  ; CHECK-NEXT:   [[COPY3:%[0-9]+]]:gpr32all = COPY %7
   ; CHECK-NEXT:   B %bb.5
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.4.direct2:
   ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:gpr32all = COPY $wzr
   ; CHECK-NEXT:   $w0 = COPY [[COPY4]]
   ; CHECK-NEXT:   RET_ReallyLR implicit $w0
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.5.indirect:
define i32 @dont_split6(i32 %0) {
   ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:gpr32common = COPY [[PHI]]
   ; CHECK-NEXT:   INLINEASM_BR &"", 0 /* attdialect */, 1507338 /* regdef:GPR32common */, def %4, 2147483657 /* reguse tiedto:$0 */, [[COPY1]](tied-def 3), 13 /* imm */, %bb.2
   ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:gpr32all = COPY %4
   ; CHECK-NEXT:   B %bb.3
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.2.loop.loop_crit_edge (machine-block-address-taken, inlineasm-br-indirect-target):
   ; CHECK-NEXT:   successors: %bb.1(0x80000000)
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   %2:gpr32all = COPY %4
+  ; CHECK-NEXT:   [[COPY3:%[0-9]+]]:gpr32all = COPY %4
   ; CHECK-NEXT:   B %bb.1
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.3.exit:
   ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:gpr32all = COPY $wzr
   ; CHECK-NEXT:   $w0 = COPY [[COPY4]]
   ; CHECK-NEXT:   RET_ReallyLR implicit $w0
 entry:
   br label %loop

llvm/test/CodeGen/PowerPC/callbr-asm-outputs-indirect-isel.ll

+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+; RUN: llc %s -o - -stop-after=finalize-isel -start-before=ppc-isel | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-n32:64-S128-v256:256:256-v512:512:512"
+target triple = "powerpc64le-unknown-linux-gnu"
+
+define void @strncpy_from_kernel_nofault_count() {
+  ; CHECK-LABEL: name: strncpy_from_kernel_nofault_count
+  ; CHECK: bb.0.entry:
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000), %bb.3(0x00000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   INLINEASM_BR &"", 0 /* attdialect */, 131082 /* regdef:GPRC */, def %1, 13 /* imm */, %bb.3
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:gprc = COPY %1
+  ; CHECK-NEXT:   B %bb.1
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1.asm.fallthrough:
+  ; CHECK-NEXT:   successors: %bb.2(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   STB [[COPY]], 0, $zero8 :: (store (s8) into `ptr null`)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2.Efault:
+  ; CHECK-NEXT:   BLR8 implicit $lr8, implicit $rm
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.3.Efault.split (machine-block-address-taken, inlineasm-br-indirect-target):
+  ; CHECK-NEXT:   successors: %bb.2(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   STB %1, 0, $zero8 :: (store (s8) into `ptr null`)
+  ; CHECK-NEXT:   B %bb.2
+entry:
+  %0 = callbr i8 asm "", "=r,!i"()
+          to label %asm.fallthrough [label %Efault.split]
+
+asm.fallthrough:
+  store i8 %0, ptr null, align 1
+  br label %Efault
+
+Efault:
+  ret void
+
+Efault.split:
+  %1 = call i8 @llvm.callbr.landingpad.i8(i8 %0)
+  store i8 %1, ptr null, align 1
+  br label %Efault
+}
+declare i8 @llvm.callbr.landingpad.i8(i8)

llvm/test/CodeGen/X86/callbr-asm-outputs-indirect-isel-m32.ll

+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+; RUN: llc -mtriple=i386-linux-gnu %s -o - -stop-after=finalize-isel \
+; RUN: -start-before=x86-isel | FileCheck %s
+
+define i8 @emulator_cmpxchg_emulated() {
+  ; CHECK-LABEL: name: emulator_cmpxchg_emulated
+  ; CHECK: bb.0.entry:
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000), %bb.2(0x00000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[MOV32rm:%[0-9]+]]:gr32 = MOV32rm $noreg, 1, $noreg, 0, $noreg :: (load (s32) from `ptr null`, align 8)
+  ; CHECK-NEXT:   INLINEASM_BR &"", 16 /* maystore attdialect */, 2359306 /* regdef:GR32 */, def %2, 2359306 /* regdef:GR32 */, def %3, 2147549193 /* reguse tiedto:$1 */, [[MOV32rm]](tied-def 5), 13 /* imm */, %bb.2
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:gr32 = COPY $eflags
+  ; CHECK-NEXT:   $eflags = COPY [[COPY]]
+  ; CHECK-NEXT:   [[SETCCr:%[0-9]+]]:gr8 = SETCCr 4, implicit $eflags
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:gr32 = COPY %3
+  ; CHECK-NEXT:   JMP_1 %bb.1
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1.asm.fallthrough:
+  ; CHECK-NEXT:   $al = COPY [[SETCCr]]
+  ; CHECK-NEXT:   RET 0, $al
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2.efaultu64.split (machine-block-address-taken, inlineasm-br-indirect-target):
+  ; CHECK-NEXT:   [[SETCCr1:%[0-9]+]]:gr8 = SETCCr 4, implicit $eflags
+  ; CHECK-NEXT:   $al = COPY [[SETCCr1]]
+  ; CHECK-NEXT:   RET 0, $al
+entry:
+  %0 = load i32, ptr null, align 8
+  %1 = callbr { i8, i32 } asm "", "={@ccz},=r,1,!i"(i32 %0)
+          to label %asm.fallthrough [label %efaultu64.split]
+
+asm.fallthrough:
+  %asmresult = extractvalue { i8, i32 } %1, 0
+  %asmresult1 = extractvalue { i8, i32 } %1, 1
+  ret i8 %asmresult
+
+efaultu64.split:
+  %2 = call { i8, i32 } @llvm.callbr.landingpad.sl_i8i32s({ i8, i32 } %1)
+  %asmresult2 = extractvalue { i8, i32 } %2, 0
+  %asmresult3 = extractvalue { i8, i32 } %2, 1
+  ret i8 %asmresult2
+}
+
+; Same test but return second value
+define i32 @emulator_cmpxchg_emulated2() {
+  ; CHECK-LABEL: name: emulator_cmpxchg_emulated2
+  ; CHECK: bb.0.entry:
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000), %bb.2(0x00000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[MOV32rm:%[0-9]+]]:gr32 = MOV32rm $noreg, 1, $noreg, 0, $noreg :: (load (s32) from `ptr null`, align 8)
+  ; CHECK-NEXT:   INLINEASM_BR &"", 16 /* maystore attdialect */, 2359306 /* regdef:GR32 */, def %2, 2359306 /* regdef:GR32 */, def %3, 2147549193 /* reguse tiedto:$1 */, [[MOV32rm]](tied-def 5), 13 /* imm */, %bb.2
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:gr32 = COPY $eflags
+  ; CHECK-NEXT:   $eflags = COPY [[COPY]]
+  ; CHECK-NEXT:   [[SETCCr:%[0-9]+]]:gr8 = SETCCr 4, implicit $eflags
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:gr32 = COPY %3
+  ; CHECK-NEXT:   JMP_1 %bb.1
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1.asm.fallthrough:
+  ; CHECK-NEXT:   $eax = COPY [[COPY1]]
+  ; CHECK-NEXT:   RET 0, $eax
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2.efaultu64.split (machine-block-address-taken, inlineasm-br-indirect-target):
+  ; CHECK-NEXT:   $eax = COPY %3
+  ; CHECK-NEXT:   RET 0, $eax
+entry:
+  %0 = load i32, ptr null, align 8
+  %1 = callbr { i8, i32 } asm "", "={@ccz},=r,1,!i"(i32 %0)
+          to label %asm.fallthrough [label %efaultu64.split]
+
+asm.fallthrough:
+  %asmresult = extractvalue { i8, i32 } %1, 0
+  %asmresult1 = extractvalue { i8, i32 } %1, 1
+  ret i32 %asmresult1
+
+efaultu64.split:
+  %2 = call { i8, i32 } @llvm.callbr.landingpad.sl_i8i32s({ i8, i32 } %1)
+  %asmresult2 = extractvalue { i8, i32 } %2, 0
+  %asmresult3 = extractvalue { i8, i32 } %2, 1
+  ret i32 %asmresult3
+}
+
+define i64 @multireg() {
+  ; CHECK-LABEL: name: multireg
+  ; CHECK: bb.0.entry:
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000), %bb.2(0x00000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   INLINEASM_BR &"", 0 /* attdialect */, 18 /* regdef */, implicit-def $eax, implicit-def $edx, 13 /* imm */, %bb.2
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:gr32 = COPY $eax
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:gr32 = COPY $edx
+  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:gr32 = COPY [[COPY1]]
+  ; CHECK-NEXT:   [[COPY3:%[0-9]+]]:gr32 = COPY [[COPY]]
+  ; CHECK-NEXT:   JMP_1 %bb.1
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1.ft:
+  ; CHECK-NEXT:   $eax = COPY [[COPY3]]
+  ; CHECK-NEXT:   $edx = COPY [[COPY2]]
+  ; CHECK-NEXT:   RET 0, $eax, $edx
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2.split (machine-block-address-taken, inlineasm-br-indirect-target):
+  ; CHECK-NEXT:   liveins: $eax, $edx
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:gr32 = COPY $eax
+  ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:gr32 = COPY $edx
+  ; CHECK-NEXT:   $eax = COPY [[COPY4]]
+  ; CHECK-NEXT:   $edx = COPY [[COPY5]]
+  ; CHECK-NEXT:   RET 0, $eax, $edx
+entry:
+  %0 = callbr i64 asm "", "=A,!i"() to label %ft [label %split]
+ft:
+  ret i64 %0
+split:
+  %1 = call i64 @llvm.callbr.landingpad.i64(i64 %0)
+  ret i64 %1
+}
+declare i64 @llvm.callbr.landingpad.i64(i64)
+declare { i8, i32 } @llvm.callbr.landingpad.sl_i8i32s({ i8, i32 })

llvm/test/CodeGen/X86/callbr-asm-outputs-indirect-isel.ll

entry:
   %direct = callbr i32 asm "# $0", "=r,!i"()
           to label %cleanup [label %cleanup]
 
 cleanup:
   %indirect = call i32 @llvm.callbr.landingpad.i32(i32 %direct)
   ret i32 %indirect
 }
 
+define i64 @condition_code() {
+  ; CHECK-LABEL: name: condition_code
+  ; CHECK: bb.0 (%ir-block.0):
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000), %bb.2(0x00000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   INLINEASM_BR &"", 16 /* maystore attdialect */, 2359306 /* regdef:GR32 */, def %1, 13 /* imm */, %bb.2
+  ; CHECK-NEXT:   [[SETCCr:%[0-9]+]]:gr8 = SETCCr 4, implicit $eflags
+  ; CHECK-NEXT:   [[MOVZX32rr8_:%[0-9]+]]:gr32 = MOVZX32rr8 killed [[SETCCr]]
+  ; CHECK-NEXT:   [[SUBREG_TO_REG:%[0-9]+]]:gr64 = SUBREG_TO_REG 0, killed [[MOVZX32rr8_]], %subreg.sub_32bit
+  ; CHECK-NEXT:   JMP_1 %bb.1
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1.b:
+  ; CHECK-NEXT:   $rax = COPY [[SUBREG_TO_REG]]
+  ; CHECK-NEXT:   RET 0, $rax
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2.c (machine-block-address-taken, inlineasm-br-indirect-target):
+  ; CHECK-NEXT:   [[SETCCr1:%[0-9]+]]:gr8 = SETCCr 4, implicit $eflags
+  ; CHECK-NEXT:   [[MOVZX32rr8_1:%[0-9]+]]:gr32 = MOVZX32rr8 killed [[SETCCr1]]
+  ; CHECK-NEXT:   [[SUBREG_TO_REG1:%[0-9]+]]:gr64 = SUBREG_TO_REG 0, killed [[MOVZX32rr8_1]], %subreg.sub_32bit
+  ; CHECK-NEXT:   $rax = COPY [[SUBREG_TO_REG1]]
+  ; CHECK-NEXT:   RET 0, $rax
+  %a = callbr i64 asm "", "={@ccz},!i"()
+       to label %b [label %c]
+
+b:
+  ret i64 %a
+
+c:
+  %1 = call i64 @llvm.callbr.landingpad.i64(i64 %a)
+  ret i64 %1
+}
+
 declare i64 @llvm.callbr.landingpad.i64(i64)
 declare i32 @llvm.callbr.landingpad.i32(i32)
 declare { i32, i32 } @llvm.callbr.landingpad.sl_i32i32s({ i32, i32 })