[X86] Disable nop padding before instruction following a prefix

[LuoYuanke]

LGTM

[skan]

Wrap X86II::isPrefix in a static function to hide some details

llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp

static bool isRIPRelative(const MCInst &MI, const MCInstrInfo &MCII) {
   int MemoryOperand = X86II::getMemoryOperandNo(TSFlags);
   if (MemoryOperand < 0)
     return false;
   unsigned BaseRegNum = MemoryOperand + CurOp + X86::AddrBaseReg;
   unsigned BaseReg = MI.getOperand(BaseRegNum).getReg();
   return (BaseReg == X86::RIP);
 }
 
+/// Check if the instruction is a prefix.
+static bool isPrefix(const MCInst &MI, const MCInstrInfo &MCII) {
+  return X86II::isPrefix(MCII.get(MI.getOpcode()).TSFlags);
+}
+
 /// Check if the instruction is valid as the first instruction in macro fusion.
 static bool isFirstMacroFusibleInst(const MCInst &Inst,
                                     const MCInstrInfo &MCII) {
   // An Intel instruction with RIP relative addressing is not macro fusible.
   if (isRIPRelative(Inst, MCII))
     return false;
   X86::FirstMacroFusionInstKind FIK =
       X86::classifyFirstOpcodeInMacroFusion(Inst.getOpcode());
void X86AsmBackend::alignBranchesBegin(MCObjectStreamer &OS,
   if (!needAlign(OS))
     return;
 
   if (hasInterruptDelaySlot(PrevInst))
     // If this instruction follows an interrupt enabling instruction with a one
     // instruction delay, inserting a nop would change behavior.
     return;
 
+  if (isPrefix(PrevInst, *MCII))
+    // If this instruction follows a prefix, inserting a nop would change
+    // semantic.
+    return;
+
   if (!isMacroFused(PrevInst, Inst))
     // Macro fusion doesn't happen indeed, clear the pending.
     PendingBoundaryAlign = nullptr;
 
   if (PendingBoundaryAlign &&
       OS.getCurrentFragment()->getPrevNode() == PendingBoundaryAlign) {
     // Macro fusion actually happens and there is no other fragment inserted
     // after the previous instruction.

llvm/test/MC/X86/align-branch-64-prefix.s

+  # RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu --x86-align-branch-boundary=32 --x86-align-branch=jmp+call %s | llvm-objdump -d --no-show-raw-insn - | FileCheck %s
+
+  # Exercise cases where prefixes are specified for instructions to be aligned
+  # and thus can't add a nop in between without changing semantic.
+
+  .text
+
+  # CHECK: 1d:       int3
+  # CHECK: 1e:       jmp
+  # CHECK: 24:       int3
+  .p2align  5
+  .rept 30
+  int3
+  .endr
+  CS
+  jmp baz
+  int3
+
+  # CHECK: 5d:       int3
+  # CHECK: 5e:       jmp
+  # CHECK: 64:       int3
+  .p2align  5
+  .rept 30
+  int3
+  .endr
+  GS
+  jmp baz
+  int3
+
+  # CHECK: 9d:       int3
+  # CHECK: 9e:       call
+  # CHECK: a6:       int3
+  .p2align  5
+  .rept 30
+  int3
+  .endr
+  data16
+  call *___tls_get_addr@GOT(%ecx)
+  int3
+
+  # CHECK: de:       lock
+  # CHECK: df:       jmp
+  # CHECK: e4:       int3
+  .p2align  5
+  .rept 30
+  int3
+  .endr
+  lock
+  jmp baz
+  int3
+
+  # CHECK: 11d:       int3
+  # CHECK: 11e:       jmp
+  # CHECK: 124:       int3
+  .p2align  5
+  .rept 30
+  int3
+  .endr
+  rex64
+  jmp baz
+  int3
+
+  # CHECK: 15d:      int3
+  # CHECK: 15e:      {{.*}} jmp
+  # CHECK: 164:      int3
+  .p2align  5
+  .rept 30
+  int3
+  .endr
+  xacquire
+  jmp baz
+  int3
+
+  .section ".text.other"
+bar:
+  retq