diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
--- a/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
@@ -970,10 +970,6 @@
       const uint64_t OrigOffset = Layout.getFragmentOffset(&F);
 #endif
       const uint64_t OrigSize = Asm.computeFragmentSize(Layout, F);
-      if (OrigSize == 0 || Relaxable.empty()) {
-        Relaxable.clear();
-        continue;
-      }
 
       // To keep the effects local, prefer to relax instructions closest to
       // the align directive.  This is purely about human understandability
diff --git a/llvm/test/MC/X86/align-branch-64-basic.s b/llvm/test/MC/X86/align-branch-64-basic.s
--- a/llvm/test/MC/X86/align-branch-64-basic.s
+++ b/llvm/test/MC/X86/align-branch-64-basic.s
@@ -103,32 +103,14 @@
 bar:
   retq
 
-  # CHECK: <test_pad_via_relax>:
-  # CHECK: 200: testq
-  # CHECK: 203: jne
-  # CHECK: 209: int3
-  # note 6 byte jne which could be a 2 byte jne, but is instead
-  # expanded for padding purposes
-  # CHECK-NOT: nop
-  # CHECK: 220: callq
-  .global test_pad_via_relax
-  .p2align  5
-test_pad_via_relax:
-  testq %rax, %rax
-  jnz bar
-  .rept 23
-  int3
-  .endr
-  callq bar
-
   # This case looks really tempting to pad, but doing so for the call causes
   # the jmp to be misaligned.
   # CHECK: <test_pad_via_relax_neg1>:
-  # CHECK: 240: int3
-  # CHECK: 25a: testq
-  # CHECK: 25d: jne
-  # CHECK: 25f: nop
-  # CHECK: 260: callq
+  # CHECK: 200: int3
+  # CHECK: 21a: testq
+  # CHECK: 21d: jne
+  # CHECK: 21f: nop
+  # CHECK: 220: callq
   .global test_pad_via_relax_neg1
   .p2align  5
 test_pad_via_relax_neg1:
@@ -141,10 +123,10 @@
 
   # Same as previous, but without fusion
   # CHECK: <test_pad_via_relax_neg2>:
-  # CHECK: 280: int3
-  # CHECK: 29d: jmp
-  # CHECK: 29f: nop
-  # CHECK: 2a0: callq
+  # CHECK: 240: int3
+  # CHECK: 25d: jmp
+  # CHECK: 25f: nop
+  # CHECK: 260: callq
   .global test_pad_via_relax_neg2
   .p2align  5
 test_pad_via_relax_neg2:
@@ -154,7 +136,7 @@
   jmp bar2
   callq bar2
 
-bar2: 
+bar2:
 
   .section "unknown"
   .p2align 4
diff --git a/llvm/test/MC/X86/align-branch-64-pad-max-prefix.s b/llvm/test/MC/X86/align-branch-64-pad-max-prefix.s
new file mode 100644
--- /dev/null
+++ b/llvm/test/MC/X86/align-branch-64-pad-max-prefix.s
@@ -0,0 +1,18 @@
+  # RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu --x86-align-branch-boundary=32 --x86-align-branch=jmp -x86-pad-max-prefix-size=5 %s | llvm-objdump -d --no-show-raw-insn - | FileCheck %s
+  # Check instructions can be aligned correctly along with option -x86-pad-max-prefix-size=5
+
+  .text
+  .p2align 5
+  .rept 24
+  int3
+  .endr
+  # We should not increase the length of this jmp to reduce the bytes of
+  # following nops, doing so would make the jmp misaligned.
+# CHECK:      18:          jmp
+  jmp bar
+# CHECK:      1d:          nop
+# CHECK:      1e:          nop
+# CHECK:      1f:          nop
+# CHECK:      20:          int3
+  .p2align 5
+  int3