diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
--- a/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
@@ -970,10 +970,6 @@
       const uint64_t OrigOffset = Layout.getFragmentOffset(&F);
 #endif
       const uint64_t OrigSize = Asm.computeFragmentSize(Layout, F);
-      if (OrigSize == 0 || Relaxable.empty()) {
-        Relaxable.clear();
-        continue;
-      }
 
       // To keep the effects local, prefer to relax instructions closest to
       // the align directive.  This is purely about human understandability
diff --git a/llvm/test/MC/X86/align-branch-64-pad-max-prefix.s b/llvm/test/MC/X86/align-branch-64-pad-max-prefix.s
new file mode 100644
--- /dev/null
+++ b/llvm/test/MC/X86/align-branch-64-pad-max-prefix.s
@@ -0,0 +1,18 @@
+  # RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu --x86-align-branch-boundary=32 --x86-align-branch=jmp -x86-pad-max-prefix-size=5 %s | llvm-objdump -d --no-show-raw-insn - | FileCheck %s
+  # Check instructions can be aligned correctly along with option -x86-pad-max-prefix-size=5
+
+  .text
+  .p2align 5
+  .rept 24
+  int3
+  .endr
+  # We should not increase the length of this jmp to reduce the bytes of
+  # following nops, doing so would make the jmp misaligned.
+# CHECK:      18:          jmp
+  jmp bar
+# CHECK:      1d:          nop
+# CHECK:      1e:          nop
+# CHECK:      1f:          nop
+# CHECK:      20:          int3
+  .p2align 5
+  int3