diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
--- a/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
@@ -566,6 +566,10 @@
     return false;
   assert(allowAutoPadding() && "incorrect initialization!");
 
+  // We only pad in text section.
+  if (!OS.getCurrentSectionOnly()->getKind().isText())
+    return false;
+
   // To be Done: Currently don't deal with Bundle cases.
   if (OS.getAssembler().isBundlingEnabled())
     return false;
diff --git a/llvm/test/MC/X86/align-branch-section-type.s b/llvm/test/MC/X86/align-branch-section-type.s
new file mode 100644
--- /dev/null
+++ b/llvm/test/MC/X86/align-branch-section-type.s
@@ -0,0 +1,18 @@
+# RUN: llvm-mc -filetype=obj -triple x86_64 --x86-align-branch-boundary=32 --x86-align-branch=ret %s | llvm-readobj -S | FileCheck %s
+
+  # Check we only pad in a text section
+
+  # CHECK:  Name: text
+  # CHECK:  AddressAlignment: 32
+  .section text, "ax"
+  ret
+
+  # CHECK:  Name: excluded
+  # CHECK:  AddressAlignment: 1
+  .section excluded, "e"
+  ret
+
+  # CHECK:  Name: tls
+  # CHECK:  AddressAlignment: 1
+  .section tls, "awT"
+  ret