diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
--- a/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
@@ -566,6 +566,10 @@
     return false;
   assert(allowAutoPadding() && "incorrect initialization!");
 
+  // We only pad in text section.
+  if (!OS.getCurrentSectionOnly()->getKind().isText())
+    return false;
+
   // To be Done: Currently don't deal with Bundle cases.
   if (OS.getAssembler().isBundlingEnabled())
     return false;
diff --git a/llvm/test/MC/X86/align-branch-section-type.s b/llvm/test/MC/X86/align-branch-section-type.s
new file mode 100644
--- /dev/null
+++ b/llvm/test/MC/X86/align-branch-section-type.s
@@ -0,0 +1,18 @@
+# RUN: llvm-mc -filetype=obj -triple x86_64 --x86-align-branch-boundary=32 --x86-align-branch=ret %s | llvm-readobj -S | FileCheck %s
+
+## Check we only pad in a text section
+
+# CHECK-LABEL:  Name: text
+# CHECK:        AddressAlignment: 32
+.section text, "ax"
+ret
+
+# CHECK-LABEL:  Name: excluded
+# CHECK:        AddressAlignment: 1
+.section excluded, "e"
+ret
+
+# CHECK-LABEL:  Name: tls
+# CHECK:        AddressAlignment: 1
+.section tls, "awT"
+ret