Index: llvm/trunk/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
===================================================================
--- llvm/trunk/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
+++ llvm/trunk/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
@@ -815,6 +815,10 @@
 class AMDGPUAsmParser : public MCTargetAsmParser {
   MCAsmParser &Parser;
 
+  // Number of extra operands parsed after the first optional operand.
+  // This may be necessary to skip hardcoded mandatory operands.
+  static const unsigned MAX_OPR_LOOKAHEAD = 1;
+
   unsigned ForcedEncodingSize = 0;
   bool ForcedDPP = false;
   bool ForcedSDWA = false;
@@ -1037,6 +1041,7 @@
 
 public:
   OperandMatchResultTy parseOptionalOperand(OperandVector &Operands);
+  OperandMatchResultTy parseOptionalOpr(OperandVector &Operands);
 
   OperandMatchResultTy parseExpTgt(OperandVector &Operands);
   OperandMatchResultTy parseSendMsgOp(OperandVector &Operands);
@@ -3859,7 +3864,7 @@
   } else {
     // Swizzle "offset" operand is optional.
     // If it is omitted, try parsing other optional operands.
-    return parseOptionalOperand(Operands);
+    return parseOptionalOpr(Operands);
   }
 }
 
@@ -4179,6 +4184,39 @@
 };
 
 OperandMatchResultTy AMDGPUAsmParser::parseOptionalOperand(OperandVector &Operands) {
+  unsigned size = Operands.size();
+  assert(size > 0);
+
+  OperandMatchResultTy res = parseOptionalOpr(Operands);
+
+  // This is a hack to enable hardcoded mandatory operands which follow
+  // optional operands.
+  //
+  // Current design assumes that all operands after the first optional operand
+  // are also optional. However implementation of some instructions violates
+  // this rule (see e.g. flat/global atomic which have hardcoded 'glc' operands).
+  //
+  // To alleviate this problem, we have to (implicitly) parse extra operands
+  // to make sure autogenerated parser of custom operands never hit hardcoded
+  // mandatory operands.
+
+  if (size == 1 || ((AMDGPUOperand &)*Operands[size - 1]).isRegKind()) {
+
+    // We have parsed the first optional operand.
+    // Parse as many operands as necessary to skip all mandatory operands.
+
+    for (unsigned i = 0; i < MAX_OPR_LOOKAHEAD; ++i) {
+      if (res != MatchOperand_Success ||
+          getLexer().is(AsmToken::EndOfStatement)) break;
+      if (getLexer().is(AsmToken::Comma)) Parser.Lex();
+      res = parseOptionalOpr(Operands);
+    }
+  }
+
+  return res;
+}
+
+OperandMatchResultTy AMDGPUAsmParser::parseOptionalOpr(OperandVector &Operands) {
   OperandMatchResultTy res;
   for (const OptionalOperand &Op : AMDGPUOptionalOperandTable) {
     // try to parse any optional operand here
Index: llvm/trunk/test/MC/AMDGPU/flat-gfx9.s
===================================================================
--- llvm/trunk/test/MC/AMDGPU/flat-gfx9.s
+++ llvm/trunk/test/MC/AMDGPU/flat-gfx9.s
@@ -35,6 +35,54 @@
 // GFX9: flat_atomic_add v[3:4], v5 offset:8 slc ; encoding: [0x08,0x00,0x0a,0xdd,0x03,0x05,0x00,0x00]
 // VIERR: :1: error: invalid operand for instruction
 
+flat_atomic_cmpswap v[1:2], v[3:4] offset:4095
+// GFX9: flat_atomic_cmpswap v[1:2], v[3:4] offset:4095 ; encoding: [0xff,0x0f,0x04,0xdd,0x01,0x03,0x00,0x00]
+// VIERR: :1: error: invalid operand for instruction
+
+flat_atomic_cmpswap v[1:2], v[3:4] offset:4095 slc
+// GFX9: flat_atomic_cmpswap v[1:2], v[3:4] offset:4095 slc ; encoding: [0xff,0x0f,0x06,0xdd,0x01,0x03,0x00,0x00]
+// VIERR: :1: error: invalid operand for instruction
+
+flat_atomic_cmpswap v[1:2], v[3:4]
+// GFX9: flat_atomic_cmpswap v[1:2], v[3:4] ; encoding: [0x00,0x00,0x04,0xdd,0x01,0x03,0x00,0x00]
+// VI:   flat_atomic_cmpswap v[1:2], v[3:4] ; encoding: [0x00,0x00,0x04,0xdd,0x01,0x03,0x00,0x00]
+
+flat_atomic_cmpswap v[1:2], v[3:4] slc
+// GFX9: flat_atomic_cmpswap v[1:2], v[3:4] slc ; encoding: [0x00,0x00,0x06,0xdd,0x01,0x03,0x00,0x00]
+// VI:   flat_atomic_cmpswap v[1:2], v[3:4] slc ; encoding: [0x00,0x00,0x06,0xdd,0x01,0x03,0x00,0x00]
+
+flat_atomic_cmpswap v[1:2], v[3:4] offset:4095 glc
+// GCNERR: error: invalid operand for instruction
+
+flat_atomic_cmpswap v[1:2], v[3:4] glc
+// GCNERR: error: invalid operand for instruction
+
+flat_atomic_cmpswap v0, v[1:2], v[3:4] offset:4095 glc
+// GFX9: flat_atomic_cmpswap v0, v[1:2], v[3:4] offset:4095 glc ; encoding: [0xff,0x0f,0x05,0xdd,0x01,0x03,0x00,0x00]
+// VIERR: :1: error: invalid operand for instruction
+
+flat_atomic_cmpswap v0, v[1:2], v[3:4] offset:4095 glc slc
+// GFX9: flat_atomic_cmpswap v0, v[1:2], v[3:4] offset:4095 glc slc ; encoding: [0xff,0x0f,0x07,0xdd,0x01,0x03,0x00,0x00]
+// VIERR: :1: error: invalid operand for instruction
+
+flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
+// GFX9: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc ; encoding: [0x00,0x00,0x05,0xdd,0x01,0x03,0x00,0x00]
+// VI:   flat_atomic_cmpswap v0, v[1:2], v[3:4] glc ; encoding: [0x00,0x00,0x05,0xdd,0x01,0x03,0x00,0x00]
+
+flat_atomic_cmpswap v0, v[1:2], v[3:4] glc slc
+// GFX9: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc slc ; encoding: [0x00,0x00,0x07,0xdd,0x01,0x03,0x00,0x00]
+// VI:   flat_atomic_cmpswap v0, v[1:2], v[3:4] glc slc ; encoding: [0x00,0x00,0x07,0xdd,0x01,0x03,0x00,0x00]
+
+flat_atomic_cmpswap v0, v[1:2], v[3:4]
+// GFX9: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc ; encoding: [0x00,0x00,0x05,0xdd,0x01,0x03,0x00,0x00]
+// VI:   flat_atomic_cmpswap v0, v[1:2], v[3:4] glc ; encoding: [0x00,0x00,0x05,0xdd,0x01,0x03,0x00,0x00]
+
+flat_atomic_cmpswap v0, v[1:2], v[3:4] offset:4095
+// GCNERR: error: too few operands for instruction
+
+flat_atomic_cmpswap v0, v[1:2], v[3:4] slc
+// GCNERR: error: invalid operand for instruction
+
 flat_atomic_swap v[3:4], v5 offset:16
 // GFX9: flat_atomic_swap v[3:4], v5 offset:16 ; encoding: [0x10,0x00,0x00,0xdd,0x03,0x05,0x00,0x00]
 // VIERR: :1: error: invalid operand for instruction
Index: llvm/trunk/test/MC/Disassembler/AMDGPU/flat_gfx9.txt
===================================================================
--- llvm/trunk/test/MC/Disassembler/AMDGPU/flat_gfx9.txt
+++ llvm/trunk/test/MC/Disassembler/AMDGPU/flat_gfx9.txt
@@ -9,6 +9,18 @@
 # CHECK: flat_atomic_add v0, v[0:1], v0 offset:4095 glc    ; encoding: [0xff,0x0f,0x09,0xdd,0x00,0x00,0x00,0x00]
 0xff,0x0f,0x09,0xdd,0x00,0x00,0x00,0x00
 
+# CHECK: flat_atomic_add v0, v[0:1], v0 offset:4095 glc slc ; encoding: [0xff,0x0f,0x0b,0xdd,0x00,0x00,0x00,0x00]
+0xff,0x0f,0x0b,0xdd,0x00,0x00,0x00,0x00
+
+# CHECK: flat_atomic_add v0, v[0:1], v0 glc ; encoding: [0x00,0x00,0x09,0xdd,0x00,0x00,0x00,0x00]
+0x00,0x00,0x09,0xdd,0x00,0x00,0x00,0x00
+
+# CHECK: flat_atomic_add v0, v[0:1], v0 glc slc ; encoding: [0x00,0x00,0x0b,0xdd,0x00,0x00,0x00,0x00]
+0x00,0x00,0x0b,0xdd,0x00,0x00,0x00,0x00
+
+# CHECK: flat_atomic_add v[0:1], v0 slc  ; encoding: [0x00,0x00,0x0a,0xdd,0x00,0x00,0x00,0x00]
+0x00,0x00,0x0a,0xdd,0x00,0x00,0x00,0x00
+
 # CHECK: flat_atomic_add v[0:1], v0 offset:4095 slc    ; encoding: [0xff,0x0f,0x0a,0xdd,0x00,0x00,0x00,0x00]
 0xff,0x0f,0x0a,0xdd,0x00,0x00,0x00,0x00