[PowerPC] Don't crash when disassembling invalid immediate

[nemanjai]

Summary

There is an assert in the disassembler functions to ensure that the immediate is the appropriate width. However, sometimes what is being disassembled is not instructions but data that happens to have the bit pattern of an existing instruction but invalid operands. It is valid for such things to exist in the text section so we don't want to crash when disassembling such a thing.

This patch removes the asserts and produces a disassembler failure for such cases.

[barannikov88]

AFAIK values >12 are all invalid, not only those that are >15. This instruction and the one which inserts elements need custom decoder for their operands. Other changes are unnecessary.
But let's see what PPC guys say.

[nemanjai]

In D140245#4002642, @barannikov88 wrote:

AFAIK values >12 are all invalid, not only those that are >15. This instruction and the one which inserts elements need custom decoder for their operands. Other changes are unnecessary.
But let's see what PPC guys say.

Thank you for your comment. However, I must clarify. Values larger than 12 produce undefined results, but they are not invalid. The instruction xxinsertw vs1, vs2, 14 is not an illegal instruction - although it does produce undefined results.

[rzurob]

LGTM

[nemanjai]

Committed in https://reviews.llvm.org/rGc86f8d4276ae

llvm/lib/Target/PowerPC/Disassembler/PPCDisassembler.cpp

 
 #define DecodeQSRCRegisterClass DecodeQFRCRegisterClass
 #define DecodeQBRCRegisterClass DecodeQFRCRegisterClass
 
 template <unsigned N>
 static DecodeStatus decodeUImmOperand(MCInst &Inst, uint64_t Imm,
                                       int64_t Address,
                                       const MCDisassembler *Decoder) {
-  assert(isUInt<N>(Imm) && "Invalid immediate");
+  if (!isUInt<N>(Imm))
+    return MCDisassembler::Fail;
   Inst.addOperand(MCOperand::createImm(Imm));
   return MCDisassembler::Success;
 }
 
 template <unsigned N>
 static DecodeStatus decodeSImmOperand(MCInst &Inst, uint64_t Imm,
                                       int64_t Address,
                                       const MCDisassembler *Decoder) {
-  assert(isUInt<N>(Imm) && "Invalid immediate");
+  if (!isUInt<N>(Imm))
+    return MCDisassembler::Fail;
   Inst.addOperand(MCOperand::createImm(SignExtend64<N>(Imm)));
   return MCDisassembler::Success;
 }
 
 static DecodeStatus decodeImmZeroOperand(MCInst &Inst, uint64_t Imm,
                                          int64_t Address,
                                          const MCDisassembler *Decoder) {
   if (Imm != 0)

llvm/lib/Target/PowerPC/PPCRegisterInfo.td

 
 def PPCU1ImmAsmOperand : AsmOperandClass {
   let Name = "U1Imm"; let PredicateMethod = "isU1Imm";
   let RenderMethod = "addImmOperands";
 }
 def u1imm   : Operand<i32> {
   let PrintMethod = "printU1ImmOperand";
   let ParserMatchClass = PPCU1ImmAsmOperand;
+  let DecoderMethod = "decodeUImmOperand<1>";
   let OperandType = "OPERAND_IMMEDIATE";
 }
 
 def PPCU2ImmAsmOperand : AsmOperandClass {
   let Name = "U2Imm"; let PredicateMethod = "isU2Imm";
   let RenderMethod = "addImmOperands";
 }
 def u2imm   : Operand<i32> {
   let PrintMethod = "printU2ImmOperand";
   let ParserMatchClass = PPCU2ImmAsmOperand;
+  let DecoderMethod = "decodeUImmOperand<2>";
   let OperandType = "OPERAND_IMMEDIATE";
 }
 
 def PPCATBitsAsHintAsmOperand : AsmOperandClass {
   let Name = "ATBitsAsHint"; let PredicateMethod = "isATBitsAsHint";
   let RenderMethod = "addImmOperands"; // Irrelevant, predicate always fails.
 }
 def atimm   : Operand<i32> {
   let PrintMethod = "printATBitsAsHint";
   let ParserMatchClass = PPCATBitsAsHintAsmOperand;
   let OperandType = "OPERAND_IMMEDIATE";
 }
 
 def PPCU3ImmAsmOperand : AsmOperandClass {
   let Name = "U3Imm"; let PredicateMethod = "isU3Imm";
   let RenderMethod = "addImmOperands";
 }
 def u3imm   : Operand<i32> {
   let PrintMethod = "printU3ImmOperand";
   let ParserMatchClass = PPCU3ImmAsmOperand;
+  let DecoderMethod = "decodeUImmOperand<3>";
   let OperandType = "OPERAND_IMMEDIATE";
 }
 
 def PPCU4ImmAsmOperand : AsmOperandClass {
   let Name = "U4Imm"; let PredicateMethod = "isU4Imm";
   let RenderMethod = "addImmOperands";
 }
 def u4imm   : Operand<i32> {
   let PrintMethod = "printU4ImmOperand";
   let ParserMatchClass = PPCU4ImmAsmOperand;
+  let DecoderMethod = "decodeUImmOperand<4>";
   let OperandType = "OPERAND_IMMEDIATE";
 }
 def PPCS5ImmAsmOperand : AsmOperandClass {
   let Name = "S5Imm"; let PredicateMethod = "isS5Imm";
   let RenderMethod = "addImmOperands";
 }
 def s5imm   : Operand<i32> {
   let PrintMethod = "printS5ImmOperand";

llvm/test/MC/Disassembler/PowerPC/ppc64-encoding-ISA31-invalid.txt

 
 # pstxssp 1, -8589934592(3), 1. However, RA is not zero with R=1
 # CHECK: warning: invalid instruction encoding
 0x04 0x12 0x00 0x00 0xbc 0x23 0x00 0x00
 
 # pstxv 31, 8589934591(3), 1. However, RA is not zero with R=1
 # CHECK: warning: invalid instruction encoding
 0x04 0x11 0xff 0xff 0xdb 0xe3 0xff 0xff
+
+# xxextractuw 52, 30, 20 (i.e. the immediate 20 is invalid)
+# CHECK: warning: invalid instruction encoding
+0xf2 0x94 0xf2 0x95