[llvm-rc] Add integer expressions parsing ability. [7/8]

[mnbvmar]

Summary

This allows the ints to be written as integer expressions evaluating to unsigned 16-bit/32-bit integers.

All the expressions may use the following operators: + - & | ~, and parentheses. Minus token - can be also unary. There is no precedence of the operators other than the unary operators binding stronger than their binary counterparts.

[ecbeckmann]

This looks fine to me.

[mnbvmar]

[Testing] Change temporary files into pipes.

[rnk]

lgtm

[rnk]

llvm/tools/llvm-rc/ResourceScriptParser.cpp

121

These comments are great, thanks!

llvm/test/tools/llvm-rc/Inputs/parser-expr-bad-binary-1.rc

+LANGUAGE 0, &

llvm/test/tools/llvm-rc/Inputs/parser-expr-bad-binary-2.rc

+LANGUAGE 3||0, 0

llvm/test/tools/llvm-rc/Inputs/parser-expr-bad-binary-3.rc

+LANGUAGE 3+-+3, 0

llvm/test/tools/llvm-rc/Inputs/parser-expr-bad-unary.rc

+LANGUAGE 1~1, 0

llvm/test/tools/llvm-rc/Inputs/parser-expr-unbalanced-1.rc

+LANGUAGE (1+2, 0

llvm/test/tools/llvm-rc/Inputs/parser-expr-unbalanced-2.rc

+LANGUAGE 1+2)+3+4(, 0

llvm/test/tools/llvm-rc/Inputs/parser-expr-unbalanced-3.rc

+LANGUAGE (1+2+3)), 0

llvm/test/tools/llvm-rc/Inputs/parser-expr.rc

+LANGUAGE 3 + 2, 3 - 2
+LANGUAGE 3 | 2, 3 & 2
+LANGUAGE -3, ~3
+LANGUAGE 1|1&0, 0&0|1
+LANGUAGE 3+4-5, 3-4+5
+LANGUAGE 1+2|3, 3|1+2
+LANGUAGE 6&~5, 6&-8
+LANGUAGE -1, --1
+LANGUAGE ----1, -----1
+LANGUAGE ~1, ~~1
+LANGUAGE ~~~~1, ~~~~~1
+LANGUAGE 5-(1+2), 1|(1&0)
+LANGUAGE ~(3-7), -(3+~7)
+LANGUAGE 0, (1+3)|(2+11)
+LANGUAGE (((((((5))))))), (((((((7)))))))

llvm/test/tools/llvm-rc/parser-expr.test

+; RUN: llvm-rc /V %p/Inputs/parser-expr.rc | FileCheck %s
+
+; CHECK:  Language: 5, Sublanguage: 1
+; CHECK-NEXT:  Language: 3, Sublanguage: 2
+; CHECK-NEXT:  Language: 4294967293, Sublanguage: 4294967292
+; CHECK-NEXT:  Language: 0, Sublanguage: 1
+; CHECK-NEXT:  Language: 2, Sublanguage: 4
+; CHECK-NEXT:  Language: 3, Sublanguage: 5
+; CHECK-NEXT:  Language: 2, Sublanguage: 0
+; CHECK-NEXT:  Language: 4294967295, Sublanguage: 1
+; CHECK-NEXT:  Language: 1, Sublanguage: 4294967295
+; CHECK-NEXT:  Language: 4294967294, Sublanguage: 1
+; CHECK-NEXT:  Language: 1, Sublanguage: 4294967294
+; CHECK-NEXT:  Language: 2, Sublanguage: 1
+; CHECK-NEXT:  Language: 3, Sublanguage: 5
+; CHECK-NEXT:  Language: 0, Sublanguage: 13
+; CHECK-NEXT:  Language: 5, Sublanguage: 7
+
+
+; RUN: not llvm-rc /V %p/Inputs/parser-expr-bad-binary-1.rc 2> %t.b1
+; RUN: FileCheck %s --check-prefix BINARY1 --input-file %t.b1
+
+; BINARY1: llvm-rc: Error parsing file: expected '-', '~', integer or '(', got &
+
+
+; RUN: not llvm-rc /V %p/Inputs/parser-expr-bad-binary-2.rc 2> %t.b2
+; RUN: FileCheck %s --check-prefix BINARY2 --input-file %t.b2
+
+; BINARY2: llvm-rc: Error parsing file: expected '-', '~', integer or '(', got |
+
+
+; RUN: not llvm-rc /V %p/Inputs/parser-expr-bad-binary-3.rc 2> %t.b3
+; RUN: FileCheck %s --check-prefix BINARY3 --input-file %t.b3
+
+; BINARY3: llvm-rc: Error parsing file: expected '-', '~', integer or '(', got +
+
+
+; RUN: not llvm-rc /V %p/Inputs/parser-expr-bad-unary.rc 2> %t.u
+; RUN: FileCheck %s --check-prefix UNARY --input-file %t.u
+
+; UNARY: llvm-rc: Error parsing file: expected ',', got ~
+
+
+; RUN: not llvm-rc /V %p/Inputs/parser-expr-unbalanced-1.rc 2> %t.p1
+; RUN: FileCheck %s --check-prefix UNBALANCED1 --input-file %t.p1
+
+; UNBALANCED1: llvm-rc: Error parsing file: expected ')', got ,
+
+
+; RUN: not llvm-rc /V %p/Inputs/parser-expr-unbalanced-2.rc 2> %t.p2
+; RUN: FileCheck %s --check-prefix UNBALANCED2 --input-file %t.p2
+
+; UNBALANCED2: llvm-rc: Error parsing file: expected ',', got )
+
+
+; RUN: not llvm-rc /V %p/Inputs/parser-expr-unbalanced-3.rc 2> %t.p3
+; RUN: FileCheck %s --check-prefix UNBALANCED3 --input-file %t.p3
+
+; UNBALANCED3: llvm-rc: Error parsing file: expected ',', got )

llvm/test/tools/llvm-rc/parser.test

 ; RUN: FileCheck %s --check-prefix PSTRINGTABLE2 --input-file %t3
 
 ; PSTRINGTABLE2:  llvm-rc: Error parsing file: expected optional statement type, BEGIN or '{', got NONSENSETYPE
 
 
 ; RUN: not llvm-rc /V %p/Inputs/parser-eof.rc 2> %t4
 ; RUN: FileCheck %s --check-prefix PEOF --input-file %t4
 
-; PEOF:  llvm-rc: Error parsing file: expected integer, got <EOF>
+; PEOF:  llvm-rc: Error parsing file: expected '-', '~', integer or '(', got <EOF>
 
 
 ; RUN: not llvm-rc /V %p/Inputs/parser-no-characteristics-arg.rc 2> %t5
 ; RUN: FileCheck %s --check-prefix PCHARACTERISTICS1 --input-file %t5
 
-; PCHARACTERISTICS1:  llvm-rc: Error parsing file: expected integer, got BEGIN
+; PCHARACTERISTICS1:  llvm-rc: Error parsing file: expected '-', '~', integer or '(', got BEGIN
 
 
 ; RUN: not llvm-rc /V %p/Inputs/parser-nonsense-token.rc 2> %t6
 ; RUN: FileCheck %s --check-prefix PNONSENSE1 --input-file %t6
 
 ; PNONSENSE1:  llvm-rc: Error parsing file: expected int or identifier, got &
 
 

 ; RUN: FileCheck %s --check-prefix PLANGUAGE1 --input-file %t9
 
 ; PLANGUAGE1:  llvm-rc: Error parsing file: expected ',', got 7
 
 
 ; RUN: not llvm-rc /V %p/Inputs/parser-language-too-many-commas.rc 2> %t10
 ; RUN: FileCheck %s --check-prefix PLANGUAGE2 --input-file %t10
 
-; PLANGUAGE2:  llvm-rc: Error parsing file: expected integer, got ,
+; PLANGUAGE2:  llvm-rc: Error parsing file: expected '-', '~', integer or '(', got ,
 
 
 ; RUN: not llvm-rc /V %p/Inputs/parser-html-bad-string.rc 2> %t11
 ; RUN: FileCheck %s --check-prefix PHTML1 --input-file %t11
 
 ; PHTML1:  llvm-rc: Error parsing file: expected string, got ThisPassesInTheOriginalToolButDocSaysItShouldBeQuoted
 
 
 ; RUN: not llvm-rc /V %p/Inputs/parser-html-extra-comma.rc 2> %t12
 ; RUN: FileCheck %s --check-prefix PHTML2 --input-file %t12
 
 ; PHTML2:  llvm-rc: Error parsing file: expected string, got ,
 
 
 ; RUN: not llvm-rc /V %p/Inputs/parser-menu-bad-id.rc 2> %t13
 ; RUN: FileCheck %s --check-prefix PMENU1 --input-file %t13
 
-; PMENU1:  llvm-rc: Error parsing file: expected integer, got A
+; PMENU1:  llvm-rc: Error parsing file: expected '-', '~', integer or '(', got A
 
 
 ; RUN: not llvm-rc /V %p/Inputs/parser-menu-bad-flag.rc 2> %t14
 ; RUN: FileCheck %s --check-prefix PMENU2 --input-file %t14
 
 ; PMENU2:  llvm-rc: Error parsing file: expected CHECKED/GRAYED/HELP/INACTIVE/MENUBARBREAK/MENUBREAK, got ERRONEOUS
 
 

 ; RUN: FileCheck %s --check-prefix PDIALOG4 --input-file %t20
 
 ; PDIALOG4:  llvm-rc: Error parsing file: expected control type, END or '}', got UNKNOWN
 
 
 ; RUN: not llvm-rc /V %p/Inputs/parser-dialog-unnecessary-string.rc 2> %t21
 ; RUN: FileCheck %s --check-prefix PDIALOG5 --input-file %t21
 
-; PDIALOG5:  llvm-rc: Error parsing file: expected integer, got "This shouldn't be here"
+; PDIALOG5:  llvm-rc: Error parsing file: expected '-', '~', integer or '(', got "This shouldn't be here"
 
 
 ; RUN: not llvm-rc /V %p/Inputs/parser-versioninfo-wrong-fixed.rc 2> %t22
 ; RUN: FileCheck %s --check-prefix PVERSIONINFO1 --input-file %t22
 
 ; PVERSIONINFO1:  llvm-rc: Error parsing file: expected fixed VERSIONINFO statement type, got WEIRDFIXED
 
 

llvm/tools/llvm-rc/ResourceScriptParser.h

private:
   const RCToken &look() const;
   // Read the current token and advance the state by one token.
   const RCToken &read();
   // Advance the state by one token, discarding the current token.
   void consume();
 
   // The following methods try to read a single token, check if it has the
   // correct type and then parse it.
+  // Each integer can be written as an arithmetic expression producing an
+  // unsigned 32-bit integer.
   Expected<uint32_t> readInt();            // Parse an integer.
   Expected<StringRef> readString();        // Parse a string.
   Expected<StringRef> readIdentifier();    // Parse an identifier.
   Expected<IntOrString> readIntOrString(); // Parse an integer or a string.
   Expected<IntOrString> readTypeOrName();  // Parse an integer or an identifier.
 
+  // Helper integer expression parsing methods.
+  Expected<uint32_t> parseIntExpr1();
+  Expected<uint32_t> parseIntExpr2();
+
   // Advance the state by one, discarding the current token.
   // If the discarded token had an incorrect type, fail.
   Error consumeType(Kind TokenKind);
 
   // Check the current token type. If it's TokenKind, discard it.
   // Return true if the parser consumed this token successfully.
   bool consumeOptionalType(Kind TokenKind);
 

llvm/tools/llvm-rc/ResourceScriptParser.cpp

const RCToken &RCParser::read() {
   return *CurLoc++;
 }
 
 void RCParser::consume() {
   assert(!isEof());
   CurLoc++;
 }
 
-Expected<uint32_t> RCParser::readInt() {
-  if (!isNextTokenKind(Kind::Int))
-    return getExpectedError("integer");
+// An integer description might consist of a single integer or
+// an arithmetic expression evaluating to the integer. The expressions
+// can contain the following tokens: <int> ( ) + - | & ~. Their meaning
+// is the same as in C++.
+// The operators in the original RC implementation have the following
+// precedence:
+//   1) Unary operators (- ~),
+//   2) Binary operators (+ - & |), with no precedence.
+//
+// The following grammar is used to parse the expressions Exp1:
+//   Exp1 ::= Exp2 || Exp1 + Exp2 || Exp1 - Exp2 || Exp1 | Exp2 || Exp1 & Exp2
+//   Exp2 ::= -Exp2 || ~Exp2 || Int || (Exp1).
+// (More conveniently, Exp1 is a non-empty sequence of Exp2 expressions,
+// separated by binary operators.)
+//
+// Expressions of type Exp1 are read by parseIntExpr1(Inner) method, while Exp2
+// is read by parseIntExpr2().
+//
+// The original Microsoft tool handles multiple unary operators incorrectly.
+// For example, in 16-bit little-endian integers:
+//    1 => 01 00, -1 => ff ff, --1 => ff ff, ---1 => 01 00;
+//    1 => 01 00, ~1 => fe ff, ~~1 => fd ff, ~~~1 => fc ff.
+// Our implementation differs from the original one and handles these
+// operators correctly:
+//    1 => 01 00, -1 => ff ff, --1 => 01 00, ---1 => ff ff;
+//    1 => 01 00, ~1 => fe ff, ~~1 => 01 00, ~~~1 => fe ff.
+
+Expected<uint32_t> RCParser::readInt() { return parseIntExpr1(); }
+
+Expected<uint32_t> RCParser::parseIntExpr1() {
+  // Exp1 ::= Exp2 || Exp1 + Exp2 || Exp1 - Exp2 || Exp1 | Exp2 || Exp1 & Exp2.
+  ASSIGN_OR_RETURN(FirstResult, parseIntExpr2());
+  uint32_t Result = *FirstResult;
+
+  while (!isEof() && look().isBinaryOp()) {
+    auto OpToken = read();
+    ASSIGN_OR_RETURN(NextResult, parseIntExpr2());
+
+    switch (OpToken.kind()) {
+    case Kind::Plus:
+      Result += *NextResult;
+      break;
+
+    case Kind::Minus:
+      Result -= *NextResult;
+      break;
+
+    case Kind::Pipe:
+      Result |= *NextResult;
+      break;
+
+    case Kind::Amp:
+      Result &= *NextResult;
+      break;
+
+    default:
+      llvm_unreachable("Already processed all binary ops.");
+    }
+  }
+
+  return Result;
+}
+
+Expected<uint32_t> RCParser::parseIntExpr2() {
+  // Exp2 ::= -Exp2 || ~Exp2 || Int || (Exp1).
+  static const char ErrorMsg[] = "'-', '~', integer or '('";
+
+  if (isEof())
+    return getExpectedError(ErrorMsg);
+
+  switch (look().kind()) {
+  case Kind::Minus: {
+    consume();
+    ASSIGN_OR_RETURN(Result, parseIntExpr2());
+    return -(*Result);
+  }
+
+  case Kind::Tilde: {
+    consume();
+    ASSIGN_OR_RETURN(Result, parseIntExpr2());
+    return ~(*Result);
+  }
+
+  case Kind::Int:
     return read().intValue();
+
+  case Kind::LeftParen: {
+    consume();
+    ASSIGN_OR_RETURN(Result, parseIntExpr1());
+    RETURN_IF_ERROR(consumeType(Kind::RightParen));
+    return *Result;
+  }
+
+  default:
+    return getExpectedError(ErrorMsg);
+  }
 }
 
 Expected<StringRef> RCParser::readString() {
   if (!isNextTokenKind(Kind::String))
     return getExpectedError("string");
   return read().value();
 }
 

llvm/tools/llvm-rc/ResourceScriptToken.h

#undef SHORT_TOKEN
   RCToken(RCToken::Kind RCTokenKind, StringRef Value);
 
   // Get an integer value of the integer token.
   uint32_t intValue() const;
 
   StringRef value() const;
   Kind kind() const;
 
+  // Check if a token describes a binary operator.
+  bool isBinaryOp() const;
+
 private:
   Kind TokenKind;
   StringRef TokenValue;
 };
 
 // Tokenize Input.
 // In case no error occured, the return value contains
 //   tokens in order they were in the input file.

llvm/tools/llvm-rc/ResourceScriptToken.cpp

uint32_t RCToken::intValue() const {
   (void)IsSuccess;  // Silence the compiler warning when -DNDEBUG flag is on.
   return Result;
 }
 
 StringRef RCToken::value() const { return TokenValue; }
 
 Kind RCToken::kind() const { return TokenKind; }
 
+bool RCToken::isBinaryOp() const {
+  switch (TokenKind) {
+  case Kind::Plus:
+  case Kind::Minus:
+  case Kind::Pipe:
+  case Kind::Amp:
+    return true;
+  default:
+    return false;
+  }
+}
+
 static Error getStringError(const Twine &message) {
   return make_error<StringError>("Error parsing file: " + message,
                                  inconvertibleErrorCode());
 }
 
 namespace {
 
 class Tokenizer {