[Bash-autocompletion] Pass all flags in shell command-line to Clang

[yamaguchi]

Summary

Previously, we passed "#" to --autocomplete to indicate to enable cc1
flags. For example, when -cc1 or -Xclang was passed to bash, bash
executed clang --autocomplete=#-<flag they want to complete>.

However, this was not a good implementation because it depends -Xclang
and -cc1 parsing to shell. So I changed this to pass all flags shell
has, so that Clang can handle them internally.

I had to change many testcases because API spec changed quite a lot.

[v.g.vassilev]

clang/lib/Driver/Driver.cpp

1171 ↗

(On Diff #120474)

Why not a while loop?

[yamaguchi]

clang/lib/Driver/Driver.cpp

1171 ↗

(On Diff #120474)

I didn't have special reason, I'll change it to while if it fits better.

[yamaguchi]

Update diff. Pass all flags just with "," including values.

[yamaguchi]

Made a trivial fix in the testcase

[teemperor]

As this will land in clang 7, can you add a note to the release notes that this breaks backwards compatibility for this interface (can be another commit)? Also, we should point out that this is the only API change this interface will have in the foreseeable future.

cfe/trunk/include/clang/Driver/Driver.h

public:
   /// GetProgramPath - Lookup \p Name in the list of program search paths.
   ///
   /// \param TC - The provided tool chain for additional information on
   /// directories to search.
   //
   // FIXME: This should be in CompilationInfo.
   std::string GetProgramPath(StringRef Name, const ToolChain &TC) const;
 
-  /// handleAutocompletions - Handle --autocomplete by searching and printing
+  /// HandleAutocompletions - Handle --autocomplete by searching and printing
   /// possible flags, descriptions, and its arguments.
-  void handleAutocompletions(StringRef PassedFlags) const;
+  void HandleAutocompletions(StringRef PassedFlags) const;
 
   /// HandleImmediateArgs - Handle any arguments which should be
   /// treated before building actions or binding tools.
   ///
   /// \return Whether any compilation should be built for this
   /// invocation.
   bool HandleImmediateArgs(const Compilation &C);
 

cfe/trunk/lib/Driver/Driver.cpp

 /// option.
 static void PrintDiagnosticCategories(raw_ostream &OS) {
   // Skip the empty category.
   for (unsigned i = 1, max = DiagnosticIDs::getNumberOfCategories(); i != max;
        ++i)
     OS << i << ',' << DiagnosticIDs::getCategoryNameFromID(i) << '\n';
 }
 
-void Driver::handleAutocompletions(StringRef PassedFlags) const {
+void Driver::HandleAutocompletions(StringRef PassedFlags) const {
+  if (PassedFlags == "") return;
   // Print out all options that start with a given argument. This is used for
   // shell autocompletion.
   std::vector<std::string> SuggestedCompletions;
+  std::vector<std::string> Flags;
 
   unsigned short DisableFlags =
       options::NoDriverOption | options::Unsupported | options::Ignored;
-  // We want to show cc1-only options only when clang is invoked as "clang
-  // -cc1". When clang is invoked as "clang -cc1", we add "#" to the beginning
-  // of an --autocomplete  option so that the clang driver can distinguish
-  // whether it is requested to show cc1-only options or not.
-  if (PassedFlags.size() > 0 && PassedFlags[0] == '#') {
+
+  // Parse PassedFlags by "," as all the command-line flags are passed to this
+  // function separated by ","
+  StringRef TargetFlags = PassedFlags;
+  while (TargetFlags != "") {
+    StringRef CurFlag;
+    std::tie(CurFlag, TargetFlags) = TargetFlags.split(",");
+    Flags.push_back(std::string(CurFlag));
+  }
+
+  // We want to show cc1-only options only when clang is invoked with -cc1 or
+  // -Xclang.
+  if (std::find(Flags.begin(), Flags.end(), "-Xclang") != Flags.end() || std::find(Flags.begin(), Flags.end(), "-cc1") != Flags.end())
     DisableFlags &= ~options::NoDriverOption;
-    PassedFlags = PassedFlags.substr(1);
+
+  StringRef Cur;
+  Cur = Flags.at(Flags.size() - 1);
+  StringRef Prev;
+  if (Flags.size() >= 2) {
+    Prev = Flags.at(Flags.size() - 2);
+    SuggestedCompletions = Opts->suggestValueCompletions(Prev, Cur);
   }
 
-  if (PassedFlags.find(',') == StringRef::npos) {
+  if (SuggestedCompletions.empty())
+    SuggestedCompletions = Opts->suggestValueCompletions(Cur, "");
+
+  if (SuggestedCompletions.empty()) {
     // If the flag is in the form of "--autocomplete=-foo",
     // we were requested to print out all option names that start with "-foo".
     // For example, "--autocomplete=-fsyn" is expanded to "-fsyntax-only".
-    SuggestedCompletions = Opts->findByPrefix(PassedFlags, DisableFlags);
+    SuggestedCompletions = Opts->findByPrefix(Cur, DisableFlags);
 
     // We have to query the -W flags manually as they're not in the OptTable.
     // TODO: Find a good way to add them to OptTable instead and them remove
     // this code.
     for (StringRef S : DiagnosticIDs::getDiagnosticFlags())
-      if (S.startswith(PassedFlags))
+      if (S.startswith(Cur))
         SuggestedCompletions.push_back(S);
-  } else {
-    // If the flag is in the form of "--autocomplete=foo,bar", we were
-    // requested to print out all option values for "-foo" that start with
-    // "bar". For example,
-    // "--autocomplete=-stdlib=,l" is expanded to "libc++" and "libstdc++".
-    StringRef Option, Arg;
-    std::tie(Option, Arg) = PassedFlags.split(',');
-    SuggestedCompletions = Opts->suggestValueCompletions(Option, Arg);
   }
 
+
   // Sort the autocomplete candidates so that shells print them out in a
   // deterministic order. We could sort in any way, but we chose
   // case-insensitive sorting for consistency with the -help option
   // which prints out options in the case-insensitive alphabetical order.
   std::sort(SuggestedCompletions.begin(), SuggestedCompletions.end(),
             [](StringRef A, StringRef B) {
               if (int X = A.compare_lower(B))
                 return X < 0;
bool Driver::HandleImmediateArgs(const Compilation &C) {
 
   if (Arg *A = C.getArgs().getLastArg(options::OPT_print_prog_name_EQ)) {
     llvm::outs() << GetProgramPath(A->getValue(), TC) << "\n";
     return false;
   }
 
   if (Arg *A = C.getArgs().getLastArg(options::OPT_autocomplete)) {
     StringRef PassedFlags = A->getValue();
-    handleAutocompletions(PassedFlags);
+    HandleAutocompletions(PassedFlags);
     return false;
   }
 
   if (C.getArgs().hasArg(options::OPT_print_libgcc_file_name)) {
     ToolChain::RuntimeLibType RLT = TC.GetRuntimeLibType(C.getArgs());
     const llvm::Triple Triple(TC.ComputeEffectiveClangTriple(C.getArgs()));
     RegisterEffectiveTriple TripleRAII(TC, Triple);
     switch (RLT) {

cfe/trunk/test/Driver/autocomplete.c

 // Test for the --autocompletion flag, which is an API used for shell
 // autocompletion. You may have to update tests in this file when you
 // add/modify flags, change HelpTexts or the values of some flags.
 
 // Some corner cases.
-// RUN: %clang --autocomplete= | FileCheck %s -check-prefix=ALL_FLAGS
-// RUN: %clang --autocomplete=# | FileCheck %s -check-prefix=ALL_FLAGS
-// Let's pick some example flags that are hopefully unlikely to change.
-// ALL_FLAGS: -fast
-// ALL_FLAGS: -fastcp
-// ALL_FLAGS: -fastf
 // Just test that this doesn't crash:
+// RUN: %clang --autocomplete=
 // RUN: %clang --autocomplete=,
 // RUN: %clang --autocomplete==
 // RUN: %clang --autocomplete=,,
 // RUN: %clang --autocomplete=-
 
 // RUN: %clang --autocomplete=-fsyn | FileCheck %s -check-prefix=FSYN
 // FSYN: -fsyntax-only
-// RUN: %clang --autocomplete=-std= | FileCheck %s -check-prefix=STD
+// RUN: %clang --autocomplete=-std | FileCheck %s -check-prefix=STD
 // STD: -std= Language standard to compile for
 // RUN: %clang --autocomplete=foo | FileCheck %s -check-prefix=FOO
 // FOO-NOT: foo
 // RUN: %clang --autocomplete=-stdlib=,l | FileCheck %s -check-prefix=STDLIB
 // STDLIB: libc++
 // STDLIB-NEXT: libstdc++
-// RUN: %clang --autocomplete=-stdlib=, | FileCheck %s -check-prefix=STDLIBALL
+// RUN: %clang --autocomplete=-stdlib= | FileCheck %s -check-prefix=STDLIBALL
 // STDLIBALL: libc++
 // STDLIBALL-NEXT: libstdc++
 // STDLIBALL-NEXT: platform
 // RUN: %clang --autocomplete=-meabi,d | FileCheck %s -check-prefix=MEABI
 // MEABI: default
-// RUN: %clang --autocomplete=-meabi, | FileCheck %s -check-prefix=MEABIALL
+// RUN: %clang --autocomplete=-meabi | FileCheck %s -check-prefix=MEABIALL
 // MEABIALL: 4
 // MEABIALL-NEXT: 5
 // MEABIALL-NEXT: default
 // MEABIALL-NEXT: gnu
 // RUN: %clang --autocomplete=-cl-std=,CL2 | FileCheck %s -check-prefix=CLSTD
 // CLSTD: CL2.0
-// RUN: %clang --autocomplete=-cl-std=, | FileCheck %s -check-prefix=CLSTDALL
+// RUN: %clang --autocomplete=-cl-std= | FileCheck %s -check-prefix=CLSTDALL
 // CLSTDALL: cl
 // CLSTDALL-NEXT: CL
 // CLSTDALL-NEXT: cl1.1
 // CLSTDALL-NEXT: CL1.1
 // CLSTDALL-NEXT: cl1.2
 // CLSTDALL-NEXT: CL1.2
 // CLSTDALL-NEXT: cl2.0
 // CLSTDALL-NEXT: CL2.0
 // RUN: %clang --autocomplete=-fno-sanitize-coverage=,f | FileCheck %s -check-prefix=FNOSANICOVER
 // FNOSANICOVER: func
-// RUN: %clang --autocomplete=-fno-sanitize-coverage=, | FileCheck %s -check-prefix=FNOSANICOVERALL
+// RUN: %clang --autocomplete=-fno-sanitize-coverage= | FileCheck %s -check-prefix=FNOSANICOVERALL
 // FNOSANICOVERALL: 8bit-counters
 // FNOSANICOVERALL-NEXT: bb
 // FNOSANICOVERALL-NEXT: edge
 // FNOSANICOVERALL-NEXT: func
 // FNOSANICOVERALL-NEXT: indirect-calls
 // FNOSANICOVERALL-NEXT: inline-8bit-counters
 // FNOSANICOVERALL-NEXT: no-prune
 // FNOSANICOVERALL-NEXT: trace-bb
 // FNOSANICOVERALL-NEXT: trace-cmp
 // FNOSANICOVERALL-NEXT: trace-div
 // FNOSANICOVERALL-NEXT: trace-gep
 // FNOSANICOVERALL-NEXT: trace-pc
 // FNOSANICOVERALL-NEXT: trace-pc-guard
-// RUN: %clang --autocomplete=-ffp-contract=, | FileCheck %s -check-prefix=FFPALL
+// RUN: %clang --autocomplete=-ffp-contract= | FileCheck %s -check-prefix=FFPALL
 // FFPALL: fast
 // FFPALL-NEXT: off
 // FFPALL-NEXT: on
-// RUN: %clang --autocomplete=-flto=, | FileCheck %s -check-prefix=FLTOALL
+// RUN: %clang --autocomplete=-flto= | FileCheck %s -check-prefix=FLTOALL
 // FLTOALL: full
 // FLTOALL-NEXT: thin
-// RUN: %clang --autocomplete=-fveclib=, | FileCheck %s -check-prefix=FVECLIBALL
+// RUN: %clang --autocomplete=-fveclib= | FileCheck %s -check-prefix=FVECLIBALL
 // FVECLIBALL: Accelerate
 // FVECLIBALL-NEXT: none
 // FVECLIBALL-NEXT: SVML
-// RUN: %clang --autocomplete=-fshow-overloads=, | FileCheck %s -check-prefix=FSOVERALL
+// RUN: %clang --autocomplete=-fshow-overloads= | FileCheck %s -check-prefix=FSOVERALL
 // FSOVERALL: all
 // FSOVERALL-NEXT: best
-// RUN: %clang --autocomplete=-fvisibility=, | FileCheck %s -check-prefix=FVISIBILITYALL
+// RUN: %clang --autocomplete=-fvisibility= | FileCheck %s -check-prefix=FVISIBILITYALL
 // FVISIBILITYALL: default
 // FVISIBILITYALL-NEXT: hidden
-// RUN: %clang --autocomplete=-mfloat-abi=, | FileCheck %s -check-prefix=MFLOATABIALL
+// RUN: %clang --autocomplete=-mfloat-abi= | FileCheck %s -check-prefix=MFLOATABIALL
 // MFLOATABIALL: hard
 // MFLOATABIALL-NEXT: soft
 // MFLOATABIALL-NEXT: softfp
-// RUN: %clang --autocomplete=-mthread-model, | FileCheck %s -check-prefix=MTHREADMODELALL
+// RUN: %clang --autocomplete=-mthread-model | FileCheck %s -check-prefix=MTHREADMODELALL
 // MTHREADMODELALL: posix
 // MTHREADMODELALL-NEXT: single
-// RUN: %clang --autocomplete=-mrelocation-model, | FileCheck %s -check-prefix=MRELOCMODELALL
+// RUN: %clang --autocomplete=-mrelocation-model | FileCheck %s -check-prefix=MRELOCMODELALL
 // MRELOCMODELALL: dynamic-no-pic
 // MRELOCMODELALL-NEXT: pic
 // MRELOCMODELALL-NEXT: ropi
 // MRELOCMODELALL-NEXT: ropi-rwpi
 // MRELOCMODELALL-NEXT: rwpi
 // MRELOCMODELALL-NEXT: static
-// RUN: %clang --autocomplete=-mrelocation-mode | FileCheck %s -check-prefix=MRELOCMODEL_CLANG
-// MRELOCMODEL_CLANG-NOT: -mrelocation-model
-// RUN: %clang --autocomplete=#-mrelocation-mode | FileCheck %s -check-prefix=MRELOCMODEL_CC1
-// MRELOCMODEL_CC1: -mrelocation-model
 // RUN: %clang --autocomplete=-Wma | FileCheck %s -check-prefix=WARNING
 // WARNING: -Wmacro-redefined
 // WARNING-NEXT: -Wmain
 // WARNING-NEXT: -Wmain-return-type
 // WARNING-NEXT: -Wmalformed-warning-check
 // WARNING-NEXT: -Wmany-braces-around-scalar-init
 // WARNING-NEXT: -Wmax-unsigned-zero
 // RUN: %clang --autocomplete=-Wno-invalid-pp- | FileCheck %s -check-prefix=NOWARNING
 // NOWARNING: -Wno-invalid-pp-token
-// RUN: %clang --autocomplete=-analyzer-checker, | FileCheck %s -check-prefix=ANALYZER
+// RUN: %clang --autocomplete=-analyzer-checker | FileCheck %s -check-prefix=ANALYZER
 // ANALYZER: unix.Malloc
-// RUN: %clang --autocomplete=-std=, | FileCheck %s -check-prefix=STDVAL
+// RUN: %clang --autocomplete=-std= | FileCheck %s -check-prefix=STDVAL
 // STDVAL: c99
+//
+// Clang shouldn't autocomplete CC1 options unless -cc1 or -Xclang were provided
+// RUN: %clang --autocomplete=-mrelocation-mode | FileCheck %s -check-prefix=MRELOCMODEL_CLANG
+// MRELOCMODEL_CLANG-NOT: -mrelocation-model
+// RUN: %clang --autocomplete=-Xclang,-mrelocation-mode | FileCheck %s -check-prefix=MRELOCMODEL_CC1
+// RUN: %clang --autocomplete=-cc1,-mrelocation-mode | FileCheck %s -check-prefix=MRELOCMODEL_CC1
+// MRELOCMODEL_CC1: -mrelocation-model
+// Make sure it ignores passed flags unlesss they are -Xclang or -cc1
+// RUN: %clang --autocomplete=foo,bar,,-fsyn | FileCheck %s -check-prefix=FSYN-CORON
+// FSYN-CORON: -fsyntax-only
+// Check if they can autocomplete values with coron
+// RUN: %clang --autocomplete=foo,bar,,,-fno-sanitize-coverage=,f | FileCheck %s -check-prefix=FNOSANICOVER-CORON
+// FNOSANICOVER-CORON: func

cfe/trunk/utils/bash-autocomplete.sh

if [[ "$?" != 0 ]]; then
     cur="${COMP_WORDS[$cword]}"
   fi
 
   w1="${COMP_WORDS[$cword - 1]}"
   if [[ $cword > 1 ]]; then
     w2="${COMP_WORDS[$cword - 2]}"
   fi
 
-  # Clang want to know if -cc1 or -Xclang option is specified or not, because we don't want to show
-  # cc1 options otherwise.
-  if [[ "${COMP_WORDS[1]}" == "-cc1" || "$w1" == "-Xclang" ]]; then
-    arg="#"
-  fi
-
-  # bash always separates '=' as a token even if there's no space before/after '='.
-  # On the other hand, '=' is just a regular character for clang options that
-  # contain '='. For example, "-stdlib=" is defined as is, instead of "-stdlib" and "=".
-  # So, we need to partially undo bash tokenization here for integrity.
-  if [[ "$cur" == -* ]]; then
-    # -foo<tab>
-    arg="$arg$cur"
-  elif [[ "$w1" == -*  && "$cur" == '=' ]]; then
-    # -foo=<tab>
-    arg="$arg$w1=,"
-  elif [[ "$cur" == -*= ]]; then
-    # -foo=<tab>
-    arg="$arg$cur,"
-  elif [[ "$w1" == -* ]]; then
-    # -foo <tab> or -foo bar<tab>
-    arg="$arg$w1,$cur"
-  elif [[ "$w2" == -* && "$w1" == '=' ]]; then
-    # -foo=bar<tab>
-    arg="$arg$w2=,$cur"
-  elif [[ ${cur: -1} != '=' && ${cur/=} != $cur ]]; then
-    # -foo=bar<tab>
-    arg="$arg${cur%=*}=,${cur#*=}"
+  # Pass all the current command-line flags to clang, so that clang can handle
+  # these internally.
+  # '=' is separated differently by bash, so we have to concat them without ','
+  for i in `seq 1 $cword`; do
+    if [[ $i == $cword || "${COMP_WORDS[$(($i+1))]}" == '=' ]]; then
+      arg="$arg${COMP_WORDS[$i]}"
+    else
+      arg="$arg${COMP_WORDS[$i]},"
     fi
+  done
 
   # expand ~ to $HOME
   eval local path=${COMP_WORDS[0]}
   flags=$( "$path" --autocomplete="$arg" 2>/dev/null | sed -e 's/\t.*//' )
   # If clang is old that it does not support --autocomplete,
   # fall back to the filename completion.
   if [[ "$?" != 0 ]]; then
     _clang_filedir
     return
   fi
 
   # When clang does not emit any possible autocompletion, or user pushed tab after " ",
   # just autocomplete files.
-  if [[ "$flags" == "$(echo -e '\n')" || "$arg" == "" ]]; then
+  if [[ "$flags" == "$(echo -e '\n')" ]]; then
     # If -foo=<tab> and there was no possible values, autocomplete files.
     [[ "$cur" == '=' || "$cur" == -*= ]] && cur=""
     _clang_filedir
   elif [[ "$cur" == '=' ]]; then
     COMPREPLY=( $( compgen -W "$flags" -- "") )
   else
     # Bash automatically appends a space after '=' by default.
     # Disable it so that it works nicely for options in the form of -foo=bar.
     [[ "${flags: -1}" == '=' ]] && compopt -o nospace 2> /dev/null
     COMPREPLY=( $( compgen -W "$flags" -- "$cur" ) )
   fi
 }
 complete -F _clang clang

llvm/trunk/lib/Option/OptTable.cpp

for (size_t I = FirstSearchableIndex, E = OptionInfos.size(); I < E; I++) {
     if (!In.Values || !optionMatches(In, Option))
       continue;
 
     SmallVector<StringRef, 8> Candidates;
     StringRef(In.Values).split(Candidates, ",", -1, false);
 
     std::vector<std::string> Result;
     for (StringRef Val : Candidates)
-      if (Val.startswith(Arg))
+      if (Val.startswith(Arg) && Arg.compare(Val))
         Result.push_back(Val);
     return Result;
   }
   return {};
 }
 
 std::vector<std::string>
 OptTable::findByPrefix(StringRef Cur, unsigned short DisableFlags) const {
   std::vector<std::string> Ret;
   for (size_t I = FirstSearchableIndex, E = OptionInfos.size(); I < E; I++) {
     const Info &In = OptionInfos[I];
     if (!In.Prefixes || (!In.HelpText && !In.GroupID))
       continue;
     if (In.Flags & DisableFlags)
       continue;
 
     for (int I = 0; In.Prefixes[I]; I++) {
       std::string S = std::string(In.Prefixes[I]) + std::string(In.Name) + "\t";
       if (In.HelpText)
         S += In.HelpText;
-      if (StringRef(S).startswith(Cur))
+      if (StringRef(S).startswith(Cur) && S.compare(std::string(Cur) + "\t"))
         Ret.push_back(S);
     }
   }
   return Ret;
 }
 
 unsigned OptTable::findNearest(StringRef Option, std::string &NearestString,
                                unsigned FlagsToInclude, unsigned FlagsToExclude,