diff --git a/clang-tools-extra/clangd/CMakeLists.txt b/clang-tools-extra/clangd/CMakeLists.txt --- a/clang-tools-extra/clangd/CMakeLists.txt +++ b/clang-tools-extra/clangd/CMakeLists.txt @@ -44,7 +44,11 @@ ) include(${CMAKE_CURRENT_SOURCE_DIR}/quality/CompletionModel.cmake) -gen_decision_forest(${CMAKE_CURRENT_SOURCE_DIR}/quality/model CompletionModel clang::clangd::Example) +gen_decision_forest(${CMAKE_CURRENT_SOURCE_DIR}/quality/model + CompletionModel + clang::clangd::Example + CompletionModel_header_file + CompletionModel_cpp_files) if(MSVC AND NOT CLANG_CL) set_source_files_properties(CompileCommands.cpp PROPERTIES COMPILE_FLAGS -wd4130) # disables C4130: logical operation on address of string constant @@ -102,7 +106,7 @@ TUScheduler.cpp URI.cpp XRefs.cpp - ${CMAKE_CURRENT_BINARY_DIR}/CompletionModel.cpp + ${CompletionModel_cpp_files} index/Background.cpp index/BackgroundIndexLoader.cpp @@ -140,9 +144,9 @@ omp_gen ) -# Include generated CompletionModel headers. +# Include generated Completion Model header. target_include_directories(clangDaemon PUBLIC - $ + $ ) clang_target_link_libraries(clangDaemon diff --git a/clang-tools-extra/clangd/quality/CompletionModel.cmake b/clang-tools-extra/clangd/quality/CompletionModel.cmake --- a/clang-tools-extra/clangd/quality/CompletionModel.cmake +++ b/clang-tools-extra/clangd/quality/CompletionModel.cmake @@ -1,18 +1,33 @@ -# Run the Completion Model Codegenerator on the model present in the +# Run the Completion Model Codegenerator on the model present in the # ${model} directory. -# Produces a pair of files called ${filename}.h and ${filename}.cpp in the -# ${CMAKE_CURRENT_BINARY_DIR}. The generated header +# Produces files called ${filename}.h and ${filename}*.cpp in the +# ${CMAKE_CURRENT_BINARY_DIR}/${filename} directory. The generated header # will define a C++ class called ${cpp_class} - which may be a # namespace-qualified class name. +# Sets variables referenced by ${output_header_file_variable} and +# ${output_cpp_files_variable}. set(CLANGD_COMPLETION_MODEL_COMPILER ${CMAKE_CURRENT_LIST_DIR}/CompletionModelCodegen.py) -function(gen_decision_forest model filename cpp_class) +function(gen_decision_forest model filename cpp_class + output_header_file_variable output_cpp_files_variable) set(model_compiler ${CLANGD_COMPLETION_MODEL_COMPILER}) - set(output_dir ${CMAKE_CURRENT_BINARY_DIR}) + set(output_dir ${CMAKE_CURRENT_BINARY_DIR}/${filename}) set(header_file ${output_dir}/${filename}.h) - set(cpp_file ${output_dir}/${filename}.cpp) - add_custom_command(OUTPUT ${header_file} ${cpp_file} + # Command for initial (CMake-invocation-time) generation of files. + message(STATUS "Generating code completion model runtime") + execute_process(COMMAND "${Python3_EXECUTABLE}" ${model_compiler} + --model ${model} + --output_dir ${output_dir} + --filename ${filename} + --cpp_class ${cpp_class}) + + file(GLOB cpp_files "${output_dir}/${filename}*.cpp") + + # Command for potential regeneration of files after changes in model compiler, + # forest.json or features.json. + # This works reliably only if list of generated files is unchanged. + add_custom_command(OUTPUT ${header_file} ${cpp_files} COMMAND "${Python3_EXECUTABLE}" ${model_compiler} --model ${model} --output_dir ${output_dir} @@ -20,19 +35,22 @@ --cpp_class ${cpp_class} COMMENT "Generating code completion model runtime..." DEPENDS ${model_compiler} ${model}/forest.json ${model}/features.json - VERBATIM ) + VERBATIM) set_source_files_properties(${header_file} PROPERTIES GENERATED 1) - set_source_files_properties(${cpp_file} PROPERTIES + set_source_files_properties(${cpp_files} PROPERTIES GENERATED 1) # Disable unused label warning for generated files. if (CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") - set_source_files_properties(${cpp_file} PROPERTIES + set_source_files_properties(${cpp_files} PROPERTIES COMPILE_FLAGS /wd4102) else() - set_source_files_properties(${cpp_file} PROPERTIES + set_source_files_properties(${cpp_files} PROPERTIES COMPILE_FLAGS -Wno-unused) endif() + + set(${output_header_file_variable} ${header_file} PARENT_SCOPE) + set(${output_cpp_files_variable} ${cpp_files} PARENT_SCOPE) endfunction() diff --git a/clang-tools-extra/clangd/quality/CompletionModelCodegen.py b/clang-tools-extra/clangd/quality/CompletionModelCodegen.py --- a/clang-tools-extra/clangd/quality/CompletionModelCodegen.py +++ b/clang-tools-extra/clangd/quality/CompletionModelCodegen.py @@ -1,13 +1,15 @@ """Code generator for Code Completion Model Inference. Tool runs on the Decision Forest model defined in {model} directory. -It generates two files: {output_dir}/{filename}.h and {output_dir}/{filename}.cpp -The generated files defines the Example class named {cpp_class} having all the features as class members. +It generates files: {output_dir}/{filename}.h, {output_dir}/{filename}.cpp, +and {output_dir}/{filename}{number}.cpp for each Decision Tree. +The generated files define the Example class named {cpp_class} having all the features as class members. The generated runtime provides an `Evaluate` function which can be used to score a code completion candidate. """ import argparse import json +import os import struct @@ -194,34 +196,36 @@ `float Evaluate(const {Example}&)` function. This function can be used to score an Example.""" - code = "" + functions_codes = {} # Generate evaluation function of each tree. - code += "namespace {\n" tree_num = 0 for tree_json in forest_json: - code += "LLVM_ATTRIBUTE_NOINLINE float EvaluateTree%d(const %s& E) {\n" % (tree_num, cpp_class.name) - code += " " + \ + functions_codes[f"{tree_num}"] = "LLVM_ATTRIBUTE_NOINLINE float EvaluateTree%d(const %s& E) {\n" % (tree_num, cpp_class.name) + functions_codes[f"{tree_num}"] += " " + \ "\n ".join( tree(tree_json, tree_num=tree_num, node_num=0)[0]) + "\n" - code += "}\n\n" + functions_codes[f"{tree_num}"] += "}\n" tree_num += 1 - code += "} // namespace\n\n" # Combine the scores of all trees in the final function. # MSAN will timeout if these functions are inlined. - code += "float Evaluate(const %s& E) {\n" % cpp_class.name - code += " float Score = 0;\n" + final_function_code = "" for tree_num in range(len(forest_json)): - code += " Score += EvaluateTree%d(E);\n" % tree_num - code += " return Score;\n" - code += "}\n" + final_function_code += "float EvaluateTree%d(const %s& E);\n" % (tree_num, cpp_class.name) + final_function_code += "\n" + final_function_code += "float Evaluate(const %s& E) {\n" % cpp_class.name + final_function_code += " float Score = 0;\n" + for tree_num in range(len(forest_json)): + final_function_code += " Score += EvaluateTree%d(E);\n" % tree_num + final_function_code += " return Score;\n" + final_function_code += "}\n" - return code + return functions_codes, final_function_code def gen_cpp_code(forest_json, features_json, filename, cpp_class): - """Generates code for the .cpp file.""" + """Generates code for the .cpp files.""" # Headers # Required by OrderEncode(float F). angled_include = [ @@ -242,11 +246,11 @@ for feature in features_json if feature["kind"] == "ENUM") nl = "\n" - return """%s -%s + functions_codes, final_function_code = evaluate_func(forest_json, cpp_class) -#define BIT(X) (1LL << X) + cpp_code = { + "": """%s %s @@ -268,8 +272,23 @@ %s %s """ % (nl.join(angled_include), nl.join(quoted_include), cpp_class.ns_begin(), - using_decls, cpp_class.name, evaluate_func(forest_json, cpp_class), - cpp_class.ns_end()) + cpp_class.name, final_function_code, cpp_class.ns_end())} + + for function_code_num, function_code in functions_codes.items(): + cpp_code[function_code_num] = """%s + +#define BIT(X) (1LL << X) + +%s + +%s + +%s +%s +""" % (nl.join(quoted_include), cpp_class.ns_begin(), using_decls, + function_code, cpp_class.ns_end()) + + return cpp_code def main(): @@ -286,7 +305,6 @@ output_dir = ns.output_dir filename = ns.filename header_file = "%s/%s.h" % (output_dir, filename) - cpp_file = "%s/%s.cpp" % (output_dir, filename) cpp_class = CppClass(cpp_class=ns.cpp_class) model_file = "%s/forest.json" % ns.model @@ -298,12 +316,15 @@ with open(model_file) as m: forest_json = json.load(m) - with open(cpp_file, 'w+t') as output_cc: - output_cc.write( - gen_cpp_code(forest_json=forest_json, - features_json=features_json, - filename=filename, - cpp_class=cpp_class)) + os.makedirs(output_dir, exist_ok=True) + + for cpp_code_num, cpp_code in gen_cpp_code(forest_json=forest_json, + features_json=features_json, + filename=filename, + cpp_class=cpp_class).items(): + cpp_file = "%s/%s%s.cpp" % (output_dir, filename, cpp_code_num) + with open(cpp_file, 'w+t') as output_cc: + output_cc.write(cpp_code) with open(header_file, 'w+t') as output_h: output_h.write(gen_header_code( diff --git a/clang-tools-extra/clangd/unittests/CMakeLists.txt b/clang-tools-extra/clangd/unittests/CMakeLists.txt --- a/clang-tools-extra/clangd/unittests/CMakeLists.txt +++ b/clang-tools-extra/clangd/unittests/CMakeLists.txt @@ -20,7 +20,11 @@ endif() include(${CMAKE_CURRENT_SOURCE_DIR}/../quality/CompletionModel.cmake) -gen_decision_forest(${CMAKE_CURRENT_SOURCE_DIR}/decision_forest_model DecisionForestRuntimeTest ::ns1::ns2::test::Example) +gen_decision_forest(${CMAKE_CURRENT_SOURCE_DIR}/decision_forest_model + DecisionForestRuntimeTest + ::ns1::ns2::test::Example + DecisionForestRuntimeTest_header_file + DecisionForestRuntimeTest_cpp_files) add_custom_target(ClangdUnitTests) add_unittest(ClangdUnitTests ClangdTests @@ -95,7 +99,7 @@ TypeHierarchyTests.cpp URITests.cpp XRefsTests.cpp - ${CMAKE_CURRENT_BINARY_DIR}/DecisionForestRuntimeTest.cpp + ${DecisionForestRuntimeTest_cpp_files} support/CancellationTests.cpp support/ContextTests.cpp @@ -134,9 +138,9 @@ $ ) -# Include generated ComletionModel headers. +# Include generated Completion Model header. target_include_directories(ClangdTests PUBLIC - $ + $ ) clang_target_link_libraries(ClangdTests