Index: debuginfo-tests/dexter/dex/debugger/Debuggers.py =================================================================== --- debuginfo-tests/dexter/dex/debugger/Debuggers.py +++ debuginfo-tests/dexter/dex/debugger/Debuggers.py @@ -151,16 +151,19 @@ dexter_version=context.version) -def get_debugger_steps(context): +def get_debugger_steps(context, parse_commands=True): step_collection = empty_debugger_steps(context) - with Timer('parsing commands'): - try: - step_collection.commands = _get_command_infos(context) - except CommandParseError as e: - msg = 'parser error: {}({}): {}\n{}\n{}\n'.format( - e.filename, e.lineno, e.info, e.src, e.caret) - raise DebuggerException(msg) + if parse_commands: + with Timer('parsing commands'): + try: + step_collection.commands = _get_command_infos(context) + except CommandParseError as e: + msg = 'parser error: {}({}): {}\n{}\n{}\n'.format( + e.filename, e.lineno, e.info, e.src, e.caret) + raise DebuggerException(msg) + else: + step_collection.commands = OrderedDict() with NamedTemporaryFile( dir=context.working_directory.path, delete=False) as fp: Index: debuginfo-tests/dexter/dex/debugger/lldb/LLDB.py =================================================================== --- debuginfo-tests/dexter/dex/debugger/lldb/LLDB.py +++ debuginfo-tests/dexter/dex/debugger/lldb/LLDB.py @@ -9,12 +9,14 @@ import imp import os +from collections import OrderedDict from subprocess import CalledProcessError, check_output, STDOUT import sys from dex.debugger.DebuggerBase import DebuggerBase from dex.dextIR import FrameIR, LocIR, StepIR, StopReason, ValueIR from dex.dextIR import StackFrame, SourceLocation, ProgramState +from dex.dextIR.ValueIR import ValueKind from dex.utils.Exceptions import DebuggerException, LoadDebuggerException from dex.utils.ReturnCode import ReturnCode @@ -158,9 +160,29 @@ frames.append(frame) + visible_variables = OrderedDict() + args = sb_frame.get_arguments() + local_variables = sb_frame.get_locals() + for value in sb_frame.get_all_variables(): + # Ignore the value if it has not been declared yet + decl = value.GetDeclaration() + if decl: + if sb_line.GetLine() <= decl.GetLine(): + continue + # We don't add pointer values into the set of visible variables as their value changes between runs + if not value.TypeIsPointerType(): + value_name = str(value.GetName()) + kind = ValueKind.GLOBAL + if args.GetFirstValueByName(value_name): + kind = ValueKind.ARG + elif local_variables.GetFirstValueByName(value_name): + kind = ValueKind.LOCAL + visible_variables[value_name] = self.evaluate_expression(value_name, 0, kind) + state_frame = StackFrame(function=frame.function, is_inlined=frame.is_inlined, location=SourceLocation(**loc_dict), + visible_variables=visible_variables, watches={}) for expr in map( lambda watch, idx=i: self.evaluate_expression(watch, idx), @@ -191,7 +213,7 @@ def frames_below_main(self): return ['__scrt_common_main_seh', '__libc_start_main'] - def evaluate_expression(self, expression, frame_idx=0) -> ValueIR: + def evaluate_expression(self, expression, frame_idx=0, kind: ValueKind = None) -> ValueIR: result = self._thread.GetFrameAtIndex(frame_idx ).EvaluateExpression(expression) error_string = str(result.error) @@ -201,6 +223,8 @@ "Can't run the expression locally", "use of undeclared identifier", "no member named", + "has unknown type", # is this a proper fix? + "indirection requires pointer operand", # is this a proper fix? "Couldn't lookup symbols", "reference to local variable", "invalid use of 'this' outside of a non-static member function", @@ -241,4 +265,5 @@ could_evaluate=could_evaluate, is_optimized_away=is_optimized_away, is_irretrievable=is_irretrievable, + kind=kind ) Index: debuginfo-tests/dexter/dex/dextIR/ProgramState.py =================================================================== --- debuginfo-tests/dexter/dex/dextIR/ProgramState.py +++ debuginfo-tests/dexter/dex/dextIR/ProgramState.py @@ -48,6 +48,7 @@ function: str = None, is_inlined: bool = None, location: SourceLocation = None, + visible_variables: OrderedDict = None, watches: OrderedDict = None): if watches is None: watches = {} @@ -55,6 +56,7 @@ self.function = function self.is_inlined = is_inlined self.location = location + self.visible_variables = visible_variables self.watches = watches def __str__(self): Index: debuginfo-tests/dexter/dex/dextIR/ValueIR.py =================================================================== --- debuginfo-tests/dexter/dex/dextIR/ValueIR.py +++ debuginfo-tests/dexter/dex/dextIR/ValueIR.py @@ -5,6 +5,14 @@ # See https://llvm.org/LICENSE.txt for license information. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +from enum import Enum + + +class ValueKind(Enum): + ARG = 0 + LOCAL = 1 + GLOBAL = 2 + class ValueIR: """Data class to store the result of an expression evaluation.""" @@ -16,7 +24,8 @@ could_evaluate: bool, error_string: str = None, is_optimized_away: bool = False, - is_irretrievable: bool = False): + is_irretrievable: bool = False, + kind: ValueKind = None): self.expression = expression self.value = value self.type_name = type_name @@ -24,6 +33,7 @@ self.error_string = error_string self.is_optimized_away = is_optimized_away self.is_irretrievable = is_irretrievable + self.kind = kind def __str__(self): prefix = '"{}": '.format(self.expression) Index: debuginfo-tests/dexter/dex/tools/Main.py =================================================================== --- debuginfo-tests/dexter/dex/tools/Main.py +++ debuginfo-tests/dexter/dex/tools/Main.py @@ -66,7 +66,7 @@ """ Returns a list of expected DExTer Tools """ return [ - 'clang-opt-bisect', 'help', 'list-debuggers', 'no-tool-', + 'clang-opt-bisect', 'help', 'gen', 'list-debuggers', 'no-tool-', 'run-debugger-internal-', 'test', 'view' ] Index: debuginfo-tests/dexter/dex/tools/TestToolBase.py =================================================================== --- debuginfo-tests/dexter/dex/tools/TestToolBase.py +++ debuginfo-tests/dexter/dex/tools/TestToolBase.py @@ -84,14 +84,19 @@ 'could not find test path "{}"'.format( options.test_path)) - options.results_directory = os.path.abspath(options.results_directory) - if not os.path.isdir(options.results_directory): - try: - os.makedirs(options.results_directory, exist_ok=True) - except OSError as e: - raise Error( - 'could not create directory "{}" ({})'. - format(options.results_directory, e.strerror)) + if self._needs_results_directory(): + options.results_directory = os.path.abspath(options.results_directory) + if not os.path.isdir(options.results_directory): + try: + os.makedirs(options.results_directory, exist_ok=True) + except OSError as e: + raise Error( + 'could not create directory "{}" ({})'. + format(options.results_directory, e.strerror)) + + # Returns true if this test tool requires a result directory + def _needs_results_directory(self) -> bool: + return True def go(self) -> ReturnCode: # noqa options = self.context.options Index: debuginfo-tests/dexter/dex/tools/gen/Tool.py =================================================================== --- /dev/null +++ debuginfo-tests/dexter/dex/tools/gen/Tool.py @@ -0,0 +1,266 @@ +# DExTer : Debugging Experience Tester +# ~~~~~~ ~ ~~ ~ ~~ +# +# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +"""Gen tool.""" + +import shutil +from collections import defaultdict + +from dex.builder import add_builder_tool_arguments, run_external_build_script +from dex.debugger.Debuggers import add_debugger_tool_arguments, get_debugger_steps +from dex.dextIR import StepKind, BuilderIR +from dex.dextIR.ValueIR import ValueKind, ValueIR +from dex.tools import TestToolBase +from dex.tools.test.Tool import TestCase +from dex.utils.Exceptions import BuildScriptException, DebuggerException, Error +from dex.utils.ReturnCode import ReturnCode + + +# The expected debugging experience +# +# Contains dictionaries for every expected value and debugger steps for one file +class ExpectedExperience: + def __init__(self, *args, **kwargs): + self._expected_values = defaultdict(list) + self._num_step_kinds = defaultdict(int) + + # Records that an expr must have a given value at some line + def record_value(self, expr: str, line: int, value: ValueIR): + self._expected_values[(expr, line)].append(value) + + # Records a new occurence of a StepKind step_kind + def record_step(self, step_kind: StepKind): + self._num_step_kinds[step_kind] += 1 + + # Returns a dictionary of (expr, line) -> [ValueIR] + def get_expected_values_dict(self): + return self._expected_values + + # Returns a dictionary of StepKind -> number of times this step is expected + def get_num_expected_steps_dict(self): + return self._expected_values + + # Generates a list of DexExpectWatchValue/DexExpectStepKind commands for this ExpectedExperience + def generate_dex_commands(self, expect_steps=True, accepted_var_kinds: [ValueKind] = None) -> [str]: + commands = [] + for keys, values in self._expected_values.items(): + if len(values) == 0: + continue + + var_kind = values[0].kind + + # Check the kind of the variable - if we don't have to expect it, ignore it + if accepted_var_kinds and var_kind not in accepted_var_kinds: + continue + + command = "DexExpectWatchValue('" + keys[0] + "', " + # FIXME: Ideally, we should get rid of the require_in_order, but for some reasons values are not always + # in the correct order so we have to use it for now + + # Note: The 'if v.value' is to ignore 'None' values in the array + values = list(v.value for v in values if v.value) + if len(values) == 0: + continue + + for value in values: + value_str = value.replace("'", "\\'") + command += "'" + value_str + "', " + command += " on_line=" + str(keys[1]) + ", require_in_order=False)" + commands.append(command) + if expect_steps: + for step_kind, num_occurences in self._num_step_kinds.items(): + commands.append("DexExpectStepKind('" + step_kind.name + "', " + str(num_occurences) + ")") + return commands + + +# DExTer generator tool test +class Tool(TestToolBase): + """DExTer generator tool test - Automatically generates DExTer commands for C/C++ files + """ + + def __init__(self, *args, **kwargs): + super(Tool, self).__init__(*args, **kwargs) + self.dextIR = None + + @property + def name(self): + return 'DExTer Generator' + + def add_tool_arguments(self, parser, defaults): + parser.description = Tool.__doc__ + add_builder_tool_arguments(parser) + add_debugger_tool_arguments(parser, self.context, defaults) + # FIXME: This shouldn't be named "test_path" - it should be something more generic, but it will require + # changes to TestToolbase + parser.add_argument( + 'test_path', + type=str, + metavar='', + nargs='?', + help='directory in which the generation tool will run') + parser.add_argument( + '--no-expect-steps', + action="store_true", + help='do not generate DexExpectStepKind commands') + parser.add_argument( + '--expect-values-of', + nargs='+', + default="ALL", + help='sets the kind of variables that should be considered by the generator. Possible values: ALL, ARG, ' + 'LOCAL and/or GLOBAL') + + # Appends the DExTer commands from the 'ExpectedExperience' to the file 'filepath' + def _write_expected_experience(self, filepath, expected_experience: ExpectedExperience): + generated_dex_commands_beg = "//===--- AUTO-GENERATED DEXTER COMMANDS ---===//" + generated_dex_commands_end = "//===--------------------------------------===//" + + # Read every line in the file and append them to a vector if they don't contain auto-generated + # commands + lines = [] + with open(filepath, "r") as file: + in_generated_dex_commands = False + for line in file.readlines(): + if generated_dex_commands_beg in line: + # Sometimes, generated_dex_commands_beg can be at the end of a file + # e.g. } //===---... + # Detect such cases and extract the part of the line before the marker. + if line.strip() != generated_dex_commands_beg: + lines.append(line.split(generated_dex_commands_beg)[0]) + in_generated_dex_commands = True + elif generated_dex_commands_end in line: + in_generated_dex_commands = False + elif not in_generated_dex_commands: + lines.append(line) + + options = self.context.options + # The set of accepted variable kinds. If set to None, every variable kind will be accepted + accepted_var_kinds = set() + for kind in options.expect_values_of: + if kind.lower() == "all": + accepted_var_kinds = None + break + if kind.lower() == "arg": + accepted_var_kinds.add(ValueKind.ARG) + elif kind.lower() == "local": + accepted_var_kinds.add(ValueKind.LOCAL) + elif kind.lower() == "global": + accepted_var_kinds.add(ValueKind.GLOBAL) + + if self.context.options.verbose: + if accepted_var_kinds: + kinds_list = list(k.name for k in accepted_var_kinds) + out = "Generating commands for the following variable kinds: " + str(kinds_list) + else: + out = "Generating commands for every variable kind" + self.context.o.auto(out + "\n\n") + + commands = expected_experience.generate_dex_commands(expect_steps=not options.no_expect_steps, + accepted_var_kinds=accepted_var_kinds) + + # If the last line doesn't have a newline at the end, insert one. + if lines[-1][-1] != '\n': + lines[-1] += '\n' + + # Add the generated DExTer commands to the array + lines.append(generated_dex_commands_beg + "\n") + for command in commands: + lines.append("//" + command + "\n") + lines.append(generated_dex_commands_end + "\n") + + if self.context.options.verbose: + self.context.o.auto("%s commands generated in file '%s'\n" % (str(len(commands)), filepath)) + + # Write everything back to the file + with open(filepath, "w") as file: + for line in lines: + file.write(line) + + def _get_steps(self, builderIR): + """Generate a list of debugger steps""" + steps = get_debugger_steps(self.context, parse_commands=False) + steps.builder = builderIR + return steps + + def _handle_results(self) -> ReturnCode: + return ReturnCode.OK + + def _build_test_case(self): + """Build an executable from the test source with the given --builder + script and flags (--cflags, --ldflags) in the working directory. + Or, if the --binary option has been given, copy the executable provided + into the working directory and rename it to match the --builder output. + + FIXME: This is duplicate code from test.Tool + """ + + options = self.context.options + if options.binary: + # Copy user's binary into the tmp working directory + shutil.copy(options.binary, options.executable) + builderIR = BuilderIR( + name='binary', + cflags=str([options.binary]), + ldflags='') + else: + options = self.context.options + compiler_options = [options.cflags for _ in options.source_files] + linker_options = options.ldflags + _, _, builderIR = run_external_build_script( + self.context, + script_path=self.build_script, + source_files=options.source_files, + compiler_options=compiler_options, + linker_options=linker_options, + executable_file=options.executable) + return builderIR + + def handle_options(self, defaults): + options = self.context.options + if options.debugger.lower() != "lldb": + raise Error("--debugger %s is not supported by the generation tool - only 'lldb' is " + "supported" % options.debugger) + super(Tool, self).handle_options(defaults) + + def _needs_results_directory(self) -> bool: + return False + + def _run_test(self, test_dir): + test_name = self._get_test_name(test_dir) + try: + # Build the test case and retrieve the debugger steps + builderIR = self._build_test_case() + steps = self._get_steps(builderIR) + if self.context.options.verbose: + self.context.o.auto(str(steps)) + + # A dictionary of file path -> ExpectedExperience + expected_experiences = defaultdict(ExpectedExperience) + + # Iterate over each debugger step + for step in steps.steps: + frame_info = step.current_frame + loc = frame_info.loc + line = loc.lineno + + # The expected experience instance for the file in which this step happens + expected_experience = expected_experiences[loc.path] + + # Record the step + expected_experience.record_step(step.step_kind) + + # Now record the expected value of every visible variable in the top frame + frame = step.program_state.frames[0] + for var, value in frame.visible_variables.items(): + expected_experience.record_value(var, line, value) + + # Finally, write the set of expected experiences as DExTer commands in the file + for filepath, expected_experience in expected_experiences.items(): + self._write_expected_experience(filepath, expected_experience) + + except (BuildScriptException, DebuggerException) as e: + # FIXME: Report errors without using TestCase! + self.context.o.auto(TestCase(self.context, test_name, None, e)) + return Index: debuginfo-tests/dexter/dex/tools/gen/__init__.py =================================================================== --- /dev/null +++ debuginfo-tests/dexter/dex/tools/gen/__init__.py @@ -0,0 +1,8 @@ +# DExTer : Debugging Experience Tester +# ~~~~~~ ~ ~~ ~ ~~ +# +# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +from dex.tools.gen.Tool import Tool