diff --git a/cross-project-tests/debuginfo-tests/dexter-tests/address.cpp b/cross-project-tests/debuginfo-tests/dexter-tests/address.cpp new file mode 100644 --- /dev/null +++ b/cross-project-tests/debuginfo-tests/dexter-tests/address.cpp @@ -0,0 +1,15 @@ +// REQUIRES: system-windows +// +// RUN: %dexter --fail-lt 1.0 -w --builder 'clang-cl_vs2015' \ +// RUN: --debugger 'dbgeng' --cflags '/Z7 /Zi' --ldflags '/Z7 /Zi' -- %s + +int main() { + int *a = nullptr; + a = new int(5); + int *b = a; + delete a; +} + +// DexDeclareAddress('addr', 'a', on_line=9) +// DexExpectWatchValue('a', 0, address('addr'), from_line=8, to_line=10) +// DexExpectWatchValue('b', address('addr'), on_line=10) diff --git a/cross-project-tests/debuginfo-tests/dexter/Commands.md b/cross-project-tests/debuginfo-tests/dexter/Commands.md --- a/cross-project-tests/debuginfo-tests/dexter/Commands.md +++ b/cross-project-tests/debuginfo-tests/dexter/Commands.md @@ -9,6 +9,7 @@ * [DexLimitSteps](Commands.md#DexLimitSteps) * [DexLabel](Commands.md#DexLabel) * [DexWatch](Commands.md#DexWatch) +* [DexDeclareAddress](Commands.md#DexDeclareAddress) * [DexDeclareFile](Commands.md#DexDeclareFile) --- @@ -229,6 +230,61 @@ DexExpectWatchValues(..., on_line=ref('my_line_name') - 5) +### Heuristic +This command does not contribute to the heuristic score. + +---- +## DexDeclareAddress + DexDeclareAddress(declared_address, expr, **on_line[, **hit_count]) + + Args: + declared_address (str): The unique name of an address, which can be used + in DexExpectWatch-commands. + expr (str): An expression to evaluate to provide the value of this + address. + on_line (int): The line at which the value of the expression will be + assigned to the address. + hit_count (int): If provided, reads the value of the source expression + after the line has been stepped onto the given number + of times ('hit_count = 0' gives default behaviour). + +### Description +Declares a variable that can be used in DexExpectWatch- commands as an expected +value by using the `address(str[, int])` function. This is primarily +useful for checking the values of pointer variables, which are generally +determined at run-time (and so cannot be consistently matched by a hard-coded +expected value), but may be consistent relative to each other. An example use of +this command is as follows, using a set of pointer variables "foo", "bar", and +"baz": + + DexDeclareAddress('my_addr', 'bar', on_line=12) + DexExpectWatchValue('foo', address('my_addr'), on_line=10) + DexExpectWatchValue('bar', address('my_addr'), on_line=12) + DexExpectWatchValue('baz', address('my_addr', 16), on_line=14) + +On the first line, we declare the name of our variable 'my_addr'. This name must +be unique (the same name cannot be declared twice), and attempting to reference +an undeclared variable with `address` will fail. The value of the address +variable will be assigned as the value of 'bar' when line 12 is first stepped +on. + +On lines 2-4, we use the `address` function to refer to our variable. The first +usage occurs on line 10, before the line where 'my_addr' is assigned its value; +this is a valid use, as we assign the address value and check for correctness +after gathering all debug information for the test. Thus the first test command +will pass if 'foo' on line 10 has the same value as 'bar' on line 12. + +The second command will pass iff 'bar' is available at line 12 - even if the +variable and lines are identical in DexDeclareAddress and DexExpectWatchValue, +the latter will still expect a valid value. Similarly, if the variable for a +DexDeclareAddress command is not available at the given line, any test against +that address will fail. + +The `address` function also accepts an optional integer argument representing an +offset (which may be negative) to be applied to the address value, so +`address('my_addr', 16)` resolves to `my_addr + 16`. In the above example, this +means that we expect `baz == bar + 16`. + ### Heuristic This command does not contribute to the heuristic score. diff --git a/cross-project-tests/debuginfo-tests/dexter/dex/command/ParseCommand.py b/cross-project-tests/debuginfo-tests/dexter/dex/command/ParseCommand.py --- a/cross-project-tests/debuginfo-tests/dexter/dex/command/ParseCommand.py +++ b/cross-project-tests/debuginfo-tests/dexter/dex/command/ParseCommand.py @@ -19,11 +19,13 @@ from dex.command.CommandBase import CommandBase from dex.command.commands.DexDeclareFile import DexDeclareFile +from dex.command.commands.DexDeclareAddress import DexDeclareAddress from dex.command.commands.DexExpectProgramState import DexExpectProgramState from dex.command.commands.DexExpectStepKind import DexExpectStepKind from dex.command.commands.DexExpectStepOrder import DexExpectStepOrder from dex.command.commands.DexExpectWatchType import DexExpectWatchType from dex.command.commands.DexExpectWatchValue import DexExpectWatchValue +from dex.command.commands.DexExpectWatchBase import AddressExpression, DexExpectWatchBase from dex.command.commands.DexLabel import DexLabel from dex.command.commands.DexLimitSteps import DexLimitSteps from dex.command.commands.DexUnreachable import DexUnreachable @@ -38,6 +40,7 @@ { name (str): command (class) } """ return { + DexDeclareAddress.get_name() : DexDeclareAddress, DexDeclareFile.get_name() : DexDeclareFile, DexExpectProgramState.get_name() : DexExpectProgramState, DexExpectStepKind.get_name() : DexExpectStepKind, @@ -71,7 +74,7 @@ return valid_commands -def _build_command(command_type, labels, raw_text: str, path: str, lineno: str) -> CommandBase: +def _build_command(command_type, labels, addresses, raw_text: str, path: str, lineno: str) -> CommandBase: """Build a command object from raw text. This function will call eval(). @@ -88,9 +91,15 @@ return line raise format_unresolved_label_err(label_name, raw_text, path, lineno) + def get_address_object(address_name: str, offset: int=0): + if address_name not in addresses: + raise format_unresolved_address_err(address_name, raw_text, path, lineno) + return AddressExpression(address_name, offset) + valid_commands = _merge_subcommands( command_type.get_name(), { 'ref': label_to_line, + 'address': get_address_object, command_type.get_name(): command_type, }) @@ -176,6 +185,14 @@ err.info = f'Unresolved label: \'{label}\'' return err +def format_unresolved_address_err(address: str, src: str, filename: str, lineno) -> CommandParseError: + err = CommandParseError() + err.src = src + err.caret = '' # Don't bother trying to point to the bad address. + err.filename = filename + err.lineno = lineno + err.info = f'Unresolved address: \'{address}\'' + return err def format_parse_err(msg: str, path: str, lines: list, point: TextPoint) -> CommandParseError: err = CommandParseError() @@ -208,9 +225,25 @@ raise err labels[label.eval()] = label.get_line() +def add_address(addresses, address, cmd_path, cmd_lineno): + # Enforce unique address variables. + address_name = address.get_address_name() + if address_name in addresses: + err = CommandParseError() + err.info = f'Found duplicate address: \'{address_name}\'' + err.lineno = cmd_lineno + err.filename = cmd_path + err.src = address.raw_text + # Don't both trying to point to it since we're only printing the raw + # command, which isn't much text. + err.caret = '' + raise err + addresses.append(address_name) def _find_all_commands_in_file(path, file_lines, valid_commands, source_root_dir): labels = {} # dict of {name: line}. + addresses = [] # list of addresses. + address_resolutions = {} cmd_path = path declared_files = set() commands = defaultdict(dict) @@ -256,6 +289,7 @@ command = _build_command( valid_commands[command_name], labels, + addresses, raw_text, cmd_path, cmd_point.get_lineno(), @@ -275,6 +309,9 @@ else: if type(command) is DexLabel: add_line_label(labels, command, path, cmd_point.get_lineno()) + elif type(command) is DexDeclareAddress: + add_address(addresses, command, path, cmd_point.get_lineno()) + command.address_resolutions = address_resolutions elif type(command) is DexDeclareFile: cmd_path = command.declared_file if not os.path.isabs(cmd_path): @@ -284,6 +321,8 @@ # TODO: keep stored paths as PurePaths for 'longer'. cmd_path = str(PurePath(cmd_path)) declared_files.add(cmd_path) + elif isinstance(command, DexExpectWatchBase): + command.address_resolutions = address_resolutions assert (path, cmd_point) not in commands[command_name], ( command_name, commands[command_name]) commands[command_name][path, cmd_point] = command diff --git a/cross-project-tests/debuginfo-tests/dexter/dex/command/commands/DexDeclareAddress.py b/cross-project-tests/debuginfo-tests/dexter/dex/command/commands/DexDeclareAddress.py new file mode 100644 --- /dev/null +++ b/cross-project-tests/debuginfo-tests/dexter/dex/command/commands/DexDeclareAddress.py @@ -0,0 +1,57 @@ +# DExTer : Debugging Experience Tester +# ~~~~~~ ~ ~~ ~ ~~ +# +# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +"""Commmand sets the path for all following commands to 'declared_file'. +""" + +import os + +from dex.command.CommandBase import CommandBase, StepExpectInfo + +class DexDeclareAddress(CommandBase): + def __init__(self, addr_name, expression, **kwargs): + + if not isinstance(addr_name, str): + raise TypeError('invalid argument type') + + self.addr_name = addr_name + self.expression = expression + self.on_line = kwargs.pop('on_line') + self.hit_count = kwargs.pop('hit_count', 0) + + self.address_resolutions = None + + super(DexDeclareAddress, self).__init__() + + @staticmethod + def get_name(): + return __class__.__name__ + + def get_watches(self): + return [StepExpectInfo(self.expression, self.path, 0, range(self.on_line, self.on_line + 1))] + + def get_address_name(self): + return self.addr_name + + def eval(self, step_collection): + assert os.path.exists(self.path) + self.address_resolutions[self.get_address_name()] = None + for step in step_collection.steps: + loc = step.current_location + + if (loc.path and os.path.exists(loc.path) and + os.path.samefile(loc.path, self.path) and + loc.lineno == self.on_line): + if self.hit_count > 0: + self.hit_count -= 1 + continue + try: + watch = step.program_state.frames[0].watches[self.expression] + except KeyError: + pass + else: + self.address_resolutions[self.get_address_name()] = int(watch.value, 16) + break diff --git a/cross-project-tests/debuginfo-tests/dexter/dex/command/commands/DexExpectWatchBase.py b/cross-project-tests/debuginfo-tests/dexter/dex/command/commands/DexExpectWatchBase.py --- a/cross-project-tests/debuginfo-tests/dexter/dex/command/commands/DexExpectWatchBase.py +++ b/cross-project-tests/debuginfo-tests/dexter/dex/command/commands/DexExpectWatchBase.py @@ -17,7 +17,39 @@ from dex.command.CommandBase import CommandBase, StepExpectInfo from dex.command.StepValueInfo import StepValueInfo - +class AddressExpression(object): + def __init__(self, name, offset=0): + self.name = name + self.offset = offset + + def is_resolved(self, resolutions): + return self.name in resolutions + + # Given the resolved value of the address, resolve the final value of + # this expression. + def resolved_value(self, resolutions): + if not self.name in resolutions: + return None + # Technically we should fill(8) if we're debugging on a 32bit architecture? + return "0x" + hex(resolutions[self.name] + self.offset)[2:].zfill(16) + + # Given the final value of this expression, determine the resolved value of + # the address that would produce that value. + def inverse_resolve(self, resolved_value): + return int(resolved_value, 16) - self.offset + +def resolved_value(value, resolutions): + return value.resolved_value(resolutions) if isinstance(value, AddressExpression) else value + +def describe_value(value): + if isinstance(value, AddressExpression): + offset = "" + if value.offset > 0: + offset = f"+{value.offset}" + elif value.offset < 0: + offset = str(value.offset) + return f"address '{value.name}'{offset}" + return value class DexExpectWatchBase(CommandBase): def __init__(self, *args, **kwargs): @@ -25,7 +57,7 @@ raise TypeError('expected at least two args') self.expression = args[0] - self.values = [str(arg) for arg in args[1:]] + self.values = [arg if isinstance(arg, AddressExpression) else str(arg) for arg in args[1:]] try: on_line = kwargs.pop('on_line') self._from_line = on_line @@ -66,6 +98,15 @@ # unexpected value. self.unexpected_watches = [] + # List of StepValueInfos for all observed watches that were not + # invalid, irretrievable, or optimized out (combines expected and + # unexpected). + self.observed_watches = [] + + # dict of address names to their final resolved values, None until it + # gets assigned externally. + self.address_resolutions = None + super(DexExpectWatchBase, self).__init__() @@ -78,11 +119,19 @@ @property def missing_values(self): - return sorted(list(self._missing_values)) + return sorted(list(describe_value(v) for v in self._missing_values)) @property def encountered_values(self): - return sorted(list(set(self.values) - self._missing_values)) + return sorted(list(set(resolved_value(v, self.address_resolutions) for v in set(self.values) - self._missing_values))) + + @property + def unresolved_addresses(self): + return [value for value in self.values if isinstance(value, AddressExpression) and value.name not in self.address_resolutions] + + @property + def used_address_names(self): + return [value.name for value in self.values if isinstance(value, AddressExpression)] @abc.abstractmethod def _get_expected_field(self, watch): @@ -104,13 +153,25 @@ self.irretrievable_watches.append(step_info) return - if step_info.expected_value not in self.values: + # Check to see if this value matches with a resolved address. + matching_address = None + for v in self.values: + if (isinstance(v, AddressExpression) and + v.name in self.address_resolutions and + v.resolved_value(self.address_resolutions) == step_info.expected_value): + matching_address = v + break + + # If this is not an expected value, either a direct value or an address, + # then this is an unexpected watch. + if step_info.expected_value not in self.values and matching_address is None: self.unexpected_watches.append(step_info) return self.expected_watches.append(step_info) + value_to_remove = matching_address if matching_address is not None else step_info.expected_value try: - self._missing_values.remove(step_info.expected_value) + self._missing_values.remove(value_to_remove) except KeyError: pass @@ -177,8 +238,9 @@ value_change_watches.append(watch) prev_value = watch.expected_value + resolved_values = [resolved_value(v, self.address_resolutions) for v in self.values] self.misordered_watches = self._check_watch_order( value_change_watches, [ - v for v in self.values if v in + v for v in resolved_values if v in [w.expected_value for w in self.expected_watches] ]) diff --git a/cross-project-tests/debuginfo-tests/dexter/dex/heuristic/Heuristic.py b/cross-project-tests/debuginfo-tests/dexter/dex/heuristic/Heuristic.py --- a/cross-project-tests/debuginfo-tests/dexter/dex/heuristic/Heuristic.py +++ b/cross-project-tests/debuginfo-tests/dexter/dex/heuristic/Heuristic.py @@ -109,6 +109,13 @@ self.penalty_missing_step, self.penalty_misordered_steps ]) + # Before evaluating scoring commands, evaluate address values. + try: + for command in steps.commands['DexDeclareAddress']: + command.eval(steps) + except KeyError: + pass + # Get DexExpectWatchType results. try: for command in steps.commands['DexExpectWatchType']: