diff --git a/debuginfo-tests/dexter/Commands.md b/debuginfo-tests/dexter/Commands.md --- a/debuginfo-tests/dexter/Commands.md +++ b/debuginfo-tests/dexter/Commands.md @@ -9,6 +9,7 @@ * [DexLimitSteps](Commands.md#DexLimitSteps) * [DexLabel](Commands.md#DexLabel) * [DexWatch](Commands.md#DexWatch) +* [DexDeclareFile](Commands.md#DexDeclareFile) --- ## DexExpectProgramState @@ -231,6 +232,23 @@ ### Heuristic This command does not contribute to the heuristic score. +---- +## DexDeclareFile + DexDeclareFile(declared_file) + + Args: + name (str): A declared file path for which all subsequent commands + will have their path attribute set too. + +### Description +Set the path attribute of all commands from this point in the test onwards. +The new path holds until the end of the test file or until a new DexDeclareFile +command is encountered. Used in conjunction with .dex files, DexDeclareFile can +be used to write your dexter commands in a separate test file avoiding inlined +Dexter commands mixed with test source. + +### Heuristic +This command does not contribute to the heuristic score. --- ## DexWatch diff --git a/debuginfo-tests/dexter/d.diff b/debuginfo-tests/dexter/d.diff new file mode 100644 --- /dev/null +++ b/debuginfo-tests/dexter/d.diff @@ -0,0 +1,463 @@ +diff --git a/debuginfo-tests/dexter/Commands.md b/debuginfo-tests/dexter/Commands.md +index 5de685906c01..da14b8d59ba7 100644 +--- a/debuginfo-tests/dexter/Commands.md ++++ b/debuginfo-tests/dexter/Commands.md +@@ -9,6 +9,7 @@ + * [DexLimitSteps](Commands.md#DexLimitSteps) + * [DexLabel](Commands.md#DexLabel) + * [DexWatch](Commands.md#DexWatch) ++* [DexDeclareFile](Commands.md#DexDeclareFile) + + --- + ## DexExpectProgramState +@@ -231,6 +232,23 @@ arithmetic operators to get offsets from labels: + ### Heuristic + This command does not contribute to the heuristic score. + ++---- ++## DexDeclareFile ++ DexDeclareFile(declared_file) ++ ++ Args: ++ name (str): A declared file path for which all subsequent commands ++ will have their path attribute set too. ++ ++### Description ++Set the path attribute of all commands from this point in the test onwards. ++The new path holds until the end of the test file or until a new DexDeclareFile ++command is encountered. Used in conjunction with .dex files, DexDeclareFile can ++be used to write your dexter commands in a separate test file avoiding inlined ++Dexter commands mixed with test source. ++ ++### Heuristic ++This command does not contribute to the heuristic score. + + --- + ## DexWatch +diff --git a/debuginfo-tests/dexter/dex/command/ParseCommand.py b/debuginfo-tests/dexter/dex/command/ParseCommand.py +index c9908ef4b399..81e5c6c117f0 100644 +--- a/debuginfo-tests/dexter/dex/command/ParseCommand.py ++++ b/debuginfo-tests/dexter/dex/command/ParseCommand.py +@@ -12,12 +12,13 @@ Python code being embedded within DExTer commands. + import os + import unittest + from copy import copy +- ++from pathlib import PurePath + from collections import defaultdict, OrderedDict + + from dex.utils.Exceptions import CommandParseError + + from dex.command.CommandBase import CommandBase ++from dex.command.commands.DexDeclareFile import DexDeclareFile + from dex.command.commands.DexExpectProgramState import DexExpectProgramState + from dex.command.commands.DexExpectStepKind import DexExpectStepKind + from dex.command.commands.DexExpectStepOrder import DexExpectStepOrder +@@ -37,6 +38,7 @@ def _get_valid_commands(): + { name (str): command (class) } + """ + return { ++ DexDeclareFile.get_name() : DexDeclareFile, + DexExpectProgramState.get_name() : DexExpectProgramState, + DexExpectStepKind.get_name() : DexExpectStepKind, + DexExpectStepOrder.get_name() : DexExpectStepOrder, +@@ -209,6 +211,8 @@ def add_line_label(labels, label, cmd_path, cmd_lineno): + + def _find_all_commands_in_file(path, file_lines, valid_commands): + labels = {} # dict of {name: line}. ++ cmd_path = path ++ declared_files = set() + commands = defaultdict(dict) + paren_balance = 0 + region_start = TextPoint(0, 0) +@@ -253,7 +257,7 @@ def _find_all_commands_in_file(path, file_lines, valid_commands): + valid_commands[command_name], + labels, + raw_text, +- path, ++ cmd_path, + cmd_point.get_lineno(), + ) + except SyntaxError as e: +@@ -271,6 +275,14 @@ def _find_all_commands_in_file(path, file_lines, valid_commands): + else: + if type(command) is DexLabel: + add_line_label(labels, command, path, cmd_point.get_lineno()) ++ elif type(command) is DexDeclareFile: ++ cmd_path = command.declared_file ++ if not os.path.isabs(cmd_path): ++ source_dir = os.path.dirname(path) ++ cmd_path = os.path.join(source_dir, cmd_path) ++ # TODO: keep stored paths as PurePaths for 'longer'. ++ cmd_path = str(PurePath(cmd_path)) ++ declared_files.add(cmd_path) + assert (path, cmd_point) not in commands[command_name], ( + command_name, commands[command_name]) + commands[command_name][path, cmd_point] = command +@@ -281,32 +293,34 @@ def _find_all_commands_in_file(path, file_lines, valid_commands): + err_point.char += len(command_name) + msg = "Unbalanced parenthesis starting here" + raise format_parse_err(msg, path, file_lines, err_point) +- return dict(commands) ++ return dict(commands), declared_files + +-def _find_all_commands(source_files): ++def _find_all_commands(test_files): + commands = defaultdict(dict) + valid_commands = _get_valid_commands() +- for source_file in source_files: +- with open(source_file) as fp: ++ new_source_files = set() ++ for test_file in test_files: ++ with open(test_file) as fp: + lines = fp.readlines() +- file_commands = _find_all_commands_in_file(source_file, lines, +- valid_commands) ++ file_commands, declared_files = _find_all_commands_in_file(test_file, ++ lines, valid_commands) + for command_name in file_commands: + commands[command_name].update(file_commands[command_name]) ++ new_source_files |= declared_files + +- return dict(commands) ++ return dict(commands), new_source_files + +-def get_command_infos(source_files): ++def get_command_infos(test_files): + with Timer('parsing commands'): + try: +- commands = _find_all_commands(source_files) ++ commands, new_source_files = _find_all_commands(test_files) + command_infos = OrderedDict() + for command_type in commands: + for command in commands[command_type].values(): + if command_type not in command_infos: + command_infos[command_type] = [] + command_infos[command_type].append(command) +- return OrderedDict(command_infos) ++ return OrderedDict(command_infos), new_source_files + except CommandParseError as e: + msg = 'parser error: {}({}): {}\n{}\n{}\n'.format( + e.filename, e.lineno, e.info, e.src, e.caret) +@@ -344,7 +358,8 @@ class TestParseCommand(unittest.TestCase): + Returns: + { cmd_name: { (path, line): command_obj } } + """ +- return _find_all_commands_in_file(__file__, lines, self.valid_commands) ++ cmds, declared_files = _find_all_commands_in_file(__file__, lines, self.valid_commands) ++ return cmds + + + def _find_all_mock_values_in_lines(self, lines): +diff --git a/debuginfo-tests/dexter/dex/command/commands/DexDeclareFile.py b/debuginfo-tests/dexter/dex/command/commands/DexDeclareFile.py +new file mode 100644 +index 000000000000..c40c854575d9 +--- /dev/null ++++ b/debuginfo-tests/dexter/dex/command/commands/DexDeclareFile.py +@@ -0,0 +1,31 @@ ++# DExTer : Debugging Experience Tester ++# ~~~~~~ ~ ~~ ~ ~~ ++# ++# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. ++# See https://llvm.org/LICENSE.txt for license information. ++# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception ++"""Commmand sets the path for all following commands to 'declared_file'. ++""" ++ ++from pathlib import PurePath ++ ++from dex.command.CommandBase import CommandBase ++ ++ ++class DexDeclareFile(CommandBase): ++ def __init__(self, declared_file): ++ ++ if not isinstance(declared_file, str): ++ raise TypeError('invalid argument type') ++ ++ # Use PurePath to create a cannonical platform path. ++ # TODO: keep paths as PurePath objects for 'longer' ++ self.declared_file = str(PurePath(declared_file)) ++ super(DexDeclareFile, self).__init__() ++ ++ @staticmethod ++ def get_name(): ++ return __class__.__name__ ++ ++ def eval(self): ++ return self.declared_file +diff --git a/debuginfo-tests/dexter/dex/tools/TestToolBase.py b/debuginfo-tests/dexter/dex/tools/TestToolBase.py +index a2d8a90c005e..cfea497124b5 100644 +--- a/debuginfo-tests/dexter/dex/tools/TestToolBase.py ++++ b/debuginfo-tests/dexter/dex/tools/TestToolBase.py +@@ -100,26 +100,38 @@ class TestToolBase(ToolBase): + options.executable = os.path.join( + self.context.working_directory.path, 'tmp.exe') + ++ # Test files contain dexter commands. ++ options.test_files = [] ++ # Source files are to be compiled by the builder script and may also ++ # contains dexter commands. ++ options.source_files = [] + if os.path.isdir(options.test_path): +- + subdirs = sorted([ + r for r, _, f in os.walk(options.test_path) + if 'test.cfg' in f + ]) + + for subdir in subdirs: +- +- # TODO: read file extensions from the test.cfg file instead so +- # that this isn't just limited to C and C++. +- options.source_files = [ +- os.path.normcase(os.path.join(subdir, f)) +- for f in os.listdir(subdir) if any( +- f.endswith(ext) for ext in ['.c', '.cpp']) +- ] ++ for f in os.listdir(subdir): ++ # TODO: read file extensions from the test.cfg file instead so ++ # that this isn't just limited to C and C++. ++ file_path = os.path.normcase(os.path.join(subdir, f)) ++ if f.endswith('.cpp'): ++ options.source_files.append(file_path) ++ elif f.endswith('.c'): ++ options.source_files.append(file_path) ++ elif f.endswith('.dex'): ++ options.test_files.append(file_path) ++ # Source files can contain dexter commands too. ++ options.test_files = options.test_files + options.source_files + + self._run_test(self._get_test_name(subdir)) + else: +- options.source_files = [options.test_path] ++ # We're dealing with a direct file path to a test file. If the file is non ++ # .dex, then it must be a source file. ++ if not options.test_path.endswith('.dex'): ++ options.source_files = [options.test_path] ++ options.test_files = [options.test_path] + self._run_test(self._get_test_name(options.test_path)) + + return self._handle_results() +diff --git a/debuginfo-tests/dexter/dex/tools/clang_opt_bisect/Tool.py b/debuginfo-tests/dexter/dex/tools/clang_opt_bisect/Tool.py +index 6e936bd98a3c..c910d9c537ca 100644 +--- a/debuginfo-tests/dexter/dex/tools/clang_opt_bisect/Tool.py ++++ b/debuginfo-tests/dexter/dex/tools/clang_opt_bisect/Tool.py +@@ -92,8 +92,9 @@ class Tool(TestToolBase): + executable_path=self.context.options.executable, + source_paths=self.context.options.source_files, + dexter_version=self.context.version) +- step_collection.commands = get_command_infos( +- self.context.options.source_files) ++ step_collection.commands, new_source_files = get_command_infos( ++ self.context.options.test_files) ++ self.context.options.source_files.extend(list(new_source_files)) + debugger_controller = DefaultController(self.context, step_collection) + return debugger_controller + +diff --git a/debuginfo-tests/dexter/dex/tools/test/Tool.py b/debuginfo-tests/dexter/dex/tools/test/Tool.py +index 43191fd44bd5..2d3ddce8f7b6 100644 +--- a/debuginfo-tests/dexter/dex/tools/test/Tool.py ++++ b/debuginfo-tests/dexter/dex/tools/test/Tool.py +@@ -138,8 +138,10 @@ class Tool(TestToolBase): + source_paths=self.context.options.source_files, + dexter_version=self.context.version) + +- step_collection.commands = get_command_infos( +- self.context.options.source_files) ++ step_collection.commands, new_source_files = get_command_infos( ++ self.context.options.test_files) ++ ++ self.context.options.source_files.extend(list(new_source_files)) + + if 'DexLimitSteps' in step_collection.commands: + debugger_controller = ConditionalController(self.context, step_collection) +diff --git a/debuginfo-tests/dexter/feature_tests/commands/penalty/dex_declare_file.cpp b/debuginfo-tests/dexter/feature_tests/commands/penalty/dex_declare_file.cpp +new file mode 100644 +index 000000000000..7860ffd5dda4 +--- /dev/null ++++ b/debuginfo-tests/dexter/feature_tests/commands/penalty/dex_declare_file.cpp +@@ -0,0 +1,17 @@ ++// Purpose: ++// Check that \DexDeclareFile causes a DexExpectWatchValue's to generate a ++// missing value penalty when the declared path is incorrect. ++// ++// UNSUPPORTED: system-darwin ++// ++// ++// RUN: not %dexter_regression_test -- %s | FileCheck %s ++// CHECK: dex_declare_file.cpp ++ ++int main() { ++ int result = 0; ++ return result; //DexLabel('return') ++} ++ ++// DexDeclareFile('this_file_does_not_exist.cpp') ++// DexExpectWatchValue('result', 0, on_line='return') +diff --git a/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/dex_and_source/commands.dex b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/dex_and_source/commands.dex +new file mode 100644 +index 000000000000..bbad7db943bf +--- /dev/null ++++ b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/dex_and_source/commands.dex +@@ -0,0 +1,2 @@ ++DexDeclareFile('test.cpp') ++DexExpectWatchValue('result', 0, on_line=14) +diff --git a/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/dex_and_source/lit.local.cfg.py b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/dex_and_source/lit.local.cfg.py +new file mode 100644 +index 000000000000..159c376beedb +--- /dev/null ++++ b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/dex_and_source/lit.local.cfg.py +@@ -0,0 +1 @@ ++config.suffixes = ['.cpp'] +diff --git a/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/dex_and_source/test.cfg b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/dex_and_source/test.cfg +new file mode 100644 +index 000000000000..e69de29bb2d1 +diff --git a/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/dex_and_source/test.cpp b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/dex_and_source/test.cpp +new file mode 100644 +index 000000000000..5f1d50efe8d0 +--- /dev/null ++++ b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/dex_and_source/test.cpp +@@ -0,0 +1,15 @@ ++// Purpose: ++// Check that \DexDeclareFile changes the path of all succeeding commands ++// to the file path it declares. Also check that dexter correctly accepts ++// files with .dex extensions. ++// ++// UNSUPPORTED: system-darwin ++// ++// ++// RUN: %dexter_regression_test -- %S | FileCheck %s ++// CHECK: dex_and_source ++ ++int main() { ++ int result = 0; ++ return result; ++} +diff --git a/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/precompiled_binary/commands.dex b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/precompiled_binary/commands.dex +new file mode 100644 +index 000000000000..1aec2f8f3b64 +--- /dev/null ++++ b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/precompiled_binary/commands.dex +@@ -0,0 +1,18 @@ ++# Purpose: ++# Check that \DexDeclareFile's file declaration can reference source files ++# in a precompiled binary. ++# ++# UNSUPPORTED: system-darwin ++# ++# RUN: %clang %S/test.cpp -O0 -g -o %t ++# RUN: %dexter_regression_test --binary %t %s | FileCheck %s ++# CHECK: commands.dex ++# ++# test.cpp ++# 1. int main() { ++# 2. int result = 0; ++# 3. return result; ++# 4. } ++ ++DexDeclareFile('test.cpp') ++DexExpectWatchValue('result', 0, on_line=3) +diff --git a/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/precompiled_binary/lit.local.cfg.py b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/precompiled_binary/lit.local.cfg.py +new file mode 100644 +index 000000000000..e65498f23dde +--- /dev/null ++++ b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/precompiled_binary/lit.local.cfg.py +@@ -0,0 +1 @@ ++config.suffixes = ['.dex'] +diff --git a/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/precompiled_binary/test.cpp b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/precompiled_binary/test.cpp +new file mode 100644 +index 000000000000..4d3cc5846e66 +--- /dev/null ++++ b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/precompiled_binary/test.cpp +@@ -0,0 +1,4 @@ ++int main() { ++ int result = 0; ++ return result; ++} +diff --git a/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/precompiled_binary_different_dir/dex_commands/commands.dex b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/precompiled_binary_different_dir/dex_commands/commands.dex +new file mode 100644 +index 000000000000..964c770d3325 +--- /dev/null ++++ b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/precompiled_binary_different_dir/dex_commands/commands.dex +@@ -0,0 +1,19 @@ ++# Purpose: ++# Check that \DexDeclareFile's file declaration can reference source files ++# not included in the test directory ++# ++# UNSUPPORTED: system-darwin ++# ++# RUN: %clang %S/../source/test.cpp -O0 -g -o %t ++# RUN: %dexter_regression_test --binary %t %s | FileCheck %s ++# RUN: rm %t ++# CHECK: commands.dex ++# ++# test.cpp ++# 1. int main() { ++# 2. int result = 0; ++# 3. return result; ++# 4. } ++ ++DexDeclareFile('../source/test.cpp') ++DexExpectWatchValue('result', 0, on_line=3) +diff --git a/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/precompiled_binary_different_dir/lit.local.cfg.py b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/precompiled_binary_different_dir/lit.local.cfg.py +new file mode 100644 +index 000000000000..e65498f23dde +--- /dev/null ++++ b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/precompiled_binary_different_dir/lit.local.cfg.py +@@ -0,0 +1 @@ ++config.suffixes = ['.dex'] +diff --git a/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/precompiled_binary_different_dir/source/test.cpp b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/precompiled_binary_different_dir/source/test.cpp +new file mode 100644 +index 000000000000..4d3cc5846e66 +--- /dev/null ++++ b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/precompiled_binary_different_dir/source/test.cpp +@@ -0,0 +1,4 @@ ++int main() { ++ int result = 0; ++ return result; ++} +diff --git a/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/windows_noncanonical_path/lit.local.cfg.py b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/windows_noncanonical_path/lit.local.cfg.py +new file mode 100644 +index 000000000000..e65498f23dde +--- /dev/null ++++ b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/windows_noncanonical_path/lit.local.cfg.py +@@ -0,0 +1 @@ ++config.suffixes = ['.dex'] +diff --git a/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/windows_noncanonical_path/source/test file.cpp b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/windows_noncanonical_path/source/test file.cpp +new file mode 100644 +index 000000000000..f6dcd82e93e7 +--- /dev/null ++++ b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/windows_noncanonical_path/source/test file.cpp +@@ -0,0 +1,4 @@ ++int main(const int argc, const char * argv[]) { ++ int result = argc; ++ return result; ++} +\ No newline at end of file +diff --git a/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/windows_noncanonical_path/test.cfg b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/windows_noncanonical_path/test.cfg +new file mode 100644 +index 000000000000..e69de29bb2d1 +diff --git a/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/windows_noncanonical_path/test.dex b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/windows_noncanonical_path/test.dex +new file mode 100644 +index 000000000000..d9c9b80044b6 +--- /dev/null ++++ b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/windows_noncanonical_path/test.dex +@@ -0,0 +1,17 @@ ++# Purpose: ++# Check that non-canonical paths resolve correctly on Windows. ++# ++# REQUIRES: system-windows ++# ++# RUN: %clang "%S/source/test file.cpp" -O0 -g -o %t ++# RUN: %dexter_regression_test --binary %t %s | FileCheck %s ++# CHECK: test.dex ++# ++# ./source/test file.cpp ++# 1 int main(const int argc, const char * argv[]) { ++# 2 int result = argc; ++# 3 return result; ++# 4 } ++ ++DexDeclareFile('./sOuRce\\test filE.cpp') ++DexExpectWatchValue('result', 1, on_line=3) diff --git a/debuginfo-tests/dexter/dex/command/ParseCommand.py b/debuginfo-tests/dexter/dex/command/ParseCommand.py --- a/debuginfo-tests/dexter/dex/command/ParseCommand.py +++ b/debuginfo-tests/dexter/dex/command/ParseCommand.py @@ -12,12 +12,13 @@ import os import unittest from copy import copy - +from pathlib import PurePath from collections import defaultdict, OrderedDict from dex.utils.Exceptions import CommandParseError from dex.command.CommandBase import CommandBase +from dex.command.commands.DexDeclareFile import DexDeclareFile from dex.command.commands.DexExpectProgramState import DexExpectProgramState from dex.command.commands.DexExpectStepKind import DexExpectStepKind from dex.command.commands.DexExpectStepOrder import DexExpectStepOrder @@ -37,6 +38,7 @@ { name (str): command (class) } """ return { + DexDeclareFile.get_name() : DexDeclareFile, DexExpectProgramState.get_name() : DexExpectProgramState, DexExpectStepKind.get_name() : DexExpectStepKind, DexExpectStepOrder.get_name() : DexExpectStepOrder, @@ -209,6 +211,8 @@ def _find_all_commands_in_file(path, file_lines, valid_commands): labels = {} # dict of {name: line}. + cmd_path = path + declared_files = set() commands = defaultdict(dict) paren_balance = 0 region_start = TextPoint(0, 0) @@ -253,7 +257,7 @@ valid_commands[command_name], labels, raw_text, - path, + cmd_path, cmd_point.get_lineno(), ) except SyntaxError as e: @@ -271,6 +275,14 @@ else: if type(command) is DexLabel: add_line_label(labels, command, path, cmd_point.get_lineno()) + elif type(command) is DexDeclareFile: + cmd_path = command.declared_file + if not os.path.isabs(cmd_path): + source_dir = os.path.dirname(path) + cmd_path = os.path.join(source_dir, cmd_path) + # TODO: keep stored paths as PurePaths for 'longer'. + cmd_path = str(PurePath(cmd_path)) + declared_files.add(cmd_path) assert (path, cmd_point) not in commands[command_name], ( command_name, commands[command_name]) commands[command_name][path, cmd_point] = command @@ -281,32 +293,34 @@ err_point.char += len(command_name) msg = "Unbalanced parenthesis starting here" raise format_parse_err(msg, path, file_lines, err_point) - return dict(commands) + return dict(commands), declared_files -def _find_all_commands(source_files): +def _find_all_commands(test_files): commands = defaultdict(dict) valid_commands = _get_valid_commands() - for source_file in source_files: - with open(source_file) as fp: + new_source_files = set() + for test_file in test_files: + with open(test_file) as fp: lines = fp.readlines() - file_commands = _find_all_commands_in_file(source_file, lines, - valid_commands) + file_commands, declared_files = _find_all_commands_in_file(test_file, + lines, valid_commands) for command_name in file_commands: commands[command_name].update(file_commands[command_name]) + new_source_files |= declared_files - return dict(commands) + return dict(commands), new_source_files -def get_command_infos(source_files): +def get_command_infos(test_files): with Timer('parsing commands'): try: - commands = _find_all_commands(source_files) + commands, new_source_files = _find_all_commands(test_files) command_infos = OrderedDict() for command_type in commands: for command in commands[command_type].values(): if command_type not in command_infos: command_infos[command_type] = [] command_infos[command_type].append(command) - return OrderedDict(command_infos) + return OrderedDict(command_infos), new_source_files except CommandParseError as e: msg = 'parser error: {}({}): {}\n{}\n{}\n'.format( e.filename, e.lineno, e.info, e.src, e.caret) @@ -344,7 +358,8 @@ Returns: { cmd_name: { (path, line): command_obj } } """ - return _find_all_commands_in_file(__file__, lines, self.valid_commands) + cmds, declared_files = _find_all_commands_in_file(__file__, lines, self.valid_commands) + return cmds def _find_all_mock_values_in_lines(self, lines): diff --git a/debuginfo-tests/dexter/dex/command/commands/DexDeclareFile.py b/debuginfo-tests/dexter/dex/command/commands/DexDeclareFile.py new file mode 100644 --- /dev/null +++ b/debuginfo-tests/dexter/dex/command/commands/DexDeclareFile.py @@ -0,0 +1,31 @@ +# DExTer : Debugging Experience Tester +# ~~~~~~ ~ ~~ ~ ~~ +# +# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +"""Commmand sets the path for all following commands to 'declared_file'. +""" + +from pathlib import PurePath + +from dex.command.CommandBase import CommandBase + + +class DexDeclareFile(CommandBase): + def __init__(self, declared_file): + + if not isinstance(declared_file, str): + raise TypeError('invalid argument type') + + # Use PurePath to create a cannonical platform path. + # TODO: keep paths as PurePath objects for 'longer' + self.declared_file = str(PurePath(declared_file)) + super(DexDeclareFile, self).__init__() + + @staticmethod + def get_name(): + return __class__.__name__ + + def eval(self): + return self.declared_file diff --git a/debuginfo-tests/dexter/dex/tools/TestToolBase.py b/debuginfo-tests/dexter/dex/tools/TestToolBase.py --- a/debuginfo-tests/dexter/dex/tools/TestToolBase.py +++ b/debuginfo-tests/dexter/dex/tools/TestToolBase.py @@ -100,26 +100,38 @@ options.executable = os.path.join( self.context.working_directory.path, 'tmp.exe') + # Test files contain dexter commands. + options.test_files = [] + # Source files are to be compiled by the builder script and may also + # contains dexter commands. + options.source_files = [] if os.path.isdir(options.test_path): - subdirs = sorted([ r for r, _, f in os.walk(options.test_path) if 'test.cfg' in f ]) for subdir in subdirs: - - # TODO: read file extensions from the test.cfg file instead so - # that this isn't just limited to C and C++. - options.source_files = [ - os.path.normcase(os.path.join(subdir, f)) - for f in os.listdir(subdir) if any( - f.endswith(ext) for ext in ['.c', '.cpp']) - ] + for f in os.listdir(subdir): + # TODO: read file extensions from the test.cfg file instead so + # that this isn't just limited to C and C++. + file_path = os.path.normcase(os.path.join(subdir, f)) + if f.endswith('.cpp'): + options.source_files.append(file_path) + elif f.endswith('.c'): + options.source_files.append(file_path) + elif f.endswith('.dex'): + options.test_files.append(file_path) + # Source files can contain dexter commands too. + options.test_files = options.test_files + options.source_files self._run_test(self._get_test_name(subdir)) else: - options.source_files = [options.test_path] + # We're dealing with a direct file path to a test file. If the file is non + # .dex, then it must be a source file. + if not options.test_path.endswith('.dex'): + options.source_files = [options.test_path] + options.test_files = [options.test_path] self._run_test(self._get_test_name(options.test_path)) return self._handle_results() diff --git a/debuginfo-tests/dexter/dex/tools/clang_opt_bisect/Tool.py b/debuginfo-tests/dexter/dex/tools/clang_opt_bisect/Tool.py --- a/debuginfo-tests/dexter/dex/tools/clang_opt_bisect/Tool.py +++ b/debuginfo-tests/dexter/dex/tools/clang_opt_bisect/Tool.py @@ -92,8 +92,9 @@ executable_path=self.context.options.executable, source_paths=self.context.options.source_files, dexter_version=self.context.version) - step_collection.commands = get_command_infos( - self.context.options.source_files) + step_collection.commands, new_source_files = get_command_infos( + self.context.options.test_files) + self.context.options.source_files.extend(list(new_source_files)) debugger_controller = DefaultController(self.context, step_collection) return debugger_controller diff --git a/debuginfo-tests/dexter/dex/tools/test/Tool.py b/debuginfo-tests/dexter/dex/tools/test/Tool.py --- a/debuginfo-tests/dexter/dex/tools/test/Tool.py +++ b/debuginfo-tests/dexter/dex/tools/test/Tool.py @@ -138,8 +138,10 @@ source_paths=self.context.options.source_files, dexter_version=self.context.version) - step_collection.commands = get_command_infos( - self.context.options.source_files) + step_collection.commands, new_source_files = get_command_infos( + self.context.options.test_files) + + self.context.options.source_files.extend(list(new_source_files)) if 'DexLimitSteps' in step_collection.commands: debugger_controller = ConditionalController(self.context, step_collection) diff --git a/debuginfo-tests/dexter/feature_tests/commands/penalty/dex_declare_file.cpp b/debuginfo-tests/dexter/feature_tests/commands/penalty/dex_declare_file.cpp new file mode 100644 --- /dev/null +++ b/debuginfo-tests/dexter/feature_tests/commands/penalty/dex_declare_file.cpp @@ -0,0 +1,17 @@ +// Purpose: +// Check that \DexDeclareFile causes a DexExpectWatchValue's to generate a +// missing value penalty when the declared path is incorrect. +// +// UNSUPPORTED: system-darwin +// +// +// RUN: not %dexter_regression_test -- %s | FileCheck %s +// CHECK: dex_declare_file.cpp + +int main() { + int result = 0; + return result; //DexLabel('return') +} + +// DexDeclareFile('this_file_does_not_exist.cpp') +// DexExpectWatchValue('result', 0, on_line='return') diff --git a/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/dex_and_source/commands.dex b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/dex_and_source/commands.dex new file mode 100644 --- /dev/null +++ b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/dex_and_source/commands.dex @@ -0,0 +1,2 @@ +DexDeclareFile('test.cpp') +DexExpectWatchValue('result', 0, on_line=14) diff --git a/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/dex_and_source/lit.local.cfg.py b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/dex_and_source/lit.local.cfg.py new file mode 100644 --- /dev/null +++ b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/dex_and_source/lit.local.cfg.py @@ -0,0 +1 @@ +config.suffixes = ['.cpp'] diff --git a/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/dex_and_source/test.cfg b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/dex_and_source/test.cfg new file mode 100644 diff --git a/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/dex_and_source/test.cpp b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/dex_and_source/test.cpp new file mode 100644 --- /dev/null +++ b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/dex_and_source/test.cpp @@ -0,0 +1,15 @@ +// Purpose: +// Check that \DexDeclareFile changes the path of all succeeding commands +// to the file path it declares. Also check that dexter correctly accepts +// files with .dex extensions. +// +// UNSUPPORTED: system-darwin +// +// +// RUN: %dexter_regression_test -- %S | FileCheck %s +// CHECK: dex_and_source + +int main() { + int result = 0; + return result; +} diff --git a/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/precompiled_binary/commands.dex b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/precompiled_binary/commands.dex new file mode 100644 --- /dev/null +++ b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/precompiled_binary/commands.dex @@ -0,0 +1,18 @@ +# Purpose: +# Check that \DexDeclareFile's file declaration can reference source files +# in a precompiled binary. +# +# UNSUPPORTED: system-darwin +# +# RUN: %clang %S/test.cpp -O0 -g -o %t +# RUN: %dexter_regression_test --binary %t %s | FileCheck %s +# CHECK: commands.dex +# +# test.cpp +# 1. int main() { +# 2. int result = 0; +# 3. return result; +# 4. } + +DexDeclareFile('test.cpp') +DexExpectWatchValue('result', 0, on_line=3) diff --git a/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/precompiled_binary/lit.local.cfg.py b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/precompiled_binary/lit.local.cfg.py new file mode 100644 --- /dev/null +++ b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/precompiled_binary/lit.local.cfg.py @@ -0,0 +1 @@ +config.suffixes = ['.dex'] diff --git a/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/precompiled_binary/test.cpp b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/precompiled_binary/test.cpp new file mode 100644 --- /dev/null +++ b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/precompiled_binary/test.cpp @@ -0,0 +1,4 @@ +int main() { + int result = 0; + return result; +} diff --git a/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/precompiled_binary_different_dir/dex_commands/commands.dex b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/precompiled_binary_different_dir/dex_commands/commands.dex new file mode 100644 --- /dev/null +++ b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/precompiled_binary_different_dir/dex_commands/commands.dex @@ -0,0 +1,19 @@ +# Purpose: +# Check that \DexDeclareFile's file declaration can reference source files +# not included in the test directory +# +# UNSUPPORTED: system-darwin +# +# RUN: %clang %S/../source/test.cpp -O0 -g -o %t +# RUN: %dexter_regression_test --binary %t %s | FileCheck %s +# RUN: rm %t +# CHECK: commands.dex +# +# test.cpp +# 1. int main() { +# 2. int result = 0; +# 3. return result; +# 4. } + +DexDeclareFile('../source/test.cpp') +DexExpectWatchValue('result', 0, on_line=3) diff --git a/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/precompiled_binary_different_dir/lit.local.cfg.py b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/precompiled_binary_different_dir/lit.local.cfg.py new file mode 100644 --- /dev/null +++ b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/precompiled_binary_different_dir/lit.local.cfg.py @@ -0,0 +1 @@ +config.suffixes = ['.dex'] diff --git a/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/precompiled_binary_different_dir/source/test.cpp b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/precompiled_binary_different_dir/source/test.cpp new file mode 100644 --- /dev/null +++ b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/precompiled_binary_different_dir/source/test.cpp @@ -0,0 +1,4 @@ +int main() { + int result = 0; + return result; +} diff --git a/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/windows_noncanonical_path/lit.local.cfg.py b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/windows_noncanonical_path/lit.local.cfg.py new file mode 100644 --- /dev/null +++ b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/windows_noncanonical_path/lit.local.cfg.py @@ -0,0 +1 @@ +config.suffixes = ['.dex'] diff --git a/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/windows_noncanonical_path/source/test file.cpp b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/windows_noncanonical_path/source/test file.cpp new file mode 100644 --- /dev/null +++ b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/windows_noncanonical_path/source/test file.cpp @@ -0,0 +1,4 @@ +int main(const int argc, const char * argv[]) { + int result = argc; + return result; +} \ No newline at end of file diff --git a/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/windows_noncanonical_path/test.cfg b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/windows_noncanonical_path/test.cfg new file mode 100644 diff --git a/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/windows_noncanonical_path/test.dex b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/windows_noncanonical_path/test.dex new file mode 100644 --- /dev/null +++ b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/windows_noncanonical_path/test.dex @@ -0,0 +1,17 @@ +# Purpose: +# Check that non-canonical paths resolve correctly on Windows. +# +# REQUIRES: system-windows +# +# RUN: %clang "%S/source/test file.cpp" -O0 -g -o %t +# RUN: %dexter_regression_test --binary %t %s | FileCheck %s +# CHECK: test.dex +# +# ./source/test file.cpp +# 1 int main(const int argc, const char * argv[]) { +# 2 int result = argc; +# 3 return result; +# 4 } + +DexDeclareFile('./sOuRce\\test filE.cpp') +DexExpectWatchValue('result', 1, on_line=3)