# Copyright 2020 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. import os import posixpath import sys import common BLINK_TOOLS_DIR = os.path.join(common.SRC_DIR, 'third_party', 'blink', 'tools') CATAPULT_DIR = os.path.join(common.SRC_DIR, 'third_party', 'catapult') OUT_DIR = os.path.join(common.SRC_DIR, "out", "{}") DEFAULT_ISOLATED_SCRIPT_TEST_OUTPUT = os.path.join(OUT_DIR, "results.json") TYP_DIR = os.path.join(CATAPULT_DIR, 'third_party', 'typ') WEB_TESTS_DIR = os.path.normpath( os.path.join(BLINK_TOOLS_DIR, os.pardir, 'web_tests')) if BLINK_TOOLS_DIR not in sys.path: sys.path.append(BLINK_TOOLS_DIR) if TYP_DIR not in sys.path: sys.path.append(TYP_DIR) from blinkpy.common.host import Host from blinkpy.common.path_finder import PathFinder class BaseWptScriptAdapter(common.BaseIsolatedScriptArgsAdapter): """The base class for script adapters that use wptrunner to execute web platform tests. This contains any code shared between these scripts, such as integrating output with the results viewer. Subclasses contain other (usually platform-specific) logic.""" def __init__(self, host=None): super(BaseWptScriptAdapter, self).__init__() if not host: host = Host() self.fs = host.filesystem self.path_finder = PathFinder(self.fs) self.port = host.port_factory.get() # Path to the output of the test run. Comes from the args passed to the # run, parsed after this constructor. Can be overwritten by tests. self.wpt_output = None self.wptreport = None self.layout_test_results_subdir = 'layout-test-results' default_wpt_binary = os.path.join( common.SRC_DIR, "third_party", "wpt_tools", "wpt", "wpt") self.wpt_binary = os.environ.get("WPT_BINARY", default_wpt_binary) @property def wpt_root_dir(self): return self.path_finder.path_from_web_tests( self.path_finder.wpt_prefix()) @property def output_directory(self): return self.path_finder.path_from_chromium_base('out', self.options.target) @property def mojo_js_directory(self): return self.fs.join(self.output_directory, 'gen') def add_extra_arguments(self, parser): parser.add_argument( '-t', '--target', default='Release', help='Target build subdirectory under //out') parser.add_argument( '--repeat', '--gtest_repeat', type=int, default=1, help='Number of times to run the tests') # TODO(crbug/1306222): wptrunner currently cannot rerun individual # failed tests, so this flag is accepted but not used. parser.add_argument( '--test-launcher-retry-limit', metavar='LIMIT', type=int, default=0, help='Maximum number of times to rerun a failed test') parser.add_argument( '--default-exclude', action='store_true', help=('Only run the tests explicitly given in arguments ' '(can run no tests, which will exit with code 0)')) parser.add_argument( '--dry-run', action='store_true', help='Do not upload results to ResultDB') # We provide an option to show wptrunner's help here because the 'wpt' # executable may not be inaccessible from the user's PATH. The top-level # 'wpt' command also needs to have virtualenv disabled. parser.add_argument( '--wpt-help', action='store_true', help="Show the wptrunner help message and exit") self.output_group = parser.add_argument_group( 'Output Logging', 'Options for controlling logging behavior.') self.output_group.add_argument( '--log-wptreport', nargs='?', const=self._default_wpt_report(), help=('Log a wptreport in JSON to the output directory ' '(default filename: %(const)s)')) self.output_group.add_argument( '-v', '--verbose', action='count', default=0, help='Increase verbosity') # Parser will format the epilog for us. parser.epilog = (parser.epilog or '') + ' ' + ( 'All unrecognized arguments are passed through to wptrunner. ' "Use '--wpt-help' to see wptrunner's usage." ) def maybe_set_default_isolated_script_test_output(self): if self.options.isolated_script_test_output: return default_value = DEFAULT_ISOLATED_SCRIPT_TEST_OUTPUT.format( self.options.target) print("--isolated-script-test-output not set, defaulting to %s" % default_value) self.options.isolated_script_test_output = default_value def generate_test_output_args(self, output): return ['--log-chromium', output] def generate_sharding_args(self, total_shards, shard_index): return ['--total-chunks=%d' % total_shards, # shard_index is 0-based but WPT's this-chunk to be 1-based '--this-chunk=%d' % (shard_index + 1), # The default sharding strategy is to shard by directory. But # we want to hash each test to determine which shard runs it. # This allows running individual directories that have few # tests across many shards. '--chunk-type=hash'] def parse_args(self, args=None): super(BaseWptScriptAdapter, self).parse_args(args) # Update the output directory to the default if it's not set. # We cannot provide a CLI arg default because the path depends on # --target. self.maybe_set_default_isolated_script_test_output() if self.options.log_wptreport: wpt_output = self.options.isolated_script_test_output self.wptreport = self.fs.join( self.fs.dirname(wpt_output), self.options.log_wptreport) def do_pre_test_run_tasks(self): super(BaseWptScriptAdapter, self).do_pre_test_run_tasks() if self.options.wpt_help: command = [ self.select_python_executable(), ] command.extend(self._wpt_run_args) command.extend(['--help']) exit_code = common.run_command(command) self.parser.exit(exit_code) @property def _wpt_run_args(self): """The start of a 'wpt run' command.""" return [ self.wpt_binary, # Use virtualenv packages installed by vpython, not wpt. '--venv=%s' % self.path_finder.chromium_base(), '--skip-venv-setup', 'run', ] @property def rest_args(self): wpt_args = super(BaseWptScriptAdapter, self).rest_args rest_args = list(self._wpt_run_args) rest_args.extend([ # By default, wpt will treat unexpected passes as errors, so we # disable that to be consistent with Chromium CI. '--no-fail-on-unexpected-pass', self.wpt_product_name(), '--no-pause-after-test', '--no-capture-stdio', '--no-manifest-download', '--tests=%s' % self.wpt_root_dir, '--mojojs-path=%s' % self.mojo_js_directory, '--repeat=%d' % self.options.repeat, ]) if self.options.default_exclude: rest_args.extend(['--default-exclude']) if self.wptreport: rest_args.extend(['--log-wptreport', self.wptreport]) if self.options.verbose >= 3: rest_args.extend([ '--log-mach=-', '--log-mach-level=debug', '--log-mach-verbose', ]) if self.options.verbose >= 4: rest_args.extend([ '--webdriver-arg=--verbose', '--webdriver-arg="--log-path=-"', ]) rest_args.extend(wpt_args) return rest_args def do_post_test_run_tasks(self): if not self.wpt_output and self.options: self.wpt_output = self.options.isolated_script_test_output command = [ self.select_python_executable(), os.path.join(BLINK_TOOLS_DIR, 'wpt_process_results.py'), '--verbose', '--target', self.options.target, '--web-tests-dir', WEB_TESTS_DIR, '--artifacts-dir', os.path.join(os.path.dirname(self.wpt_output), self.layout_test_results_subdir), '--wpt-results', self.wpt_output, ] if self.wptreport: command.extend(['--wpt-report', self.wptreport]) common.run_command(command) @classmethod def wpt_product_name(cls): raise NotImplementedError def _default_wpt_report(self): product = self.wpt_product_name() shard_index = os.environ.get('GTEST_SHARD_INDEX') if shard_index is not None: return 'wpt_reports_%s_%02d.json' % (product, int(shard_index)) return 'wpt_reports_%s.json' % product