1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
|
# Copyright 2020 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import os
import posixpath
import sys
import common
BLINK_TOOLS_DIR = os.path.join(common.SRC_DIR, 'third_party', 'blink', 'tools')
CATAPULT_DIR = os.path.join(common.SRC_DIR, 'third_party', 'catapult')
OUT_DIR = os.path.join(common.SRC_DIR, "out", "{}")
DEFAULT_ISOLATED_SCRIPT_TEST_OUTPUT = os.path.join(OUT_DIR, "results.json")
TYP_DIR = os.path.join(CATAPULT_DIR, 'third_party', 'typ')
WEB_TESTS_DIR = os.path.normpath(
os.path.join(BLINK_TOOLS_DIR, os.pardir, 'web_tests'))
if BLINK_TOOLS_DIR not in sys.path:
sys.path.append(BLINK_TOOLS_DIR)
if TYP_DIR not in sys.path:
sys.path.append(TYP_DIR)
from blinkpy.common.host import Host
from blinkpy.common.path_finder import PathFinder
class BaseWptScriptAdapter(common.BaseIsolatedScriptArgsAdapter):
"""The base class for script adapters that use wptrunner to execute web
platform tests. This contains any code shared between these scripts, such
as integrating output with the results viewer. Subclasses contain other
(usually platform-specific) logic."""
def __init__(self, host=None):
super(BaseWptScriptAdapter, self).__init__()
if not host:
host = Host()
self.fs = host.filesystem
self.path_finder = PathFinder(self.fs)
self.port = host.port_factory.get()
# Path to the output of the test run. Comes from the args passed to the
# run, parsed after this constructor. Can be overwritten by tests.
self.wpt_output = None
self.wptreport = None
self.layout_test_results_subdir = 'layout-test-results'
default_wpt_binary = os.path.join(
common.SRC_DIR, "third_party", "wpt_tools", "wpt", "wpt")
self.wpt_binary = os.environ.get("WPT_BINARY", default_wpt_binary)
@property
def wpt_root_dir(self):
return self.path_finder.path_from_web_tests(
self.path_finder.wpt_prefix())
@property
def output_directory(self):
return self.path_finder.path_from_chromium_base('out',
self.options.target)
@property
def mojo_js_directory(self):
return self.fs.join(self.output_directory, 'gen')
def add_extra_arguments(self, parser):
parser.add_argument(
'-t',
'--target',
default='Release',
help='Target build subdirectory under //out')
parser.add_argument(
'--repeat',
'--gtest_repeat',
type=int,
default=1,
help='Number of times to run the tests')
# TODO(crbug/1306222): wptrunner currently cannot rerun individual
# failed tests, so this flag is accepted but not used.
parser.add_argument(
'--test-launcher-retry-limit',
metavar='LIMIT',
type=int,
default=0,
help='Maximum number of times to rerun a failed test')
parser.add_argument(
'--default-exclude',
action='store_true',
help=('Only run the tests explicitly given in arguments '
'(can run no tests, which will exit with code 0)'))
parser.add_argument(
'--dry-run',
action='store_true',
help='Do not upload results to ResultDB')
# We provide an option to show wptrunner's help here because the 'wpt'
# executable may not be inaccessible from the user's PATH. The top-level
# 'wpt' command also needs to have virtualenv disabled.
parser.add_argument(
'--wpt-help',
action='store_true',
help="Show the wptrunner help message and exit")
self.output_group = parser.add_argument_group(
'Output Logging',
'Options for controlling logging behavior.')
self.output_group.add_argument(
'--log-wptreport',
nargs='?',
const=self._default_wpt_report(),
help=('Log a wptreport in JSON to the output directory '
'(default filename: %(const)s)'))
self.output_group.add_argument(
'-v',
'--verbose',
action='count',
default=0,
help='Increase verbosity')
# Parser will format the epilog for us.
parser.epilog = (parser.epilog or '') + ' ' + (
'All unrecognized arguments are passed through to wptrunner. '
"Use '--wpt-help' to see wptrunner's usage."
)
def maybe_set_default_isolated_script_test_output(self):
if self.options.isolated_script_test_output:
return
default_value = DEFAULT_ISOLATED_SCRIPT_TEST_OUTPUT.format(
self.options.target)
print("--isolated-script-test-output not set, defaulting to %s" %
default_value)
self.options.isolated_script_test_output = default_value
def generate_test_output_args(self, output):
return ['--log-chromium', output]
def generate_sharding_args(self, total_shards, shard_index):
return ['--total-chunks=%d' % total_shards,
# shard_index is 0-based but WPT's this-chunk to be 1-based
'--this-chunk=%d' % (shard_index + 1),
# The default sharding strategy is to shard by directory. But
# we want to hash each test to determine which shard runs it.
# This allows running individual directories that have few
# tests across many shards.
'--chunk-type=hash']
def parse_args(self, args=None):
super(BaseWptScriptAdapter, self).parse_args(args)
# Update the output directory to the default if it's not set.
# We cannot provide a CLI arg default because the path depends on
# --target.
self.maybe_set_default_isolated_script_test_output()
if self.options.log_wptreport:
wpt_output = self.options.isolated_script_test_output
self.wptreport = self.fs.join(
self.fs.dirname(wpt_output),
self.options.log_wptreport)
def do_pre_test_run_tasks(self):
super(BaseWptScriptAdapter, self).do_pre_test_run_tasks()
if self.options.wpt_help:
command = [
self.select_python_executable(),
]
command.extend(self._wpt_run_args)
command.extend(['--help'])
exit_code = common.run_command(command)
self.parser.exit(exit_code)
@property
def _wpt_run_args(self):
"""The start of a 'wpt run' command."""
return [
self.wpt_binary,
# Use virtualenv packages installed by vpython, not wpt.
'--venv=%s' % self.path_finder.chromium_base(),
'--skip-venv-setup',
'run',
]
@property
def rest_args(self):
wpt_args = super(BaseWptScriptAdapter, self).rest_args
rest_args = list(self._wpt_run_args)
rest_args.extend([
# By default, wpt will treat unexpected passes as errors, so we
# disable that to be consistent with Chromium CI.
'--no-fail-on-unexpected-pass',
self.wpt_product_name(),
'--no-pause-after-test',
'--no-capture-stdio',
'--no-manifest-download',
'--tests=%s' % self.wpt_root_dir,
'--mojojs-path=%s' % self.mojo_js_directory,
'--repeat=%d' % self.options.repeat,
])
if self.options.default_exclude:
rest_args.extend(['--default-exclude'])
if self.wptreport:
rest_args.extend(['--log-wptreport', self.wptreport])
if self.options.verbose >= 3:
rest_args.extend([
'--log-mach=-',
'--log-mach-level=debug',
'--log-mach-verbose',
])
if self.options.verbose >= 4:
rest_args.extend([
'--webdriver-arg=--verbose',
'--webdriver-arg="--log-path=-"',
])
rest_args.extend(wpt_args)
return rest_args
def do_post_test_run_tasks(self):
if not self.wpt_output and self.options:
self.wpt_output = self.options.isolated_script_test_output
command = [
self.select_python_executable(),
os.path.join(BLINK_TOOLS_DIR, 'wpt_process_results.py'),
'--verbose',
'--target',
self.options.target,
'--web-tests-dir',
WEB_TESTS_DIR,
'--artifacts-dir',
os.path.join(os.path.dirname(self.wpt_output),
self.layout_test_results_subdir),
'--wpt-results',
self.wpt_output,
]
if self.wptreport:
command.extend(['--wpt-report', self.wptreport])
common.run_command(command)
@classmethod
def wpt_product_name(cls):
raise NotImplementedError
def _default_wpt_report(self):
product = self.wpt_product_name()
shard_index = os.environ.get('GTEST_SHARD_INDEX')
if shard_index is not None:
return 'wpt_reports_%s_%02d.json' % (product, int(shard_index))
return 'wpt_reports_%s.json' % product
|
