forked from mozilla/version-control-tools
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathrun-tests
executable file
·392 lines (320 loc) · 14.6 KB
/
run-tests
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
#!/usr/bin/env python2.7
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# This file is used to run all Mercurial-related tests in this repository.
from __future__ import print_function
import argparse
import errno
import math
import multiprocessing
import os
import subprocess
import sys
# Mercurial's run-tests.py isn't meant to be loaded as a module. We do it
# anyway.
HERE = os.path.dirname(os.path.abspath(__file__))
RUNTESTS = os.path.join(HERE, 'pylib', 'mercurial-support', 'run-tests.py')
if __name__ == '__main__':
if 'VIRTUAL_ENV' not in os.environ:
activate = os.path.join(HERE, 'venv', 'bin', 'activate_this.py')
execfile(activate, dict(__file__=activate))
sys.executable = os.path.join(HERE, 'venv', 'bin', 'python')
os.environ['VIRTUAL_ENV'] = os.path.join(HERE, 'venv')
venv_name = os.path.relpath(os.environ['VIRTUAL_ENV'],
os.path.join(HERE, 'venv'))
if venv_name == '.':
venv_name = 'global'
if venv_name not in ('global', 'hgdev', 'vcssync', 'cross-channel-l10n'):
print('unknown virtualenv: %s (this should not happen)' % venv_name)
sys.exit(1)
try:
import vcttesting.docker as vctdocker
except ImportError:
vctdocker = None
from vcttesting.testing import (
get_docker_state,
get_extensions,
get_hg_version,
get_test_files,
run_nose_tests,
produce_coverage_reports,
remove_err_files,
)
# Unbuffer stdout.
sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
parser = argparse.ArgumentParser()
parser.add_argument('--with-hg')
parser.add_argument('-C', '--cover', action='store_true')
parser.add_argument('-j', '--jobs', type=int)
parser.add_argument('--all-hg-versions', action='store_true',
help='Test against all marked compatible Mercurial versions')
parser.add_argument('--no-hg-tip', action='store_true',
help='Do not run tests against the @ bookmark of hg')
parser.add_argument('--no-unit', action='store_true',
help='Do not run Python unit tests')
parser.add_argument('--use-last-images', action='store_true',
help='Skip building new images when possible')
parser.add_argument('--headless', action='store_true',
help='Only run tests that do not require a framebuffer '
'(skips Selenium tests)')
parser.add_argument('--no-docker', action='store_true',
help='Only run tests that do not require Docker')
options, extra = parser.parse_known_args(sys.argv)
if not vctdocker:
options.no_docker = True
# This is somewhat arbitrary. It makes version management logic a bit
# more complicated and isn't worth supporting.
if options.all_hg_versions and options.with_hg:
print('abort: --all-hg-versions and --with-hg are mutually exclusive')
sys.exit(1)
# some arguments belong to us only. Don't pass it along to run-tests.py.
filter_args = {
'--all-hg-versions',
'--no-hg-tip',
'--use-last-images',
'--headless',
'--no-docker',
}
hg_harness_args = [a for a in sys.argv if a not in filter_args]
hg_harness_args[0] = RUNTESTS
coveragerc = os.path.join(HERE, '.coveragerc')
cover_dir = os.path.join(HERE, 'coverage')
cover_data_dir = os.path.join(cover_dir, 'data')
try:
os.makedirs(cover_data_dir)
except OSError as e:
if e.errno != errno.EEXIST:
raise
# run-tests.py's coverage options don't work for us... yet. So, we hack
# in code coverage manually.
if options.cover:
os.environ['COVERAGE_DIR'] = cover_dir
os.environ['CODE_COVERAGE'] = '1'
if options.headless:
os.environ['ONLY_HEADLESS_TESTS'] = '1'
if options.no_docker:
os.environ['SKIP_DOCKER_TESTS'] = '1'
# We do our own code coverage. Strip it so run-tests.py doesn't try to do
# it's own.
hg_harness_args = [a for a in hg_harness_args if a != '--cover']
verbose = '-v' in hg_harness_args or '--verbose' in hg_harness_args
os.environ['BUGZILLA_USERNAME'] = '[email protected]'
os.environ['BUGZILLA_PASSWORD'] = 'password'
orig_args = list(hg_harness_args)
# Explicitly use our own HG from the virtualenv so other installs
# on the system don't interfere.
if not options.with_hg:
hg = os.path.join(os.path.dirname(sys.executable), 'hg')
hg_harness_args.extend(['--with-hg', hg])
# Always produce an XUnit result file.
hg_harness_args.extend(['--xunit',
os.path.join(HERE, 'coverage', 'results.xml')])
extensions = get_extensions()
test_files = get_test_files(extensions, venv_name)
extension_tests = test_files['extension']
unit_tests = test_files['unit']
hg_tests = test_files['hg']
# Tests requiring Docker in some form.
docker_tests = {t for t, reqs in test_files['docker_requirements'].items()
if reqs}
possible_tests = [os.path.normpath(os.path.abspath(a))
for a in extra[1:] if not a.startswith('-')]
# Filter out arguments that might be tests.
hg_harness_args = [a for a in hg_harness_args
if os.path.normpath(os.path.abspath(a)) not in possible_tests]
requested_tests = []
for t in possible_tests:
if t in test_files['all']:
requested_tests.append(t)
continue
if os.path.isdir(t):
t = os.path.normpath(t)
for test in test_files['all']:
common = os.path.commonprefix([t, test])
common = os.path.normpath(common)
if common == t and test not in requested_tests:
requested_tests.append(test)
continue
run_hg_tests = []
run_unit_tests = []
# All tests unless we got an argument that is a test.
if not requested_tests:
run_hg_tests.extend(extension_tests)
run_hg_tests.extend(hg_tests)
if not options.no_unit:
run_unit_tests.extend(unit_tests)
else:
for t in requested_tests:
if t in unit_tests:
if not options.no_unit:
run_unit_tests.append(t)
else:
run_hg_tests.append(t)
run_all_tests = run_hg_tests + run_unit_tests
run_hg_tests_docker = [t for t in run_hg_tests if t in docker_tests]
run_hg_tests_no_docker = [t for t in run_hg_tests if t not in docker_tests]
run_unit_tests_docker = [t for t in run_unit_tests if t in docker_tests]
run_unit_tests_no_docker = [t for t in run_unit_tests if t not in docker_tests]
# By default, run 1.25 * CPU count non-Docker tests in parallel.
# Docker tests are much more expensive. So give them 2 cores each.
# Memory constrained environments may not be able to handle the Docker
# default. We may want to consider taking memory capacity into account
# as well.
no_docker_jobs = max(int(math.floor(float(multiprocessing.cpu_count()) * 1.25)),
1)
docker_jobs = max(multiprocessing.cpu_count() / 2, 1)
if options.jobs:
no_docker_jobs = options.jobs
docker_jobs = options.jobs
# Enable tests to interact with our Docker controlling script.
have_docker = False
if not options.no_docker:
docker_state = os.path.join(HERE, '.dockerstate')
docker_url, docker_tls = vctdocker.params_from_env(os.environ)
docker = vctdocker.Docker(docker_state, docker_url, tls=docker_tls)
if docker.is_alive():
have_docker = True
os.environ['DOCKER_HOSTNAME'] = docker.docker_hostname
# We build the base BMO images in the test runner because doing it
# from tests would be racey. It is easier to do it here instead of
# complicating code with locks.
#
# But only do this if a test we are running utilizes Docker.
os.environ.update(
get_docker_state(docker, venv_name, run_all_tests,
verbose=verbose,
use_last=options.use_last_images))
print('Will run %d non-Docker tests concurrently' % no_docker_jobs)
if have_docker:
print('Will run %d Docker tests concurrently' % docker_jobs)
# Our custom HGHAVE introduces a check for running Mercurial version.
# This is done by consulting a HGVERSION environment variable.
hg = os.path.join(os.environ['VIRTUAL_ENV'], 'bin', 'hg')
if options.with_hg:
hg = options.with_hg
hgversion = get_hg_version(hg)
if hgversion is None:
print('Unable to determine Mercurial version')
sys.exit(1)
os.environ['HGVERSION'] = hgversion
res = 0
# Don't run with main Mercurial if --all-hg-versions is used because
# --all-hg-versions should run the same version that is installed in the
# virtualenv.
if run_hg_tests and not options.all_hg_versions:
# The Mercurial test harness has been observed to not remove the .err
# files after execution. This is probably a result of us using
# separate directories. Manually remove .err files.
remove_err_files(run_hg_tests)
# Docker tests are a bit heavyweight and tend to fail more than
# non-Docker tests. We perform multiple invocations. First without
# Docker. Then with.
print('Running non-Docker Mercurial tests')
args = hg_harness_args + ['-j', str(no_docker_jobs)] + run_hg_tests_no_docker
res = subprocess.call(args, cwd=HERE)
if have_docker and run_hg_tests_docker:
# We should ideally not leak containers and images. But until
# the Mercurial test harness allows tests to register cleanup
# actions, there is only so much we can do.
print('Running Docker Mercurial tests')
with docker.auto_clean_orphans():
args = hg_harness_args + ['-j', str(docker_jobs)] + run_hg_tests_docker
res2 = subprocess.call(args, cwd=HERE)
if res2:
res = res2
elif run_hg_tests_docker:
print('Skipping %d Docker Mercurial tests because Docker not '
'available' % len(run_hg_tests_docker))
if run_unit_tests:
nose_res = 0
if run_unit_tests_no_docker:
print('Running non-Docker unit tests')
nose_res = run_nose_tests(run_unit_tests_no_docker, no_docker_jobs,
verbose=verbose)
if nose_res:
res = nose_res
if have_docker and run_unit_tests_docker:
print('Running Docker unit tests')
with docker.auto_clean_orphans():
nose_res2 = run_nose_tests(run_unit_tests_docker, docker_jobs,
verbose=verbose)
if nose_res2:
res = nose_res2
elif run_unit_tests_docker:
print('Skipping %d Docker unit tests because Docker not available' %
len(run_unit_tests_docker))
# If we're running the full compatibility run, figure out what versions
# apply to what and run them.
if options.all_hg_versions:
# No need to grab code coverage for legacy versions - it just slows
# us down.
# Assertion: we have no tests that only work on legacy Mercurial
# versions.
try:
del os.environ['CODE_COVERAGE']
except KeyError:
pass
mercurials_dir = os.path.join(os.environ['VIRTUAL_ENV'], 'mercurials')
# Maps directories/versions to lists of tests to run.
# We normalize X.Y.Z to X.Y for compatibility because the monthly
# minor releases of Mercurial shouldn't change behavior. If an
# extension is marked as compatible with X.Y, we run its tests
# against all X.Y and X.Y.Z releases seen on disk.
versions = {}
for dirver in os.listdir(mercurials_dir):
if dirver.startswith('.') or dirver == '@':
continue
normdirver = '.'.join(dirver.split('.')[0:2])
tests = versions.setdefault(dirver, set())
tests |= set(hg_tests)
for m in extensions:
for extver in m['testedwith']:
normever = '.'.join(extver.split('.')[0:2])
if extver == dirver or normever == normdirver:
tests |= m['tests']
def run_hg_tests(version, tests):
if requested_tests:
tests = [t for t in tests if t in requested_tests]
if not tests:
return
remove_err_files(tests)
hg_path = os.path.join(mercurials_dir, version, 'bin', 'hg')
common_args = [RUNTESTS] + ['--with-hg', hg_path]
tests_docker = sorted(t for t in tests if t in docker_tests)
tests_no_docker = sorted(t for t in tests if t not in docker_tests)
real_version = get_hg_version(hg_path)
print('Testing with Mercurial %s (resolved to %s)' % (
version, real_version))
sys.stdout.flush()
os.environ['HGVERSION'] = real_version
print('Running non-Docker Mercurial tests')
args = common_args + ['-j', str(no_docker_jobs)] + tests_no_docker
subprocess.call(args, cwd=HERE)
sys.stdout.flush()
sys.stderr.flush()
if have_docker and tests_docker:
print('Running Docker Mercurial tests')
with docker.auto_clean_orphans():
args = common_args + ['-j', str(docker_jobs)] + tests_docker
subprocess.call(args, cwd=HERE)
sys.stdout.flush()
sys.stderr.flush()
elif tests_docker:
print('Skipping %d Docker Mercurial tests because Docker not '
'available' % len(tests_docker))
for version, tests in sorted(versions.items()):
res2 = run_hg_tests(version, tests)
if res2:
res = res2
# Run all tests against @ because we always want to be compatible
# with the bleeding edge of development.
if not options.no_hg_tip:
all_hg_tests = []
for m in extensions:
all_hg_tests.extend(sorted(m['tests']))
all_hg_tests.extend(hg_tests)
run_hg_tests('@', all_hg_tests)
if options.cover:
produce_coverage_reports(cover_dir)
sys.exit(res)