mirror of
https://github.com/Samsung/escargot.git
synced 2026-06-22 10:01:50 +00:00
e9833cd791
Delete debugger_tester.sh script and rewrite it in python. Also add option to run individial tests. Signed-off-by: Ádám László Kulcsár <adam.kulcsar@szteszoftver.hu>
1107 lines
43 KiB
Python
Executable file
1107 lines
43 KiB
Python
Executable file
#!/usr/bin/env python3
|
|
|
|
# Copyright 2018-present Samsung Electronics Co., Ltd.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
from __future__ import print_function
|
|
|
|
import os
|
|
import traceback
|
|
import sys
|
|
import time
|
|
import re, fnmatch
|
|
import shlex
|
|
|
|
from argparse import ArgumentParser
|
|
from difflib import unified_diff
|
|
from glob import glob
|
|
from os.path import abspath, basename, dirname, join, relpath
|
|
from shutil import copy
|
|
from subprocess import Popen, PIPE, DEVNULL
|
|
|
|
PROJECT_SOURCE_DIR = dirname(dirname(abspath(__file__)))
|
|
DEFAULT_ESCARGOT = join(PROJECT_SOURCE_DIR, 'escargot')
|
|
|
|
|
|
COLOR_RED = '\033[31m'
|
|
COLOR_GREEN = '\033[32m'
|
|
COLOR_YELLOW = '\033[33m'
|
|
COLOR_BLUE = '\033[34m'
|
|
COLOR_PURPLE = '\033[35m'
|
|
COLOR_RESET = '\033[0m'
|
|
|
|
|
|
RUNNERS = {}
|
|
DEFAULT_RUNNERS = []
|
|
|
|
|
|
class runner(object):
|
|
|
|
def __init__(self, suite, default=False):
|
|
self.suite = suite
|
|
self.default = default
|
|
|
|
def __call__(self, fn):
|
|
RUNNERS[self.suite] = fn
|
|
if self.default:
|
|
DEFAULT_RUNNERS.append(self.suite)
|
|
return fn
|
|
|
|
|
|
def run(args, cwd=None, env=None, stdout=None, checkresult=True, report=False):
|
|
if cwd:
|
|
print(COLOR_BLUE + 'cd ' + cwd + ' && \\' + COLOR_RESET)
|
|
if env:
|
|
for var, val in sorted(env.items()):
|
|
print(COLOR_BLUE + var + '=' + val + ' \\' + COLOR_RESET)
|
|
print(COLOR_BLUE + ' '.join(args) + COLOR_RESET)
|
|
|
|
if env is not None:
|
|
full_env = dict(os.environ)
|
|
full_env.update(env)
|
|
env = full_env
|
|
|
|
proc = Popen(args, cwd=cwd, env=env, stdout=PIPE if report else stdout)
|
|
|
|
counter = 0
|
|
|
|
while report:
|
|
nextline = proc.stdout.readline()
|
|
if nextline == '' and proc.poll() is not None:
|
|
break
|
|
stdout.write(nextline)
|
|
stdout.flush()
|
|
if counter % 250 == 0:
|
|
print('Ran %d tests..' % (counter))
|
|
counter += 1
|
|
|
|
out, _ = proc.communicate()
|
|
|
|
if out:
|
|
print(out.decode("utf-8"))
|
|
|
|
if checkresult and proc.returncode:
|
|
raise Exception('command `%s` exited with %s' % (' '.join(args), proc.returncode))
|
|
return out
|
|
|
|
|
|
def readfile(filename):
|
|
with open(filename, 'r') as f:
|
|
return f.readlines()
|
|
|
|
|
|
@runner('sunspider')
|
|
def run_sunspider(engine, arch, extra_arg):
|
|
run([join('.', 'sunspider'),
|
|
'--shell', engine,
|
|
'--suite', 'sunspider-1.0.2'],
|
|
cwd=join(PROJECT_SOURCE_DIR, 'test', 'vendortest', 'SunSpider'))
|
|
|
|
|
|
@runner('sunspider-js', default=True)
|
|
def run_sunspider_js(engine, arch, extra_arg):
|
|
run([engine] + sorted(glob(join(PROJECT_SOURCE_DIR, 'test', 'vendortest', 'SunSpider', 'tests', 'sunspider-1.0.2', '*.js'))))
|
|
|
|
|
|
@runner('octane', default=True)
|
|
def run_octane(engine, arch, extra_arg):
|
|
max_retry_count = 5
|
|
try_count = 0
|
|
last_error = None
|
|
while try_count < max_retry_count:
|
|
try:
|
|
OCTANE_DIR = join(PROJECT_SOURCE_DIR, 'test', 'octane')
|
|
|
|
stdout = run(['/usr/bin/time', '-f', '%M', '-o', 'mem.txt', engine, 'run.js'],
|
|
cwd=OCTANE_DIR,
|
|
stdout=PIPE)
|
|
f = open(join(OCTANE_DIR, 'mem.txt'))
|
|
for s in f:
|
|
rss = s.strip("\n")
|
|
mem = int(rss)
|
|
print('Octane maximum resident set size: ' + str(mem) + 'KB')
|
|
|
|
if arch == str('x86_64') and mem > 250000:
|
|
raise Exception('Exceed memory consumption')
|
|
if arch == str('x86') and mem > 150000:
|
|
raise Exception('Exceed memory consumption')
|
|
|
|
if 'Score' not in stdout.decode("utf-8"):
|
|
raise Exception('no "Score" in stdout')
|
|
return
|
|
except Exception as e:
|
|
last_error = e
|
|
try_count = try_count + 1
|
|
|
|
|
|
raise last_error
|
|
|
|
|
|
@runner('octane-loading', default=True)
|
|
def run_octane_loading(engine, arch, extra_arg):
|
|
OCTANE_OVERRIDE_DIR = join(PROJECT_SOURCE_DIR, 'tools', 'test', 'octane')
|
|
OCTANE_DIR = join(PROJECT_SOURCE_DIR, 'test', 'octane')
|
|
copy(join(OCTANE_OVERRIDE_DIR, 'runLoading.js'), join(OCTANE_DIR, 'runLoading.js'))
|
|
|
|
run([engine, 'runLoading.js'],
|
|
cwd=OCTANE_DIR)
|
|
|
|
|
|
@runner('modifiedVendorTest', default=True)
|
|
def run_internal_test(engine, arch, extra_arg):
|
|
INTERNAL_OVERRIDE_DIR = join(PROJECT_SOURCE_DIR, 'tools', 'test', 'ModifiedVendorTest')
|
|
INTERNAL_DIR = join(PROJECT_SOURCE_DIR, 'test', 'vendortest', 'ModifiedVendorTest')
|
|
|
|
copy(join(INTERNAL_OVERRIDE_DIR, 'internal-test-cases.txt'), join(INTERNAL_DIR, 'internal-test-cases.txt'))
|
|
copy(join(INTERNAL_OVERRIDE_DIR, 'internal-test-driver.py'), join(INTERNAL_DIR, 'driver.py'))
|
|
|
|
run(['python', 'driver.py', engine, 'internal-test-cases.txt'],
|
|
cwd=INTERNAL_DIR)
|
|
|
|
def copy_test262_files():
|
|
TEST262_OVERRIDE_DIR = join(PROJECT_SOURCE_DIR, 'tools', 'test', 'test262')
|
|
TEST262_DIR = join(PROJECT_SOURCE_DIR, 'test', 'test262')
|
|
|
|
copy(join(TEST262_OVERRIDE_DIR, 'excludelist.orig.xml'), join(TEST262_DIR, 'excludelist.xml'))
|
|
copy(join(TEST262_OVERRIDE_DIR, 'cth.js'), join(TEST262_DIR, 'harness', 'cth.js'))
|
|
copy(join(TEST262_OVERRIDE_DIR, 'testIntl.js'), join(TEST262_DIR, 'harness', 'testIntl.js'))
|
|
copy(join(TEST262_OVERRIDE_DIR, 'asyncHelpers.js'), join(TEST262_DIR, 'harness', 'asyncHelpers.js'))
|
|
|
|
copy(join(TEST262_OVERRIDE_DIR, 'parseTestRecord.py'), join(TEST262_DIR, 'tools', 'packaging', 'parseTestRecord.py'))
|
|
copy(join(TEST262_OVERRIDE_DIR, 'test262.py'), join(TEST262_DIR, 'tools', 'packaging', 'test262.py')) # for parallel running (we should re-implement this for es6 suite)
|
|
|
|
@runner('test262', default=True)
|
|
def run_test262(engine, arch, extra_arg):
|
|
copy_test262_files()
|
|
TEST262_DIR = join(PROJECT_SOURCE_DIR, 'test', 'test262')
|
|
args = ['python3', join('tools', 'packaging', 'test262.py'),
|
|
'--command', engine,
|
|
'--summary']
|
|
if len(extra_arg["test262_extra_arg"]):
|
|
args.extend(extra_arg["test262_extra_arg"].split(" "))
|
|
stdout = run(args,
|
|
cwd=TEST262_DIR,
|
|
env={'TZ': 'US/Pacific'},
|
|
stdout=PIPE)
|
|
|
|
summary = stdout.decode("utf-8").split('=== Test262 Summary ===')[1]
|
|
if summary.find('- All tests succeeded') < 0:
|
|
raise Exception('test262 failed')
|
|
print('test262: All tests passed')
|
|
|
|
@runner('test262-strict', default=False)
|
|
def run_test262_strict(engine, arch, extra_arg):
|
|
copy_test262_files()
|
|
TEST262_DIR = join(PROJECT_SOURCE_DIR, 'test', 'test262')
|
|
out = open('test262-strict_out', 'w')
|
|
|
|
args = ['python3', join('tools', 'packaging', 'test262.py'),
|
|
'--command', engine,
|
|
'--full-summary',
|
|
'--strict_only'],
|
|
if len(extra_arg["test262_extra_arg"]):
|
|
args.extend(extra_arg["test262_extra_arg"].split(" "))
|
|
run(args,
|
|
cwd=TEST262_DIR,
|
|
env={'TZ': 'US/Pacific'},
|
|
stdout=out,
|
|
report=True)
|
|
|
|
out.close()
|
|
|
|
with open('test262-strict_out', 'r') as out:
|
|
full = out.read()
|
|
summary = full.split('=== Summary ===')[1]
|
|
if summary.find('- All tests succeeded') < 0:
|
|
print(summary)
|
|
raise Exception('test262-strict failed')
|
|
|
|
print('test262-strict: All tests passed')
|
|
|
|
|
|
@runner('test262-nonstrict', default=False)
|
|
def run_test262_nonstrict(engine, arch, extra_arg):
|
|
copy_test262_files()
|
|
TEST262_DIR = join(PROJECT_SOURCE_DIR, 'test', 'test262')
|
|
out = open('test262-nonstrict_out', 'w')
|
|
|
|
args = ['python3', join('tools', 'packaging', 'test262.py'),
|
|
'--command', engine,
|
|
'--full-summary',
|
|
'--non_strict_only'],
|
|
if len(extra_arg["test262_extra_arg"]):
|
|
args.extend(extra_arg["test262_extra_arg"].split(" "))
|
|
run(args,
|
|
cwd=TEST262_DIR,
|
|
env={'TZ': 'US/Pacific'},
|
|
stdout=out,
|
|
report=True)
|
|
|
|
out.close()
|
|
|
|
with open('test262-nonstrict_out', 'r') as out:
|
|
full = out.read()
|
|
summary = full.split('=== Summary ===')[1]
|
|
if summary.find('- All tests succeeded') < 0:
|
|
print(summary)
|
|
raise Exception('test262-nonstrict failed')
|
|
|
|
print('test262-nonstrict: All tests passed')
|
|
|
|
def compile_test_data_runner(extra_arg):
|
|
if extra_arg["skip_build_test_data_runner"]:
|
|
return
|
|
run(['g++', "./tools/test/test-data-runner/test-data-runner.cpp", "-o", "./tools/test/test-data-runner/test-data-runner", "-g3", "-std=c++11", "-lpthread"],
|
|
cwd=PROJECT_SOURCE_DIR,
|
|
stdout=PIPE)
|
|
|
|
@runner('test262-dump', default=False)
|
|
def run_test262_dump(engine, arch, extra_arg):
|
|
copy_test262_files()
|
|
TEST262_DIR = join(PROJECT_SOURCE_DIR, 'test', 'test262')
|
|
|
|
args = ['python3', join('tools', 'packaging', 'test262.py'),
|
|
'--command', engine,
|
|
'--summary']
|
|
if len(extra_arg["test262_extra_arg"]):
|
|
args.extend(extra_arg["test262_extra_arg"].split(" "))
|
|
stdout = run(args,
|
|
cwd=TEST262_DIR,
|
|
env={'TZ': 'US/Pacific', 'ESCARGOT_DUMP262DATA': '1'},
|
|
stdout=PIPE)
|
|
|
|
summary = stdout.decode("utf-8").split('=== Test262 Summary ===')[1]
|
|
if summary.find('- All tests succeeded') < 0:
|
|
raise Exception('test262 failed')
|
|
print('test262: All tests passed and dumped')
|
|
|
|
@runner('test262-dump-run', default=False)
|
|
def run_test262_dump_run(engine, arch, extra_arg):
|
|
compile_test_data_runner(extra_arg)
|
|
|
|
stdout = run([extra_arg["test_data_runner_path"], "--shell", engine,
|
|
"--test", "test262", "--test-data", join(PROJECT_SOURCE_DIR, 'test', 'test262', 'test262_data')],
|
|
cwd=join(PROJECT_SOURCE_DIR, 'test', 'test262'),
|
|
stdout=PIPE)
|
|
|
|
stdout = stdout.decode("utf-8")
|
|
if stdout.find("Passed") < 0:
|
|
raise Exception('failed')
|
|
print(stdout)
|
|
print('test262-dump-passed')
|
|
|
|
@runner('spidermonkey', default=True)
|
|
def run_spidermonkey(engine, arch, extra_arg):
|
|
SPIDERMONKEY_OVERRIDE_DIR = join(PROJECT_SOURCE_DIR, 'tools', 'test', 'spidermonkey')
|
|
SPIDERMONKEY_DIR = join(PROJECT_SOURCE_DIR, 'test', 'vendortest', 'SpiderMonkey')
|
|
|
|
run([join(SPIDERMONKEY_DIR, 'jstests.py'),
|
|
'--no-progress', '-s',
|
|
'--timeout', '500',
|
|
'--xul-info', '%s-gcc3:Linux:false' % arch,
|
|
'--exclude-file', join(SPIDERMONKEY_OVERRIDE_DIR, 'excludelist.txt'),
|
|
engine,
|
|
'--output-file', join(SPIDERMONKEY_OVERRIDE_DIR, '%s.log.txt' % arch),
|
|
'--failure-file', join(SPIDERMONKEY_OVERRIDE_DIR, '%s.gen.txt' % arch),
|
|
'non262/Array', 'non262/ArrayBuffer', 'non262/arrow-functions', 'non262/BigInt', 'non262/Boolean', 'non262/class', 'non262/comprehensions', 'non262/DataView', 'non262/Date',
|
|
'non262/destructuring', 'non262/Error', 'non262/eval', 'non262/Exceptions', 'non262/execution-contexts', 'non262/expressions', 'non262/extensions', 'non262/fields', 'non262/Function',
|
|
'non262/GC', 'non262/generators', 'non262/get-set', 'non262/global', 'non262/Intl', 'non262/iterable', 'non262/jit', 'non262/JSON', 'non262/lexical', 'non262/lexical-conventions',
|
|
'non262/lexical-environment', 'non262/literals', 'non262/Map', 'non262/Math', 'non262/misc', 'non262/module', 'non262/Number', 'non262/object', 'non262/operators', 'non262/pipeline',
|
|
'non262/Promise', 'non262/Proxy', 'non262/Reflect', 'non262/reflect-parse', 'non262/RegExp', 'non262/regress', 'non262/Scope', 'non262/Script', 'non262/Set', 'non262/statements',
|
|
'non262/strict', 'non262/String', 'non262/Symbol', 'non262/syntax', 'non262/template-strings', 'non262/TypedArray', 'non262/TypedObject', 'non262/types', 'non262/Unicode', 'non262/WeakMap'],
|
|
env={'LOCALE': 'en_US'})
|
|
|
|
orig = sorted(readfile(join(SPIDERMONKEY_OVERRIDE_DIR, '%s.orig.txt' % arch)))
|
|
gen = sorted(readfile(join(SPIDERMONKEY_OVERRIDE_DIR, '%s.gen.txt' % arch)))
|
|
diff = list(unified_diff(orig, gen))
|
|
if diff:
|
|
for diffline in diff:
|
|
print(diffline)
|
|
raise Exception('failure files differ')
|
|
|
|
@runner('spidermonkey-dump', default=False)
|
|
def run_spidermonkey_dump(engine, arch, extra_arg):
|
|
SPIDERMONKEY_OVERRIDE_DIR = join(PROJECT_SOURCE_DIR, 'tools', 'test', 'spidermonkey')
|
|
SPIDERMONKEY_DIR = join(PROJECT_SOURCE_DIR, 'test', 'vendortest', 'SpiderMonkey')
|
|
|
|
log_path = join(SPIDERMONKEY_OVERRIDE_DIR, '%s.log.txt' % arch)
|
|
if not os.path.exists(log_path):
|
|
run_spidermonkey(engine, arch, extra_arg)
|
|
|
|
log = sorted(readfile(log_path))
|
|
for idx, x in enumerate(log):
|
|
text = log[idx]
|
|
text = text.split(" ")
|
|
text.pop(0)
|
|
text = " ".join(text)
|
|
text = text.replace(PROJECT_SOURCE_DIR + "/", "")
|
|
text = text.replace("-f ", "")
|
|
text = text.replace("\n", "")
|
|
log[idx] = text
|
|
|
|
with open(join(SPIDERMONKEY_OVERRIDE_DIR, '%s.data.txt' % arch), "w") as output:
|
|
for idx, x in enumerate(log):
|
|
output.write(log[idx] + "\n")
|
|
if log[idx].endswith("-n.js"):
|
|
output.write("3\n")
|
|
else:
|
|
output.write("0\n")
|
|
output.write("\n")
|
|
|
|
@runner('spidermonkey-dump-run', default=False)
|
|
def run_spidermonkey_dump_run(engine, arch, extra_arg):
|
|
compile_test_data_runner(extra_arg)
|
|
|
|
SPIDERMONKEY_OVERRIDE_DIR = join(PROJECT_SOURCE_DIR, 'tools', 'test', 'spidermonkey')
|
|
stdout = run([extra_arg["test_data_runner_path"], '--test-data', join(SPIDERMONKEY_OVERRIDE_DIR, '%s.data.txt' % arch),
|
|
"--shell", engine, "--env", "LOCALE=en_US"],
|
|
cwd=PROJECT_SOURCE_DIR,
|
|
stdout=PIPE)
|
|
|
|
stdout = stdout.decode("utf-8")
|
|
if stdout.find("Passed") < 0:
|
|
raise Exception('failed')
|
|
print(stdout)
|
|
print('spidermonkey-dump-passed')
|
|
|
|
|
|
@runner('jsc-stress', default=True)
|
|
def run_jsc_stress(engine, arch, extra_arg):
|
|
JSC_DIR = join('test', 'vendortest', 'driver')
|
|
|
|
run([join(JSC_DIR, 'driver.py'),
|
|
'-s', 'stress',
|
|
'-e', engine,
|
|
'-a', arch],
|
|
cwd=PROJECT_SOURCE_DIR,
|
|
env={'PYTHONPATH': '.'})
|
|
|
|
@runner('jsc-stress-dump', default=False)
|
|
def run_jsc_stress_dump(engine, arch, extra_arg):
|
|
JSC_DIR = join('test', 'vendortest', 'driver')
|
|
|
|
log_path = join(JSC_DIR, 'jsc.stress.%s.gen.txt' % arch)
|
|
if not os.path.exists(log_path):
|
|
run_jsc_stress(engine, arch, extra_arg)
|
|
|
|
log = readfile(log_path)
|
|
with open(join(JSC_DIR, 'jsc.stress.%s.data.txt' % arch), "w") as output:
|
|
for idx, x in enumerate(log):
|
|
text = log[idx]
|
|
if ".... Success\n" in text:
|
|
stext = text.split(" ")
|
|
output.write(stext[1] + " " + stext[2] + "\n")
|
|
output.write("0\n")
|
|
output.write("\n")
|
|
|
|
@runner('jsc-stress-dump-run', default=False)
|
|
def run_jsc_stress_dump_run(engine, arch, extra_arg):
|
|
JSC_DIR = join('test', 'vendortest', 'driver')
|
|
data_path = join(JSC_DIR, 'jsc.stress.%s.data.txt' % arch)
|
|
|
|
compile_test_data_runner(extra_arg)
|
|
|
|
stdout = run([extra_arg["test_data_runner_path"], "--shell", engine,
|
|
"--test-data", data_path, "--env", "GC_FREE_SPACE_DIVISOR=1"],
|
|
cwd=PROJECT_SOURCE_DIR,
|
|
stdout=PIPE)
|
|
|
|
stdout = stdout.decode("utf-8")
|
|
if stdout.find("Passed") < 0:
|
|
raise Exception('failed')
|
|
print(stdout)
|
|
print('jsc-stress-dump-passed')
|
|
|
|
def _run_regression_tests(engine, assert_js, files, is_fail):
|
|
fails = 0
|
|
for file in files:
|
|
proc = Popen([engine, assert_js, file], stdout=PIPE)
|
|
out, _ = proc.communicate()
|
|
|
|
if is_fail and proc.returncode or not is_fail and not proc.returncode:
|
|
print('%sOK: %s%s' % (COLOR_GREEN, file, COLOR_RESET))
|
|
else:
|
|
print('%sFAIL(%d): %s%s' % (COLOR_RED, proc.returncode, file, COLOR_RESET))
|
|
print(out.decode("utf-8"))
|
|
|
|
fails += 1
|
|
|
|
return fails
|
|
|
|
|
|
@runner('regression-tests', default=True)
|
|
def run_regression_tests(engine, arch, extra_arg):
|
|
REGRESSION_DIR = join(PROJECT_SOURCE_DIR, 'test', 'vendortest', 'Escargot', 'regression-tests')
|
|
REGRESSION_XFAIL_DIR = join(REGRESSION_DIR, 'xfail')
|
|
REGRESSION_ASSERT_JS = join(REGRESSION_DIR, 'assert.js')
|
|
|
|
print('Running regression tests:')
|
|
xpass = glob(join(REGRESSION_DIR, 'issue-*.js')) + glob(join(REGRESSION_DIR, 'issue-*.mjs'))
|
|
xpass_result = _run_regression_tests(engine, REGRESSION_ASSERT_JS, xpass, False)
|
|
|
|
print('Running regression tests expected to fail:')
|
|
xfail = glob(join(REGRESSION_XFAIL_DIR, 'issue-*.js')) + glob(join(REGRESSION_XFAIL_DIR, 'issue-*.mjs'))
|
|
xfail_result = _run_regression_tests(engine, REGRESSION_ASSERT_JS, xfail, True)
|
|
|
|
tests_total = len(xpass) + len(xfail)
|
|
fail_total = xfail_result + xpass_result
|
|
print('TOTAL: %d' % (tests_total))
|
|
print('%sPASS : %d%s' % (COLOR_GREEN, tests_total - fail_total, COLOR_RESET))
|
|
print('%sFAIL : %d%s' % (COLOR_RED, fail_total, COLOR_RESET))
|
|
|
|
if fail_total > 0:
|
|
raise Exception("Regression tests failed")
|
|
|
|
|
|
def _run_jetstream(engine, target_test):
|
|
JETSTREAM_OVERRIDE_DIR = join(PROJECT_SOURCE_DIR, 'tools', 'test', 'jetstream')
|
|
JETSTREAM_DIR = join(PROJECT_SOURCE_DIR, 'test', 'vendortest', 'JetStream-1.1')
|
|
|
|
copy(join(JETSTREAM_OVERRIDE_DIR, 'jetstream.CDjsSetup.js'), join(JETSTREAM_DIR, 'CDjsSetup.js'))
|
|
copy(join(JETSTREAM_OVERRIDE_DIR, 'jetstream.OctaneSetup.js'), join(JETSTREAM_DIR, 'OctaneSetup.js'))
|
|
copy(join(JETSTREAM_OVERRIDE_DIR, 'jetstream.Octane2Setup.js'), join(JETSTREAM_DIR, 'Octane2Setup.js'))
|
|
copy(join(JETSTREAM_OVERRIDE_DIR, 'jetstream.SimpleSetup.js'), join(JETSTREAM_DIR, 'SimpleSetup.js'))
|
|
copy(join(JETSTREAM_OVERRIDE_DIR, 'jetstream.SunSpiderSetup.js'), join(JETSTREAM_DIR, 'SunSpiderSetup.js'))
|
|
copy(join(JETSTREAM_OVERRIDE_DIR, 'jetstream.cdjs.util.js'), join(JETSTREAM_DIR, 'cdjs', 'util.js'))
|
|
copy(join(JETSTREAM_OVERRIDE_DIR, 'jetstream.runOnePlan.js'), join(JETSTREAM_DIR, 'runOnePlan.js'))
|
|
copy(join(JETSTREAM_OVERRIDE_DIR, 'jetstream.run.sh'), join(JETSTREAM_DIR, 'run.sh'))
|
|
|
|
run([join('.', 'run.sh'), engine, target_test],
|
|
cwd=JETSTREAM_DIR)
|
|
run(['python', join(JETSTREAM_OVERRIDE_DIR, 'parsingResults.py'),
|
|
join(JETSTREAM_OVERRIDE_DIR, 'jetstream-result-raw.res'),
|
|
target_test])
|
|
if 'NaN' in ''.join(readfile(join(JETSTREAM_OVERRIDE_DIR, 'jetstream-result-raw.res'))):
|
|
raise Exception('result contains "NaN"')
|
|
|
|
|
|
@runner('jetstream-only-simple-parallel-1')
|
|
def run_jetstream_only_simple_parallel_1(engine, arch, extra_arg):
|
|
_run_jetstream(engine, 'simple-1')
|
|
|
|
|
|
@runner('jetstream-only-simple-parallel-2')
|
|
def run_jetstream_only_simple_parallel_2(engine, arch, extra_arg):
|
|
_run_jetstream(engine, 'simple-2')
|
|
|
|
|
|
@runner('jetstream-only-simple-parallel-3')
|
|
def run_jetstream_only_simple_parallel_3(engine, arch, extra_arg):
|
|
_run_jetstream(engine, 'simple-3')
|
|
|
|
|
|
@runner('jetstream-only-simple', default=True)
|
|
def run_jetstream_only_simple(engine, arch, extra_arg):
|
|
_run_jetstream(engine, 'simple')
|
|
|
|
|
|
@runner('jetstream-only-cdjs', default=True)
|
|
def run_jetstream_only_cdjs(engine, arch, extra_arg):
|
|
_run_jetstream(engine, 'cdjs')
|
|
|
|
|
|
@runner('jetstream-only-sunspider')
|
|
def run_jetstream_only_sunspider(engine, arch, extra_arg):
|
|
_run_jetstream(engine, 'sunspider')
|
|
|
|
|
|
@runner('jetstream-only-octane')
|
|
def run_jetstream_only_octane(engine, arch, extra_arg):
|
|
_run_jetstream(engine, 'octane')
|
|
|
|
|
|
@runner('chakracore', default=True)
|
|
def run_chakracore(engine, arch, extra_arg):
|
|
CHAKRACORE_OVERRIDE_DIR = join(PROJECT_SOURCE_DIR, 'tools', 'test', 'chakracore')
|
|
CHAKRACORE_DIR = join(PROJECT_SOURCE_DIR, 'test', 'vendortest', 'ChakraCore')
|
|
|
|
copy(join(CHAKRACORE_OVERRIDE_DIR, 'chakracore.run.sh'), join(CHAKRACORE_DIR, 'run.sh'))
|
|
copy(join(CHAKRACORE_OVERRIDE_DIR, 'chakracore.include.js'), join(CHAKRACORE_DIR, 'include.js'))
|
|
copy(join(CHAKRACORE_OVERRIDE_DIR, 'chakracore.rlexedirs.xml'), join(CHAKRACORE_DIR, 'rlexedirs.xml'))
|
|
for rlexexml in glob(join(CHAKRACORE_OVERRIDE_DIR, '*.rlexe.xml')):
|
|
dirname, _, filename = basename(rlexexml).partition('.')
|
|
copy(rlexexml, join(CHAKRACORE_DIR, dirname, filename))
|
|
for js in [join(CHAKRACORE_DIR, 'DynamicCode', 'eval-nativecodedata.js'),
|
|
join(CHAKRACORE_DIR, 'utf8', 'unicode_digit_as_identifier_should_work.js')]:
|
|
with open(js, 'a') as f:
|
|
f.write("WScript.Echo('PASS');")
|
|
|
|
stdout = run(['bash', join(CHAKRACORE_DIR, 'run.sh'), relpath(engine)],
|
|
stdout=PIPE)
|
|
with open(join(PROJECT_SOURCE_DIR, 'test', 'vendortest', 'driver', 'chakracore.%s.gen.txt' % arch), 'w') as gen_txt:
|
|
result = stdout.decode("utf-8")
|
|
gen_txt.write(result)
|
|
run(['diff',
|
|
join(CHAKRACORE_OVERRIDE_DIR, 'chakracore.%s.orig.txt' % arch),
|
|
join(PROJECT_SOURCE_DIR, 'test', 'vendortest', 'driver', 'chakracore.%s.gen.txt' % arch)])
|
|
|
|
@runner('chakracore-dump', default=False)
|
|
def run_chakracore_dump(engine, arch, extra_arg):
|
|
CHAKRACORE_OVERRIDE_DIR = join(PROJECT_SOURCE_DIR, 'tools', 'test', 'chakracore')
|
|
CHAKRACORE_DIR = join(PROJECT_SOURCE_DIR, 'test', 'vendortest', 'ChakraCore')
|
|
|
|
log_path = join(CHAKRACORE_OVERRIDE_DIR, 'chakracore.%s.orig.txt' % arch)
|
|
|
|
driver_file = "tools/test/chakracore/chakracore.include.js"
|
|
log = readfile(log_path)
|
|
with open(join(CHAKRACORE_OVERRIDE_DIR, 'chakracore.%s.data.txt' % arch), "w") as output:
|
|
for idx, x in enumerate(log):
|
|
if ".......... Pass\n" in x:
|
|
segs = x.split(" ")
|
|
kind = segs[0]
|
|
kind = kind[1:]
|
|
kind = kind[:-1]
|
|
tc = join('test', 'vendortest', 'ChakraCore', kind, segs[1])
|
|
output.write(driver_file + " " + tc + "\n")
|
|
output.write("0\n")
|
|
|
|
baseline = tc[:-2] + "baseline"
|
|
try:
|
|
# some test has invalid unicode
|
|
tc_contents = "".join(readfile(tc))
|
|
except:
|
|
tc_contents = ""
|
|
if os.path.exists(baseline) and "testRunner" not in tc_contents:
|
|
baseline = baseline + "\n"
|
|
else:
|
|
baseline = "\n"
|
|
|
|
output.write(baseline)
|
|
|
|
@runner('chakracore-dump-run', default=False)
|
|
def run_chakracore_dump_run(engine, arch, extra_arg):
|
|
compile_test_data_runner(extra_arg)
|
|
|
|
CHAKRACORE_OVERRIDE_DIR = join(PROJECT_SOURCE_DIR, 'tools', 'test', 'chakracore')
|
|
stdout = run([extra_arg["test_data_runner_path"], "--shell", engine,
|
|
"--test-data", join(CHAKRACORE_OVERRIDE_DIR, 'chakracore.%s.data.txt' % arch),
|
|
"--env", "GC_FREE_SPACE_DIVISOR=1 TZ=US/Pacific",
|
|
"--treat-global-tostring-as-object"],
|
|
cwd=join(PROJECT_SOURCE_DIR),
|
|
stdout=PIPE)
|
|
|
|
stdout = stdout.decode("utf-8")
|
|
if stdout.find("Passed") < 0:
|
|
raise Exception('failed')
|
|
|
|
print('chakracore-dump-passed')
|
|
|
|
@runner('v8', default=True)
|
|
def run_v8(engine, arch, extra_arg):
|
|
V8_OVERRIDE_DIR = join(PROJECT_SOURCE_DIR, 'tools', 'test', 'v8')
|
|
V8_DIR = join(PROJECT_SOURCE_DIR, 'test', 'vendortest', 'v8')
|
|
|
|
copy(join(V8_OVERRIDE_DIR, 'v8.mjsunit.status'), join(V8_DIR, 'test', 'mjsunit', 'mjsunit.status'))
|
|
copy(join(V8_OVERRIDE_DIR, 'v8.mjsunit.js'), join(V8_DIR, 'test', 'mjsunit', 'mjsunit.js'))
|
|
copy(join(V8_OVERRIDE_DIR, 'v8.run-tests.py'), join(V8_DIR, 'tools', 'run-tests.py'))
|
|
copy(join(V8_OVERRIDE_DIR, 'v8.testsuite.py'), join(V8_DIR, 'tools', 'testrunner', 'local', 'testsuite.py'))
|
|
copy(join(V8_OVERRIDE_DIR, 'v8.execution.py'), join(V8_DIR, 'tools', 'testrunner', 'local', 'execution.py'))
|
|
copy(join(V8_OVERRIDE_DIR, 'v8.progress.py'), join(V8_DIR, 'tools', 'testrunner', 'local', 'progress.py'))
|
|
|
|
arch = {'x86': 'x32', 'x86_64': 'x64'}[arch]
|
|
|
|
if (engine == "escargot"):
|
|
shell_str = "../../../escargot"
|
|
else:
|
|
shell_str = engine
|
|
|
|
stdout = run([join(V8_DIR, 'tools', 'run-tests.py'),
|
|
'--timeout=120',
|
|
'--quickcheck',
|
|
'--no-presubmit', '--no-variants',
|
|
'--arch-and-mode=%s.release' % arch,
|
|
'--shell', shell_str,
|
|
'--escargot',
|
|
'--report',
|
|
'-p', 'verbose',
|
|
'--no-sorting',
|
|
'--no-network',
|
|
'mjsunit'],
|
|
stdout=PIPE,
|
|
env={'GC_FREE_SPACE_DIVISOR': '1'},
|
|
checkresult=False)
|
|
# FIXME: V8 test runner tends to exit with 2 (most probably the result of
|
|
# a `self.remaining != 0` in `testrunner.local.execution.Runner.Run()`.
|
|
# Couldn't find out yet why that happens.
|
|
# NOTE: This has also been suppressed in cmake/make-based executors by
|
|
# piping the output of run-test.py into a tee. In case of pipes, the
|
|
# observed exit code is that of the last element of the pipe, which is
|
|
# tee in that case (which is always 0).
|
|
|
|
if '=== All tests succeeded' not in stdout.decode("utf-8"):
|
|
raise Exception('Not all tests succeeded')
|
|
|
|
return stdout
|
|
|
|
|
|
@runner('v8-dump', default=False)
|
|
def run_v8_dump(engine, arch, extra_arg):
|
|
stdout = run_v8(engine, arch, extra_arg)
|
|
stdout = stdout.decode("utf-8").split("\n")
|
|
|
|
TOOL_V8_DIR = join(PROJECT_SOURCE_DIR, 'tools', 'test', 'v8')
|
|
STATUS_FILE_CONTENTS = readfile(join(TOOL_V8_DIR, 'v8.mjsunit.status'))
|
|
|
|
driver_path = "tools/test/v8/v8.mjsunit.js";
|
|
with open(join(TOOL_V8_DIR, '%s.data.txt' % arch), "w") as output:
|
|
for line in stdout:
|
|
if "Done running " in line:
|
|
casename = line.split(" ")[2].split(":")[0]
|
|
skipped = False
|
|
for status_file_content in STATUS_FILE_CONTENTS:
|
|
if casename[8:] in status_file_content:
|
|
if "PASS" not in status_file_content or "FAIL" in status_file_content:
|
|
skipped = True
|
|
if not skipped and "bugs/" not in casename:
|
|
print("TEST " + casename)
|
|
filename = "test/vendortest/v8/test/" + casename + ".js"
|
|
output.write(driver_path)
|
|
output.write(" ")
|
|
output.write(filename)
|
|
output.write("\n")
|
|
output.write("0\n")
|
|
output.write("\n")
|
|
else:
|
|
print("SKIP " + casename)
|
|
|
|
@runner('v8-dump-run', default=False)
|
|
def run_v8_dump_run(engine, arch, extra_arg):
|
|
compile_test_data_runner(extra_arg)
|
|
|
|
TOOL_V8_DIR = join(PROJECT_SOURCE_DIR, 'tools', 'test', 'v8')
|
|
stdout = run([extra_arg["test_data_runner_path"], "--shell", engine,
|
|
"--test-data", join(TOOL_V8_DIR, '%s.data.txt' % arch), "--env", "GC_FREE_SPACE_DIVISOR=1"],
|
|
cwd=join(PROJECT_SOURCE_DIR),
|
|
stdout=PIPE)
|
|
|
|
stdout = stdout.decode("utf-8")
|
|
if stdout.find("Passed") < 0:
|
|
raise Exception('failed')
|
|
|
|
print('v8-dump-passed')
|
|
|
|
@runner('new-es', default=True)
|
|
def run_new_es(engine, arch, extra_arg):
|
|
NEW_ES_DIR = join(PROJECT_SOURCE_DIR, 'test', 'vendortest', 'Escargot', 'new-es')
|
|
NEW_ES_ASSERT_JS = join(NEW_ES_DIR, 'assert.js')
|
|
|
|
print('Running new-es test:')
|
|
files = glob(join(NEW_ES_DIR, '*.js'))
|
|
files.remove(NEW_ES_ASSERT_JS)
|
|
fail_total = 0
|
|
fail_files = []
|
|
for file in files:
|
|
proc = Popen([engine, NEW_ES_ASSERT_JS, file], stdout=PIPE)
|
|
out, _ = proc.communicate()
|
|
|
|
if not proc.returncode:
|
|
print('%sOK: %s%s' % (COLOR_GREEN, file, COLOR_RESET))
|
|
else:
|
|
print('%sFAIL(%d): %s%s' % (COLOR_RED, proc.returncode, file, COLOR_RESET))
|
|
print(out.decode("utf-8"))
|
|
fail_total += 1
|
|
fail_files.append(file)
|
|
|
|
|
|
tests_total = len(files)
|
|
print('TOTAL: %d' % (tests_total))
|
|
print('%sPASS : %d%s' % (COLOR_GREEN, tests_total - fail_total, COLOR_RESET))
|
|
print('%sFAIL : %d%s' % (COLOR_RED, fail_total, COLOR_RESET))
|
|
for fail in fail_files:
|
|
print('%sFAIL : %s%s' % (COLOR_RED, fail, COLOR_RESET))
|
|
|
|
if fail_total > 0:
|
|
raise Exception('new-es tests failed')
|
|
|
|
@runner('intl', default=True)
|
|
def run_intl(engine, arch, extra_arg):
|
|
INTL_DIR = join(PROJECT_SOURCE_DIR, 'test', 'vendortest', 'Escargot', 'intl')
|
|
INTL_ASSERT_JS = join(INTL_DIR, 'assert.js')
|
|
|
|
print('Running Intl test:')
|
|
files = glob(join(INTL_DIR, '*.js'))
|
|
files.remove(INTL_ASSERT_JS)
|
|
fails = 0
|
|
for file in files:
|
|
proc = Popen([engine, INTL_ASSERT_JS, file], stdout=PIPE)
|
|
out, _ = proc.communicate()
|
|
|
|
if not proc.returncode:
|
|
print('%sOK: %s%s' % (COLOR_GREEN, file, COLOR_RESET))
|
|
else:
|
|
print('%sFAIL(%d): %s%s' % (COLOR_RED, proc.returncode, file, COLOR_RESET))
|
|
print(out.decode("utf-8"))
|
|
fails += 1
|
|
|
|
if fails > 0:
|
|
raise Exception('Intl tests failed')
|
|
|
|
@runner('escargot-test-dump', default=False)
|
|
def run_escargot_test_dump(engine, arch, extra_arg):
|
|
NEW_ES_DIR = join(PROJECT_SOURCE_DIR, 'test', 'vendortest', 'Escargot', 'new-es')
|
|
NEW_ES_ASSERT_JS = join(NEW_ES_DIR, 'assert.js')
|
|
|
|
newes = glob(join(NEW_ES_DIR, '*.js'))
|
|
newes.remove(NEW_ES_ASSERT_JS)
|
|
|
|
REGRESSION_DIR = join(PROJECT_SOURCE_DIR, 'test', 'vendortest', 'Escargot', 'regression-tests')
|
|
REGRESSION_XFAIL_DIR = join(REGRESSION_DIR, 'xfail')
|
|
REGRESSION_ASSERT_JS = join(REGRESSION_DIR, 'assert.js')
|
|
xpass = glob(join(REGRESSION_DIR, 'issue-*.js'))
|
|
xfail = glob(join(REGRESSION_XFAIL_DIR, 'issue-*.js'))
|
|
|
|
INTL_DIR = join(PROJECT_SOURCE_DIR, 'test', 'vendortest', 'Escargot', 'intl')
|
|
INTL_ASSERT_JS = join(INTL_DIR, 'assert.js')
|
|
|
|
intl = glob(join(INTL_DIR, '*.js'))
|
|
intl.remove(INTL_ASSERT_JS)
|
|
|
|
with open(join(PROJECT_SOURCE_DIR, 'test', 'vendortest', 'Escargot', 'data.txt'), "w") as output:
|
|
for file in newes:
|
|
output.write(NEW_ES_ASSERT_JS.replace(PROJECT_SOURCE_DIR + "/", "") + " "
|
|
+ file.replace(PROJECT_SOURCE_DIR + "/", "") + " " + "\n")
|
|
output.write("0\n")
|
|
output.write("\n")
|
|
|
|
for file in xpass:
|
|
output.write(REGRESSION_ASSERT_JS.replace(PROJECT_SOURCE_DIR + "/", "") + " "
|
|
+ file.replace(PROJECT_SOURCE_DIR + "/", "") + " " + "\n")
|
|
output.write("0\n")
|
|
output.write("\n")
|
|
|
|
for file in xfail:
|
|
output.write(INTL_ASSERT_JS.replace(PROJECT_SOURCE_DIR + "/", "") + " "
|
|
+ file.replace(PROJECT_SOURCE_DIR + "/", "") + " " + "\n")
|
|
output.write("3\n")
|
|
output.write("\n")
|
|
|
|
for file in intl:
|
|
output.write(INTL_ASSERT_JS.replace(PROJECT_SOURCE_DIR + "/", "") + " "
|
|
+ file.replace(PROJECT_SOURCE_DIR + "/", "") + " " + "\n")
|
|
output.write("0\n")
|
|
output.write("\n")
|
|
|
|
@runner('escargot-test-dump-run', default=False)
|
|
def run_escargot_test_dump_run(engine, arch, extra_arg):
|
|
compile_test_data_runner(extra_arg)
|
|
|
|
stdout = run([extra_arg["test_data_runner_path"], "--shell", engine,
|
|
"--test-data", join(PROJECT_SOURCE_DIR, 'test', 'vendortest', 'Escargot', 'data.txt')],
|
|
cwd=join(PROJECT_SOURCE_DIR),
|
|
stdout=PIPE)
|
|
|
|
stdout = stdout.decode("utf-8")
|
|
if stdout.find("Passed") < 0:
|
|
raise Exception('failed')
|
|
print(stdout)
|
|
print('escargot-dump-passed')
|
|
|
|
@runner('wasm-js', default=False)
|
|
def run_wasm_js(engine, arch, extra_arg):
|
|
WASM_TEST_ROOT = join(PROJECT_SOURCE_DIR, 'test', 'vendortest', 'wasm-js')
|
|
WASM_TEST_DIR = join(WASM_TEST_ROOT, 'tests')
|
|
WASM_TEST_MJS = join(WASM_TEST_ROOT, 'mjsunit.js')
|
|
WASM_TEST_HARNESS = join(WASM_TEST_ROOT, 'testharness.js')
|
|
|
|
EXCLUDE_LIST_FILE = join(PROJECT_SOURCE_DIR, 'tools', 'test', 'wasm-js', 'exclude_list.txt')
|
|
exclude_list = []
|
|
with open(EXCLUDE_LIST_FILE) as f:
|
|
exclude_list = f.read().replace('\n', ' ').split()
|
|
|
|
files = []
|
|
for root, dirnames, filenames in os.walk(join(WASM_TEST_DIR)):
|
|
if "proposals" in root:
|
|
# skip proposals test
|
|
continue
|
|
for filename in fnmatch.filter(filenames, '*.any.js'):
|
|
full_path = join(root, filename)
|
|
rel_path = full_path[len(WASM_TEST_DIR)+1:]
|
|
if rel_path in exclude_list:
|
|
continue
|
|
files.append(join(root, filename))
|
|
files = sorted(files)
|
|
|
|
WPT_ROOT = "/wasm/jsapi/"
|
|
META_SCRIPT_REGEXP = re.compile(r"META:\s*script=(.*)")
|
|
fail_total = 0
|
|
fail_files = []
|
|
for file in files:
|
|
source = ""
|
|
with open(file) as f:
|
|
source = f.read()
|
|
|
|
script_files = []
|
|
for script in META_SCRIPT_REGEXP.findall(source):
|
|
if (script.startswith(WPT_ROOT)):
|
|
script = join(WASM_TEST_DIR, script[len(WPT_ROOT):])
|
|
else:
|
|
script = join(os.path.dirname(file), script)
|
|
script_files.append(script)
|
|
|
|
script_files.append(file)
|
|
|
|
stdout = run([engine, WASM_TEST_MJS, WASM_TEST_HARNESS] + script_files, stdout=PIPE)
|
|
out = stdout.decode("utf-8")
|
|
|
|
if out.find("FAIL") >= 0:
|
|
fail_total += 1
|
|
fail_files.append(file)
|
|
else:
|
|
print('%sOK: %s%s' % (COLOR_GREEN, file, COLOR_RESET))
|
|
|
|
tests_total = len(files)
|
|
print('TOTAL: %d' % (tests_total))
|
|
print('%sPASS : %d%s' % (COLOR_GREEN, tests_total - fail_total, COLOR_RESET))
|
|
print('%sFAIL : %d%s' % (COLOR_RED, fail_total, COLOR_RESET))
|
|
for fail in fail_files:
|
|
print('%sFAIL : %s%s' % (COLOR_RED, fail, COLOR_RESET))
|
|
|
|
if fail_total > 0:
|
|
raise Exception('wasm-js tests failed')
|
|
|
|
@runner('cctest', default=False)
|
|
def run_cctest(engine, arch, extra_arg):
|
|
proc = Popen([engine], stdout=PIPE)
|
|
out, _ = proc.communicate()
|
|
|
|
outStr = out.decode("utf-8")
|
|
print(outStr)
|
|
result = outStr.lower()
|
|
if 'fail' in result:
|
|
raise Exception('Not all tests succeeded')
|
|
|
|
@runner('debugger-server-source', default=True)
|
|
def run_escargot_debugger(engine, arch, extra_arg):
|
|
ESCARGOT_DEBUGGER_TEST_DIR = join(PROJECT_SOURCE_DIR, 'tools', 'debugger', 'tests')
|
|
ESCARGOT_DEBUGGER = join(PROJECT_SOURCE_DIR, 'tools', 'debugger', 'debugger.py')
|
|
print('Running Escargot-Debugger-Server-Source test:')
|
|
fails = 0
|
|
|
|
files = os.listdir(ESCARGOT_DEBUGGER_TEST_DIR)
|
|
if extra_arg['debugger_extra']:
|
|
specific_tests = list(extra_arg['debugger_extra'])
|
|
files = [file for test in specific_tests for file in files if test in file and file.endswith(".cmd")]
|
|
else:
|
|
files = os.listdir(ESCARGOT_DEBUGGER_TEST_DIR)
|
|
|
|
for file in files:
|
|
if file.endswith(".cmd"):
|
|
commands = ""
|
|
with open(ESCARGOT_DEBUGGER_TEST_DIR + "/" + file, 'r') as f:
|
|
commands = f.read()
|
|
|
|
additional_options = ""
|
|
if "do_wait_exit2" in file:
|
|
additional_options += " --wait-before-exit "
|
|
|
|
args = shlex.split(engine + " --start-debug-server --debugger-wait-source" + additional_options)
|
|
escargot_client = Popen(args, stdin=PIPE, stdout=PIPE, stderr=PIPE, text=True)
|
|
|
|
debugger_options = ""
|
|
if "do_command" in file:
|
|
debugger_options += " --command \"b do_command.js:2;c;e i;c\""
|
|
elif "do_wait_exit1" in file:
|
|
debugger_options += " --wait-before-exit 1"
|
|
|
|
test_base, _ = os.path.splitext(file.removesuffix(".cmd"))
|
|
test_case = ESCARGOT_DEBUGGER_TEST_DIR + "/" + test_base
|
|
|
|
if "client_source_multiple" in file:
|
|
test_case = test_case + "_2.js " + test_case + "_1.js"
|
|
elif "module_debug" not in file:
|
|
test_case += ".js"
|
|
else:
|
|
test_case += "_1.mjs"
|
|
|
|
args = shlex.split(ESCARGOT_DEBUGGER + " --non-interactive --client-source " + test_case + debugger_options)
|
|
debugger = Popen(args, stdin=PIPE, stdout=PIPE, stderr=DEVNULL, text=True)
|
|
stdout, stderr = debugger.communicate(commands)
|
|
output = stdout.splitlines()
|
|
|
|
expected = ""
|
|
expected_file = file.removesuffix(".cmd") + ".expected"
|
|
with open(ESCARGOT_DEBUGGER_TEST_DIR + "/" + expected_file, "r") as f:
|
|
expected = f.read()
|
|
expected = expected.splitlines()
|
|
|
|
if "module_debug" in file:
|
|
output.pop(2)
|
|
|
|
escargot_client.communicate()
|
|
return_code = escargot_client.returncode
|
|
output = [x.replace(PROJECT_SOURCE_DIR + "/", "") for x in output]
|
|
|
|
if output == expected:
|
|
print('%sOK: %s%s' % (COLOR_GREEN, test_case, COLOR_RESET))
|
|
else:
|
|
print('%sFAIL(%d): %s%s' % (COLOR_RED, return_code, test_case, COLOR_RESET))
|
|
fails += 1
|
|
|
|
if fails > 0:
|
|
raise Exception('Escargot-Debugger-Server-Source tests failed')
|
|
|
|
@runner('debugger-client-source', default=True)
|
|
def run_escargot_debugger2(engine, arch, extra_arg):
|
|
ESCARGOT_DEBUGGER_TEST_DIR = join(PROJECT_SOURCE_DIR, 'tools', 'debugger', 'tests')
|
|
ESCARGOT_DEBUGGER = join(PROJECT_SOURCE_DIR, 'tools', 'debugger', 'debugger.py')
|
|
print('Running Escargot-Debugger-Client-Source test:')
|
|
fails = 0
|
|
|
|
files = os.listdir(ESCARGOT_DEBUGGER_TEST_DIR)
|
|
if extra_arg['debugger_extra']:
|
|
specific_tests = list(extra_arg['debugger_extra'])
|
|
files = [file for test in specific_tests for file in files if test in file and file.endswith(".cmd")]
|
|
else:
|
|
files = os.listdir(ESCARGOT_DEBUGGER_TEST_DIR)
|
|
|
|
for file in files:
|
|
if "client_source" in file:
|
|
continue
|
|
|
|
if file.endswith(".cmd"):
|
|
commands = ""
|
|
with open(ESCARGOT_DEBUGGER_TEST_DIR + "/" + file, 'r') as f:
|
|
commands = f.read()
|
|
|
|
test_case, _ = os.path.splitext(file.removesuffix(".cmd"))
|
|
test_case = ESCARGOT_DEBUGGER_TEST_DIR + "/" + file.removesuffix(".cmd")
|
|
additional_options = ""
|
|
if "do_wait_exit2" in file:
|
|
additional_options += " --wait-before-exit "
|
|
elif "module_debug" in file:
|
|
additional_options += " --module "
|
|
|
|
if "module_debug" not in file:
|
|
test_case += ".js"
|
|
else:
|
|
test_case += "_1.mjs"
|
|
|
|
args = shlex.split(engine + " --start-debug-server " + additional_options + os.path.relpath(test_case, PROJECT_SOURCE_DIR))
|
|
escargot_client = Popen(args, stdin=PIPE, stdout=PIPE, stderr=PIPE, text=True)
|
|
|
|
debugger_options = ""
|
|
if "do_command" in file:
|
|
debugger_options += " --command \"b do_command.js:2;c;e i;c\""
|
|
elif "do_wait_exit1" in file:
|
|
debugger_options += " --wait-before-exit 1"
|
|
|
|
args = shlex.split(ESCARGOT_DEBUGGER + " --non-interactive"+ debugger_options)
|
|
debugger = Popen(args, stdin=PIPE, stdout=PIPE, stderr=DEVNULL, text=True)
|
|
stdout, stderr = debugger.communicate(commands)
|
|
output = stdout.splitlines()
|
|
|
|
expected = ""
|
|
expected_file = file.removesuffix(".cmd") + ".expected"
|
|
with open(ESCARGOT_DEBUGGER_TEST_DIR + "/" + expected_file, "r") as f:
|
|
expected = f.read()
|
|
expected = expected.splitlines()
|
|
|
|
if "module_debug" in file:
|
|
output.pop(2)
|
|
|
|
stdout, stderr = escargot_client.communicate()
|
|
return_code = escargot_client.returncode
|
|
|
|
if output == expected:
|
|
print('%sOK: %s%s' % (COLOR_GREEN, test_case, COLOR_RESET))
|
|
else:
|
|
print('%sFAIL(%d): %s%s' % (COLOR_RED, return_code, test_case, COLOR_RESET))
|
|
fails += 1
|
|
|
|
if fails > 0:
|
|
raise Exception('Escargot-Debugger-Client-Source tests failed')
|
|
|
|
@runner('web-tooling-benchmark', default=False)
|
|
def run_web_tooling_benchmark(engine, arch, extra_arg):
|
|
WEB_TOOLING_DIR = join(PROJECT_SOURCE_DIR, 'test', 'web-tooling-benchmark')
|
|
WEB_TOOLING_SRC_DIR = join(WEB_TOOLING_DIR, 'src')
|
|
|
|
suite_file = join(WEB_TOOLING_SRC_DIR, 'suite.js')
|
|
with open(suite_file, "r") as f:
|
|
lines = f.readlines()
|
|
with open(suite_file, "w") as f:
|
|
for line in lines:
|
|
# minimize sample count to 1
|
|
f.write(re.sub(r'minSamples: 20', 'minSamples: 1', line))
|
|
|
|
flag_file = join(WEB_TOOLING_SRC_DIR, 'cli-flags-helper.js')
|
|
with open(flag_file, "r") as f:
|
|
lines = f.readlines()
|
|
with open(flag_file, "w") as f:
|
|
for line in lines:
|
|
# exclude long running tests
|
|
if "babel" not in line and "babylon" not in line and "chai" not in line:
|
|
f.write(line)
|
|
|
|
proc = Popen(['npx', 'npm@6', 'install'], cwd=WEB_TOOLING_DIR, stdout=PIPE)
|
|
proc.wait()
|
|
|
|
run([engine, 'dist/cli.js'], cwd=WEB_TOOLING_DIR)
|
|
|
|
@runner('dump-all', default=False)
|
|
def run_dump_all(engine, arch, extra_arg):
|
|
for test in RUNNERS:
|
|
if test.endswith("-dump"):
|
|
RUNNERS[test](engine, arch, extra_arg)
|
|
|
|
@runner('dump-run-all', default=False)
|
|
def run_dump_run_all(engine, arch, extra_arg):
|
|
for test in RUNNERS:
|
|
if test.endswith("-dump-run"):
|
|
RUNNERS[test](engine, arch, extra_arg)
|
|
|
|
def main():
|
|
parser = ArgumentParser(description='Escargot Test Suite Runner')
|
|
parser.add_argument('--engine', metavar='PATH', default=DEFAULT_ESCARGOT,
|
|
help='path to the engine to be tested (default: %(default)s)')
|
|
parser.add_argument('--arch', metavar='NAME', choices=['x86', 'x86_64'], default='x86_64',
|
|
help='architecture the engine was built for (%(choices)s; default: %(default)s)')
|
|
parser.add_argument('--test262-extra-arg', default='',
|
|
help='extra argument variable to test262')
|
|
parser.add_argument('--test-data-runner-path', metavar='PATH', default=(PROJECT_SOURCE_DIR + '/tools/test/test-data-runner/test-data-runner'),
|
|
help='test-data-runner executable path')
|
|
parser.add_argument('--skip-build-test-data-runner', default=False, action="store_true",
|
|
help='Skip build test-data-runner executable')
|
|
parser.add_argument('suite', metavar='SUITE', nargs='*', default=sorted(DEFAULT_RUNNERS),
|
|
help='test suite to run (%s; default: %s)' % (', '.join(sorted(RUNNERS.keys())), ' '.join(sorted(DEFAULT_RUNNERS))))
|
|
parser.add_argument('--debugger-test', default="", help='Run specific debugger tests based on given substring(s)', nargs='+')
|
|
args = parser.parse_args()
|
|
|
|
for suite in args.suite:
|
|
if suite not in RUNNERS:
|
|
parser.error('invalid test suite: %s' % suite)
|
|
|
|
success, fail = [], []
|
|
extra_arg = {
|
|
'test262_extra_arg' : args.test262_extra_arg,
|
|
'skip_build_test_data_runner' : args.skip_build_test_data_runner,
|
|
'test_data_runner_path' : args.test_data_runner_path,
|
|
'debugger_extra': args.debugger_test
|
|
}
|
|
for suite in args.suite:
|
|
print(COLOR_PURPLE + 'running test suite: ' + suite + COLOR_RESET)
|
|
try:
|
|
RUNNERS[suite](args.engine, args.arch, extra_arg)
|
|
success += [suite]
|
|
except Exception as e:
|
|
print('\n'.join(COLOR_YELLOW + line + COLOR_RESET for line in traceback.format_exc().splitlines()))
|
|
fail += [suite]
|
|
|
|
if success:
|
|
print(COLOR_GREEN + sys.argv[0] + ': success: ' + ', '.join(success) + COLOR_RESET)
|
|
sys.exit(COLOR_RED + sys.argv[0] + ': fail: ' + ', '.join(fail) + COLOR_RESET if fail else None)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|