#!/usr/bin/env python # Copyright JS Foundation and other contributors, http://js.foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from __future__ import print_function import argparse import os import re import shutil import subprocess import sys import util def get_platform_cmd_prefix(): if sys.platform == 'win32': return ['cmd', '/S', '/C'] return ['python2'] # The official test262.py isn't python3 compatible, but has python shebang. def get_arguments(): execution_runtime = os.environ.get('RUNTIME', '') parser = argparse.ArgumentParser() parser.add_argument('--runtime', metavar='FILE', default=execution_runtime, help='Execution runtime (e.g. qemu)') parser.add_argument('--engine', metavar='FILE', required=True, help='JerryScript binary to run tests with') parser.add_argument('--test-dir', metavar='DIR', required=True, help='Directory contains test262 test suite') group = parser.add_mutually_exclusive_group(required=True) group.add_argument('--es51', action='store_true', help='Run test262 ES5.1 version') group.add_argument('--es2015', default=False, const='default', nargs='?', choices=['default', 'all', 'update'], help='Run test262 - ES2015. default: all tests except excludelist, ' + 'all: all tests, update: all tests and update excludelist') group.add_argument('--esnext', default=False, const='default', nargs='?', choices=['default', 'all', 'update'], help='Run test262 - ES.next. default: all tests except excludelist, ' + 'all: all tests, update: all tests and update excludelist') parser.add_argument('--test262-test-list', metavar='LIST', help='Add a comma separated list of tests or directories to run in test262 test suite') args = parser.parse_args() if args.es2015: args.test_dir = os.path.join(args.test_dir, 'es2015') args.test262_harness_dir = os.path.abspath(os.path.dirname(__file__)) args.test262_git_hash = 'fd44cd73dfbce0b515a2474b7cd505d6176a9eb5' args.excludelist_path = os.path.join('tests', 'test262-es6-excludelist.xml') elif args.esnext: args.test_dir = os.path.join(args.test_dir, 'esnext') args.test262_harness_dir = os.path.abspath(os.path.dirname(__file__)) args.test262_git_hash = '281eb10b2844929a7c0ac04527f5b42ce56509fd' args.excludelist_path = os.path.join('tests', 'test262-esnext-excludelist.xml') else: args.test_dir = os.path.join(args.test_dir, 'es51') args.test262_harness_dir = args.test_dir args.test262_git_hash = 'es5-tests' args.mode = args.es2015 or args.esnext return args def prepare_test262_test_suite(args): if os.path.isdir(os.path.join(args.test_dir, '.git')): return 0 return_code = subprocess.call(['git', 'clone', '--no-checkout', 'https://github.com/tc39/test262.git', args.test_dir]) if return_code: print('Cloning test262 repository failed.') return return_code return_code = subprocess.call(['git', 'checkout', args.test262_git_hash], cwd=args.test_dir) assert not return_code, 'Cloning test262 repository failed - invalid git revision.' if args.es51: path_to_remove = os.path.join(args.test_dir, 'test', 'suite', 'bestPractice') if os.path.isdir(path_to_remove): shutil.rmtree(path_to_remove) path_to_remove = os.path.join(args.test_dir, 'test', 'suite', 'intl402') if os.path.isdir(path_to_remove): shutil.rmtree(path_to_remove) return 0 def update_exclude_list(args): print("=== Summary - updating excludelist ===\n") passing_tests = set() failing_tests = set() new_passing_tests = set() with open(os.path.join(os.path.dirname(args.engine), 'test262.report'), 'r') as report_file: for line in report_file: match = re.match('(=== )?(.*) (?:failed|passed) in (?:non-strict|strict)', line) if match: (unexpected, test) = match.groups() test = test.replace('\\', '/') if unexpected: failing_tests.add(test + '.js') else: passing_tests.add(test + '.js') # Tests pass in strict-mode but fail in non-strict-mode (or vice versa) should be considered as failures passing_tests = passing_tests - failing_tests with open(args.excludelist_path, 'r+') as exclude_file: lines = exclude_file.readlines() exclude_file.seek(0) exclude_file.truncate() # Skip the last line "" to be able to insert new failing tests. for line in lines[:-1]: match = re.match(r" ", line) if match: test = match.group(1) if test in failing_tests: failing_tests.remove(test) exclude_file.write(line) elif test in passing_tests: new_passing_tests.add(test) else: exclude_file.write(line) else: exclude_file.write(line) if failing_tests: print("New failing tests added to the excludelist") for test in sorted(failing_tests): exclude_file.write(' \n') print(" " + test) print("") exclude_file.write('\n') if new_passing_tests: print("New passing tests removed from the excludelist") for test in sorted(new_passing_tests): print(" " + test) print("") if failing_tests or new_passing_tests: print("Excludelist was updated succesfully.") return 1 print("Excludelist was already up-to-date.") return 0 def main(args): return_code = prepare_test262_test_suite(args) if return_code: return return_code if sys.platform == 'win32': original_timezone = util.get_timezone() util.set_sighdl_to_reset_timezone(original_timezone) util.set_timezone('Pacific Standard Time') command = (args.runtime + ' ' + args.engine).strip() kwargs = {} if sys.version_info.major >= 3: kwargs['errors'] = 'ignore' if args.es51: test262_harness_path = os.path.join(args.test262_harness_dir, 'tools/packaging/test262.py') else: test262_harness_path = os.path.join(args.test262_harness_dir, 'test262-harness.py') test262_command = get_platform_cmd_prefix() + \ [test262_harness_path, '--command', command, '--tests', args.test_dir, '--summary'] if 'excludelist_path' in args and args.mode == 'default': test262_command.extend(['--exclude-list', args.excludelist_path]) if args.test262_test_list: test262_command.extend(args.test262_test_list.split(',')) proc = subprocess.Popen(test262_command, universal_newlines=True, stdout=subprocess.PIPE, **kwargs) return_code = 1 with open(os.path.join(os.path.dirname(args.engine), 'test262.report'), 'w') as output_file: counter = 0 summary_found = False summary_end_found = False while True: output = proc.stdout.readline() if not output: break output_file.write(output) if output.startswith('=== Summary ==='): summary_found = True print('') if summary_found: if not summary_end_found: print(output, end='') if not output.strip(): summary_end_found = True if 'All tests succeeded' in output: return_code = 0 elif re.search('in (non-)?strict mode', output): counter += 1 if (counter % 100) == 0: print(".", end='') if (counter % 5000) == 0: print(" Executed %d tests." % counter) proc.wait() if sys.platform == 'win32': util.set_timezone(original_timezone) if args.mode == 'update': return_code = update_exclude_list(args) return return_code if __name__ == "__main__": sys.exit(main(get_arguments()))