diff --git a/test/subset/meson.build b/test/subset/meson.build index 7bbdc5c90..7037223c7 100644 --- a/test/subset/meson.build +++ b/test/subset/meson.build @@ -111,6 +111,7 @@ foreach t : tests # ideally better to break and let meson handles them in parallel timeout: 500, workdir: meson.current_build_dir() / '..' / '..', + protocol: 'tap', suite: 'subset', ) endforeach @@ -126,6 +127,7 @@ foreach t : repack_tests meson.current_source_dir() / 'data' / 'repack_tests' / fname, ], workdir: meson.current_build_dir() / '..' / '..', + protocol: 'tap', suite: ['subset', 'repack'], ) endforeach diff --git a/test/subset/run-repack-tests.py b/test/subset/run-repack-tests.py index a0389e5aa..8003e7163 100755 --- a/test/subset/run-repack-tests.py +++ b/test/subset/run-repack-tests.py @@ -17,8 +17,8 @@ from repack_test import RepackTest try: from fontTools.ttLib import TTFont except ImportError: - print("fonttools is not present, skipping test.") - sys.exit(77) + print("# fonttools is not present, skipping test.") + sys.exit() ots_sanitize = shutil.which("ots-sanitize") @@ -27,7 +27,7 @@ EXE_WRAPPER = os.environ.get("MESON_EXE_WRAPPER") def subset_cmd(command): global hb_subset, process - print(hb_subset + " " + " ".join(command)) + print("# " + hb_subset + " " + " ".join(command)) process.stdin.write((";".join(command) + "\n").encode("utf-8")) process.stdin.flush() return process.stdout.readline().decode("utf-8").strip() @@ -43,11 +43,20 @@ def cmd(command): def fail_test(test, cli_args, message): + global fails + fails += 1 + print("ERROR: %s" % message) print("Test State:") print(" test.font_name %s" % test.font_name) print(" test.test_path %s" % os.path.abspath(test.test_path)) - return 1 + print("not ok -", test) + print(" ---", file=sys.stderr) + print(" message: \"%s\"" % message, file=sys.stderr) + print(" test.font_name: \"%s\"" % test.font_name) + print(" test.test_path: \"%s\"" % os.path.abspath(test.test_path)) + print(" ...", file=sys.stderr) + return False def run_test(test, should_check_ots): @@ -58,7 +67,7 @@ def run_test(test, should_check_ots): "--unicodes=%s" % test.codepoints_string(), "--drop-tables-=GPOS,GSUB,GDEF", ] - print(" ".join(cli_args)) + print("#", " ".join(cli_args)) ret = subset_cmd(cli_args) if ret != "success": @@ -72,16 +81,16 @@ def run_test(test, should_check_ots): return fail_test(test, cli_args, "ttx failed to parse the result") if should_check_ots: - print("Checking output with ots-sanitize.") + print("# Checking output with ots-sanitize.") if not check_ots(out_file): return fail_test(test, cli_args, "ots for subsetted file fails.") - return 0 + return True def has_ots(): if not ots_sanitize: - print("OTS is not present, skipping all ots checks.") + print("# OTS is not present, skipping all ots checks.") return False return True @@ -89,7 +98,7 @@ def has_ots(): def check_ots(path): ots_report, returncode = cmd([ots_sanitize, path]) if returncode: - print("OTS Failure: %s" % ots_report) + print("# OTS Failure: %s" % ots_report) return False return True @@ -102,8 +111,6 @@ hb_subset, args = args[0], args[1:] if len(args) != 1: sys.exit("No tests supplied.") -has_ots = has_ots() - batch_cmd = [hb_subset, "--batch"] if EXE_WRAPPER: batch_cmd = [EXE_WRAPPER] + batch_cmd @@ -111,6 +118,10 @@ process = subprocess.Popen( batch_cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=sys.stdout ) +print("TAP version 14") + +has_ots = has_ots() + fails = 0 path = args[0] @@ -119,13 +130,17 @@ if not path.endswith(".tests"): out_dir = tempfile.mkdtemp() +number = 0 with open(path, mode="r", encoding="utf-8") as f: # TODO(garretrieger): re-enable OTS checking. - fails += run_test(RepackTest(path, f.read()), False) + if run_test(RepackTest(path, f.read()), False): + number += 1 + print("ok %d - %s" % (number, os.path.basename(path))) +print("1..%d" % number) if fails != 0: - sys.exit("%d test(s) failed; output left in %s" % (fails, out_dir)) + print("# %d test(s) failed; output left in %s" % (fails, out_dir)) else: - print("All tests passed.") + print("# All tests passed.") shutil.rmtree(out_dir) diff --git a/test/subset/run-tests.py b/test/subset/run-tests.py index b403bfab9..96dda6ed5 100755 --- a/test/subset/run-tests.py +++ b/test/subset/run-tests.py @@ -29,7 +29,7 @@ def subset_cmd(command): if subset_process.poll() is not None: subset_process = open_subset_batch_process() - print(hb_subset + " " + " ".join(command)) + print("# " + hb_subset + " " + " ".join(command)) subset_process.stdin.write((";".join(command) + "\n").encode("utf-8")) subset_process.stdin.flush() return subset_process.stdout.readline().decode("utf-8").strip() @@ -45,19 +45,28 @@ def cmd(command): def fail_test(test, cli_args, message): - print("ERROR: %s" % message) - print("Test State:") - print(" test.font_path %s" % os.path.abspath(test.font_path)) - print(" test.profile_path %s" % os.path.abspath(test.profile_path)) - print(" test.unicodes %s" % test.unicodes()) + global fails + fails += 1 + expected_file = os.path.join( test_suite.get_output_directory(), test.get_font_name() ) - print(" expected_file %s" % os.path.abspath(expected_file)) - return 1 + + print("not ok -", test) + print(" ---", file=sys.stderr) + print(" message: \"%s\"" % message, file=sys.stderr) + print(" test.font_path: \"%s\"" % os.path.abspath(test.font_path), file=sys.stderr) + print(" test.profile_path: \"%s\"" % os.path.abspath(test.profile_path), file=sys.stderr) + print(" test.unicodes: \"%s\"" % test.unicodes(), file=sys.stderr) + print(" expected_file: \"%s\"" % os.path.abspath(expected_file), file=sys.stderr) + print(" ...", file=sys.stderr) + return False def run_test(test, should_check_ots, preprocess): + global number + number += 1 + out_file = os.path.join( out_dir, test.get_font_name() + "-subset" + test.get_font_extension() ) @@ -99,13 +108,13 @@ def run_test(test, should_check_ots, preprocess): if expected_contents == actual_contents: if should_check_ots: - print("Checking output with ots-sanitize.") + print("# Checking output with ots-sanitize.") if not check_ots(out_file): return fail_test(test, cli_args, "ots for subsetted file fails.") - return 0 + return True if TTFont is None: - print("fonttools is not present, skipping TTX diff.") + print("# fonttools is not present, skipping TTX diff.") return fail_test(test, cli_args, "hash for expected and actual does not match.") with io.StringIO() as fp: @@ -142,7 +151,7 @@ def run_test(test, should_check_ots, preprocess): def has_ots(): if not ots_sanitize: - print("OTS is not present, skipping all ots checks.") + print("# OTS is not present, skipping all ots checks.") return False return True @@ -150,7 +159,7 @@ def has_ots(): def check_ots(path): ots_report, returncode = cmd([ots_sanitize, path]) if returncode: - print("OTS Failure: %s" % ots_report) + print("# OTS Failure: %s" % ots_report) return False return True @@ -163,6 +172,8 @@ hb_subset, args = args[0], args[1:] if not len(args): sys.exit("No tests supplied.") +print("TAP version 14") + has_ots = has_ots() env = os.environ.copy() @@ -189,19 +200,23 @@ def open_subset_batch_process(): subset_process = open_subset_batch_process() out_dir = tempfile.mkdtemp() +number = 0 fails = 0 for path in args: with open(path, mode="r", encoding="utf-8") as f: - print("Running tests in " + path) + print("# Running tests in " + path) test_suite = SubsetTestSuite(path, f.read()) for test in test_suite.tests(): # Tests are run with and without preprocessing, results should be the # same between them. - fails += run_test(test, has_ots, False) - fails += run_test(test, has_ots, True) + for preprocess in [False, True]: + if run_test(test, has_ots, preprocess): + print("ok %d - %s" % (number, test)) + +print("1..%d" % number) if fails != 0: - sys.exit("%d test(s) failed; output left in %s" % (fails, out_dir)) + print("# %d test(s) failed; output left in %s" % (fails, out_dir), file=sys.stderr) else: - print("All tests passed.") + print("# All tests passed.") shutil.rmtree(out_dir)