From be22e43d7d70b280a4dc1afeed51199e98978556 Mon Sep 17 00:00:00 2001 From: Behdad Esfahbod Date: Sun, 9 Feb 2025 14:55:44 +0000 Subject: [PATCH 1/7] [test/fuzzing] Run each fuzzer on all fonts in one process Much much faster, specially under valgrind, than spawning one process per font. Fixes https://github.com/harfbuzz/harfbuzz/issues/5061 --- test/fuzzing/meson.build | 6 +- test/fuzzing/run-draw-fuzzer-tests.py | 99 +++++++++---------- test/fuzzing/run-repacker-fuzzer-tests.py | 107 ++++++++++---------- test/fuzzing/run-shape-fuzzer-tests.py | 102 ++++++++++--------- test/fuzzing/run-subset-fuzzer-tests.py | 113 +++++++++++----------- 5 files changed, 205 insertions(+), 222 deletions(-) diff --git a/test/fuzzing/meson.build b/test/fuzzing/meson.build index f6ebbddcf..cada41948 100644 --- a/test/fuzzing/meson.build +++ b/test/fuzzing/meson.build @@ -41,23 +41,21 @@ test('shape-fuzzer', find_program('run-shape-fuzzer-tests.py'), args: [ hb_shape_fuzzer_exe, ], - timeout: 90, depends: [hb_shape_fuzzer_exe, libharfbuzz, libharfbuzz_subset], workdir: meson.current_build_dir() / '..' / '..', env: env, priority: 1, - suite: ['fuzzing', 'slow'], + suite: ['fuzzing'], ) test('subset-fuzzer', find_program('run-subset-fuzzer-tests.py'), args: [ hb_subset_fuzzer_exe, ], - timeout: 90, workdir: meson.current_build_dir() / '..' / '..', env: env, priority: 1, - suite: ['fuzzing', 'slow'], + suite: ['fuzzing'], ) test('repacker-fuzzer', find_program('run-repacker-fuzzer-tests.py'), diff --git a/test/fuzzing/run-draw-fuzzer-tests.py b/test/fuzzing/run-draw-fuzzer-tests.py index 8b5a2e82d..33b265e25 100755 --- a/test/fuzzing/run-draw-fuzzer-tests.py +++ b/test/fuzzing/run-draw-fuzzer-tests.py @@ -1,66 +1,59 @@ #!/usr/bin/env python3 -import sys, os, subprocess, tempfile, shutil +import sys +import os +import subprocess +import tempfile +def run_command(command): + with tempfile.TemporaryFile() as tempf: + p = subprocess.Popen(command, stdout=tempf, stderr=tempf) + p.wait() + tempf.seek(0) + output = tempf.read().decode('utf-8', errors='replace') + return output, p.returncode -def cmd (command): - # https://stackoverflow.com/a/4408409 as we might have huge output sometimes - with tempfile.TemporaryFile () as tempf: - p = subprocess.Popen (command, stderr=tempf) +srcdir = os.getenv("srcdir", ".") +EXEEXT = os.getenv("EXEEXT", "") +top_builddir = os.getenv("top_builddir", ".") - try: - p.wait () - tempf.seek (0) - text = tempf.read () +hb_draw_fuzzer = os.path.join(top_builddir, "hb-draw-fuzzer" + EXEEXT) +# If not found automatically, try sys.argv[1] +if not os.path.exists(hb_draw_fuzzer): + if len(sys.argv) < 2 or not os.path.exists(sys.argv[1]): + sys.exit( + "Failed to find hb-draw-fuzzer binary automatically.\n" + "Please provide it as the first argument to the tool." + ) + hb_draw_fuzzer = sys.argv[1] - #TODO: Detect debug mode with a better way - is_debug_mode = b"SANITIZE" in text +print("Using hb_draw_fuzzer:", hb_draw_fuzzer) - return ("" if is_debug_mode else text.decode ("utf-8").strip ()), p.returncode - except subprocess.TimeoutExpired: - return 'error: timeout, ' + ' '.join (command), 1 +# Collect all files from the fonts/ directory +parent_path = os.path.join(srcdir, "fonts") +if not os.path.isdir(parent_path): + sys.exit(f"Directory {parent_path} not found or not a directory.") +files_to_check = [ + os.path.join(parent_path, f) for f in os.listdir(parent_path) + if os.path.isfile(os.path.join(parent_path, f)) +] -srcdir = os.getenv ("srcdir", ".") -EXEEXT = os.getenv ("EXEEXT", "") -top_builddir = os.getenv ("top_builddir", ".") -hb_draw_fuzzer = os.path.join (top_builddir, "hb-draw-fuzzer" + EXEEXT) +if not files_to_check: + print(f"No files found in {parent_path}") + sys.exit(1) -if not os.path.exists (hb_draw_fuzzer): - if len (sys.argv) == 1 or not os.path.exists (sys.argv[1]): - sys.exit ("""Failed to find hb-draw-fuzzer binary automatically, -please provide it as the first argument to the tool""") +# Single invocation passing all files +cmd_line = [hb_draw_fuzzer] + files_to_check +output, returncode = run_command(cmd_line) - hb_draw_fuzzer = sys.argv[1] +# Print output if not empty +if output.strip(): + print(output) -print ('hb_draw_fuzzer:', hb_draw_fuzzer) -fails = 0 +# If there's an error, print a message and exit non-zero +if returncode != 0: + print("Failure while processing these files:", ", ".join(os.path.basename(f) for f in files_to_check)) + sys.exit(returncode) -valgrind = None -if os.getenv ('RUN_VALGRIND', ''): - valgrind = shutil.which ('valgrind') - if valgrind is None: - sys.exit ("""Valgrind requested but not found.""") - -parent_path = os.path.join (srcdir, "fonts") -for file in os.listdir (parent_path): - if "draw" not in file: continue - path = os.path.join (parent_path, file) - - if valgrind: - text, returncode = cmd ([valgrind, '--leak-check=full', '--error-exitcode=1', hb_draw_fuzzer, path]) - else: - text, returncode = cmd ([hb_draw_fuzzer, path]) - if 'error' in text: - returncode = 1 - - if (not valgrind or returncode) and text.strip (): - print (text) - - if returncode != 0: - print ('failure on %s' % file) - fails = fails + 1 - - -if fails: - sys.exit ("%d draw fuzzer related tests failed." % fails) +print("All files processed successfully.") diff --git a/test/fuzzing/run-repacker-fuzzer-tests.py b/test/fuzzing/run-repacker-fuzzer-tests.py index 85a23e13e..81971531f 100755 --- a/test/fuzzing/run-repacker-fuzzer-tests.py +++ b/test/fuzzing/run-repacker-fuzzer-tests.py @@ -1,68 +1,65 @@ #!/usr/bin/env python3 -import sys, os, subprocess, tempfile, shutil +import sys +import os +import subprocess +import tempfile + +def run_command(command): + with tempfile.TemporaryFile() as tempf: + p = subprocess.Popen(command, stdout=tempf, stderr=tempf) + p.wait() + tempf.seek(0) + output = tempf.read().decode('utf-8', errors='replace') + return output, p.returncode -def cmd (command): - # https://stackoverflow.com/a/4408409 as we might have huge output sometimes - with tempfile.TemporaryFile () as tempf: - p = subprocess.Popen (command, stderr=tempf) +# Environment and binary location +srcdir = os.getenv("srcdir", ".") +EXEEXT = os.getenv("EXEEXT", "") +top_builddir = os.getenv("top_builddir", ".") - try: - p.wait () - tempf.seek (0) - text = tempf.read () +hb_repacker_fuzzer = os.path.join(top_builddir, "hb-repacker-fuzzer" + EXEEXT) +# If the binary isn't found, try sys.argv[1] +if not os.path.exists(hb_repacker_fuzzer): + if len(sys.argv) < 2 or not os.path.exists(sys.argv[1]): + sys.exit( + "Failed to find hb-repacker-fuzzer binary automatically.\n" + "Please provide it as the first argument to the tool." + ) + hb_repacker_fuzzer = sys.argv[1] - #TODO: Detect debug mode with a better way - is_debug_mode = b"SANITIZE" in text +print("hb_repacker_fuzzer:", hb_repacker_fuzzer) - return ("" if is_debug_mode else text.decode ("utf-8").strip ()), p.returncode - except subprocess.TimeoutExpired: - return 'error: timeout, ' + ' '.join (command), 1 +# Collect all files from graphs/ +graphs_path = os.path.join(srcdir, "graphs") +if not os.path.isdir(graphs_path): + sys.exit(f"No 'graphs' directory found at {graphs_path}.") +files_to_check = [ + os.path.join(graphs_path, f) + for f in os.listdir(graphs_path) + if os.path.isfile(os.path.join(graphs_path, f)) +] -srcdir = os.getenv ("srcdir", ".") -EXEEXT = os.getenv ("EXEEXT", "") -top_builddir = os.getenv ("top_builddir", ".") -hb_repacker_fuzzer = os.path.join (top_builddir, "hb-repacker-fuzzer" + EXEEXT) +if not files_to_check: + print("No files found in the 'graphs' directory.") + sys.exit(1) -if not os.path.exists (hb_repacker_fuzzer): - if len (sys.argv) < 2 or not os.path.exists (sys.argv[1]): - sys.exit ("""Failed to find hb-repacker-fuzzer binary automatically, -please provide it as the first argument to the tool""") +# Single invocation passing all files +print(f"Running repacker fuzzer against {len(files_to_check)} file(s) in 'graphs'...") +cmd_line = [hb_repacker_fuzzer] + files_to_check +output, returncode = run_command(cmd_line) - hb_repacker_fuzzer = sys.argv[1] +# Print the output if present +if output.strip(): + print(output) -print ('hb_repacker_fuzzer:', hb_repacker_fuzzer) -fails = 0 +# Exit if there's an error +if returncode != 0: + print("Failed for these files:") + for f in files_to_check: + print(" ", f) + sys.exit("1 repacker fuzzer related test(s) failed.") -valgrind = None -if os.getenv ('RUN_VALGRIND', ''): - valgrind = shutil.which ('valgrind') - if valgrind is None: - sys.exit ("""Valgrind requested but not found.""") - -def run_dir (parent_path): - global fails - for file in os.listdir (parent_path): - path = os.path.join(parent_path, file) - print ("running repacker fuzzer against %s" % path) - if valgrind: - text, returncode = cmd ([valgrind, '--leak-check=full', '--error-exitcode=1', hb_repacker_fuzzer, path]) - else: - text, returncode = cmd ([hb_repacker_fuzzer, path]) - if 'error' in text: - returncode = 1 - - if (not valgrind or returncode) and text.strip (): - print (text) - - if returncode != 0: - print ("failed for %s" % path) - fails = fails + 1 - - -run_dir (os.path.join (srcdir, "graphs")) - -if fails: - sys.exit ("%d repacker fuzzer related tests failed." % fails) +print("All repacker fuzzer tests passed successfully.") diff --git a/test/fuzzing/run-shape-fuzzer-tests.py b/test/fuzzing/run-shape-fuzzer-tests.py index 382f60929..6ce7c1c47 100755 --- a/test/fuzzing/run-shape-fuzzer-tests.py +++ b/test/fuzzing/run-shape-fuzzer-tests.py @@ -1,65 +1,63 @@ #!/usr/bin/env python3 -import sys, os, subprocess, tempfile, shutil +import sys +import os +import subprocess +import tempfile +def run_command(command): + """Run a command, capturing potentially large output.""" + with tempfile.TemporaryFile() as tempf: + p = subprocess.Popen(command, stdout=tempf, stderr=tempf) + p.wait() + tempf.seek(0) + output = tempf.read().decode('utf-8', errors='replace') + return output, p.returncode -def cmd (command): - # https://stackoverflow.com/a/4408409 as we might have huge output sometimes - with tempfile.TemporaryFile () as tempf: - p = subprocess.Popen (command, stderr=tempf) +srcdir = os.getenv("srcdir", ".") +EXEEXT = os.getenv("EXEEXT", "") +top_builddir = os.getenv("top_builddir", ".") - try: - p.wait () - tempf.seek (0) - text = tempf.read () +hb_shape_fuzzer = os.path.join(top_builddir, "hb-shape-fuzzer" + EXEEXT) +if not os.path.exists(hb_shape_fuzzer): + # If not found automatically, fall back to the first CLI argument. + if len(sys.argv) < 2 or not os.path.exists(sys.argv[1]): + sys.exit( + "Failed to find hb-shape-fuzzer binary automatically.\n" + "Please provide it as the first argument to the tool." + ) + hb_shape_fuzzer = sys.argv[1] - #TODO: Detect debug mode with a better way - is_debug_mode = b"SANITIZE" in text +print("hb_shape_fuzzer:", hb_shape_fuzzer) - return ("" if is_debug_mode else text.decode ("utf-8").strip ()), p.returncode - except subprocess.TimeoutExpired: - return 'error: timeout, ' + ' '.join (command), 1 +fonts_dir = os.path.join(srcdir, "fonts") +if not os.path.isdir(fonts_dir): + sys.exit(f"Fonts directory not found at: {fonts_dir}") +# Gather all files in `fonts_dir` +files_to_test = [ + os.path.join(fonts_dir, f) + for f in os.listdir(fonts_dir) + if os.path.isfile(os.path.join(fonts_dir, f)) +] -srcdir = os.getenv ("srcdir", ".") -EXEEXT = os.getenv ("EXEEXT", "") -top_builddir = os.getenv ("top_builddir", ".") -hb_shape_fuzzer = os.path.join (top_builddir, "hb-shape-fuzzer" + EXEEXT) +if not files_to_test: + print(f"No files found in {fonts_dir}") + sys.exit(1) -if not os.path.exists (hb_shape_fuzzer): - if len (sys.argv) == 1 or not os.path.exists (sys.argv[1]): - sys.exit ("""Failed to find hb-shape-fuzzer binary automatically, -please provide it as the first argument to the tool""") +# Single invocation with all test files +cmd_line = [hb_shape_fuzzer] + files_to_test +output, returncode = run_command(cmd_line) - hb_shape_fuzzer = sys.argv[1] +# Print output if any +if output.strip(): + print(output) -print ('hb_shape_fuzzer:', hb_shape_fuzzer) -fails = 0 +# Fail if return code is non-zero +if returncode != 0: + print("Failure on the following file(s):") + for f in files_to_test: + print(" ", f) + sys.exit("1 shape fuzzer test failed.") -valgrind = None -if os.getenv ('RUN_VALGRIND', ''): - valgrind = shutil.which ('valgrind') - if valgrind is None: - sys.exit ("""Valgrind requested but not found.""") - -parent_path = os.path.join (srcdir, "fonts") -for file in os.listdir (parent_path): - path = os.path.join (parent_path, file) - - if valgrind: - text, returncode = cmd ([valgrind, '--leak-check=full', '--error-exitcode=1', hb_shape_fuzzer, path]) - else: - text, returncode = cmd ([hb_shape_fuzzer, path]) - if 'error' in text: - returncode = 1 - - if (not valgrind or returncode) and text.strip (): - print (text) - - if returncode != 0: - print ('failure on %s' % file) - fails = fails + 1 - - -if fails: - sys.exit ("%d shape fuzzer related tests failed." % fails) +print("All shape fuzzer tests passed successfully.") diff --git a/test/fuzzing/run-subset-fuzzer-tests.py b/test/fuzzing/run-subset-fuzzer-tests.py index da7d1e570..a3fdee75f 100755 --- a/test/fuzzing/run-subset-fuzzer-tests.py +++ b/test/fuzzing/run-subset-fuzzer-tests.py @@ -1,72 +1,69 @@ #!/usr/bin/env python3 -import sys, os, subprocess, tempfile, shutil +import sys +import os +import subprocess +import tempfile +def run_command(command): + """Run a command, capturing potentially large output.""" + with tempfile.TemporaryFile() as tempf: + p = subprocess.Popen(command, stdout=tempf, stderr=tempf) + p.wait() + tempf.seek(0) + output = tempf.read().decode("utf-8", errors="replace") + return output, p.returncode -def cmd (command): - # https://stackoverflow.com/a/4408409 as we might have huge output sometimes - with tempfile.TemporaryFile () as tempf: - p = subprocess.Popen (command, stderr=tempf) +# Environment variables and binary location +srcdir = os.getenv("srcdir", ".") +EXEEXT = os.getenv("EXEEXT", "") +top_builddir = os.getenv("top_builddir", ".") - try: - p.wait () - tempf.seek (0) - text = tempf.read () +hb_subset_fuzzer = os.path.join(top_builddir, "hb-subset-fuzzer" + EXEEXT) +# If not found automatically, fall back to the first CLI argument +if not os.path.exists(hb_subset_fuzzer): + if len(sys.argv) < 2 or not os.path.exists(sys.argv[1]): + sys.exit( + "Failed to find hb-subset-fuzzer binary automatically.\n" + "Please provide it as the first argument to the tool." + ) + hb_subset_fuzzer = sys.argv[1] - #TODO: Detect debug mode with a better way - is_debug_mode = b"SANITIZE" in text +print("hb_subset_fuzzer:", hb_subset_fuzzer) - return ("" if is_debug_mode else text.decode ("utf-8").strip ()), p.returncode - except subprocess.TimeoutExpired: - return 'error: timeout, ' + ' '.join (command), 1 +# Gather all files from both directories +dir1 = os.path.join(srcdir, "..", "subset", "data", "fonts") +dir2 = os.path.join(srcdir, "fonts") +files_to_test = [] -srcdir = os.getenv ("srcdir", ".") -EXEEXT = os.getenv ("EXEEXT", "") -top_builddir = os.getenv ("top_builddir", ".") -hb_subset_fuzzer = os.path.join (top_builddir, "hb-subset-fuzzer" + EXEEXT) +for d in [dir1, dir2]: + if not os.path.isdir(d): + # Skip if the directory doesn't exist + continue + for f in os.listdir(d): + file_path = os.path.join(d, f) + if os.path.isfile(file_path): + files_to_test.append(file_path) -if not os.path.exists (hb_subset_fuzzer): - if len (sys.argv) < 2 or not os.path.exists (sys.argv[1]): - sys.exit ("""Failed to find hb-subset-fuzzer binary automatically, -please provide it as the first argument to the tool""") +if not files_to_test: + print("No fonts found in either directory.") + sys.exit(1) - hb_subset_fuzzer = sys.argv[1] +# Run the fuzzer once, passing all collected files +print(f"Running subset fuzzer on {len(files_to_test)} file(s).") +cmd_line = [hb_subset_fuzzer] + files_to_test +output, returncode = run_command(cmd_line) -print ('hb_subset_fuzzer:', hb_subset_fuzzer) -fails = 0 +# Print any output +if output.strip(): + print(output) -valgrind = None -if os.getenv ('RUN_VALGRIND', ''): - valgrind = shutil.which ('valgrind') - if valgrind is None: - sys.exit ("""Valgrind requested but not found.""") +# If there's an error, exit non-zero +if returncode != 0: + print("Failure while processing these files:") + for f in files_to_test: + print(" ", f) + sys.exit("1 subset fuzzer test failed.") -def run_dir (parent_path): - global fails - for file in os.listdir (parent_path): - path = os.path.join(parent_path, file) - # TODO: Run on all the fonts not just subset related ones - if "subset" not in path: continue - - print ("running subset fuzzer against %s" % path) - if valgrind: - text, returncode = cmd ([valgrind, '--leak-check=full', '--error-exitcode=1', hb_subset_fuzzer, path]) - else: - text, returncode = cmd ([hb_subset_fuzzer, path]) - if 'error' in text: - returncode = 1 - - if (not valgrind or returncode) and text.strip (): - print (text) - - if returncode != 0: - print ("failed for %s" % path) - fails = fails + 1 - - -run_dir (os.path.join (srcdir, "..", "subset", "data", "fonts")) -run_dir (os.path.join (srcdir, "fonts")) - -if fails: - sys.exit ("%d subset fuzzer related tests failed." % fails) +print("All subset fuzzer tests passed successfully.") From 86329643fd7d3bd564b52330af440ed8fc4a8042 Mon Sep 17 00:00:00 2001 From: Behdad Esfahbod Date: Sun, 9 Feb 2025 15:12:03 +0000 Subject: [PATCH 2/7] [test/fuzzing] Call binaries with 64 fonts at a time Second try... Previous attempt caused a too-many-command-line-args on Windows. https://github.com/harfbuzz/harfbuzz/issues/5061 --- test/fuzzing/hb_fuzzer_tools.py | 54 +++++++++++++ test/fuzzing/run-draw-fuzzer-tests.py | 82 ++++++++++---------- test/fuzzing/run-repacker-fuzzer-tests.py | 86 +++++++++------------ test/fuzzing/run-shape-fuzzer-tests.py | 85 +++++++++------------ test/fuzzing/run-subset-fuzzer-tests.py | 92 ++++++++++------------- 5 files changed, 205 insertions(+), 194 deletions(-) create mode 100644 test/fuzzing/hb_fuzzer_tools.py diff --git a/test/fuzzing/hb_fuzzer_tools.py b/test/fuzzing/hb_fuzzer_tools.py new file mode 100644 index 000000000..8176e59c4 --- /dev/null +++ b/test/fuzzing/hb_fuzzer_tools.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +import os +import sys +import subprocess +import tempfile + +def run_command(command): + """ + Run a command, capturing potentially large output in a temp file. + Returns (output_string, exit_code). + """ + with tempfile.TemporaryFile() as tempf: + p = subprocess.Popen(command, stdout=tempf, stderr=tempf) + p.wait() + tempf.seek(0) + output = tempf.read().decode("utf-8", errors="replace") + return output, p.returncode + +def chunkify(lst, chunk_size=64): + """ + Yield successive chunk_size-sized slices from lst. + """ + for i in range(0, len(lst), chunk_size): + yield lst[i:i + chunk_size] + +def find_fuzzer_binary(default_path, argv): + """ + If default_path exists, return it; + otherwise check argv[1] for a user-supplied binary path; + otherwise exit with an error. + """ + if os.path.exists(default_path): + return default_path + + if len(argv) > 1 and os.path.exists(argv[1]): + return argv[1] + + sys.exit( + f"Failed to find {os.path.basename(default_path)} binary.\n" + "Please provide it as the first argument to the tool." + ) + +def gather_files(directory): + """ + Return a list of *all* files (not subdirs) in `directory`. + If `directory` doesn’t exist, returns an empty list. + """ + if not os.path.isdir(directory): + return [] + return [ + os.path.join(directory, f) + for f in os.listdir(directory) + if os.path.isfile(os.path.join(directory, f)) + ] diff --git a/test/fuzzing/run-draw-fuzzer-tests.py b/test/fuzzing/run-draw-fuzzer-tests.py index 33b265e25..da4420e3f 100755 --- a/test/fuzzing/run-draw-fuzzer-tests.py +++ b/test/fuzzing/run-draw-fuzzer-tests.py @@ -2,58 +2,52 @@ import sys import os -import subprocess -import tempfile +from hb_fuzzer_tools import ( + run_command, + chunkify, + find_fuzzer_binary, + gather_files +) -def run_command(command): - with tempfile.TemporaryFile() as tempf: - p = subprocess.Popen(command, stdout=tempf, stderr=tempf) - p.wait() - tempf.seek(0) - output = tempf.read().decode('utf-8', errors='replace') - return output, p.returncode +def main(): + srcdir = os.getenv("srcdir", ".") + EXEEXT = os.getenv("EXEEXT", "") + top_builddir = os.getenv("top_builddir", ".") -srcdir = os.getenv("srcdir", ".") -EXEEXT = os.getenv("EXEEXT", "") -top_builddir = os.getenv("top_builddir", ".") + # Find the fuzzer binary + default_bin = os.path.join(top_builddir, "hb-draw-fuzzer" + EXEEXT) + hb_draw_fuzzer = find_fuzzer_binary(default_bin, sys.argv) -hb_draw_fuzzer = os.path.join(top_builddir, "hb-draw-fuzzer" + EXEEXT) -# If not found automatically, try sys.argv[1] -if not os.path.exists(hb_draw_fuzzer): - if len(sys.argv) < 2 or not os.path.exists(sys.argv[1]): - sys.exit( - "Failed to find hb-draw-fuzzer binary automatically.\n" - "Please provide it as the first argument to the tool." - ) - hb_draw_fuzzer = sys.argv[1] + print("Using hb_draw_fuzzer:", hb_draw_fuzzer) -print("Using hb_draw_fuzzer:", hb_draw_fuzzer) + # Gather all files from fonts/ + fonts_dir = os.path.join(srcdir, "fonts") + files_to_test = gather_files(fonts_dir) -# Collect all files from the fonts/ directory -parent_path = os.path.join(srcdir, "fonts") -if not os.path.isdir(parent_path): - sys.exit(f"Directory {parent_path} not found or not a directory.") + if not files_to_test: + print("No files found in", fonts_dir) + sys.exit(0) -files_to_check = [ - os.path.join(parent_path, f) for f in os.listdir(parent_path) - if os.path.isfile(os.path.join(parent_path, f)) -] + fails = 0 + batch_index = 0 -if not files_to_check: - print(f"No files found in {parent_path}") - sys.exit(1) + # Run in batches of up to 64 files + for chunk in chunkify(files_to_test, 64): + batch_index += 1 + cmd_line = [hb_draw_fuzzer] + chunk + output, returncode = run_command(cmd_line) -# Single invocation passing all files -cmd_line = [hb_draw_fuzzer] + files_to_check -output, returncode = run_command(cmd_line) + if output.strip(): + print(output) -# Print output if not empty -if output.strip(): - print(output) + if returncode != 0: + print(f"Failure in batch #{batch_index}") + fails += 1 -# If there's an error, print a message and exit non-zero -if returncode != 0: - print("Failure while processing these files:", ", ".join(os.path.basename(f) for f in files_to_check)) - sys.exit(returncode) + if fails > 0: + sys.exit(f"{fails} draw fuzzer batch(es) failed.") -print("All files processed successfully.") + print("All draw fuzzer tests passed successfully.") + +if __name__ == "__main__": + main() diff --git a/test/fuzzing/run-repacker-fuzzer-tests.py b/test/fuzzing/run-repacker-fuzzer-tests.py index 81971531f..110366645 100755 --- a/test/fuzzing/run-repacker-fuzzer-tests.py +++ b/test/fuzzing/run-repacker-fuzzer-tests.py @@ -2,64 +2,52 @@ import sys import os -import subprocess -import tempfile +from hb_fuzzer_tools import ( + run_command, + chunkify, + find_fuzzer_binary, + gather_files +) -def run_command(command): - with tempfile.TemporaryFile() as tempf: - p = subprocess.Popen(command, stdout=tempf, stderr=tempf) - p.wait() - tempf.seek(0) - output = tempf.read().decode('utf-8', errors='replace') - return output, p.returncode +def main(): + srcdir = os.getenv("srcdir", ".") + EXEEXT = os.getenv("EXEEXT", "") + top_builddir = os.getenv("top_builddir", ".") + # Find the fuzzer binary + default_bin = os.path.join(top_builddir, "hb-repacker-fuzzer" + EXEEXT) + hb_repacker_fuzzer = find_fuzzer_binary(default_bin, sys.argv) -# Environment and binary location -srcdir = os.getenv("srcdir", ".") -EXEEXT = os.getenv("EXEEXT", "") -top_builddir = os.getenv("top_builddir", ".") + print("Using hb_repacker_fuzzer:", hb_repacker_fuzzer) -hb_repacker_fuzzer = os.path.join(top_builddir, "hb-repacker-fuzzer" + EXEEXT) -# If the binary isn't found, try sys.argv[1] -if not os.path.exists(hb_repacker_fuzzer): - if len(sys.argv) < 2 or not os.path.exists(sys.argv[1]): - sys.exit( - "Failed to find hb-repacker-fuzzer binary automatically.\n" - "Please provide it as the first argument to the tool." - ) - hb_repacker_fuzzer = sys.argv[1] + # Gather all files from graphs/ + graphs_dir = os.path.join(srcdir, "graphs") + files_to_test = gather_files(graphs_dir) -print("hb_repacker_fuzzer:", hb_repacker_fuzzer) + if not files_to_test: + print("No files found in", graphs_dir) + sys.exit(0) -# Collect all files from graphs/ -graphs_path = os.path.join(srcdir, "graphs") -if not os.path.isdir(graphs_path): - sys.exit(f"No 'graphs' directory found at {graphs_path}.") + fails = 0 + batch_index = 0 -files_to_check = [ - os.path.join(graphs_path, f) - for f in os.listdir(graphs_path) - if os.path.isfile(os.path.join(graphs_path, f)) -] + # Run in batches of up to 64 files + for chunk in chunkify(files_to_test, 64): + batch_index += 1 + cmd_line = [hb_repacker_fuzzer] + chunk + output, returncode = run_command(cmd_line) -if not files_to_check: - print("No files found in the 'graphs' directory.") - sys.exit(1) + if output.strip(): + print(output) -# Single invocation passing all files -print(f"Running repacker fuzzer against {len(files_to_check)} file(s) in 'graphs'...") -cmd_line = [hb_repacker_fuzzer] + files_to_check -output, returncode = run_command(cmd_line) + if returncode != 0: + print(f"Failure in batch #{batch_index}") + fails += 1 -# Print the output if present -if output.strip(): - print(output) + if fails > 0: + sys.exit(f"{fails} repacker fuzzer batch(es) failed.") -# Exit if there's an error -if returncode != 0: - print("Failed for these files:") - for f in files_to_check: - print(" ", f) - sys.exit("1 repacker fuzzer related test(s) failed.") + print("All repacker fuzzer tests passed successfully.") -print("All repacker fuzzer tests passed successfully.") +if __name__ == "__main__": + main() diff --git a/test/fuzzing/run-shape-fuzzer-tests.py b/test/fuzzing/run-shape-fuzzer-tests.py index 6ce7c1c47..aaed2794c 100755 --- a/test/fuzzing/run-shape-fuzzer-tests.py +++ b/test/fuzzing/run-shape-fuzzer-tests.py @@ -2,62 +2,51 @@ import sys import os -import subprocess -import tempfile +from hb_fuzzer_tools import ( + run_command, + chunkify, + find_fuzzer_binary, + gather_files +) -def run_command(command): - """Run a command, capturing potentially large output.""" - with tempfile.TemporaryFile() as tempf: - p = subprocess.Popen(command, stdout=tempf, stderr=tempf) - p.wait() - tempf.seek(0) - output = tempf.read().decode('utf-8', errors='replace') - return output, p.returncode +def main(): + srcdir = os.getenv("srcdir", ".") + EXEEXT = os.getenv("EXEEXT", "") + top_builddir = os.getenv("top_builddir", ".") -srcdir = os.getenv("srcdir", ".") -EXEEXT = os.getenv("EXEEXT", "") -top_builddir = os.getenv("top_builddir", ".") + default_bin = os.path.join(top_builddir, "hb-shape-fuzzer" + EXEEXT) + hb_shape_fuzzer = find_fuzzer_binary(default_bin, sys.argv) -hb_shape_fuzzer = os.path.join(top_builddir, "hb-shape-fuzzer" + EXEEXT) -if not os.path.exists(hb_shape_fuzzer): - # If not found automatically, fall back to the first CLI argument. - if len(sys.argv) < 2 or not os.path.exists(sys.argv[1]): - sys.exit( - "Failed to find hb-shape-fuzzer binary automatically.\n" - "Please provide it as the first argument to the tool." - ) - hb_shape_fuzzer = sys.argv[1] + print("Using hb_shape_fuzzer:", hb_shape_fuzzer) -print("hb_shape_fuzzer:", hb_shape_fuzzer) + # Gather all files from fonts/ + fonts_dir = os.path.join(srcdir, "fonts") + files_to_test = gather_files(fonts_dir) -fonts_dir = os.path.join(srcdir, "fonts") -if not os.path.isdir(fonts_dir): - sys.exit(f"Fonts directory not found at: {fonts_dir}") + if not files_to_test: + print("No files found in", fonts_dir) + sys.exit(0) -# Gather all files in `fonts_dir` -files_to_test = [ - os.path.join(fonts_dir, f) - for f in os.listdir(fonts_dir) - if os.path.isfile(os.path.join(fonts_dir, f)) -] + fails = 0 + batch_index = 0 -if not files_to_test: - print(f"No files found in {fonts_dir}") - sys.exit(1) + # Batch up to 64 files at a time + for chunk in chunkify(files_to_test, 64): + batch_index += 1 + cmd_line = [hb_shape_fuzzer] + chunk + output, returncode = run_command(cmd_line) -# Single invocation with all test files -cmd_line = [hb_shape_fuzzer] + files_to_test -output, returncode = run_command(cmd_line) + if output.strip(): + print(output) -# Print output if any -if output.strip(): - print(output) + if returncode != 0: + print(f"Failure in batch #{batch_index}") + fails += 1 -# Fail if return code is non-zero -if returncode != 0: - print("Failure on the following file(s):") - for f in files_to_test: - print(" ", f) - sys.exit("1 shape fuzzer test failed.") + if fails > 0: + sys.exit(f"{fails} shape fuzzer batch(es) failed.") -print("All shape fuzzer tests passed successfully.") + print("All shape fuzzer tests passed successfully.") + +if __name__ == "__main__": + main() diff --git a/test/fuzzing/run-subset-fuzzer-tests.py b/test/fuzzing/run-subset-fuzzer-tests.py index a3fdee75f..c8426c150 100755 --- a/test/fuzzing/run-subset-fuzzer-tests.py +++ b/test/fuzzing/run-subset-fuzzer-tests.py @@ -2,68 +2,54 @@ import sys import os -import subprocess -import tempfile +from hb_fuzzer_tools import ( + run_command, + chunkify, + find_fuzzer_binary, + gather_files +) -def run_command(command): - """Run a command, capturing potentially large output.""" - with tempfile.TemporaryFile() as tempf: - p = subprocess.Popen(command, stdout=tempf, stderr=tempf) - p.wait() - tempf.seek(0) - output = tempf.read().decode("utf-8", errors="replace") - return output, p.returncode +def main(): + srcdir = os.getenv("srcdir", ".") + EXEEXT = os.getenv("EXEEXT", "") + top_builddir = os.getenv("top_builddir", ".") -# Environment variables and binary location -srcdir = os.getenv("srcdir", ".") -EXEEXT = os.getenv("EXEEXT", "") -top_builddir = os.getenv("top_builddir", ".") + # Locate the binary + default_bin = os.path.join(top_builddir, "hb-subset-fuzzer" + EXEEXT) + hb_subset_fuzzer = find_fuzzer_binary(default_bin, sys.argv) -hb_subset_fuzzer = os.path.join(top_builddir, "hb-subset-fuzzer" + EXEEXT) -# If not found automatically, fall back to the first CLI argument -if not os.path.exists(hb_subset_fuzzer): - if len(sys.argv) < 2 or not os.path.exists(sys.argv[1]): - sys.exit( - "Failed to find hb-subset-fuzzer binary automatically.\n" - "Please provide it as the first argument to the tool." - ) - hb_subset_fuzzer = sys.argv[1] + print("Using hb_subset_fuzzer:", hb_subset_fuzzer) -print("hb_subset_fuzzer:", hb_subset_fuzzer) + # Gather from two directories, then combine + dir1 = os.path.join(srcdir, "..", "subset", "data", "fonts") + dir2 = os.path.join(srcdir, "fonts") -# Gather all files from both directories -dir1 = os.path.join(srcdir, "..", "subset", "data", "fonts") -dir2 = os.path.join(srcdir, "fonts") + files_to_test = gather_files(dir1) + gather_files(dir2) -files_to_test = [] + if not files_to_test: + print("No files found in either directory.") + sys.exit(0) -for d in [dir1, dir2]: - if not os.path.isdir(d): - # Skip if the directory doesn't exist - continue - for f in os.listdir(d): - file_path = os.path.join(d, f) - if os.path.isfile(file_path): - files_to_test.append(file_path) + fails = 0 + batch_index = 0 -if not files_to_test: - print("No fonts found in either directory.") - sys.exit(1) + # Batch the tests in up to 64 files per run + for chunk in chunkify(files_to_test, 64): + batch_index += 1 + cmd_line = [hb_subset_fuzzer] + chunk + output, returncode = run_command(cmd_line) -# Run the fuzzer once, passing all collected files -print(f"Running subset fuzzer on {len(files_to_test)} file(s).") -cmd_line = [hb_subset_fuzzer] + files_to_test -output, returncode = run_command(cmd_line) + if output.strip(): + print(output) -# Print any output -if output.strip(): - print(output) + if returncode != 0: + print(f"Failure in batch #{batch_index}") + fails += 1 -# If there's an error, exit non-zero -if returncode != 0: - print("Failure while processing these files:") - for f in files_to_test: - print(" ", f) - sys.exit("1 subset fuzzer test failed.") + if fails > 0: + sys.exit(f"{fails} subset fuzzer batch(es) failed.") -print("All subset fuzzer tests passed successfully.") + print("All subset fuzzer tests passed successfully.") + +if __name__ == "__main__": + main() From 1e3f59a79f028e99c3bdee394f8de788ebc08bc7 Mon Sep 17 00:00:00 2001 From: Behdad Esfahbod Date: Sun, 9 Feb 2025 15:21:18 +0000 Subject: [PATCH 3/7] [ci] Give sanitizers more time --- .circleci/config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 51ed1a95e..cbfea656b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -85,7 +85,7 @@ jobs: pip3 install meson==0.56.0 CC=clang CXX=clang++ meson setup build --default-library=static -Db_sanitize=address,undefined --buildtype=debugoptimized --wrap-mode=nodownload -Dexperimental_api=true meson compile -Cbuild -j9 - meson test -Cbuild --print-errorlogs | asan_symbolize | c++filt + meson test -Cbuild -t 10 --print-errorlogs | asan_symbolize | c++filt tsan: docker: @@ -100,7 +100,7 @@ jobs: pip3 install meson==0.56.0 CC=clang CXX=clang++ meson setup build --default-library=static -Db_sanitize=thread --buildtype=debugoptimized --wrap-mode=nodownload -Dexperimental_api=true meson compile -Cbuild -j9 - meson test -Cbuild --print-errorlogs | asan_symbolize | c++filt + meson test -Cbuild -t 10 --print-errorlogs | asan_symbolize | c++filt msan: docker: @@ -116,7 +116,7 @@ jobs: # msan, needs --force-fallback-for=glib,freetype2 also which doesn't work yet but runs fuzzer cases at least CC=clang CXX=clang++ meson setup build --default-library=static -Db_sanitize=memory --buildtype=debugoptimized --wrap-mode=nodownload -Dauto_features=disabled -Dtests=enabled -Dexperimental_api=true meson compile -Cbuild -j9 - meson test -Cbuild --print-errorlogs | asan_symbolize | c++filt + meson test -Cbuild -t 10 --print-errorlogs | asan_symbolize | c++filt clang-cxx2a: docker: From c29b1de39f0308a835fbdf0d97e049270e9fcb6f Mon Sep 17 00:00:00 2001 From: Behdad Esfahbod Date: Sun, 9 Feb 2025 15:38:18 +0000 Subject: [PATCH 4/7] [test/fuzzing] Remove old cruft --- test/fuzzing/run-draw-fuzzer-tests.py | 3 +-- test/fuzzing/run-repacker-fuzzer-tests.py | 3 +-- test/fuzzing/run-shape-fuzzer-tests.py | 3 +-- test/fuzzing/run-subset-fuzzer-tests.py | 3 +-- 4 files changed, 4 insertions(+), 8 deletions(-) diff --git a/test/fuzzing/run-draw-fuzzer-tests.py b/test/fuzzing/run-draw-fuzzer-tests.py index da4420e3f..b8e2e1d01 100755 --- a/test/fuzzing/run-draw-fuzzer-tests.py +++ b/test/fuzzing/run-draw-fuzzer-tests.py @@ -11,11 +11,10 @@ from hb_fuzzer_tools import ( def main(): srcdir = os.getenv("srcdir", ".") - EXEEXT = os.getenv("EXEEXT", "") top_builddir = os.getenv("top_builddir", ".") # Find the fuzzer binary - default_bin = os.path.join(top_builddir, "hb-draw-fuzzer" + EXEEXT) + default_bin = os.path.join(top_builddir, "hb-draw-fuzzer") hb_draw_fuzzer = find_fuzzer_binary(default_bin, sys.argv) print("Using hb_draw_fuzzer:", hb_draw_fuzzer) diff --git a/test/fuzzing/run-repacker-fuzzer-tests.py b/test/fuzzing/run-repacker-fuzzer-tests.py index 110366645..6aafc742c 100755 --- a/test/fuzzing/run-repacker-fuzzer-tests.py +++ b/test/fuzzing/run-repacker-fuzzer-tests.py @@ -11,11 +11,10 @@ from hb_fuzzer_tools import ( def main(): srcdir = os.getenv("srcdir", ".") - EXEEXT = os.getenv("EXEEXT", "") top_builddir = os.getenv("top_builddir", ".") # Find the fuzzer binary - default_bin = os.path.join(top_builddir, "hb-repacker-fuzzer" + EXEEXT) + default_bin = os.path.join(top_builddir, "hb-repacker-fuzzer") hb_repacker_fuzzer = find_fuzzer_binary(default_bin, sys.argv) print("Using hb_repacker_fuzzer:", hb_repacker_fuzzer) diff --git a/test/fuzzing/run-shape-fuzzer-tests.py b/test/fuzzing/run-shape-fuzzer-tests.py index aaed2794c..6eaea6652 100755 --- a/test/fuzzing/run-shape-fuzzer-tests.py +++ b/test/fuzzing/run-shape-fuzzer-tests.py @@ -11,10 +11,9 @@ from hb_fuzzer_tools import ( def main(): srcdir = os.getenv("srcdir", ".") - EXEEXT = os.getenv("EXEEXT", "") top_builddir = os.getenv("top_builddir", ".") - default_bin = os.path.join(top_builddir, "hb-shape-fuzzer" + EXEEXT) + default_bin = os.path.join(top_builddir, "hb-shape-fuzzer") hb_shape_fuzzer = find_fuzzer_binary(default_bin, sys.argv) print("Using hb_shape_fuzzer:", hb_shape_fuzzer) diff --git a/test/fuzzing/run-subset-fuzzer-tests.py b/test/fuzzing/run-subset-fuzzer-tests.py index c8426c150..e2030d476 100755 --- a/test/fuzzing/run-subset-fuzzer-tests.py +++ b/test/fuzzing/run-subset-fuzzer-tests.py @@ -11,11 +11,10 @@ from hb_fuzzer_tools import ( def main(): srcdir = os.getenv("srcdir", ".") - EXEEXT = os.getenv("EXEEXT", "") top_builddir = os.getenv("top_builddir", ".") # Locate the binary - default_bin = os.path.join(top_builddir, "hb-subset-fuzzer" + EXEEXT) + default_bin = os.path.join(top_builddir, "hb-subset-fuzzer") hb_subset_fuzzer = find_fuzzer_binary(default_bin, sys.argv) print("Using hb_subset_fuzzer:", hb_subset_fuzzer) From 4c43fdcd07f53e7b39371fbe87454b0b138a89c4 Mon Sep 17 00:00:00 2001 From: Khaled Hosny Date: Sun, 9 Feb 2025 17:52:13 +0200 Subject: [PATCH 5/7] [test/fuzzing] Simplify Python scripts further MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We always path the fuzzer path in meson, so we don’t need to search for fuzzer path in the scripts, and then we can use one script for all the fuzzers. --- test/fuzzing/hb_fuzzer_tools.py | 18 +------ test/fuzzing/meson.build | 19 +++---- ...aw-fuzzer-tests.py => run-fuzzer-tests.py} | 29 +++++----- test/fuzzing/run-repacker-fuzzer-tests.py | 52 ------------------ test/fuzzing/run-shape-fuzzer-tests.py | 51 ------------------ test/fuzzing/run-subset-fuzzer-tests.py | 54 ------------------- 6 files changed, 23 insertions(+), 200 deletions(-) rename test/fuzzing/{run-draw-fuzzer-tests.py => run-fuzzer-tests.py} (54%) delete mode 100755 test/fuzzing/run-repacker-fuzzer-tests.py delete mode 100755 test/fuzzing/run-shape-fuzzer-tests.py delete mode 100755 test/fuzzing/run-subset-fuzzer-tests.py diff --git a/test/fuzzing/hb_fuzzer_tools.py b/test/fuzzing/hb_fuzzer_tools.py index 8176e59c4..d10e8ab5b 100644 --- a/test/fuzzing/hb_fuzzer_tools.py +++ b/test/fuzzing/hb_fuzzer_tools.py @@ -4,6 +4,7 @@ import sys import subprocess import tempfile + def run_command(command): """ Run a command, capturing potentially large output in a temp file. @@ -16,6 +17,7 @@ def run_command(command): output = tempf.read().decode("utf-8", errors="replace") return output, p.returncode + def chunkify(lst, chunk_size=64): """ Yield successive chunk_size-sized slices from lst. @@ -23,22 +25,6 @@ def chunkify(lst, chunk_size=64): for i in range(0, len(lst), chunk_size): yield lst[i:i + chunk_size] -def find_fuzzer_binary(default_path, argv): - """ - If default_path exists, return it; - otherwise check argv[1] for a user-supplied binary path; - otherwise exit with an error. - """ - if os.path.exists(default_path): - return default_path - - if len(argv) > 1 and os.path.exists(argv[1]): - return argv[1] - - sys.exit( - f"Failed to find {os.path.basename(default_path)} binary.\n" - "Please provide it as the first argument to the tool." - ) def gather_files(directory): """ diff --git a/test/fuzzing/meson.build b/test/fuzzing/meson.build index cada41948..62cc7f2e5 100644 --- a/test/fuzzing/meson.build +++ b/test/fuzzing/meson.build @@ -34,45 +34,42 @@ foreach file_name : tests set_variable('@0@_exe'.format(test_name.underscorify()), exe) endforeach -env = environment() -env.set('srcdir', meson.current_source_dir()) - -test('shape-fuzzer', find_program('run-shape-fuzzer-tests.py'), +test('shape-fuzzer', find_program('run-fuzzer-tests.py'), args: [ hb_shape_fuzzer_exe, + meson.current_source_dir() / 'fonts', ], depends: [hb_shape_fuzzer_exe, libharfbuzz, libharfbuzz_subset], workdir: meson.current_build_dir() / '..' / '..', - env: env, priority: 1, suite: ['fuzzing'], ) -test('subset-fuzzer', find_program('run-subset-fuzzer-tests.py'), +test('subset-fuzzer', find_program('run-fuzzer-tests.py'), args: [ hb_subset_fuzzer_exe, + meson.current_source_dir() / 'fonts', ], workdir: meson.current_build_dir() / '..' / '..', - env: env, priority: 1, suite: ['fuzzing'], ) -test('repacker-fuzzer', find_program('run-repacker-fuzzer-tests.py'), +test('repacker-fuzzer', find_program('run-fuzzer-tests.py'), args: [ hb_repacker_fuzzer_exe, + meson.current_source_dir() / 'fonts', ], workdir: meson.current_build_dir() / '..' / '..', - env: env, priority: 1, suite: ['fuzzing'], ) -test('draw-fuzzer', find_program('run-draw-fuzzer-tests.py'), +test('draw-fuzzer', find_program('run-fuzzer-tests.py'), args: [ hb_draw_fuzzer_exe, + meson.current_source_dir() / 'fonts', ], workdir: meson.current_build_dir() / '..' / '..', - env: env, suite: ['fuzzing'], ) diff --git a/test/fuzzing/run-draw-fuzzer-tests.py b/test/fuzzing/run-fuzzer-tests.py similarity index 54% rename from test/fuzzing/run-draw-fuzzer-tests.py rename to test/fuzzing/run-fuzzer-tests.py index b8e2e1d01..4b7515588 100755 --- a/test/fuzzing/run-draw-fuzzer-tests.py +++ b/test/fuzzing/run-fuzzer-tests.py @@ -2,25 +2,21 @@ import sys import os -from hb_fuzzer_tools import ( - run_command, - chunkify, - find_fuzzer_binary, - gather_files -) +from hb_fuzzer_tools import run_command, chunkify, gather_files + def main(): - srcdir = os.getenv("srcdir", ".") - top_builddir = os.getenv("top_builddir", ".") - # Find the fuzzer binary - default_bin = os.path.join(top_builddir, "hb-draw-fuzzer") - hb_draw_fuzzer = find_fuzzer_binary(default_bin, sys.argv) + assert len(sys.argv) > 2, "Please provide fuzzer binary and fonts directory paths." + assert os.path.exists(sys.argv[1]), "The fuzzer binary does not exist." + assert os.path.exists(sys.argv[2]), "The fonts directory does not exist." - print("Using hb_draw_fuzzer:", hb_draw_fuzzer) + fuzzer = sys.argv[1] + fonts_dir = sys.argv[2] + + print("Using fuzzer:", fuzzer) # Gather all files from fonts/ - fonts_dir = os.path.join(srcdir, "fonts") files_to_test = gather_files(fonts_dir) if not files_to_test: @@ -33,7 +29,7 @@ def main(): # Run in batches of up to 64 files for chunk in chunkify(files_to_test, 64): batch_index += 1 - cmd_line = [hb_draw_fuzzer] + chunk + cmd_line = [fuzzer] + chunk output, returncode = run_command(cmd_line) if output.strip(): @@ -44,9 +40,10 @@ def main(): fails += 1 if fails > 0: - sys.exit(f"{fails} draw fuzzer batch(es) failed.") + sys.exit(f"{fails} fuzzer batch(es) failed.") + + print("All fuzzer tests passed successfully.") - print("All draw fuzzer tests passed successfully.") if __name__ == "__main__": main() diff --git a/test/fuzzing/run-repacker-fuzzer-tests.py b/test/fuzzing/run-repacker-fuzzer-tests.py deleted file mode 100755 index 6aafc742c..000000000 --- a/test/fuzzing/run-repacker-fuzzer-tests.py +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/env python3 - -import sys -import os -from hb_fuzzer_tools import ( - run_command, - chunkify, - find_fuzzer_binary, - gather_files -) - -def main(): - srcdir = os.getenv("srcdir", ".") - top_builddir = os.getenv("top_builddir", ".") - - # Find the fuzzer binary - default_bin = os.path.join(top_builddir, "hb-repacker-fuzzer") - hb_repacker_fuzzer = find_fuzzer_binary(default_bin, sys.argv) - - print("Using hb_repacker_fuzzer:", hb_repacker_fuzzer) - - # Gather all files from graphs/ - graphs_dir = os.path.join(srcdir, "graphs") - files_to_test = gather_files(graphs_dir) - - if not files_to_test: - print("No files found in", graphs_dir) - sys.exit(0) - - fails = 0 - batch_index = 0 - - # Run in batches of up to 64 files - for chunk in chunkify(files_to_test, 64): - batch_index += 1 - cmd_line = [hb_repacker_fuzzer] + chunk - output, returncode = run_command(cmd_line) - - if output.strip(): - print(output) - - if returncode != 0: - print(f"Failure in batch #{batch_index}") - fails += 1 - - if fails > 0: - sys.exit(f"{fails} repacker fuzzer batch(es) failed.") - - print("All repacker fuzzer tests passed successfully.") - -if __name__ == "__main__": - main() diff --git a/test/fuzzing/run-shape-fuzzer-tests.py b/test/fuzzing/run-shape-fuzzer-tests.py deleted file mode 100755 index 6eaea6652..000000000 --- a/test/fuzzing/run-shape-fuzzer-tests.py +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env python3 - -import sys -import os -from hb_fuzzer_tools import ( - run_command, - chunkify, - find_fuzzer_binary, - gather_files -) - -def main(): - srcdir = os.getenv("srcdir", ".") - top_builddir = os.getenv("top_builddir", ".") - - default_bin = os.path.join(top_builddir, "hb-shape-fuzzer") - hb_shape_fuzzer = find_fuzzer_binary(default_bin, sys.argv) - - print("Using hb_shape_fuzzer:", hb_shape_fuzzer) - - # Gather all files from fonts/ - fonts_dir = os.path.join(srcdir, "fonts") - files_to_test = gather_files(fonts_dir) - - if not files_to_test: - print("No files found in", fonts_dir) - sys.exit(0) - - fails = 0 - batch_index = 0 - - # Batch up to 64 files at a time - for chunk in chunkify(files_to_test, 64): - batch_index += 1 - cmd_line = [hb_shape_fuzzer] + chunk - output, returncode = run_command(cmd_line) - - if output.strip(): - print(output) - - if returncode != 0: - print(f"Failure in batch #{batch_index}") - fails += 1 - - if fails > 0: - sys.exit(f"{fails} shape fuzzer batch(es) failed.") - - print("All shape fuzzer tests passed successfully.") - -if __name__ == "__main__": - main() diff --git a/test/fuzzing/run-subset-fuzzer-tests.py b/test/fuzzing/run-subset-fuzzer-tests.py deleted file mode 100755 index e2030d476..000000000 --- a/test/fuzzing/run-subset-fuzzer-tests.py +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env python3 - -import sys -import os -from hb_fuzzer_tools import ( - run_command, - chunkify, - find_fuzzer_binary, - gather_files -) - -def main(): - srcdir = os.getenv("srcdir", ".") - top_builddir = os.getenv("top_builddir", ".") - - # Locate the binary - default_bin = os.path.join(top_builddir, "hb-subset-fuzzer") - hb_subset_fuzzer = find_fuzzer_binary(default_bin, sys.argv) - - print("Using hb_subset_fuzzer:", hb_subset_fuzzer) - - # Gather from two directories, then combine - dir1 = os.path.join(srcdir, "..", "subset", "data", "fonts") - dir2 = os.path.join(srcdir, "fonts") - - files_to_test = gather_files(dir1) + gather_files(dir2) - - if not files_to_test: - print("No files found in either directory.") - sys.exit(0) - - fails = 0 - batch_index = 0 - - # Batch the tests in up to 64 files per run - for chunk in chunkify(files_to_test, 64): - batch_index += 1 - cmd_line = [hb_subset_fuzzer] + chunk - output, returncode = run_command(cmd_line) - - if output.strip(): - print(output) - - if returncode != 0: - print(f"Failure in batch #{batch_index}") - fails += 1 - - if fails > 0: - sys.exit(f"{fails} subset fuzzer batch(es) failed.") - - print("All subset fuzzer tests passed successfully.") - -if __name__ == "__main__": - main() From c404d8fc70a66297a9ab47e939d0da6125dda8cc Mon Sep 17 00:00:00 2001 From: Khaled Hosny Date: Sun, 9 Feb 2025 18:08:50 +0200 Subject: [PATCH 6/7] [test/fuzzing] Merge hb_fuzzer_tools.py back and simplify --- test/fuzzing/hb_fuzzer_tools.py | 40 -------------------------------- test/fuzzing/run-fuzzer-tests.py | 40 +++++++++++++++++++++++++------- 2 files changed, 31 insertions(+), 49 deletions(-) delete mode 100644 test/fuzzing/hb_fuzzer_tools.py diff --git a/test/fuzzing/hb_fuzzer_tools.py b/test/fuzzing/hb_fuzzer_tools.py deleted file mode 100644 index d10e8ab5b..000000000 --- a/test/fuzzing/hb_fuzzer_tools.py +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env python3 -import os -import sys -import subprocess -import tempfile - - -def run_command(command): - """ - Run a command, capturing potentially large output in a temp file. - Returns (output_string, exit_code). - """ - with tempfile.TemporaryFile() as tempf: - p = subprocess.Popen(command, stdout=tempf, stderr=tempf) - p.wait() - tempf.seek(0) - output = tempf.read().decode("utf-8", errors="replace") - return output, p.returncode - - -def chunkify(lst, chunk_size=64): - """ - Yield successive chunk_size-sized slices from lst. - """ - for i in range(0, len(lst), chunk_size): - yield lst[i:i + chunk_size] - - -def gather_files(directory): - """ - Return a list of *all* files (not subdirs) in `directory`. - If `directory` doesn’t exist, returns an empty list. - """ - if not os.path.isdir(directory): - return [] - return [ - os.path.join(directory, f) - for f in os.listdir(directory) - if os.path.isfile(os.path.join(directory, f)) - ] diff --git a/test/fuzzing/run-fuzzer-tests.py b/test/fuzzing/run-fuzzer-tests.py index 4b7515588..3ff45ccee 100755 --- a/test/fuzzing/run-fuzzer-tests.py +++ b/test/fuzzing/run-fuzzer-tests.py @@ -1,23 +1,45 @@ #!/usr/bin/env python3 +import pathlib +import subprocess import sys -import os -from hb_fuzzer_tools import run_command, chunkify, gather_files +import tempfile + + +def run_command(command): + """ + Run a command, capturing potentially large output in a temp file. + Returns (output_string, exit_code). + """ + with tempfile.TemporaryFile() as tempf: + p = subprocess.Popen(command, stdout=tempf, stderr=tempf) + p.wait() + tempf.seek(0) + output = tempf.read().decode("utf-8", errors="replace").strip() + return output, p.returncode + + +def chunkify(lst, chunk_size=64): + """ + Yield successive chunk_size-sized slices from lst. + """ + for i in range(0, len(lst), chunk_size): + yield lst[i : i + chunk_size] def main(): - # Find the fuzzer binary assert len(sys.argv) > 2, "Please provide fuzzer binary and fonts directory paths." - assert os.path.exists(sys.argv[1]), "The fuzzer binary does not exist." - assert os.path.exists(sys.argv[2]), "The fonts directory does not exist." - fuzzer = sys.argv[1] - fonts_dir = sys.argv[2] + fuzzer = pathlib.Path(sys.argv[1]) + fonts_dir = pathlib.Path(sys.argv[2]) + + assert fuzzer.is_file(), f"Fuzzer binary not found: {fuzzer}" + assert fonts_dir.is_dir(), f"Fonts directory not found: {fonts_dir}" print("Using fuzzer:", fuzzer) # Gather all files from fonts/ - files_to_test = gather_files(fonts_dir) + files_to_test = [str(f) for f in fonts_dir.iterdir() if f.is_file()] if not files_to_test: print("No files found in", fonts_dir) @@ -32,7 +54,7 @@ def main(): cmd_line = [fuzzer] + chunk output, returncode = run_command(cmd_line) - if output.strip(): + if output: print(output) if returncode != 0: From 7ba3efa5c699295150840e9dbfb448404ca80502 Mon Sep 17 00:00:00 2001 From: Khaled Hosny Date: Sun, 9 Feb 2025 18:42:45 +0200 Subject: [PATCH 7/7] [tests/fuzzing] Use the correct dirs for subset and repacker fuzzers --- test/fuzzing/meson.build | 3 ++- test/fuzzing/run-fuzzer-tests.py | 16 +++++++++------- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/test/fuzzing/meson.build b/test/fuzzing/meson.build index 62cc7f2e5..65ce12ff1 100644 --- a/test/fuzzing/meson.build +++ b/test/fuzzing/meson.build @@ -49,6 +49,7 @@ test('subset-fuzzer', find_program('run-fuzzer-tests.py'), args: [ hb_subset_fuzzer_exe, meson.current_source_dir() / 'fonts', + meson.current_source_dir() / '..' / 'subset' / 'data' / 'fonts', ], workdir: meson.current_build_dir() / '..' / '..', priority: 1, @@ -58,7 +59,7 @@ test('subset-fuzzer', find_program('run-fuzzer-tests.py'), test('repacker-fuzzer', find_program('run-fuzzer-tests.py'), args: [ hb_repacker_fuzzer_exe, - meson.current_source_dir() / 'fonts', + meson.current_source_dir() / 'graphs', ], workdir: meson.current_build_dir() / '..' / '..', priority: 1, diff --git a/test/fuzzing/run-fuzzer-tests.py b/test/fuzzing/run-fuzzer-tests.py index 3ff45ccee..7f9671096 100755 --- a/test/fuzzing/run-fuzzer-tests.py +++ b/test/fuzzing/run-fuzzer-tests.py @@ -31,18 +31,20 @@ def main(): assert len(sys.argv) > 2, "Please provide fuzzer binary and fonts directory paths." fuzzer = pathlib.Path(sys.argv[1]) - fonts_dir = pathlib.Path(sys.argv[2]) - assert fuzzer.is_file(), f"Fuzzer binary not found: {fuzzer}" - assert fonts_dir.is_dir(), f"Fonts directory not found: {fonts_dir}" - print("Using fuzzer:", fuzzer) - # Gather all files from fonts/ - files_to_test = [str(f) for f in fonts_dir.iterdir() if f.is_file()] + # Gather all test files + files_to_test = [] + for fonts_dir in sys.argv[2:]: + fonts_dir = pathlib.Path(fonts_dir) + assert fonts_dir.is_dir(), f"Fonts directory not found: {fonts_dir}" + test_files = [str(f) for f in fonts_dir.iterdir() if f.is_file()] + assert test_files, f"No files found in {fonts_dir}" + files_to_test += test_files if not files_to_test: - print("No files found in", fonts_dir) + print("No test files found") sys.exit(0) fails = 0