diff --git a/.circleci/config.yml b/.circleci/config.yml index 51ed1a95e..cbfea656b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -85,7 +85,7 @@ jobs: pip3 install meson==0.56.0 CC=clang CXX=clang++ meson setup build --default-library=static -Db_sanitize=address,undefined --buildtype=debugoptimized --wrap-mode=nodownload -Dexperimental_api=true meson compile -Cbuild -j9 - meson test -Cbuild --print-errorlogs | asan_symbolize | c++filt + meson test -Cbuild -t 10 --print-errorlogs | asan_symbolize | c++filt tsan: docker: @@ -100,7 +100,7 @@ jobs: pip3 install meson==0.56.0 CC=clang CXX=clang++ meson setup build --default-library=static -Db_sanitize=thread --buildtype=debugoptimized --wrap-mode=nodownload -Dexperimental_api=true meson compile -Cbuild -j9 - meson test -Cbuild --print-errorlogs | asan_symbolize | c++filt + meson test -Cbuild -t 10 --print-errorlogs | asan_symbolize | c++filt msan: docker: @@ -116,7 +116,7 @@ jobs: # msan, needs --force-fallback-for=glib,freetype2 also which doesn't work yet but runs fuzzer cases at least CC=clang CXX=clang++ meson setup build --default-library=static -Db_sanitize=memory --buildtype=debugoptimized --wrap-mode=nodownload -Dauto_features=disabled -Dtests=enabled -Dexperimental_api=true meson compile -Cbuild -j9 - meson test -Cbuild --print-errorlogs | asan_symbolize | c++filt + meson test -Cbuild -t 10 --print-errorlogs | asan_symbolize | c++filt clang-cxx2a: docker: diff --git a/test/fuzzing/meson.build b/test/fuzzing/meson.build index f6ebbddcf..65ce12ff1 100644 --- a/test/fuzzing/meson.build +++ b/test/fuzzing/meson.build @@ -34,47 +34,43 @@ foreach file_name : tests set_variable('@0@_exe'.format(test_name.underscorify()), exe) endforeach -env = environment() -env.set('srcdir', meson.current_source_dir()) - -test('shape-fuzzer', find_program('run-shape-fuzzer-tests.py'), +test('shape-fuzzer', find_program('run-fuzzer-tests.py'), args: [ hb_shape_fuzzer_exe, + meson.current_source_dir() / 'fonts', ], - timeout: 90, depends: [hb_shape_fuzzer_exe, libharfbuzz, libharfbuzz_subset], workdir: meson.current_build_dir() / '..' / '..', - env: env, priority: 1, - suite: ['fuzzing', 'slow'], + suite: ['fuzzing'], ) -test('subset-fuzzer', find_program('run-subset-fuzzer-tests.py'), +test('subset-fuzzer', find_program('run-fuzzer-tests.py'), args: [ hb_subset_fuzzer_exe, + meson.current_source_dir() / 'fonts', + meson.current_source_dir() / '..' / 'subset' / 'data' / 'fonts', ], - timeout: 90, workdir: meson.current_build_dir() / '..' / '..', - env: env, priority: 1, - suite: ['fuzzing', 'slow'], + suite: ['fuzzing'], ) -test('repacker-fuzzer', find_program('run-repacker-fuzzer-tests.py'), +test('repacker-fuzzer', find_program('run-fuzzer-tests.py'), args: [ hb_repacker_fuzzer_exe, + meson.current_source_dir() / 'graphs', ], workdir: meson.current_build_dir() / '..' / '..', - env: env, priority: 1, suite: ['fuzzing'], ) -test('draw-fuzzer', find_program('run-draw-fuzzer-tests.py'), +test('draw-fuzzer', find_program('run-fuzzer-tests.py'), args: [ hb_draw_fuzzer_exe, + meson.current_source_dir() / 'fonts', ], workdir: meson.current_build_dir() / '..' / '..', - env: env, suite: ['fuzzing'], ) diff --git a/test/fuzzing/run-draw-fuzzer-tests.py b/test/fuzzing/run-draw-fuzzer-tests.py deleted file mode 100755 index 8b5a2e82d..000000000 --- a/test/fuzzing/run-draw-fuzzer-tests.py +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/env python3 - -import sys, os, subprocess, tempfile, shutil - - -def cmd (command): - # https://stackoverflow.com/a/4408409 as we might have huge output sometimes - with tempfile.TemporaryFile () as tempf: - p = subprocess.Popen (command, stderr=tempf) - - try: - p.wait () - tempf.seek (0) - text = tempf.read () - - #TODO: Detect debug mode with a better way - is_debug_mode = b"SANITIZE" in text - - return ("" if is_debug_mode else text.decode ("utf-8").strip ()), p.returncode - except subprocess.TimeoutExpired: - return 'error: timeout, ' + ' '.join (command), 1 - - -srcdir = os.getenv ("srcdir", ".") -EXEEXT = os.getenv ("EXEEXT", "") -top_builddir = os.getenv ("top_builddir", ".") -hb_draw_fuzzer = os.path.join (top_builddir, "hb-draw-fuzzer" + EXEEXT) - -if not os.path.exists (hb_draw_fuzzer): - if len (sys.argv) == 1 or not os.path.exists (sys.argv[1]): - sys.exit ("""Failed to find hb-draw-fuzzer binary automatically, -please provide it as the first argument to the tool""") - - hb_draw_fuzzer = sys.argv[1] - -print ('hb_draw_fuzzer:', hb_draw_fuzzer) -fails = 0 - -valgrind = None -if os.getenv ('RUN_VALGRIND', ''): - valgrind = shutil.which ('valgrind') - if valgrind is None: - sys.exit ("""Valgrind requested but not found.""") - -parent_path = os.path.join (srcdir, "fonts") -for file in os.listdir (parent_path): - if "draw" not in file: continue - path = os.path.join (parent_path, file) - - if valgrind: - text, returncode = cmd ([valgrind, '--leak-check=full', '--error-exitcode=1', hb_draw_fuzzer, path]) - else: - text, returncode = cmd ([hb_draw_fuzzer, path]) - if 'error' in text: - returncode = 1 - - if (not valgrind or returncode) and text.strip (): - print (text) - - if returncode != 0: - print ('failure on %s' % file) - fails = fails + 1 - - -if fails: - sys.exit ("%d draw fuzzer related tests failed." % fails) diff --git a/test/fuzzing/run-fuzzer-tests.py b/test/fuzzing/run-fuzzer-tests.py new file mode 100755 index 000000000..7f9671096 --- /dev/null +++ b/test/fuzzing/run-fuzzer-tests.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 + +import pathlib +import subprocess +import sys +import tempfile + + +def run_command(command): + """ + Run a command, capturing potentially large output in a temp file. + Returns (output_string, exit_code). + """ + with tempfile.TemporaryFile() as tempf: + p = subprocess.Popen(command, stdout=tempf, stderr=tempf) + p.wait() + tempf.seek(0) + output = tempf.read().decode("utf-8", errors="replace").strip() + return output, p.returncode + + +def chunkify(lst, chunk_size=64): + """ + Yield successive chunk_size-sized slices from lst. + """ + for i in range(0, len(lst), chunk_size): + yield lst[i : i + chunk_size] + + +def main(): + assert len(sys.argv) > 2, "Please provide fuzzer binary and fonts directory paths." + + fuzzer = pathlib.Path(sys.argv[1]) + assert fuzzer.is_file(), f"Fuzzer binary not found: {fuzzer}" + print("Using fuzzer:", fuzzer) + + # Gather all test files + files_to_test = [] + for fonts_dir in sys.argv[2:]: + fonts_dir = pathlib.Path(fonts_dir) + assert fonts_dir.is_dir(), f"Fonts directory not found: {fonts_dir}" + test_files = [str(f) for f in fonts_dir.iterdir() if f.is_file()] + assert test_files, f"No files found in {fonts_dir}" + files_to_test += test_files + + if not files_to_test: + print("No test files found") + sys.exit(0) + + fails = 0 + batch_index = 0 + + # Run in batches of up to 64 files + for chunk in chunkify(files_to_test, 64): + batch_index += 1 + cmd_line = [fuzzer] + chunk + output, returncode = run_command(cmd_line) + + if output: + print(output) + + if returncode != 0: + print(f"Failure in batch #{batch_index}") + fails += 1 + + if fails > 0: + sys.exit(f"{fails} fuzzer batch(es) failed.") + + print("All fuzzer tests passed successfully.") + + +if __name__ == "__main__": + main() diff --git a/test/fuzzing/run-repacker-fuzzer-tests.py b/test/fuzzing/run-repacker-fuzzer-tests.py deleted file mode 100755 index 85a23e13e..000000000 --- a/test/fuzzing/run-repacker-fuzzer-tests.py +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env python3 - -import sys, os, subprocess, tempfile, shutil - - -def cmd (command): - # https://stackoverflow.com/a/4408409 as we might have huge output sometimes - with tempfile.TemporaryFile () as tempf: - p = subprocess.Popen (command, stderr=tempf) - - try: - p.wait () - tempf.seek (0) - text = tempf.read () - - #TODO: Detect debug mode with a better way - is_debug_mode = b"SANITIZE" in text - - return ("" if is_debug_mode else text.decode ("utf-8").strip ()), p.returncode - except subprocess.TimeoutExpired: - return 'error: timeout, ' + ' '.join (command), 1 - - -srcdir = os.getenv ("srcdir", ".") -EXEEXT = os.getenv ("EXEEXT", "") -top_builddir = os.getenv ("top_builddir", ".") -hb_repacker_fuzzer = os.path.join (top_builddir, "hb-repacker-fuzzer" + EXEEXT) - -if not os.path.exists (hb_repacker_fuzzer): - if len (sys.argv) < 2 or not os.path.exists (sys.argv[1]): - sys.exit ("""Failed to find hb-repacker-fuzzer binary automatically, -please provide it as the first argument to the tool""") - - hb_repacker_fuzzer = sys.argv[1] - -print ('hb_repacker_fuzzer:', hb_repacker_fuzzer) -fails = 0 - -valgrind = None -if os.getenv ('RUN_VALGRIND', ''): - valgrind = shutil.which ('valgrind') - if valgrind is None: - sys.exit ("""Valgrind requested but not found.""") - -def run_dir (parent_path): - global fails - for file in os.listdir (parent_path): - path = os.path.join(parent_path, file) - print ("running repacker fuzzer against %s" % path) - if valgrind: - text, returncode = cmd ([valgrind, '--leak-check=full', '--error-exitcode=1', hb_repacker_fuzzer, path]) - else: - text, returncode = cmd ([hb_repacker_fuzzer, path]) - if 'error' in text: - returncode = 1 - - if (not valgrind or returncode) and text.strip (): - print (text) - - if returncode != 0: - print ("failed for %s" % path) - fails = fails + 1 - - -run_dir (os.path.join (srcdir, "graphs")) - -if fails: - sys.exit ("%d repacker fuzzer related tests failed." % fails) diff --git a/test/fuzzing/run-shape-fuzzer-tests.py b/test/fuzzing/run-shape-fuzzer-tests.py deleted file mode 100755 index 382f60929..000000000 --- a/test/fuzzing/run-shape-fuzzer-tests.py +++ /dev/null @@ -1,65 +0,0 @@ -#!/usr/bin/env python3 - -import sys, os, subprocess, tempfile, shutil - - -def cmd (command): - # https://stackoverflow.com/a/4408409 as we might have huge output sometimes - with tempfile.TemporaryFile () as tempf: - p = subprocess.Popen (command, stderr=tempf) - - try: - p.wait () - tempf.seek (0) - text = tempf.read () - - #TODO: Detect debug mode with a better way - is_debug_mode = b"SANITIZE" in text - - return ("" if is_debug_mode else text.decode ("utf-8").strip ()), p.returncode - except subprocess.TimeoutExpired: - return 'error: timeout, ' + ' '.join (command), 1 - - -srcdir = os.getenv ("srcdir", ".") -EXEEXT = os.getenv ("EXEEXT", "") -top_builddir = os.getenv ("top_builddir", ".") -hb_shape_fuzzer = os.path.join (top_builddir, "hb-shape-fuzzer" + EXEEXT) - -if not os.path.exists (hb_shape_fuzzer): - if len (sys.argv) == 1 or not os.path.exists (sys.argv[1]): - sys.exit ("""Failed to find hb-shape-fuzzer binary automatically, -please provide it as the first argument to the tool""") - - hb_shape_fuzzer = sys.argv[1] - -print ('hb_shape_fuzzer:', hb_shape_fuzzer) -fails = 0 - -valgrind = None -if os.getenv ('RUN_VALGRIND', ''): - valgrind = shutil.which ('valgrind') - if valgrind is None: - sys.exit ("""Valgrind requested but not found.""") - -parent_path = os.path.join (srcdir, "fonts") -for file in os.listdir (parent_path): - path = os.path.join (parent_path, file) - - if valgrind: - text, returncode = cmd ([valgrind, '--leak-check=full', '--error-exitcode=1', hb_shape_fuzzer, path]) - else: - text, returncode = cmd ([hb_shape_fuzzer, path]) - if 'error' in text: - returncode = 1 - - if (not valgrind or returncode) and text.strip (): - print (text) - - if returncode != 0: - print ('failure on %s' % file) - fails = fails + 1 - - -if fails: - sys.exit ("%d shape fuzzer related tests failed." % fails) diff --git a/test/fuzzing/run-subset-fuzzer-tests.py b/test/fuzzing/run-subset-fuzzer-tests.py deleted file mode 100755 index da7d1e570..000000000 --- a/test/fuzzing/run-subset-fuzzer-tests.py +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env python3 - -import sys, os, subprocess, tempfile, shutil - - -def cmd (command): - # https://stackoverflow.com/a/4408409 as we might have huge output sometimes - with tempfile.TemporaryFile () as tempf: - p = subprocess.Popen (command, stderr=tempf) - - try: - p.wait () - tempf.seek (0) - text = tempf.read () - - #TODO: Detect debug mode with a better way - is_debug_mode = b"SANITIZE" in text - - return ("" if is_debug_mode else text.decode ("utf-8").strip ()), p.returncode - except subprocess.TimeoutExpired: - return 'error: timeout, ' + ' '.join (command), 1 - - -srcdir = os.getenv ("srcdir", ".") -EXEEXT = os.getenv ("EXEEXT", "") -top_builddir = os.getenv ("top_builddir", ".") -hb_subset_fuzzer = os.path.join (top_builddir, "hb-subset-fuzzer" + EXEEXT) - -if not os.path.exists (hb_subset_fuzzer): - if len (sys.argv) < 2 or not os.path.exists (sys.argv[1]): - sys.exit ("""Failed to find hb-subset-fuzzer binary automatically, -please provide it as the first argument to the tool""") - - hb_subset_fuzzer = sys.argv[1] - -print ('hb_subset_fuzzer:', hb_subset_fuzzer) -fails = 0 - -valgrind = None -if os.getenv ('RUN_VALGRIND', ''): - valgrind = shutil.which ('valgrind') - if valgrind is None: - sys.exit ("""Valgrind requested but not found.""") - -def run_dir (parent_path): - global fails - for file in os.listdir (parent_path): - path = os.path.join(parent_path, file) - # TODO: Run on all the fonts not just subset related ones - if "subset" not in path: continue - - print ("running subset fuzzer against %s" % path) - if valgrind: - text, returncode = cmd ([valgrind, '--leak-check=full', '--error-exitcode=1', hb_subset_fuzzer, path]) - else: - text, returncode = cmd ([hb_subset_fuzzer, path]) - if 'error' in text: - returncode = 1 - - if (not valgrind or returncode) and text.strip (): - print (text) - - if returncode != 0: - print ("failed for %s" % path) - fails = fails + 1 - - -run_dir (os.path.join (srcdir, "..", "subset", "data", "fonts")) -run_dir (os.path.join (srcdir, "fonts")) - -if fails: - sys.exit ("%d subset fuzzer related tests failed." % fails)