organicmaps/harfbuzz
13
Fork 0
mirror of https://github.com/harfbuzz/harfbuzz.git synced 2025-04-13 08:42:59 +00:00
Code Issues 1 Projects Releases Wiki Activity Actions 7

Merge pull request #5062 from harfbuzz/test-fuzzer-one-go

Browse source 
[test/fuzzing] Run each fuzzer on all fonts in one process
This commit is contained in:
Behdad Esfahbod 2025-02-09 17:54:46 +00:00 • committed by GitHub
commit 39fec7b11d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 87 additions and 289 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
.circleci/config.yml
View file

@ -85,7 +85,7 @@ jobs:
pip3 install meson==0.56.0
CC=clang CXX=clang++ meson setup build --default-library=static -Db_sanitize=address,undefined --buildtype=debugoptimized --wrap-mode=nodownload -Dexperimental_api=true
meson compile -Cbuild -j9
meson test -Cbuild --print-errorlogs | asan_symbolize | c++filt
meson test -Cbuild -t 10 --print-errorlogs | asan_symbolize | c++filt
tsan:
docker:
@ -100,7 +100,7 @@ jobs:
pip3 install meson==0.56.0
CC=clang CXX=clang++ meson setup build --default-library=static -Db_sanitize=thread --buildtype=debugoptimized --wrap-mode=nodownload -Dexperimental_api=true
meson compile -Cbuild -j9
meson test -Cbuild --print-errorlogs | asan_symbolize | c++filt
meson test -Cbuild -t 10 --print-errorlogs | asan_symbolize | c++filt
msan:
docker:
@ -116,7 +116,7 @@ jobs:
# msan, needs --force-fallback-for=glib,freetype2 also which doesn't work yet but runs fuzzer cases at least
CC=clang CXX=clang++ meson setup build --default-library=static -Db_sanitize=memory --buildtype=debugoptimized --wrap-mode=nodownload -Dauto_features=disabled -Dtests=enabled -Dexperimental_api=true
meson compile -Cbuild -j9
meson test -Cbuild --print-errorlogs | asan_symbolize | c++filt
meson test -Cbuild -t 10 --print-errorlogs | asan_symbolize | c++filt
clang-cxx2a:
docker:

26
test/fuzzing/meson.build
View file

@ -34,47 +34,43 @@ foreach file_name : tests
set_variable('@0@_exe'.format(test_name.underscorify()), exe)
endforeach
env = environment()
env.set('srcdir', meson.current_source_dir())
test('shape-fuzzer', find_program('run-shape-fuzzer-tests.py'),
test('shape-fuzzer', find_program('run-fuzzer-tests.py'),
args: [
hb_shape_fuzzer_exe,
meson.current_source_dir() / 'fonts',
],
timeout: 90,
depends: [hb_shape_fuzzer_exe, libharfbuzz, libharfbuzz_subset],
workdir: meson.current_build_dir() / '..' / '..',
env: env,
priority: 1,
suite: ['fuzzing', 'slow'],
suite: ['fuzzing'],
)
test('subset-fuzzer', find_program('run-subset-fuzzer-tests.py'),
test('subset-fuzzer', find_program('run-fuzzer-tests.py'),
args: [
hb_subset_fuzzer_exe,
meson.current_source_dir() / 'fonts',
meson.current_source_dir() / '..' / 'subset' / 'data' / 'fonts',
],
timeout: 90,
workdir: meson.current_build_dir() / '..' / '..',
env: env,
priority: 1,
suite: ['fuzzing', 'slow'],
suite: ['fuzzing'],
)
test('repacker-fuzzer', find_program('run-repacker-fuzzer-tests.py'),
test('repacker-fuzzer', find_program('run-fuzzer-tests.py'),
args: [
hb_repacker_fuzzer_exe,
meson.current_source_dir() / 'graphs',
],
workdir: meson.current_build_dir() / '..' / '..',
env: env,
priority: 1,
suite: ['fuzzing'],
)
test('draw-fuzzer', find_program('run-draw-fuzzer-tests.py'),
test('draw-fuzzer', find_program('run-fuzzer-tests.py'),
args: [
hb_draw_fuzzer_exe,
meson.current_source_dir() / 'fonts',
],
workdir: meson.current_build_dir() / '..' / '..',
env: env,
suite: ['fuzzing'],
)

66
test/fuzzing/run-draw-fuzzer-tests.py
View file

@ -1,66 +0,0 @@
#!/usr/bin/env python3
import sys, os, subprocess, tempfile, shutil
def cmd (command):
# https://stackoverflow.com/a/4408409 as we might have huge output sometimes
with tempfile.TemporaryFile () as tempf:
p = subprocess.Popen (command, stderr=tempf)
try:
p.wait ()
tempf.seek (0)
text = tempf.read ()
#TODO: Detect debug mode with a better way
is_debug_mode = b"SANITIZE" in text
return ("" if is_debug_mode else text.decode ("utf-8").strip ()), p.returncode
except subprocess.TimeoutExpired:
return 'error: timeout, ' + ' '.join (command), 1
srcdir = os.getenv ("srcdir", ".")
EXEEXT = os.getenv ("EXEEXT", "")
top_builddir = os.getenv ("top_builddir", ".")
hb_draw_fuzzer = os.path.join (top_builddir, "hb-draw-fuzzer" + EXEEXT)
if not os.path.exists (hb_draw_fuzzer):
if len (sys.argv) == 1 or not os.path.exists (sys.argv[1]):
sys.exit ("""Failed to find hb-draw-fuzzer binary automatically,
please provide it as the first argument to the tool""")
hb_draw_fuzzer = sys.argv[1]
print ('hb_draw_fuzzer:', hb_draw_fuzzer)
fails = 0
valgrind = None
if os.getenv ('RUN_VALGRIND', ''):
valgrind = shutil.which ('valgrind')
if valgrind is None:
sys.exit ("""Valgrind requested but not found.""")
parent_path = os.path.join (srcdir, "fonts")
for file in os.listdir (parent_path):
if "draw" not in file: continue
path = os.path.join (parent_path, file)
if valgrind:
text, returncode = cmd ([valgrind, '--leak-check=full', '--error-exitcode=1', hb_draw_fuzzer, path])
else:
text, returncode = cmd ([hb_draw_fuzzer, path])
if 'error' in text:
returncode = 1
if (not valgrind or returncode) and text.strip ():
print (text)
if returncode != 0:
print ('failure on %s' % file)
fails = fails + 1
if fails:
sys.exit ("%d draw fuzzer related tests failed." % fails)

73
test/fuzzing/run-fuzzer-tests.py Executable file
View file

@ -0,0 +1,73 @@
#!/usr/bin/env python3
import pathlib
import subprocess
import sys
import tempfile
def run_command(command):
"""
Run a command, capturing potentially large output in a temp file.
Returns (output_string, exit_code).
"""
with tempfile.TemporaryFile() as tempf:
p = subprocess.Popen(command, stdout=tempf, stderr=tempf)
p.wait()
tempf.seek(0)
output = tempf.read().decode("utf-8", errors="replace").strip()
return output, p.returncode
def chunkify(lst, chunk_size=64):
"""
Yield successive chunk_size-sized slices from lst.
"""
for i in range(0, len(lst), chunk_size):
yield lst[i : i + chunk_size]
def main():
assert len(sys.argv) > 2, "Please provide fuzzer binary and fonts directory paths."
fuzzer = pathlib.Path(sys.argv[1])
assert fuzzer.is_file(), f"Fuzzer binary not found: {fuzzer}"
print("Using fuzzer:", fuzzer)
# Gather all test files
files_to_test = []
for fonts_dir in sys.argv[2:]:
fonts_dir = pathlib.Path(fonts_dir)
assert fonts_dir.is_dir(), f"Fonts directory not found: {fonts_dir}"
test_files = [str(f) for f in fonts_dir.iterdir() if f.is_file()]
assert test_files, f"No files found in {fonts_dir}"
files_to_test += test_files
if not files_to_test:
print("No test files found")
sys.exit(0)
fails = 0
batch_index = 0
# Run in batches of up to 64 files
for chunk in chunkify(files_to_test, 64):
batch_index += 1
cmd_line = [fuzzer] + chunk
output, returncode = run_command(cmd_line)
if output:
print(output)
if returncode != 0:
print(f"Failure in batch #{batch_index}")
fails += 1
if fails > 0:
sys.exit(f"{fails} fuzzer batch(es) failed.")
print("All fuzzer tests passed successfully.")
if __name__ == "__main__":
main()

68
test/fuzzing/run-repacker-fuzzer-tests.py
View file

@ -1,68 +0,0 @@
#!/usr/bin/env python3
import sys, os, subprocess, tempfile, shutil
def cmd (command):
# https://stackoverflow.com/a/4408409 as we might have huge output sometimes
with tempfile.TemporaryFile () as tempf:
p = subprocess.Popen (command, stderr=tempf)
try:
p.wait ()
tempf.seek (0)
text = tempf.read ()
#TODO: Detect debug mode with a better way
is_debug_mode = b"SANITIZE" in text
return ("" if is_debug_mode else text.decode ("utf-8").strip ()), p.returncode
except subprocess.TimeoutExpired:
return 'error: timeout, ' + ' '.join (command), 1
srcdir = os.getenv ("srcdir", ".")
EXEEXT = os.getenv ("EXEEXT", "")
top_builddir = os.getenv ("top_builddir", ".")
hb_repacker_fuzzer = os.path.join (top_builddir, "hb-repacker-fuzzer" + EXEEXT)
if not os.path.exists (hb_repacker_fuzzer):
if len (sys.argv) < 2 or not os.path.exists (sys.argv[1]):
sys.exit ("""Failed to find hb-repacker-fuzzer binary automatically,
please provide it as the first argument to the tool""")
hb_repacker_fuzzer = sys.argv[1]
print ('hb_repacker_fuzzer:', hb_repacker_fuzzer)
fails = 0
valgrind = None
if os.getenv ('RUN_VALGRIND', ''):
valgrind = shutil.which ('valgrind')
if valgrind is None:
sys.exit ("""Valgrind requested but not found.""")
def run_dir (parent_path):
global fails
for file in os.listdir (parent_path):
path = os.path.join(parent_path, file)
print ("running repacker fuzzer against %s" % path)
if valgrind:
text, returncode = cmd ([valgrind, '--leak-check=full', '--error-exitcode=1', hb_repacker_fuzzer, path])
else:
text, returncode = cmd ([hb_repacker_fuzzer, path])
if 'error' in text:
returncode = 1
if (not valgrind or returncode) and text.strip ():
print (text)
if returncode != 0:
print ("failed for %s" % path)
fails = fails + 1
run_dir (os.path.join (srcdir, "graphs"))
if fails:
sys.exit ("%d repacker fuzzer related tests failed." % fails)

65
test/fuzzing/run-shape-fuzzer-tests.py
View file

@ -1,65 +0,0 @@
#!/usr/bin/env python3
import sys, os, subprocess, tempfile, shutil
def cmd (command):
# https://stackoverflow.com/a/4408409 as we might have huge output sometimes
with tempfile.TemporaryFile () as tempf:
p = subprocess.Popen (command, stderr=tempf)
try:
p.wait ()
tempf.seek (0)
text = tempf.read ()
#TODO: Detect debug mode with a better way
is_debug_mode = b"SANITIZE" in text
return ("" if is_debug_mode else text.decode ("utf-8").strip ()), p.returncode
except subprocess.TimeoutExpired:
return 'error: timeout, ' + ' '.join (command), 1
srcdir = os.getenv ("srcdir", ".")
EXEEXT = os.getenv ("EXEEXT", "")
top_builddir = os.getenv ("top_builddir", ".")
hb_shape_fuzzer = os.path.join (top_builddir, "hb-shape-fuzzer" + EXEEXT)
if not os.path.exists (hb_shape_fuzzer):
if len (sys.argv) == 1 or not os.path.exists (sys.argv[1]):
sys.exit ("""Failed to find hb-shape-fuzzer binary automatically,
please provide it as the first argument to the tool""")
hb_shape_fuzzer = sys.argv[1]
print ('hb_shape_fuzzer:', hb_shape_fuzzer)
fails = 0
valgrind = None
if os.getenv ('RUN_VALGRIND', ''):
valgrind = shutil.which ('valgrind')
if valgrind is None:
sys.exit ("""Valgrind requested but not found.""")
parent_path = os.path.join (srcdir, "fonts")
for file in os.listdir (parent_path):
path = os.path.join (parent_path, file)
if valgrind:
text, returncode = cmd ([valgrind, '--leak-check=full', '--error-exitcode=1', hb_shape_fuzzer, path])
else:
text, returncode = cmd ([hb_shape_fuzzer, path])
if 'error' in text:
returncode = 1
if (not valgrind or returncode) and text.strip ():
print (text)
if returncode != 0:
print ('failure on %s' % file)
fails = fails + 1
if fails:
sys.exit ("%d shape fuzzer related tests failed." % fails)

72
test/fuzzing/run-subset-fuzzer-tests.py
View file

@ -1,72 +0,0 @@
#!/usr/bin/env python3
import sys, os, subprocess, tempfile, shutil
def cmd (command):
# https://stackoverflow.com/a/4408409 as we might have huge output sometimes
with tempfile.TemporaryFile () as tempf:
p = subprocess.Popen (command, stderr=tempf)
try:
p.wait ()
tempf.seek (0)
text = tempf.read ()
#TODO: Detect debug mode with a better way
is_debug_mode = b"SANITIZE" in text
return ("" if is_debug_mode else text.decode ("utf-8").strip ()), p.returncode
except subprocess.TimeoutExpired:
return 'error: timeout, ' + ' '.join (command), 1
srcdir = os.getenv ("srcdir", ".")
EXEEXT = os.getenv ("EXEEXT", "")
top_builddir = os.getenv ("top_builddir", ".")
hb_subset_fuzzer = os.path.join (top_builddir, "hb-subset-fuzzer" + EXEEXT)
if not os.path.exists (hb_subset_fuzzer):
if len (sys.argv) < 2 or not os.path.exists (sys.argv[1]):
sys.exit ("""Failed to find hb-subset-fuzzer binary automatically,
please provide it as the first argument to the tool""")
hb_subset_fuzzer = sys.argv[1]
print ('hb_subset_fuzzer:', hb_subset_fuzzer)
fails = 0
valgrind = None
if os.getenv ('RUN_VALGRIND', ''):
valgrind = shutil.which ('valgrind')
if valgrind is None:
sys.exit ("""Valgrind requested but not found.""")
def run_dir (parent_path):
global fails
for file in os.listdir (parent_path):
path = os.path.join(parent_path, file)
# TODO: Run on all the fonts not just subset related ones
if "subset" not in path: continue
print ("running subset fuzzer against %s" % path)
if valgrind:
text, returncode = cmd ([valgrind, '--leak-check=full', '--error-exitcode=1', hb_subset_fuzzer, path])
else:
text, returncode = cmd ([hb_subset_fuzzer, path])
if 'error' in text:
returncode = 1
if (not valgrind or returncode) and text.strip ():
print (text)
if returncode != 0:
print ("failed for %s" % path)
fails = fails + 1
run_dir (os.path.join (srcdir, "..", "subset", "data", "fonts"))
run_dir (os.path.join (srcdir, "fonts"))
if fails:
sys.exit ("%d subset fuzzer related tests failed." % fails)