From b603285082a56046ce00351c7dbdbd765d0a860f Mon Sep 17 00:00:00 2001 From: Shane Carr Date: Fri, 3 May 2019 19:08:14 -0700 Subject: [PATCH] ICU-20593 Buildtool: adding some options and improving exec mode. --- icu4c/source/data/buildtool/__main__.py | 46 ++++++++--- .../data/buildtool/renderers/common_exec.py | 76 ++++++++++++++----- .../data/buildtool/renderers/unix_exec.py | 7 -- .../data/buildtool/renderers/windows_exec.py | 7 -- 4 files changed, 95 insertions(+), 41 deletions(-) delete mode 100644 icu4c/source/data/buildtool/renderers/unix_exec.py delete mode 100644 icu4c/source/data/buildtool/renderers/windows_exec.py diff --git a/icu4c/source/data/buildtool/__main__.py b/icu4c/source/data/buildtool/__main__.py index 52d869c8951..75993a82c6a 100644 --- a/icu4c/source/data/buildtool/__main__.py +++ b/icu4c/source/data/buildtool/__main__.py @@ -14,7 +14,7 @@ import sys from . import * from .comment_stripper import CommentStripper from .request_types import CopyRequest -from .renderers import makefile, unix_exec, windows_exec +from .renderers import makefile, common_exec from . import filtration, utils import BUILDRULES @@ -63,7 +63,7 @@ arg_group_required = flag_parser.add_argument_group("required arguments") arg_group_required.add_argument( "--mode", help = "What to do with the generated rules.", - choices = ["gnumake", "unix-exec", "windows-exec"], + choices = ["gnumake", "unix-exec", "windows-exec", "bazel-exec"], required = True ) @@ -84,12 +84,24 @@ flag_parser.add_argument( default = False, action = "store_true" ) +flag_parser.add_argument( + "--ignore_xml_deprecates", + help = "Whether to ignore XML deprecates files for building res_index.", + default = False, + action = "store_true" +) flag_parser.add_argument( "--seqmode", help = "Whether to optimize rules to be run sequentially (fewer threads) or in parallel (many threads). Defaults to 'sequential', which is better for unix-exec and windows-exec modes. 'parallel' is often better for massively parallel build systems.", choices = ["sequential", "parallel"], default = "sequential" ) +flag_parser.add_argument( + "--verbose", + help = "Print more verbose output (default false).", + default = False, + action = "store_true" +) arg_group_exec = flag_parser.add_argument_group("arguments for unix-exec and windows-exec modes") arg_group_exec.add_argument( @@ -123,6 +135,9 @@ class Config(object): # Boolean: Whether to include core Unicode data files in the .dat file self.include_uni_core_data = args.include_uni_core_data + # Boolean: Whether to ignore the XML files + self.ignore_xml_deprecates = args.ignore_xml_deprecates + # Default fields before processing filter file self.filters_json_data = {} @@ -209,8 +224,8 @@ def add_copy_input_requests(requests, config, common_vars): return result -def main(): - args = flag_parser.parse_args() +def main(argv): + args = flag_parser.parse_args(argv) config = Config(args) if args.mode == "gnumake": @@ -269,19 +284,32 @@ def main(): common_vars = common )) elif args.mode == "windows-exec": - return windows_exec.run( + return common_exec.run( + platform = "windows", build_dirs = build_dirs, requests = requests, common_vars = common, tool_dir = args.tool_dir, - tool_cfg = args.tool_cfg + tool_cfg = args.tool_cfg, + verbose = args.verbose, ) elif args.mode == "unix-exec": - return unix_exec.run( + return common_exec.run( + platform = "unix", build_dirs = build_dirs, requests = requests, common_vars = common, - tool_dir = args.tool_dir + tool_dir = args.tool_dir, + verbose = args.verbose, + ) + elif args.mode == "bazel-exec": + return common_exec.run( + platform = "bazel", + build_dirs = build_dirs, + requests = requests, + common_vars = common, + tool_dir = args.tool_dir, + verbose = args.verbose, ) else: print("Mode not supported: %s" % args.mode) @@ -289,4 +317,4 @@ def main(): return 0 if __name__ == "__main__": - exit(main()) + exit(main(sys.argv[1:])) diff --git a/icu4c/source/data/buildtool/renderers/common_exec.py b/icu4c/source/data/buildtool/renderers/common_exec.py index 803d4162969..26c4c8ae55c 100644 --- a/icu4c/source/data/buildtool/renderers/common_exec.py +++ b/icu4c/source/data/buildtool/renderers/common_exec.py @@ -9,25 +9,41 @@ from ..request_types import * import os import shutil import subprocess +import sys -def run(build_dirs, requests, common_vars, **kwargs): +def run(build_dirs, requests, common_vars, verbose=True, **kwargs): for bd in build_dirs: - os.makedirs(bd.format(**common_vars), exist_ok=True) + makedirs(bd.format(**common_vars)) for request in requests: - status = run_helper(request, common_vars, **kwargs) + status = run_helper(request, common_vars, verbose=verbose, **kwargs) if status != 0: print("!!! ERROR executing above command line: exit code %d" % status) return 1 - print("All data build commands executed") + if verbose: + print("All data build commands executed") return 0 -def run_helper(request, common_vars, is_windows, tool_dir, tool_cfg=None, **kwargs): +def makedirs(dirs): + """makedirs compatible between Python 2 and 3""" + try: + # Python 3 version + os.makedirs(dirs, exist_ok=True) + except TypeError as e: + # Python 2 version + try: + os.makedirs(dirs) + except OSError as e: + if e.errno != errno.EEXIST: + raise e + +def run_helper(request, common_vars, platform, tool_dir, verbose, tool_cfg=None, **kwargs): if isinstance(request, PrintFileRequest): output_path = "{DIRNAME}/{FILENAME}".format( DIRNAME = utils.dir_for(request.output_file).format(**common_vars), FILENAME = request.output_file.filename, ) - print("Printing to file: %s" % output_path) + if verbose: + print("Printing to file: %s" % output_path) with open(output_path, "w") as f: f.write(request.content) return 0 @@ -40,7 +56,8 @@ def run_helper(request, common_vars, is_windows, tool_dir, tool_cfg=None, **kwar DIRNAME = utils.dir_for(request.output_file).format(**common_vars), FILENAME = request.output_file.filename, ) - print("Copying file to: %s" % output_path) + if verbose: + print("Copying file to: %s" % output_path) shutil.copyfile(input_path, output_path) return 0 if isinstance(request, VariableRequest): @@ -48,19 +65,27 @@ def run_helper(request, common_vars, is_windows, tool_dir, tool_cfg=None, **kwar return 0 assert isinstance(request.tool, IcuTool) - if is_windows: + if platform == "windows": cmd_template = "{TOOL_DIR}/{TOOL}/{TOOL_CFG}/{TOOL}.exe {{ARGS}}".format( TOOL_DIR = tool_dir, TOOL_CFG = tool_cfg, TOOL = request.tool.name, **common_vars ) - else: + elif platform == "unix": cmd_template = "{TOOL_DIR}/{TOOL} {{ARGS}}".format( TOOL_DIR = tool_dir, TOOL = request.tool.name, **common_vars ) + elif platform == "bazel": + cmd_template = "{TOOL_DIR}/{TOOL}/{TOOL} {{ARGS}}".format( + TOOL_DIR = tool_dir, + TOOL = request.tool.name, + **common_vars + ) + else: + raise ValueError("Unknown platform: %s" % platform) if isinstance(request, RepeatedExecutionRequest): for loop_vars in utils.repeated_execution_request_looper(request): @@ -70,13 +95,12 @@ def run_helper(request, common_vars, is_windows, tool_dir, tool_cfg=None, **kwar loop_vars, common_vars ) - if is_windows: + if platform == "windows": # Note: this / to \ substitution may be too aggressive? command_line = command_line.replace("/", "\\") - print("Running: %s" % command_line) - res = subprocess.run(command_line, shell=True) - if res.returncode != 0: - return res.returncode + returncode = run_shell_command(command_line, verbose) + if returncode != 0: + return returncode return 0 if isinstance(request, SingleExecutionRequest): command_line = utils.format_single_request_command( @@ -84,10 +108,26 @@ def run_helper(request, common_vars, is_windows, tool_dir, tool_cfg=None, **kwar cmd_template, common_vars ) - if is_windows: + if platform == "windows": # Note: this / to \ substitution may be too aggressive? command_line = command_line.replace("/", "\\") - print("Running: %s" % command_line) - res = subprocess.run(command_line, shell=True) - return res.returncode + returncode = run_shell_command(command_line, verbose) + return returncode assert False + +def run_shell_command(command_line, verbose): + if verbose: + print("Running: %s" % command_line) + return subprocess.call( + command_line, + shell = True + ) + else: + # Pipe output to /dev/null in quiet mode + with open(os.devnull, "w") as devnull: + return subprocess.call( + command_line, + shell = True, + stdout = devnull, + stderr = devnull + ) diff --git a/icu4c/source/data/buildtool/renderers/unix_exec.py b/icu4c/source/data/buildtool/renderers/unix_exec.py deleted file mode 100644 index 468f41322cb..00000000000 --- a/icu4c/source/data/buildtool/renderers/unix_exec.py +++ /dev/null @@ -1,7 +0,0 @@ -# Copyright (C) 2018 and later: Unicode, Inc. and others. -# License & terms of use: http://www.unicode.org/copyright.html - -from . import common_exec - -def run(**kwargs): - return common_exec.run(is_windows=False, **kwargs) diff --git a/icu4c/source/data/buildtool/renderers/windows_exec.py b/icu4c/source/data/buildtool/renderers/windows_exec.py deleted file mode 100644 index 731801e580d..00000000000 --- a/icu4c/source/data/buildtool/renderers/windows_exec.py +++ /dev/null @@ -1,7 +0,0 @@ -# Copyright (C) 2018 and later: Unicode, Inc. and others. -# License & terms of use: http://www.unicode.org/copyright.html - -from . import common_exec - -def run(**kwargs): - return common_exec.run(is_windows=True, **kwargs)