ICU-20593 Buildtool: adding some options and improving exec mode.

icu4c/source/data/buildtool/__main__.py

@@ -14,7 +14,7 @@ import sys
 from . import *
 from .comment_stripper import CommentStripper
 from .request_types import CopyRequest
-from .renderers import makefile, unix_exec, windows_exec
+from .renderers import makefile, common_exec
 from . import filtration, utils
 import BUILDRULES
 
@@ -63,7 +63,7 @@ arg_group_required = flag_parser.add_argument_group("required arguments")
 arg_group_required.add_argument(
     "--mode",
     help = "What to do with the generated rules.",
-    choices = ["gnumake", "unix-exec", "windows-exec"],
+    choices = ["gnumake", "unix-exec", "windows-exec", "bazel-exec"],
     required = True
 )
 
@@ -84,12 +84,24 @@ flag_parser.add_argument(
     default = False,
     action = "store_true"
 )
+flag_parser.add_argument(
+    "--ignore_xml_deprecates",
+    help = "Whether to ignore XML deprecates files for building res_index.",
+    default = False,
+    action = "store_true"
+)
 flag_parser.add_argument(
     "--seqmode",
     help = "Whether to optimize rules to be run sequentially (fewer threads) or in parallel (many threads). Defaults to 'sequential', which is better for unix-exec and windows-exec modes. 'parallel' is often better for massively parallel build systems.",
     choices = ["sequential", "parallel"],
     default = "sequential"
 )
+flag_parser.add_argument(
+    "--verbose",
+    help = "Print more verbose output (default false).",
+    default = False,
+    action = "store_true"
+)
 
 arg_group_exec = flag_parser.add_argument_group("arguments for unix-exec and windows-exec modes")
 arg_group_exec.add_argument(
@@ -123,6 +135,9 @@ class Config(object):
         # Boolean: Whether to include core Unicode data files in the .dat file
         self.include_uni_core_data = args.include_uni_core_data
 
+        # Boolean: Whether to ignore the XML files
+        self.ignore_xml_deprecates = args.ignore_xml_deprecates
+
         # Default fields before processing filter file
         self.filters_json_data = {}
 
@@ -209,8 +224,8 @@ def add_copy_input_requests(requests, config, common_vars):
     return result
 
 
-def main():
+def main(argv):
-    args = flag_parser.parse_args()
+    args = flag_parser.parse_args(argv)
     config = Config(args)
 
     if args.mode == "gnumake":
@@ -269,19 +284,32 @@ def main():
             common_vars = common
         ))
     elif args.mode == "windows-exec":
-        return windows_exec.run(
+        return common_exec.run(
+            platform = "windows",
             build_dirs = build_dirs,
             requests = requests,
             common_vars = common,
             tool_dir = args.tool_dir,
-            tool_cfg = args.tool_cfg
+            tool_cfg = args.tool_cfg,
+            verbose = args.verbose,
         )
     elif args.mode == "unix-exec":
-        return unix_exec.run(
+        return common_exec.run(
+            platform = "unix",
             build_dirs = build_dirs,
             requests = requests,
             common_vars = common,
-            tool_dir = args.tool_dir
+            tool_dir = args.tool_dir,
+            verbose = args.verbose,
+        )
+    elif args.mode == "bazel-exec":
+        return common_exec.run(
+            platform = "bazel",
+            build_dirs = build_dirs,
+            requests = requests,
+            common_vars = common,
+            tool_dir = args.tool_dir,
+            verbose = args.verbose,
         )
     else:
         print("Mode not supported: %s" % args.mode)
@@ -289,4 +317,4 @@ def main():
     return 0
 
 if __name__ == "__main__":
-    exit(main())
+    exit(main(sys.argv[1:]))

icu4c/source/data/buildtool/renderers/common_exec.py

@@ -9,25 +9,41 @@ from ..request_types import *
 import os
 import shutil
 import subprocess
+import sys
 
-def run(build_dirs, requests, common_vars, **kwargs):
+def run(build_dirs, requests, common_vars, verbose=True, **kwargs):
     for bd in build_dirs:
-        os.makedirs(bd.format(**common_vars), exist_ok=True)
+        makedirs(bd.format(**common_vars))
     for request in requests:
-        status = run_helper(request, common_vars, **kwargs)
+        status = run_helper(request, common_vars, verbose=verbose, **kwargs)
         if status != 0:
             print("!!! ERROR executing above command line: exit code %d" % status)
             return 1
-    print("All data build commands executed")
+    if verbose:
+        print("All data build commands executed")
     return 0
 
-def run_helper(request, common_vars, is_windows, tool_dir, tool_cfg=None, **kwargs):
+def makedirs(dirs):
+    """makedirs compatible between Python 2 and 3"""
+    try:
+        # Python 3 version
+        os.makedirs(dirs, exist_ok=True)
+    except TypeError as e:
+        # Python 2 version
+        try:
+            os.makedirs(dirs)
+        except OSError as e:
+            if e.errno != errno.EEXIST:
+                raise e
+
+def run_helper(request, common_vars, platform, tool_dir, verbose, tool_cfg=None, **kwargs):
     if isinstance(request, PrintFileRequest):
         output_path = "{DIRNAME}/{FILENAME}".format(
             DIRNAME = utils.dir_for(request.output_file).format(**common_vars),
             FILENAME = request.output_file.filename,
         )
-        print("Printing to file: %s" % output_path)
+        if verbose:
+            print("Printing to file: %s" % output_path)
         with open(output_path, "w") as f:
             f.write(request.content)
         return 0
@@ -40,7 +56,8 @@ def run_helper(request, common_vars, is_windows, tool_dir, tool_cfg=None, **kwar
             DIRNAME = utils.dir_for(request.output_file).format(**common_vars),
             FILENAME = request.output_file.filename,
         )
-        print("Copying file to: %s" % output_path)
+        if verbose:
+            print("Copying file to: %s" % output_path)
         shutil.copyfile(input_path, output_path)
         return 0
     if isinstance(request, VariableRequest):
@@ -48,19 +65,27 @@ def run_helper(request, common_vars, is_windows, tool_dir, tool_cfg=None, **kwar
         return 0
 
     assert isinstance(request.tool, IcuTool)
-    if is_windows:
+    if platform == "windows":
         cmd_template = "{TOOL_DIR}/{TOOL}/{TOOL_CFG}/{TOOL}.exe {{ARGS}}".format(
             TOOL_DIR = tool_dir,
             TOOL_CFG = tool_cfg,
             TOOL = request.tool.name,
             **common_vars
         )
-    else:
+    elif platform == "unix":
         cmd_template = "{TOOL_DIR}/{TOOL} {{ARGS}}".format(
             TOOL_DIR = tool_dir,
             TOOL = request.tool.name,
             **common_vars
         )
+    elif platform == "bazel":
+        cmd_template = "{TOOL_DIR}/{TOOL}/{TOOL} {{ARGS}}".format(
+            TOOL_DIR = tool_dir,
+            TOOL = request.tool.name,
+            **common_vars
+        )
+    else:
+        raise ValueError("Unknown platform: %s" % platform)
 
     if isinstance(request, RepeatedExecutionRequest):
         for loop_vars in utils.repeated_execution_request_looper(request):
@@ -70,13 +95,12 @@ def run_helper(request, common_vars, is_windows, tool_dir, tool_cfg=None, **kwar
                 loop_vars,
                 common_vars
             )
-            if is_windows:
+            if platform == "windows":
                 # Note: this / to \ substitution may be too aggressive?
                 command_line = command_line.replace("/", "\\")
-            print("Running: %s" % command_line)
+            returncode = run_shell_command(command_line, verbose)
-            res = subprocess.run(command_line, shell=True)
+            if returncode != 0:
-            if res.returncode != 0:
+                return returncode
-                return res.returncode
         return 0
     if isinstance(request, SingleExecutionRequest):
         command_line = utils.format_single_request_command(
@@ -84,10 +108,26 @@ def run_helper(request, common_vars, is_windows, tool_dir, tool_cfg=None, **kwar
             cmd_template,
             common_vars
         )
-        if is_windows:
+        if platform == "windows":
             # Note: this / to \ substitution may be too aggressive?
             command_line = command_line.replace("/", "\\")
-        print("Running: %s" % command_line)
+        returncode = run_shell_command(command_line, verbose)
-        res = subprocess.run(command_line, shell=True)
+        return returncode
-        return res.returncode
     assert False
+
+def run_shell_command(command_line, verbose):
+    if verbose:
+        print("Running: %s" % command_line)
+        return subprocess.call(
+            command_line,
+            shell = True
+        )
+    else:
+        # Pipe output to /dev/null in quiet mode
+        with open(os.devnull, "w") as devnull:
+            return subprocess.call(
+                command_line,
+                shell = True,
+                stdout = devnull,
+                stderr = devnull
+            )

icu4c/source/data/buildtool/renderers/unix_exec.py

@@ -1,7 +0,0 @@
-# Copyright (C) 2018 and later: Unicode, Inc. and others.
-# License & terms of use: http://www.unicode.org/copyright.html
-
-from . import common_exec
-
-def run(**kwargs):
-    return common_exec.run(is_windows=False, **kwargs)

icu4c/source/data/buildtool/renderers/windows_exec.py

@@ -1,7 +0,0 @@
-# Copyright (C) 2018 and later: Unicode, Inc. and others.
-# License & terms of use: http://www.unicode.org/copyright.html
-
-from . import common_exec
-
-def run(**kwargs):
-    return common_exec.run(is_windows=True, **kwargs)