From 2b3e3e3ce6f65bb15b84d837f857ecb93e176f7f Mon Sep 17 00:00:00 2001 From: Shane Carr Date: Tue, 13 Aug 2019 02:51:18 -0700 Subject: [PATCH] ICU-20524 Adding variables for relative file replacement directory. Also fixes some bugs with file replacement. --- .ci-builds/data-filter.json | 13 ++++++++- docs/userguide/icu_data/buildtool.md | 15 +++++++++- .../python/icutools/databuilder/__main__.py | 29 +++++++++++++++---- .../databuilder/filtration_schema.json | 24 ++++++++++----- .../icutools/databuilder/request_types.py | 1 + .../python/icutools/databuilder/utils.py | 28 +++++++++++++++++- 6 files changed, 94 insertions(+), 16 deletions(-) diff --git a/.ci-builds/data-filter.json b/.ci-builds/data-filter.json index 72d5aa4abd9..177478acff2 100644 --- a/.ci-builds/data-filter.json +++ b/.ci-builds/data-filter.json @@ -21,5 +21,16 @@ }, "rules": ["+/*"] } - ] + ], +// Test fileReplacements (relative path) + "fileReplacements": { + "directory": "$SRC", + "replacements": [ + { + "src": "translit/Zawgyi_my.txt", + "dest": "translit/Zawgyi_my.txt" + }, + "misc/dayPeriods.txt" + ] + } } diff --git a/docs/userguide/icu_data/buildtool.md b/docs/userguide/icu_data/buildtool.md index b0930bdf0a4..da4c859fa7e 100644 --- a/docs/userguide/icu_data/buildtool.md +++ b/docs/userguide/icu_data/buildtool.md @@ -702,8 +702,21 @@ configuration file: { src: "Zawgyi_my.txt" dest: "translit/Zawgyi_my.txt" - } + }, + "misc/dayPeriods.txt" ] } +`directory` should either be an absolute path, or a path starting with one of +the following, and it should not contain a trailing slash: + +- "$SRC" for the *icu4c/source/data* directory in the source tree +- "$FILTERS" for the directory containing filters.json +- "$CWD" for your current working directory + +When the entry in the `replacements` array is an object, the `src` and `dest` +fields indicate, for each file in the source directory (`src`), what file in +the ICU hierarchy it should replace (`dest`). When the entry is a string, the +same relative path is used for both `src` and `dest`. + Whole-file substitution happens before all other filters are applied. diff --git a/icu4c/source/python/icutools/databuilder/__main__.py b/icu4c/source/python/icutools/databuilder/__main__.py index 7cfec1453be..5d843d9c081 100644 --- a/icu4c/source/python/icutools/databuilder/__main__.py +++ b/icu4c/source/python/icutools/databuilder/__main__.py @@ -130,6 +130,7 @@ class Config(object): # Default fields before processing filter file self.filters_json_data = {} + self.filter_dir = "ERROR_NO_FILTER_FILE" # Process filter file if args.filter_file: @@ -140,6 +141,7 @@ class Config(object): except IOError: print("Error: Could not read filter file %s." % args.filter_file, file=sys.stderr) exit(1) + self.filter_dir = os.path.abspath(os.path.dirname(args.filter_file)) # Either "unihan" or "implicithan" self.coll_han_type = "unihan" @@ -188,7 +190,14 @@ class Config(object): def add_copy_input_requests(requests, config, common_vars): files_to_copy = set() for request in requests: - for f in request.all_input_files(): + request_files = request.all_input_files() + # Also add known dependency txt files as possible inputs. + # This is required for translit rule files. + if hasattr(request, "dep_targets"): + request_files += [ + f for f in request.dep_targets if isinstance(f, InFile) + ] + for f in request_files: if isinstance(f, InFile): files_to_copy.add(f) @@ -198,8 +207,12 @@ def add_copy_input_requests(requests, config, common_vars): json_data = config.filters_json_data["fileReplacements"] dirname = json_data["directory"] for directive in json_data["replacements"]: - input_file = LocalFile(dirname, directive["src"]) - output_file = InFile(directive["dest"]) + if type(directive) == str: + input_file = LocalFile(dirname, directive) + output_file = InFile(directive) + else: + input_file = LocalFile(dirname, directive["src"]) + output_file = InFile(directive["dest"]) result += [ CopyRequest( name = "input_copy_%d" % id, @@ -240,7 +253,10 @@ def main(argv): for key in list(makefile_vars.keys()) + makefile_env } common["GLOB_DIR"] = args.src_dir + common["FILTERS_DIR"] = config.filter_dir + common["CWD_DIR"] = os.getcwd() else: + makefile_vars = None common = { # GLOB_DIR is used now, whereas IN_DIR is used during execution phase. # There is no useful distinction in unix-exec or windows-exec mode. @@ -249,6 +265,8 @@ def main(argv): "IN_DIR": args.src_dir, "OUT_DIR": args.out_dir, "TMP_DIR": args.tmp_dir, + "FILTERS_DIR": config.filter_dir, + "CWD_DIR": os.getcwd(), "INDEX_NAME": "res_index", # TODO: Pull this from configure script: "ICUDATA_CHAR": "l" @@ -271,8 +289,6 @@ def main(argv): sys.exit(1) requests = BUILDRULES.generate(config, glob, common) - requests = filtration.apply_filters(requests, config) - requests = utils.flatten_requests(requests, config, common) if "fileReplacements" in config.filters_json_data: tmp_in_dir = "{TMP_DIR}/in".format(**common) @@ -282,6 +298,9 @@ def main(argv): common["IN_DIR"] = tmp_in_dir requests = add_copy_input_requests(requests, config, common) + requests = filtration.apply_filters(requests, config) + requests = utils.flatten_requests(requests, config, common) + build_dirs = utils.compute_directories(requests) if args.mode == "gnumake": diff --git a/icu4c/source/python/icutools/databuilder/filtration_schema.json b/icu4c/source/python/icutools/databuilder/filtration_schema.json index ba027edce11..2b7ff998999 100644 --- a/icu4c/source/python/icutools/databuilder/filtration_schema.json +++ b/icu4c/source/python/icutools/databuilder/filtration_schema.json @@ -49,17 +49,25 @@ "fileReplacements": { "type": "object", "properties": { - "directory": { "type": "string" }, + "directory": { + "type": "string", + "pattern": "^(\\$SRC|\\$FILTERS|\\$CWD|/$|/[^/]+)(/[^/]+)*$" + }, "replacements": { "type": "array", "items": { - "type": "object", - "properties": { - "src": { "type": "string" }, - "dest": { "type": "string" } - }, - "additionalProperties": false, - "required": ["src", "dest"] + "oneOf": [ + { "type": "string" }, + { + "type": "object", + "properties": { + "src": { "type": "string" }, + "dest": { "type": "string" } + }, + "additionalProperties": false, + "required": ["src", "dest"] + } + ] } } }, diff --git a/icu4c/source/python/icutools/databuilder/request_types.py b/icu4c/source/python/icutools/databuilder/request_types.py index 9a973faec1b..d7ffd799d19 100644 --- a/icu4c/source/python/icutools/databuilder/request_types.py +++ b/icu4c/source/python/icutools/databuilder/request_types.py @@ -147,6 +147,7 @@ class AbstractExecutionRequest(AbstractRequest): dep_target.name, self.name ), file=sys.stderr) + self.dep_targets = [] def all_input_files(self): return self.common_dep_files + self.input_files diff --git a/icu4c/source/python/icutools/databuilder/utils.py b/icu4c/source/python/icutools/databuilder/utils.py index 196a7996701..c5965906c6b 100644 --- a/icu4c/source/python/icutools/databuilder/utils.py +++ b/icu4c/source/python/icutools/databuilder/utils.py @@ -12,7 +12,7 @@ from . import * def dir_for(file): if isinstance(file, LocalFile): - return file.dirname + return get_local_dirname(file.dirname) if isinstance(file, SrcFile): return "{SRC_DIR}" if isinstance(file, InFile): @@ -26,6 +26,32 @@ def dir_for(file): assert False +LOCAL_DIRNAME_SUBSTITUTIONS = { + "SRC": "{SRC_DIR}", + "FILTERS": "{FILTERS_DIR}", + "CWD": "{CWD_DIR}" +} + + +def get_local_dirname(dirname): + if dirname.startswith("/"): + return dirname + elif dirname.startswith("$"): + # Note: directory separator substitution happens later + sep_idx = dirname.find("/") + if sep_idx == -1: + sep_idx = len(dirname) + variable = dirname[1:sep_idx] + if variable in LOCAL_DIRNAME_SUBSTITUTIONS: + return LOCAL_DIRNAME_SUBSTITUTIONS[variable] + dirname[sep_idx:] + print( + "Error: Local directory must be absolute, or relative to one of: " + + (", ".join("$%s" % v for v in LOCAL_DIRNAME_SUBSTITUTIONS.keys())), + file=sys.stderr + ) + exit(1) + + def concat_dicts(*dicts): # There is not a super great way to do this in Python: new_dict = {}