From 4b04d703ab12747bf44dc923cccabb1098c4298c Mon Sep 17 00:00:00 2001 From: Shane Carr Date: Fri, 3 May 2019 19:09:43 -0700 Subject: [PATCH] ICU-20593 Buildtool: Implementing new "additive" mode. --- icu4c/source/data/buildtool/__main__.py | 5 ++++ icu4c/source/data/buildtool/filtration.py | 30 ++++++++++++------- .../data/buildtool/filtration_schema.json | 14 ++++++++- 3 files changed, 38 insertions(+), 11 deletions(-) diff --git a/icu4c/source/data/buildtool/__main__.py b/icu4c/source/data/buildtool/__main__.py index 75993a82c6a..36d99207e55 100644 --- a/icu4c/source/data/buildtool/__main__.py +++ b/icu4c/source/data/buildtool/__main__.py @@ -156,6 +156,11 @@ class Config(object): if "collationUCAData" in self.filters_json_data: self.coll_han_type = self.filters_json_data["collationUCAData"] + # Either "additive" or "subtractive" + self.strategy = "subtractive" + if "strategy" in self.filters_json_data: + self.strategy = self.filters_json_data["strategy"] + def _parse_filter_file(self, f): # Use the Hjson parser if it is available; otherwise, use vanilla JSON. try: diff --git a/icu4c/source/data/buildtool/filtration.py b/icu4c/source/data/buildtool/filtration.py index 48d78e3c4f3..a74a7a743c6 100644 --- a/icu4c/source/data/buildtool/filtration.py +++ b/icu4c/source/data/buildtool/filtration.py @@ -247,15 +247,21 @@ def _preprocess_file_filters(requests, config): all_categories = list(sorted(all_categories)) json_data = config.filters_json_data filters = {} + default_filter_json = "exclude" if config.strategy == "additive" else "include" for category in all_categories: + filter_json = default_filter_json + # Figure out the correct filter to create if "featureFilters" in json_data and category in json_data["featureFilters"]: - filters[category] = Filter.create_from_json( - json_data["featureFilters"][category] - ) - elif "localeFilter" in json_data and category[-5:] == "_tree": - filters[category] = Filter.create_from_json( - json_data["localeFilter"] - ) + filter_json = json_data["featureFilters"][category] + if filter_json == "include" and "localeFilter" in json_data and category.endswith("_tree"): + filter_json = json_data["localeFilter"] + # Resolve the filter JSON into a filter object + if filter_json == "exclude": + filters[category] = ExclusionFilter() + elif filter_json == "include": + pass # no-op + else: + filters[category] = Filter.create_from_json(filter_json) if "featureFilters" in json_data: for category in json_data["featureFilters"]: if category not in all_categories: @@ -264,8 +270,9 @@ def _preprocess_file_filters(requests, config): class ResourceFilterInfo(object): - def __init__(self, category): + def __init__(self, category, strategy): self.category = category + self.strategy = strategy self.filter_tmp_dir = "filters/%s" % category self.input_files = None self.filter_files = None @@ -311,7 +318,10 @@ class ResourceFilterInfo(object): for file in files ) ] - self.rules_by_file = [[] for _ in range(len(files))] + if self.strategy == "additive": + self.rules_by_file = [["-/*"] for _ in range(len(files))] + else: + self.rules_by_file = [["+/*"] for _ in range(len(files))] def add_rules(self, file_filter, rules): for file, rule_list in zip(self.input_files, self.rules_by_file): @@ -369,7 +379,7 @@ def _apply_resource_filters(all_requests, config): for category in entry["categories"]: # not defaultdict because we need to pass arguments to the constructor if category not in collected: - filter_info = ResourceFilterInfo(category) + filter_info = ResourceFilterInfo(category, config.strategy) filter_info.apply_to_requests(all_requests) collected[category] = filter_info else: diff --git a/icu4c/source/data/buildtool/filtration_schema.json b/icu4c/source/data/buildtool/filtration_schema.json index 479c65affe6..929cdb5ed66 100644 --- a/icu4c/source/data/buildtool/filtration_schema.json +++ b/icu4c/source/data/buildtool/filtration_schema.json @@ -7,10 +7,22 @@ "description": "JSON Schema for an ICU data filter file", "type": "object", "properties": { + "strategy": { + "type": "string", + "enum": ["additive", "subtractive"] + }, "localeFilter": { "$ref": "#/definitions/filter" }, "featureFilters": { "type": "object", - "additionalProperties": { "$ref": "#/definitions/filter" } + "additionalProperties": { + "oneOf": [ + { "$ref": "#/definitions/filter" }, + { + "type": "string", + "enum": ["include", "exclude"] + } + ] + } }, "resourceFilters": { "type": "array",