organicmaps/icu
13
Fork 0
mirror of https://github.com/unicode-org/icu.git synced 2025-04-07 14:31:31 +00:00
Code Issues 1 Projects Releases Wiki Activity Actions 2

ICU-20627 Adding alias locales to new structure in res_index.

Browse source
This commit is contained in:
Shane Carr 2019-06-05 21:03:55 +00:00 • committed by Shane F. Carr
parent 8cf9d43541
commit fc11a7a8cb
3 changed files with 52 additions and 42 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

42
icu4c/source/data/BUILDRULES.py
View file

@ -12,7 +12,6 @@ from icutools.databuilder.request_types import *
import os
import sys
import xml.etree.ElementTree as ET
def generate(config, glob, common_vars):
@ -517,26 +516,28 @@ def generate_tree(
]
# Generate res_index file
synthetic_locales = set()
if not config.ignore_xml_deprecates:
deprecates_xml_path = os.path.join(
os.path.dirname(__file__), xml_filename)
deprecates_xml = ET.parse(deprecates_xml_path)
for child in deprecates_xml.getroot():
if child.tag == "alias":
synthetic_locales.add(child.attrib["from"])
elif child.tag == "emptyLocale":
synthetic_locales.add(child.attrib["locale"])
else:
raise ValueError("Unknown tag in deprecates XML: %s" % child.tag)
index_input_files = []
# Exclude the deprecated locale variants and root; see ICU-20628. This
# could be data-driven, but we do not want to perform I/O in this script
# (for example, we do not want to read from an XML file).
excluded_locales = set([
"ja_JP_TRADITIONAL",
"th_TH_TRADITIONAL",
"de_",
"de__PHONEBOOK",
"es_",
"es__TRADITIONAL",
"root",
])
# Put alias locales in a separate structure; see ICU-20627
alias_locales = set(locale_dependencies.data["aliases"].keys())
alias_files = []
installed_files = []
for f in input_files:
file_stem = f.filename[f.filename.rfind("/")+1:-4]
if file_stem == "root":
file_stem = IndexRequest.locale_file_stem(f)
if file_stem in excluded_locales:
continue
if file_stem in synthetic_locales:
continue
index_input_files.append(f)
destination = alias_files if file_stem in alias_locales else installed_files
destination.append(f)
cldr_version = locale_dependencies.data["cldrVersion"] if sub_dir == "locales" else None
index_file_txt = TmpFile("{IN_SUB_DIR}/{INDEX_NAME}.txt".format(
IN_SUB_DIR = sub_dir,
@ -551,7 +552,8 @@ def generate_tree(
IndexRequest(
name = index_file_target_name,
category = category,
input_files = index_input_files,
installed_files = installed_files,
alias_files = alias_files,
txt_file = index_file_txt,
output_file = index_res_file,
cldr_version = cldr_version,

9
icu4c/source/python/icutools/databuilder/__main__.py
View file

@ -83,12 +83,6 @@ flag_parser.add_argument(
default = False,
action = "store_true"
)
flag_parser.add_argument(
"--ignore_xml_deprecates",
help = "Whether to ignore XML deprecates files for building res_index.",
default = False,
action = "store_true"
)
flag_parser.add_argument(
"--seqmode",
help = "Whether to optimize rules to be run sequentially (fewer threads) or in parallel (many threads). Defaults to 'sequential', which is better for unix-exec and windows-exec modes. 'parallel' is often better for massively parallel build systems.",
@ -134,9 +128,6 @@ class Config(object):
# Boolean: Whether to include core Unicode data files in the .dat file
self.include_uni_core_data = args.include_uni_core_data
# Boolean: Whether to ignore the XML files
self.ignore_xml_deprecates = args.ignore_xml_deprecates
# Default fields before processing filter file
self.filters_json_data = {}

43
icu4c/source/python/icutools/databuilder/request_types.py
View file

@ -100,8 +100,8 @@ class AbstractExecutionRequest(AbstractRequest):
while i < len(self.input_files):
if filter.match(self.input_files[i]):
i += 1
continue
self._del_at(i)
else:
self._del_at(i)
return i > 0
def _del_at(self, i):
@ -286,7 +286,8 @@ class ListRequest(AbstractRequest):
class IndexRequest(AbstractRequest):
def __init__(self, **kwargs):
self.input_files = []
self.installed_files = []
self.alias_files = []
self.txt_file = None
self.output_file = None
self.cldr_version = ""
@ -296,12 +297,18 @@ class IndexRequest(AbstractRequest):
def apply_file_filter(self, filter):
i = 0
while i < len(self.input_files):
if filter.match(self.input_files[i]):
while i < len(self.installed_files):
if filter.match(self.installed_files[i]):
i += 1
continue
del self.input_files[i]
return i > 0
else:
del self.installed_files[i]
j = 0
while j < len(self.alias_files):
if filter.match(self.alias_files[i]):
j += 1
else:
del self.alias_files[j]
return i + j > 0
def flatten(self, config, all_requests, common_vars):
return (
@ -322,24 +329,34 @@ class IndexRequest(AbstractRequest):
)
def _generate_index_file(self, common_vars):
locales = [f.filename[f.filename.rfind("/")+1:-4] for f in self.input_files]
installed_locales = [IndexRequest.locale_file_stem(f) for f in self.installed_files]
alias_locales = [IndexRequest.locale_file_stem(f) for f in self.alias_files]
formatted_version = " CLDRVersion { \"%s\" }\n" % self.cldr_version if self.cldr_version else ""
formatted_locales = "\n".join([" %s {\"\"}" % v for v in locales])
formatted_installed_locales = "\n".join([" %s {\"\"}" % v for v in installed_locales])
formatted_alias_locales = "\n".join([" %s {\"\"}" % v for v in alias_locales])
# TODO: CLDRVersion is required only in the base file
return ("// Warning this file is automatically generated\n"
"{INDEX_NAME}:table(nofallback) {{\n"
"{FORMATTED_VERSION}"
" InstalledLocales {{\n"
"{FORMATTED_LOCALES}\n"
"{FORMATTED_INSTALLED_LOCALES}\n"
" }}\n"
" AliasLocales {{\n"
"{FORMATTED_ALIAS_LOCALES}\n"
" }}\n"
"}}").format(
FORMATTED_VERSION = formatted_version,
FORMATTED_LOCALES = formatted_locales,
FORMATTED_INSTALLED_LOCALES = formatted_installed_locales,
FORMATTED_ALIAS_LOCALES = formatted_alias_locales,
**common_vars
)
def all_input_files(self):
return self.input_files
return self.installed_files + self.alias_files
def all_output_files(self):
return [self.output_file]
@staticmethod
def locale_file_stem(f):
return f.filename[f.filename.rfind("/")+1:-4]