diff --git a/.gitignore b/.gitignore index 37cd7b65cf5..3e39c5e8e22 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,8 @@ +# Global ignores +__pycache__/ +*.pyc + +# Specific file name ignores /.git icu4c/bin icu4c/bin32uwp @@ -77,6 +82,7 @@ icu4c/source/data/obj icu4c/source/data/out icu4c/source/data/packagedata icu4c/source/data/pkgdataMakefile +icu4c/source/data/rules.mk icu4c/source/data/uni-core-data icu4c/source/data/x64 icu4c/source/data/x86 @@ -621,6 +627,7 @@ icu4c/source/test/testdata/Makefile icu4c/source/test/testdata/out icu4c/source/test/testdata/pkgdata.inc icu4c/source/test/testdata/pkgdataMakefile +icu4c/source/test/testdata/rules.mk icu4c/source/test/testmap/*.d icu4c/source/test/testmap/Debug icu4c/source/test/testmap/Makefile diff --git a/icu4c/source/configure b/icu4c/source/configure index 1ed93bfcd78..0e354e2fcb1 100755 --- a/icu4c/source/configure +++ b/icu4c/source/configure @@ -4200,7 +4200,7 @@ fi #AC_CHECK_PROG(AUTOCONF, autoconf, autoconf, true) #AC_CHECK_PROG(STRIP, strip, strip, true) -for ac_prog in python3 +for ac_prog in python3 "py -3" do # Extract the first word of "$ac_prog", so it can be a program name with args. set dummy $ac_prog; ac_word=$2 @@ -9121,6 +9121,24 @@ $as_echo "$as_me: WARNING: unrecognized options: $ac_unrecognized_opts" >&2;} fi +if test -z "$PYTHON3"; +then + echo "" > data/rules.mk +else + echo "Spawning Python to generate data/rules.mk..." + PYTHONPATH="$srcdir/data" $PYTHON3 -m buildtool \ + --format gnumake \ + --seqmode parallel \ + --glob_dir "$srcdir/data" \ + > data/rules.mk + echo "Spawning Python to generate test/testdata/rules.mk..." + PYTHONPATH="$srcdir/test/testdata:$srcdir/data" $PYTHON3 -m buildtool \ + --format gnumake \ + --seqmode parallel \ + --glob_dir "$srcdir/test/testdata" \ + > test/testdata/rules.mk +fi + echo echo "ICU for C/C++ $VERSION is ready to be built." echo "=== Important Notes: ===" @@ -9205,15 +9223,7 @@ fi if test -z "$PYTHON3"; then - echo "" - echo "NOTICE: Unable to find Python 3. ICU versions 64 and later will require Python 3 to build." - echo "See ICU-10923 for more information: https://unicode-org.atlassian.net/browse/ICU-10923" - echo "" -else - echo "" - echo "Found Python 3. You are all set for ICU 64, which will require Python 3 to build." - echo "For more info on Python 3 requirement, see: https://unicode-org.atlassian.net/browse/ICU-10923" - echo "" + echo "** Note: Python 3 not found. You will not be able to build data from source." fi $as_unset _CXX_CXXSUFFIX diff --git a/icu4c/source/configure.ac b/icu4c/source/configure.ac index 3b01890996e..eeff53543e7 100644 --- a/icu4c/source/configure.ac +++ b/icu4c/source/configure.ac @@ -195,7 +195,7 @@ fi #AC_CHECK_PROG(AUTOCONF, autoconf, autoconf, true) #AC_CHECK_PROG(STRIP, strip, strip, true) -AC_CHECK_PROGS(PYTHON3, python3) +AC_CHECK_PROGS(PYTHON3, python3 "py -3") # Check for the platform make AC_PATH_PROGS(U_MAKE, gmake gnumake, make) @@ -1385,6 +1385,24 @@ AC_CONFIG_FILES([icudefs.mk \ samples/cal/Makefile samples/layout/Makefile]) AC_OUTPUT +if test -z "$PYTHON3"; +then + echo "" > data/rules.mk +else + echo "Spawning Python to generate data/rules.mk..." + PYTHONPATH="$srcdir/data" $PYTHON3 -m buildtool \ + --format gnumake \ + --seqmode parallel \ + --glob_dir "$srcdir/data" \ + > data/rules.mk + echo "Spawning Python to generate test/testdata/rules.mk..." + PYTHONPATH="$srcdir/test/testdata:$srcdir/data" $PYTHON3 -m buildtool \ + --format gnumake \ + --seqmode parallel \ + --glob_dir "$srcdir/test/testdata" \ + > test/testdata/rules.mk +fi + echo echo "ICU for C/C++ $VERSION is ready to be built." echo "=== Important Notes: ===" @@ -1466,15 +1484,7 @@ fi if test -z "$PYTHON3"; then - echo "" - echo "NOTICE: Unable to find Python 3. ICU versions 64 and later will require Python 3 to build." - echo "See ICU-10923 for more information: https://unicode-org.atlassian.net/browse/ICU-10923" - echo "" -else - echo "" - echo "Found Python 3. You are all set for ICU 64, which will require Python 3 to build." - echo "For more info on Python 3 requirement, see: https://unicode-org.atlassian.net/browse/ICU-10923" - echo "" + echo "** Note: Python 3 not found. You will not be able to build data from source." fi $as_unset _CXX_CXXSUFFIX diff --git a/icu4c/source/data/BUILDRULES.py b/icu4c/source/data/BUILDRULES.py new file mode 100644 index 00000000000..a0489414528 --- /dev/null +++ b/icu4c/source/data/BUILDRULES.py @@ -0,0 +1,399 @@ +# Copyright (C) 2018 and later: Unicode, Inc. and others. +# License & terms of use: http://www.unicode.org/copyright.html + +from distutils.sysconfig import parse_makefile + +from buildtool import * +from buildtool import utils + +import sys + + +def generate(config, glob, common_vars): + requests = [] + pkg_exclusions = set() + + if len(glob("misc/*")) == 0: + print("Error: Cannot find data directory; please specify --glob_dir", file=sys.stderr) + exit(1) + + # DIRECTORIES + build_dirs = [ + "{OUT_DIR}", + "{OUT_DIR}/curr", + "{OUT_DIR}/lang", + "{OUT_DIR}/region", + "{OUT_DIR}/zone", + "{OUT_DIR}/unit", + "{OUT_DIR}/brkitr", + "{OUT_DIR}/coll", + "{OUT_DIR}/rbnf", + "{OUT_DIR}/translit", + "{TMP_DIR}", + "{TMP_DIR}/curr", + "{TMP_DIR}/lang", + "{TMP_DIR}/locales", + "{TMP_DIR}/region", + "{TMP_DIR}/zone", + "{TMP_DIR}/unit", + "{TMP_DIR}/coll", + "{TMP_DIR}/rbnf", + "{TMP_DIR}/translit", + "{TMP_DIR}/brkitr" + ] + + # UConv Name Aliases + if config.has_feature("cnvalias"): + input_file = InFile("mappings/convrtrs.txt") + output_file = OutFile("cnvalias.icu") + requests += [ + SingleExecutionRequest( + name = "cnvalias", + input_files = [input_file], + output_files = [output_file], + tool = IcuTool("gencnval"), + args = "-s {IN_DIR} -d {OUT_DIR} " + "{INPUT_FILES[0]}", + format_with = {} + ) + ] + + # CONFUSABLES + if config.has_feature("confusables"): + txt1 = InFile("unidata/confusables.txt") + txt2 = InFile("unidata/confusablesWholeScript.txt") + cfu = OutFile("confusables.cfu") + requests += [ + SingleExecutionRequest( + name = "confusables", + input_files = [txt1, txt2, OutFile("cnvalias.icu")], + output_files = [cfu], + tool = IcuTool("gencfu"), + args = "-d {OUT_DIR} -i {OUT_DIR} " + "-c -r {IN_DIR}/{INPUT_FILES[0]} -w {IN_DIR}/{INPUT_FILES[1]} " + "-o {OUTPUT_FILES[0]}", + format_with = {} + ) + ] + + # UConv Conversion Table Files + if config.has_feature("uconv"): + input_files = [InFile(filename) for filename in glob("mappings/*.ucm")] + output_files = [OutFile("%s.cnv" % v.filename[9:-4]) for v in input_files] + # TODO: handle BUILD_SPECIAL_CNV_FILES? Means to add --ignore-siso-check flag to makeconv + requests += [ + RepeatedOrSingleExecutionRequest( + name = "uconv", + dep_files = [], + input_files = input_files, + output_files = output_files, + tool = IcuTool("makeconv"), + args = "-s {IN_DIR} -d {OUT_DIR} -c {INPUT_FILE_PLACEHOLDER}", + format_with = {}, + repeat_with = { + "INPUT_FILE_PLACEHOLDER": [file.filename for file in input_files] + }, + flatten_with = { + "INPUT_FILE_PLACEHOLDER": " ".join(file.filename for file in input_files) + } + ) + ] + + # BRK Files + brkitr_brk_files = [] + if config.has_feature("brkitr"): + input_files = [InFile(filename) for filename in glob("brkitr/rules/*.txt")] + output_files = [OutFile("brkitr/%s.brk" % v.filename[13:-4]) for v in input_files] + brkitr_brk_files += output_files + requests += [ + RepeatedExecutionRequest( + name = "brkitr_brk", + dep_files = [OutFile("cnvalias.icu")], + input_files = input_files, + output_files = output_files, + tool = IcuTool("genbrk"), + args = "-d {OUT_DIR} -i {OUT_DIR} " + "-c -r {IN_DIR}/{INPUT_FILE} " + "-o {OUTPUT_FILE}", + format_with = {}, + repeat_with = {} + ) + ] + + # SPP FILES + if config.has_feature("stringprep"): + input_files = [InFile(filename) for filename in glob("sprep/*.txt")] + output_files = [OutFile("%s.spp" % v.filename[6:-4]) for v in input_files] + bundle_names = [v.filename[6:-4] for v in input_files] + requests += [ + RepeatedExecutionRequest( + name = "stringprep", + dep_files = [], + input_files = input_files, + output_files = output_files, + tool = IcuTool("gensprep"), + args = "-s {IN_DIR}/sprep -d {OUT_DIR} -i {OUT_DIR} " + "-b {BUNDLE_NAME} -m {IN_DIR}/unidata -u 3.2.0 {BUNDLE_NAME}.txt", + format_with = {}, + repeat_with = { + "BUNDLE_NAME": bundle_names + } + ) + ] + + # Dict Files + dict_files = [] + if config.has_feature("dictionaries"): + input_files = [InFile(filename) for filename in glob("brkitr/dictionaries/*.txt")] + output_files = [OutFile("brkitr/%s.dict" % v.filename[20:-4]) for v in input_files] + dict_files += output_files + extra_options_map = { + "brkitr/dictionaries/burmesedict.txt": "--bytes --transform offset-0x1000", + "brkitr/dictionaries/cjdict.txt": "--uchars", + "brkitr/dictionaries/khmerdict.txt": "--bytes --transform offset-0x1780", + "brkitr/dictionaries/laodict.txt": "--bytes --transform offset-0x0e80", + "brkitr/dictionaries/thaidict.txt": "--bytes --transform offset-0x0e00" + } + extra_optionses = [extra_options_map[v.filename] for v in input_files] + requests += [ + RepeatedExecutionRequest( + name = "dictionaries", + dep_files = [], + input_files = input_files, + output_files = output_files, + tool = IcuTool("gendict"), + args = "-i {OUT_DIR} " + "-c {EXTRA_OPTIONS} " + "{IN_DIR}/{INPUT_FILE} {OUT_DIR}/{OUTPUT_FILE}", + format_with = {}, + repeat_with = { + "EXTRA_OPTIONS": extra_optionses + } + ) + ] + + # NRM Files + if config.has_feature("normalization"): + input_files = [InFile(filename) for filename in glob("in/*.nrm")] + input_files.remove(InFile("in/nfc.nrm")) # nfc.nrm is pre-compiled into C++ + output_files = [OutFile(v.filename[3:]) for v in input_files] + requests += [ + RepeatedExecutionRequest( + name = "normalization", + dep_files = [], + input_files = input_files, + output_files = output_files, + tool = IcuTool("icupkg"), + args = "-t{ICUDATA_CHAR} {IN_DIR}/{INPUT_FILE} {OUT_DIR}/{OUTPUT_FILE}", + format_with = {}, + repeat_with = {} + ) + ] + + # Collation Dependency File (ucadata.icu) + if config.has_feature("coll"): + input_file = InFile("in/coll/ucadata-%s.icu" % config.coll_han_type()) + output_file = OutFile("coll/ucadata.icu") + requests += [ + SingleExecutionRequest( + name = "coll_ucadata", + input_files = [input_file], + output_files = [output_file], + tool = IcuTool("icupkg"), + args = "-t{ICUDATA_CHAR} {IN_DIR}/{INPUT_FILES[0]} {OUT_DIR}/{OUTPUT_FILES[0]}", + format_with = {} + ) + ] + + # Unicode Character Names + if config.has_feature("unames"): + input_file = InFile("in/unames.icu") + output_file = OutFile("unames.icu") + requests += [ + SingleExecutionRequest( + name = "unames", + input_files = [input_file], + output_files = [output_file], + tool = IcuTool("icupkg"), + args = "-t{ICUDATA_CHAR} {IN_DIR}/{INPUT_FILES[0]} {OUT_DIR}/{OUTPUT_FILES[0]}", + format_with = {} + ) + ] + + # Misc Data Res Files + if config.has_feature("misc"): + # TODO: Treat each misc file separately + input_files = [InFile(filename) for filename in glob("misc/*.txt")] + input_basenames = [v.filename[5:] for v in input_files] + output_files = [OutFile("%s.res" % v[:-4]) for v in input_basenames] + requests += [ + RepeatedExecutionRequest( + name = "misc", + dep_files = [], + input_files = input_files, + output_files = output_files, + tool = IcuTool("genrb"), + args = "-s {IN_DIR}/misc -d {OUT_DIR} -i {OUT_DIR} " + "-k -q " + "{INPUT_BASENAME}", + format_with = {}, + repeat_with = { + "INPUT_BASENAME": input_basenames + } + ) + ] + + # Specialized Locale Data Res Files + specialized_sub_dirs = [ + # (input dirname, output dirname, resfiles.mk path, mk version var, mk source var, use pool file, dep files) + ("locales", None, "resfiles.mk", "GENRB_CLDR_VERSION", "GENRB_SOURCE", True, + []), + ("curr", "curr", "resfiles.mk", "CURR_CLDR_VERSION", "CURR_SOURCE", True, + []), + ("lang", "lang", "resfiles.mk", "LANG_CLDR_VERSION", "LANG_SOURCE", True, + []), + ("region", "region", "resfiles.mk", "REGION_CLDR_VERSION", "REGION_SOURCE", True, + []), + ("zone", "zone", "resfiles.mk", "ZONE_CLDR_VERSION", "ZONE_SOURCE", True, + []), + ("unit", "unit", "resfiles.mk", "UNIT_CLDR_VERSION", "UNIT_SOURCE", True, + []), + # TODO: We should not need timezoneTypes.res to build collation resource bundles. + # TODO: Maybe keyTypeData.res should be baked into the common library. + ("coll", "coll", "colfiles.mk", "COLLATION_CLDR_VERSION", "COLLATION_SOURCE", False, + [OutFile("coll/ucadata.icu"), OutFile("timezoneTypes.res"), OutFile("keyTypeData.res")]), + ("brkitr", "brkitr", "brkfiles.mk", "BRK_RES_CLDR_VERSION", "BRK_RES_SOURCE", False, + brkitr_brk_files + dict_files), + ("rbnf", "rbnf", "rbnffiles.mk", "RBNF_CLDR_VERSION", "RBNF_SOURCE", False, + []), + ("translit", "translit", "trnsfiles.mk", None, "TRANSLIT_SOURCE", False, + []) + ] + + for sub_dir, out_sub_dir, resfile_name, version_var, source_var, use_pool_bundle, dep_files in specialized_sub_dirs: + out_prefix = "%s/" % out_sub_dir if out_sub_dir else "" + if config.has_feature(sub_dir): + # TODO: Clean this up for translit + if sub_dir == "translit": + input_files = [ + InFile("translit/root.txt"), + InFile("translit/en.txt"), + InFile("translit/el.txt") + ] + else: + input_files = [InFile(filename) for filename in glob("%s/*.txt" % sub_dir)] + input_basenames = [v.filename[len(sub_dir)+1:] for v in input_files] + output_files = [ + OutFile("%s%s.res" % (out_prefix, v[:-4])) + for v in input_basenames + ] + if use_pool_bundle: + input_pool_files = [OutFile("%spool.res" % out_prefix)] + use_pool_bundle_option = "--usePoolBundle {OUT_DIR}/{OUT_PREFIX}".format( + OUT_PREFIX = out_prefix, + **common_vars + ) + requests += [ + SingleExecutionRequest( + name = "%s_pool_write" % sub_dir, + input_files = dep_files + input_files, + output_files = input_pool_files, + tool = IcuTool("genrb"), + args = "-s {IN_DIR}/{IN_SUB_DIR} -d {OUT_DIR}/{OUT_PREFIX} -i {OUT_DIR} " + "--writePoolBundle -k " + "{INPUT_BASENAMES_SPACED}", + format_with = { + "IN_SUB_DIR": sub_dir, + "OUT_PREFIX": out_prefix, + "INPUT_BASENAMES_SPACED": " ".join(input_basenames) + } + ), + ] + else: + input_pool_files = [] + use_pool_bundle_option = "" + requests += [ + RepeatedOrSingleExecutionRequest( + name = "%s_res" % sub_dir, + dep_files = dep_files + input_pool_files, + input_files = input_files, + output_files = output_files, + tool = IcuTool("genrb"), + args = "-s {IN_DIR}/{IN_SUB_DIR} -d {OUT_DIR}/{OUT_PREFIX} -i {OUT_DIR} " + "{EXTRA_OPTION} -k " + "{INPUT_BASENAME}", + format_with = { + "IN_SUB_DIR": sub_dir, + "OUT_PREFIX": out_prefix, + "EXTRA_OPTION": use_pool_bundle_option + }, + repeat_with = { + "INPUT_BASENAME": input_basenames, + }, + flatten_with = { + "INPUT_BASENAME": " ".join(input_basenames) + } + ) + ] + # Generate index txt file + if sub_dir != "translit": + # TODO: Change .mk files to .py files so they can be loaded directly. + # Alternatively, figure out a way to require reading this file altogether. + # Right now, it is required for the index list file. + # Reading these files as .py will be required for Bazel. + mk_values = parse_makefile("{GLOB_DIR}/{IN_SUB_DIR}/{RESFILE_NAME}".format( + IN_SUB_DIR = sub_dir, + RESFILE_NAME = resfile_name, + **common_vars + )) + cldr_version = mk_values[version_var] if version_var and sub_dir == "locales" else None + locales = [v[:-4] for v in mk_values[source_var].split()] + pkg_exclusions |= set(output_files) - set(OutFile("%s%s.res" % (out_prefix, locale)) for locale in locales) + index_file_txt = TmpFile("{IN_SUB_DIR}/{INDEX_NAME}.txt".format( + IN_SUB_DIR = sub_dir, + **common_vars + )) + requests += [ + PrintFileRequest( + name = "%s_index_txt" % sub_dir, + output_file = index_file_txt, + content = utils.generate_index_file(locales, cldr_version, common_vars) + ) + ] + # Generate index res file + index_res_file = OutFile("{OUT_PREFIX}{INDEX_NAME}.res".format( + OUT_PREFIX = out_prefix, + **common_vars + )) + requests += [ + SingleExecutionRequest( + name = "%s_index_res" % sub_dir, + input_files = [index_file_txt], + output_files = [index_res_file], + tool = IcuTool("genrb"), + args = "-s {TMP_DIR}/{IN_SUB_DIR} -d {OUT_DIR}/{OUT_PREFIX} -i {OUT_DIR} " + "-k " + "{INDEX_NAME}.txt", + format_with = { + "IN_SUB_DIR": sub_dir, + "OUT_PREFIX": out_prefix + } + ) + ] + + # Finally, make the package. + all_output_files = list(sorted(utils.get_all_output_files(requests))) + icudata_list_file = TmpFile("icudata.lst") + requests += [ + PrintFileRequest( + name = "icudata_list", + output_file = icudata_list_file, + content = "\n".join(file.filename for file in all_output_files) + ), + VariableRequest( + name = "icudata_all_output_files", + input_files = all_output_files + [icudata_list_file] + ) + ] + + return (build_dirs, requests) diff --git a/icu4c/source/data/Makefile.in b/icu4c/source/data/Makefile.in index 688bcae3f15..d96351164f9 100644 --- a/icu4c/source/data/Makefile.in +++ b/icu4c/source/data/Makefile.in @@ -112,6 +112,10 @@ COMINCDIR=$(top_srcdir)/common/unicode SRCLISTDEPS=Makefile $(srcdir)/Makefile.in BUILD_DIRS=$(OUTDIR) $(MAINBUILDDIR) $(BUILDDIR) $(CURRBLDDIR) $(LANGBLDDIR) $(REGIONBLDDIR) $(ZONEBLDDIR) $(UNITBLDDIR) $(BRKBLDDIR) $(COLBLDDIR) $(RBNFBLDDIR) $(TRANSLITBLDDIR) $(OUTTMPDIR) $(OUTTMPDIR_390STUB) $(OUTTMPDIR)/$(CURR_TREE) $(OUTTMPDIR)/$(LANG_TREE) $(OUTTMPDIR)/$(REGION_TREE) $(OUTTMPDIR)/$(ZONE_TREE) $(OUTTMPDIR)/$(UNIT_TREE) $(OUTTMPDIR)/$(COLLATION_TREE) $(OUTTMPDIR)/$(RBNF_TREE) $(OUTTMPDIR)/$(TRANSLIT_TREE) $(OUTTMPDIR)/$(BREAK_TREE) +# Variable names for rules.mk +OUT_DIR=$(BUILDDIR) +TMP_DIR=$(OUTTMPDIR) + # relative lib links from pkgdata are the same as for tmp TOOLDIR=$(top_builddir)/tools @@ -240,128 +244,9 @@ package390: $(OUTTMPDIR)/icudata390.lst $(PKGDATA_LIST) ./icupkg.inc packagedata cp $(ICUPKGDATA_OUTDIR)/$(LIB_ICUDATA_NAME)$(STUB_SUFFIX).$(SO) $(top_builddir)/stubdata/$(LIB_ICUDATA_NAME)$(STUB_SUFFIX).$(SO) -##### Define all the data files. the build rule that depends on them is below. -# X_FILES_SHORT = just the base names (for lists) -# X_FILES = full paths (for dependency) +## Include the main build rules for data files +include $(top_builddir)/$(subdir)/rules.mk -## DAT files - Misc. data files. -# 2005-may-05 Removed Unicode properties files (unorm.icu, uprops.icu, ucase.icu, ubidi.icu) -# from data build. See Jitterbug 4497. (makedata.mak revision 1.117) -# 2010-dec Removed pnames.icu. -# These are now hardcoded in ICU4C and only loaded in ICU4J. -# -DAT_FILES_SHORT=unames.icu cnvalias.icu coll/ucadata.icu nfkc.nrm nfkc_cf.nrm uts46.nrm -DAT_FILES=$(DAT_FILES_SHORT:%=$(BUILDDIR)/%) - -## BRK files -BREAK_TREE=brkitr --include $(BRKSRCDIR)/brkfiles.mk --include $(BRKSRCDIR)/brklocal.mk -ALL_BRK_SOURCE= $(BRK_SOURCE) $(BRK_SOURCE_LOCAL) -BRK_FILES_SHORT=$(ALL_BRK_SOURCE:%.txt=$(BREAK_TREE)/%.brk) -BRK_FILES=$(ALL_BRK_SOURCE:%.txt=$(BRKBLDDIR)/%.brk) -ifdef BRK_DICT_SOURCE -ALL_DICT_SOURCE=$(BRK_DICT_SOURCE) $(BRK_DICT_SOURCE_LOCAL) -DICT_FILES_SHORT=$(ALL_DICT_SOURCE:%.txt=$(BREAK_TREE)/%.dict) -DICT_FILES=$(ALL_DICT_SOURCE:%.txt=$(BRKBLDDIR)/%.dict) -endif -ifdef BRK_RES_SOURCE -BRS_SRC= root.txt $(BRK_RES_SOURCE) $(BRK_RES_SOURCE_LOCAL) -BRS_SRC_FILES = $(BRS_SRC:%=$(BRKSRCDIR)/%) -INSTALLED_BRS_FILES = $(BRK_RES_SOURCE:%.txt=%) $(BRK_RES_SOURCE_LOCAL:%.txt=%) -endif - -## Confusables (Spoofing) files -ALL_CFU_SOURCE=$(UNICODEDATADIR)/confusables.txt $(UNICODEDATADIR)/confusablesWholeScript.txt -CFU_FILES_SHORT=confusables.cfu -CFU_FILES=$(BUILDDIR)/$(CFU_FILES_SHORT) - -## UCM files --include $(UCMSRCDIR)/ucmcore.mk --include $(UCMSRCDIR)/ucmfiles.mk --include $(UCMSRCDIR)/ucmebcdic.mk --include $(UCMSRCDIR)/ucmlocal.mk -ALL_UCM_SOURCE=ibm-37_P100-1995.ucm ibm-1047_P100-1995.ucm $(UCM_SOURCE_CORE) $(UCM_SOURCE_FILES) $(UCM_SOURCE_EBCDIC) $(UCM_SOURCE_LOCAL) -UCM_FILES = $(ALL_UCM_SOURCE:%=$(SRCDATADIR)/%) -CNV_FILES = $(ALL_UCM_SOURCE:%.ucm=$(BUILDDIR)/%.cnv) -CNV_FILES_SHORT = $(ALL_UCM_SOURCE:%.ucm=%.cnv) -UCM_SOURCE_SPECIAL=$(UCM_SOURCE_EBCDIC_IGNORE_SISO) $(UCM_SOURCE_EBCDIC_IGNORE_SISO_LOCAL) -UCM_FILES_SPECIAL=$(UCM_SOURCE_SPECIAL:%=$(UCMSRCDIR)/%) -CNV_FILES_SPECIAL=$(UCM_SOURCE_SPECIAL:%.ucm=$(BUILDDIR)/%.cnv) -CNV_FILES_SHORT_SPECIAL=$(UCM_SOURCE_SPECIAL:%.ucm=%.cnv) - -## RES files --include $(LOCSRCDIR)/resfiles.mk --include $(CURRSRCDIR)/resfiles.mk --include $(LANGSRCDIR)/resfiles.mk --include $(REGIONSRCDIR)/resfiles.mk --include $(ZONESRCDIR)/resfiles.mk --include $(UNITSRCDIR)/resfiles.mk --include $(COLSRCDIR)/colfiles.mk --include $(RBNFSRCDIR)/rbnffiles.mk --include $(TRANSLITSRCDIR)/trnsfiles.mk --include $(LOCSRCDIR)/reslocal.mk --include $(CURRSRCDIR)/reslocal.mk --include $(LANGSRCDIR)/reslocal.mk --include $(REGIONSRCDIR)/reslocal.mk --include $(ZONESRCDIR)/reslocal.mk --include $(UNITSRCDIR)/reslocal.mk --include $(COLSRCDIR)/collocal.mk --include $(BRKSRCDIR)/brslocal.mk --include $(RBNFSRCDIR)/rbnflocal.mk --include $(TRANSLITSRCDIR)/trnslocal.mk -ifdef GENRB_SOURCE -RES_SRC= root.txt $(GENRB_SOURCE) $(GENRB_ALIAS_SOURCE) $(GENRB_SOURCE_LOCAL) $(GENRB_ALIAS_SOURCE_LOCAL) -RES_SRC_FILES = $(RES_SRC:%=$(LOCSRCDIR)/%) -INSTALLED_RB_FILES = $(GENRB_SOURCE:%.txt=%) $(GENRB_SOURCE_LOCAL:%.txt=%) -endif -ifdef CURR_SOURCE -CURR_SRC= root.txt supplementalData.txt $(CURR_SOURCE) $(CURR_ALIAS_SOURCE) $(CURR_SOURCE_LOCAL) -CURR_SRC_FILES = $(CURR_SRC:%=$(CURRSRCDIR)/%) -INSTALLED_CURR_FILES = $(CURR_SOURCE:%.txt=%) $(CURR_SOURCE_LOCAL:%.txt=%) -endif -ifdef LANG_SOURCE -LANG_SRC= root.txt $(LANG_SOURCE) $(LANG_ALIAS_SOURCE) $(LANG_SOURCE_LOCAL) -LANG_SRC_FILES = $(LANG_SRC:%=$(LANGSRCDIR)/%) -INSTALLED_LANG_FILES = $(LANG_SOURCE:%.txt=%) $(LANG_SOURCE_LOCAL:%.txt=%) -endif -ifdef REGION_SOURCE -REGION_SRC= root.txt $(REGION_SOURCE) $(REGION_ALIAS_SOURCE) $(REGION_SOURCE_LOCAL) -REGION_SRC_FILES = $(REGION_SRC:%=$(REGIONSRCDIR)/%) -INSTALLED_REGION_FILES = $(REGION_SOURCE:%.txt=%) $(REGION_SOURCE_LOCAL:%.txt=%) -endif -ifdef ZONE_SOURCE -ZONE_SRC= root.txt $(ZONE_SOURCE) $(ZONE_ALIAS_SOURCE) $(ZONE_SOURCE_LOCAL) tzdbNames.txt -ZONE_SRC_FILES = $(ZONE_SRC:%=$(ZONESRCDIR)/%) -INSTALLED_ZONE_FILES = $(ZONE_SOURCE:%.txt=%) $(ZONE_SOURCE_LOCAL:%.txt=%) -endif -ifdef UNIT_SOURCE -UNIT_SRC= root.txt $(UNIT_SOURCE) $(UNIT_ALIAS_SOURCE) $(UNIT_SOURCE_LOCAL) -UNIT_SRC_FILES = $(UNIT_SRC:%=$(UNITSRCDIR)/%) -INSTALLED_UNIT_FILES = $(UNIT_SOURCE:%.txt=%) $(UNIT_SOURCE_LOCAL:%.txt=%) -endif -ifdef COLLATION_SOURCE -COL_SRC= root.txt $(COLLATION_SOURCE) $(COLLATION_ALIAS_SOURCE) $(COLLATION_SOURCE_LOCAL) -COL_SRC_FILES = $(COL_SRC:%=$(COLSRCDIR)/%) -INSTALLED_COL_FILES = $(COLLATION_SOURCE:%.txt=%) $(COLLATION_SOURCE_LOCAL:%.txt=%) -endif -ifdef RBNF_SOURCE -RBNF_SRC= root.txt $(RBNF_SOURCE) $(RBNF_ALIAS_SOURCE) $(RBNF_SOURCE_LOCAL) -RBNF_SRC_FILES = $(RBNF_SRC:%=$(RBNFSRCDIR)/%) -INSTALLED_RBNF_FILES = $(RBNF_SOURCE:%.txt=%) $(RBNF_SOURCE_LOCAL:%.txt=%) -endif -ifdef TRANSLIT_SOURCE -TRANSLIT_SRC= $(TRANSLIT_SOURCE) $(TRANSLIT_ALIAS_SOURCE) $(TRANSLIT_SOURCE_LOCAL) -TRANSLIT_SRC_FILES = $(TRANSLIT_SRC:%=$(TRANSLITSRCDIR)/%) -INSTALLED_TRANSLIT_FILES = $(TRANSLIT_SOURCE:%.txt=%) $(TRANSLIT_SOURCE_LOCAL:%.txt=%) -endif -GENRBOPTS=-k - -## MISC files --include $(MISCSRCDIR)/miscfiles.mk --include $(MISCSRCDIR)/misclocal.mk -MSC_SOURCE= $(MISC_SOURCE) $(MISC_SOURCE_LOCAL) -MSC_SRC_FILES=$(MSC_SOURCE:%=$(MISCSRCDIR)/%) ifeq ($(ENABLE_SO_VERSION_DATA),1) ifeq ($(PKGDATA_MODE),dll) @@ -375,72 +260,21 @@ endif endif endif -INDEX_NAME=res_index -INDEX_FILE=$(OUTTMPDIR)/$(INDEX_NAME).txt +PKGDATA_LIST = $(TMP_DIR)/icudata.lst -ALL_RES_SRC= $(RES_SRC) $(TRNS_SOURCE) $(MSC_SOURCE) -RES_FILES = $(ALL_RES_SRC:%.txt=$(BUILDDIR)/%.res) $(BUILDDIR)/$(INDEX_NAME).res $(BUILDDIR)/pool.res -RES_FILES_SHORT = $(ALL_RES_SRC:%.txt=%.res) $(INDEX_NAME).res pool.res -PKGDATA_LIST = $(OUTTMPDIR)/icudata.lst -CURR_TREE=curr -CURR_INDEX_FILE=$(OUTTMPDIR)/$(CURR_TREE)/$(INDEX_NAME).txt -CURR_FILES = $(CURR_SRC:%.txt=$(CURRBLDDIR)/%.res) $(CURRBLDDIR)/$(INDEX_NAME).res $(CURRBLDDIR)/pool.res -CURR_FILES_SHORT = $(CURR_SRC:%.txt=$(CURR_TREE)/%.res) $(CURR_TREE)/$(INDEX_NAME).res $(CURR_TREE)/pool.res +##### Define all the data files. the build rule that depends on them is below. +# X_FILES_SHORT = just the base names (for lists) +# X_FILES = full paths (for dependency) -LANG_TREE=lang -LANG_INDEX_FILE=$(OUTTMPDIR)/$(LANG_TREE)/$(INDEX_NAME).txt -LANG_FILES = $(LANG_SRC:%.txt=$(LANGBLDDIR)/%.res) $(LANGBLDDIR)/$(INDEX_NAME).res $(LANGBLDDIR)/pool.res -LANG_FILES_SHORT = $(LANG_SRC:%.txt=$(LANG_TREE)/%.res) $(LANG_TREE)/$(INDEX_NAME).res $(LANG_TREE)/pool.res - -REGION_TREE=region -REGION_INDEX_FILE=$(OUTTMPDIR)/$(REGION_TREE)/$(INDEX_NAME).txt -REGION_FILES = $(REGION_SRC:%.txt=$(REGIONBLDDIR)/%.res) $(REGIONBLDDIR)/$(INDEX_NAME).res $(REGIONBLDDIR)/pool.res -REGION_FILES_SHORT = $(REGION_SRC:%.txt=$(REGION_TREE)/%.res) $(REGION_TREE)/$(INDEX_NAME).res $(REGION_TREE)/pool.res - -ZONE_TREE=zone -ZONE_INDEX_FILE=$(OUTTMPDIR)/$(ZONE_TREE)/$(INDEX_NAME).txt -ZONE_FILES = $(ZONE_SRC:%.txt=$(ZONEBLDDIR)/%.res) $(ZONEBLDDIR)/$(INDEX_NAME).res $(ZONEBLDDIR)/pool.res -ZONE_FILES_SHORT = $(ZONE_SRC:%.txt=$(ZONE_TREE)/%.res) $(ZONE_TREE)/$(INDEX_NAME).res $(ZONE_TREE)/pool.res - -UNIT_TREE=unit -UNIT_INDEX_FILE=$(OUTTMPDIR)/$(UNIT_TREE)/$(INDEX_NAME).txt -UNIT_FILES = $(UNIT_SRC:%.txt=$(UNITBLDDIR)/%.res) $(UNITBLDDIR)/$(INDEX_NAME).res $(UNITBLDDIR)/pool.res -UNIT_FILES_SHORT = $(UNIT_SRC:%.txt=$(UNIT_TREE)/%.res) $(UNIT_TREE)/$(INDEX_NAME).res $(UNIT_TREE)/pool.res - -COLLATION_TREE=coll -COLLATION_INDEX_FILE=$(OUTTMPDIR)/$(COLLATION_TREE)/$(INDEX_NAME).txt -COLLATION_INDEX_RES=$(COLBLDDIR)/$(INDEX_NAME).res -COLLATION_INDEX_RES_SHORT=$(COLLATION_TREE)/$(INDEX_NAME).res -COLLATION_FILES = $(COL_SRC:%.txt=$(COLBLDDIR)/%.res) $(COLLATION_INDEX_RES) -COLLATION_FILES_SHORT = $(COL_SRC:%.txt=$(COLLATION_TREE)/%.res) - -BRK_RES_INDEX_FILE=$(OUTTMPDIR)/$(BREAK_TREE)/$(INDEX_NAME).txt -BRK_RES_INDEX_RES=$(BRKBLDDIR)/$(INDEX_NAME).res -BRK_RES_INDEX_RES_SHORT=$(BREAK_TREE)/$(INDEX_NAME).res -BRK_RES_FILES = $(BRS_SRC:%.txt=$(BRKBLDDIR)/%.res) $(BRK_RES_INDEX_RES) -BRK_RES_FILES_SHORT = $(BRS_SRC:%.txt=$(BREAK_TREE)/%.res) - -RBNF_TREE=rbnf -RBNF_INDEX_FILE=$(OUTTMPDIR)/$(RBNF_TREE)/$(INDEX_NAME).txt -RBNF_INDEX_RES=$(RBNFBLDDIR)/$(INDEX_NAME).res -RBNF_INDEX_RES_SHORT=$(RBNF_TREE)/$(INDEX_NAME).res -RBNF_FILES = $(RBNF_SRC:%.txt=$(RBNFBLDDIR)/%.res) $(RBNF_INDEX_RES) -RBNF_FILES_SHORT = $(RBNF_SRC:%.txt=$(RBNF_TREE)/%.res) - -TRANSLIT_TREE=translit -#TRANSLIT_INDEX_FILE=$(OUTTMPDIR)/$(TRANSLIT_TREE)/$(INDEX_NAME).txt -#TRANSLIT_INDEX_RES=$(TRANSLITBLDDIR)/$(INDEX_NAME).res -#TRANSLIT_INDEX_RES_SHORT=$(TRANSLIT_TREE)/$(INDEX_NAME).res -TRANSLIT_FILES = $(TRANSLIT_SRC:%.txt=$(TRANSLITBLDDIR)/%.res) -TRANSLIT_FILES_SHORT = $(TRANSLIT_SRC:%.txt=$(TRANSLIT_TREE)/%.res) - -## SPP files --include $(SPREPSRCDIR)/sprepfiles.mk --include $(SPREPSRCDIR)/spreplocal.mk -ALL_SPREP_SOURCE=$(SPREP_SOURCE) $(SPREP_SOURCE_LOCAL) -SPREP_FILES = $(ALL_SPREP_SOURCE:%.txt=$(BUILDDIR)/%.spp) -SPREP_FILES_SHORT = $(ALL_SPREP_SOURCE:%.txt=%.spp) +## DAT files - Misc. data files. +# 2005-may-05 Removed Unicode properties files (unorm.icu, uprops.icu, ucase.icu, ubidi.icu) +# from data build. See Jitterbug 4497. (makedata.mak revision 1.117) +# 2010-dec Removed pnames.icu. +# These are now hardcoded in ICU4C and only loaded in ICU4J. +# +DAT_FILES_SHORT=unames.icu cnvalias.icu coll/ucadata.icu nfkc.nrm nfkc_cf.nrm uts46.nrm +DAT_FILES=$(DAT_FILES_SHORT:%=$(BUILDDIR)/%) ## All generated files ALL_FILES = $(DAT_FILES) $(CNV_FILES) $(CNV_FILES_SPECIAL) $(BRK_FILES) $(DICT_FILES) $(RES_FILES) $(INDEX_RES_FILE) $(CURR_FILES) $(LANG_FILES) $(REGION_FILES) $(ZONE_FILES) $(UNIT_FILES) $(COLLATION_FILES) $(BRK_RES_FILES) $(RBNF_FILES) $(TRANSLIT_FILES) $(SPREP_FILES) $(CFU_FILES) @@ -466,23 +300,8 @@ endif CLEANFILES = *~ icupkg.inc *.x ifeq ($(ICUDATA_SOURCE_ARCHIVE),) -build-local: build-dir $(SO_VERSION_DATA) $(ALL_FILES) $(PKGDATA_LIST) $(OS390LIST) +build-local: build-dir $(SO_VERSION_DATA) $(ICUDATA_ALL_OUTPUT_FILES) $(PKGDATA_LIST) $(OS390LIST) echo timestamp > $@ -$(PKGDATA_LIST): $(SRCLISTDEPS) - @echo "generating $@ (list of data files)" - @-$(RMV) $@ - @for file in $(COLL_FILES_LIST); do \ - echo $$file >> $@; \ - done; - @for file in $(BRK_FILES_LIST); do \ - echo $$file >> $@; \ - done; - @for file in $(LOCALE_FILES_LIST); do \ - echo $$file >> $@; \ - done; - @for file in $(MISC_FILES_LIST); do \ - echo $$file >> $@; \ - done; else build-local: build-dir $(SO_VERSION_DATA) $(PKGDATA_LIST) $(OS390LIST) echo timestamp > $@ @@ -517,279 +336,12 @@ ifneq ($(filter order-only,$(.FEATURES)),) $(ALL_FILES) $(ALL_INDEX_SRC_FILES): | build-dir endif -# Now, sections for building each kind of data. - -#################################################### DAT -# DAT FILES - -# cnvalias.icu -$(BUILDDIR)/cnvalias.icu: $(UCMSRCDIR)/convrtrs.txt $(TOOLBINDIR)/gencnval$(TOOLEXEEXT) - $(INVOKE) $(TOOLBINDIR)/gencnval -d $(BUILDDIR) $(UCMSRCDIR)/convrtrs.txt - -# Targets for prebuilt Unicode data -$(BUILDDIR)/%.icu: $(SRCDATADIR)/in/%.icu - $(INVOKE) $(TOOLBINDIR)/icupkg -t$(ICUDATA_CHAR) $< $@ - -$(BUILDDIR)/%.nrm: $(SRCDATADIR)/in/%.nrm - $(INVOKE) $(TOOLBINDIR)/icupkg -t$(ICUDATA_CHAR) $< $@ - -$(BUILDDIR)/coll/ucadata.icu: $(SRCDATADIR)/in/coll/ucadata-unihan.icu - $(INVOKE) $(TOOLBINDIR)/icupkg -t$(ICUDATA_CHAR) $< $@ - -#################################################### SPP -# SPP FILES - -$(BUILDDIR)/%.spp: $(SPREPSRCDIR)/%.txt $(TOOLBINDIR)/gensprep$(TOOLEXEEXT) $(BUILDDIR)/unames.icu - $(INVOKE) $(TOOLBINDIR)/gensprep -d $(BUILDDIR) -i $(BUILDDIR) -s $(SPREPSRCDIR) -b $(@F:%.spp=%) -m $(UNICODEDATADIR) -u 3.2.0 $( $@; \ - echo "$(INDEX_NAME):table(nofallback) {" >> $@; \ - echo " InstalledLocales {" >> $@; \ - for file in $(INSTALLED_CURR_FILES); do \ - echo " $$file {\"\"}" >> $@; \ - done; \ - echo " }" >> $@; \ - echo "}" >> $@; - -### lang res -$(LANGBLDDIR)/%.res: $(LANGSRCDIR)/%.txt $(TOOLBINDIR)/genrb$(TOOLEXEEXT) $(DAT_FILES) | $(BUILDDIR)/cnvalias.icu - $(INVOKE) $(TOOLBINDIR)/genrb --usePoolBundle $(GENRBOPTS) -i $(BUILDDIR) -s $(LANGSRCDIR) -d $(LANGBLDDIR) $( $@; \ - echo "$(INDEX_NAME):table(nofallback) {" >> $@; \ - echo " InstalledLocales {" >> $@; \ - for file in $(INSTALLED_LANG_FILES); do \ - echo " $$file {\"\"}" >> $@; \ - done; \ - echo " }" >> $@; \ - echo "}" >> $@; - -### region res -$(REGIONBLDDIR)/%.res: $(REGIONSRCDIR)/%.txt $(TOOLBINDIR)/genrb$(TOOLEXEEXT) $(DAT_FILES) | $(BUILDDIR)/cnvalias.icu - $(INVOKE) $(TOOLBINDIR)/genrb --usePoolBundle $(GENRBOPTS) -i $(BUILDDIR) -s $(REGIONSRCDIR) -d $(REGIONBLDDIR) $( $@; \ - echo "$(INDEX_NAME):table(nofallback) {" >> $@; \ - echo " InstalledLocales {" >> $@; \ - for file in $(INSTALLED_REGION_FILES); do \ - echo " $$file {\"\"}" >> $@; \ - done; \ - echo " }" >> $@; \ - echo "}" >> $@; - -### zone res -$(ZONEBLDDIR)/%.res: $(ZONESRCDIR)/%.txt $(TOOLBINDIR)/genrb$(TOOLEXEEXT) $(DAT_FILES) | $(BUILDDIR)/cnvalias.icu - $(INVOKE) $(TOOLBINDIR)/genrb --usePoolBundle $(GENRBOPTS) -i $(BUILDDIR) -s $(ZONESRCDIR) -d $(ZONEBLDDIR) $( $@; \ - echo "$(INDEX_NAME):table(nofallback) {" >> $@; \ - echo " InstalledLocales {" >> $@; \ - for file in $(INSTALLED_ZONE_FILES); do \ - echo " $$file {\"\"}" >> $@; \ - done; \ - echo " }" >> $@; \ - echo "}" >> $@; - -### unit res -$(UNITBLDDIR)/%.res: $(UNITSRCDIR)/%.txt $(TOOLBINDIR)/genrb$(TOOLEXEEXT) $(DAT_FILES) - $(INVOKE) $(TOOLBINDIR)/genrb --usePoolBundle $(GENRBOPTS) -i $(BUILDDIR) -s $(UNITSRCDIR) -d $(UNITBLDDIR) $( $@; \ - echo "$(INDEX_NAME):table(nofallback) {" >> $@; \ - echo " InstalledLocales {" >> $@; \ - for file in $(INSTALLED_UNIT_FILES); do \ - echo " $$file {\"\"}" >> $@; \ - done; \ - echo " }" >> $@; \ - echo "}" >> $@; - -### collation res -$(COLBLDDIR)/%.res: $(COLSRCDIR)/%.txt $(TOOLBINDIR)/genrb$(TOOLEXEEXT) $(DAT_FILES) | $(BUILDDIR)/cnvalias.icu - $(INVOKE) $(TOOLBINDIR)/genrb $(GENRBOPTS) -i $(BUILDDIR) -s $(COLSRCDIR) -d $(COLBLDDIR) $( $@; \ - echo "$(INDEX_NAME):table(nofallback) {" >> $@; \ - echo " InstalledLocales {" >> $@; \ - for file in $(INSTALLED_COL_FILES); do \ - echo " $$file {\"\"}" >> $@; \ - done; \ - echo " }" >> $@; \ - echo "}" >> $@; - -### brk res -$(BRKBLDDIR)/%.res: $(BRKSRCDIR)/%.txt $(TOOLBINDIR)/genrb$(TOOLEXEEXT) $(BRK_FILES) $(DICT_FILES) $(DAT_FILES) | $(BUILDDIR)/cnvalias.icu - $(INVOKE) $(TOOLBINDIR)/genrb $(GENRBOPTS) -i $(BUILDDIR) -s $(BRKSRCDIR) -d $(BRKBLDDIR) $( $@; \ - echo "$(INDEX_NAME):table(nofallback) {" >> $@; \ - echo " InstalledLocales {" >> $@; \ - for file in $(INSTALLED_BRS_FILES); do \ - echo " $$file {\"\"}" >> $@; \ - done; \ - echo " }" >> $@; \ - echo "}" >> $@; - -### RBNF res -$(RBNFBLDDIR)/%.res: $(RBNFSRCDIR)/%.txt $(TOOLBINDIR)/genrb$(TOOLEXEEXT) $(DAT_FILES) | $(BUILDDIR)/cnvalias.icu - $(INVOKE) $(TOOLBINDIR)/genrb $(GENRBOPTS) -i $(BUILDDIR) -s $(RBNFSRCDIR) -d $(RBNFBLDDIR) $( $@; \ - echo "$(INDEX_NAME):table(nofallback) {" >> $@; \ - echo " InstalledLocales {" >> $@; \ - for file in $(INSTALLED_RBNF_FILES); do \ - echo " $$file {\"\"}" >> $@; \ - done; \ - echo " }" >> $@; \ - echo "}" >> $@; - -### TRANSLIT res -$(TRANSLITBLDDIR)/%.res: $(TRANSLITSRCDIR)/%.txt $(TOOLBINDIR)/genrb$(TOOLEXEEXT) $(DAT_FILES) | $(BUILDDIR)/cnvalias.icu - $(INVOKE) $(TOOLBINDIR)/genrb $(GENRBOPTS) -i $(BUILDDIR) -s $(TRANSLITSRCDIR) -d $(TRANSLITBLDDIR) $( $@; \ - echo "$(INDEX_NAME):table(nofallback) {" >> $@; \ - echo " CLDRVersion { \"$(GENRB_CLDR_VERSION)\" }" >> $@; \ - echo " InstalledLocales {" >> $@; \ - for file in $(INSTALLED_RB_FILES); do \ - echo " $$file {\"\"}" >> $@; \ - done; \ - echo " }" >> $@; \ - echo "}" >> $@; - -clean-resindex: - -$(RMV) $(BUILDDIR)/$(INDEX_NAME).txt $(PKGDATA_LIST) - -$(BUILDDIR)/$(INDEX_NAME).res: $(INDEX_FILE) $(TOOLBINDIR)/genrb$(TOOLEXEEXT) | $(BUILDDIR)/cnvalias.icu - $(INVOKE) $(TOOLBINDIR)/genrb $(GENRBOPTS) -i $(BUILDDIR) -d $(BUILDDIR) $(INDEX_FILE) - # The core Unicode properties files (pnames.icu, uprops.icu, ucase.icu, ubidi.icu) # are hardcoded in the common DLL and therefore not included in the data package any more. # They are not built by default but need to be built for ICU4J data and for getting the .c source files @@ -857,6 +384,13 @@ ICU4J_TZDATA_FILES=zoneinfo64 metaZones timezoneTypes windowsZones ICU4J_DATA_DIRNAME=com/ibm/icu/impl/data/$(ICUDATA_BASENAME_VERSION)b ICU4J_TZDATA_PATHS=$(ICU4J_TZDATA_FILES:%="$(ICU4J_DATA_DIRNAME)/%.res") +# Targets for prebuilt Unicode data +$(BUILDDIR)/%.icu: $(SRCDATADIR)/in/%.icu | $(DIRS) + $(INVOKE) $(TOOLBINDIR)/icupkg -t$(ICUDATA_CHAR) $< $@ + +$(BUILDDIR)/nfc.nrm: $(SRCDATADIR)/in/nfc.nrm | $(DIRS) + $(INVOKE) $(TOOLBINDIR)/icupkg -t$(ICUDATA_CHAR) $< $@ + # generate icu4j-related data to $(OUTDIR)/icu4j/com/ibm/icu/impl/data/... generate-data: build-dir packagedata $(OUTTMPDIR)/$(ICUDATA_PLATFORM_NAME).dat uni-core-data mkdir -p $(OUTDIR)/icu4j/$(ICU4J_DATA_DIRNAME) diff --git a/icu4c/source/data/buildtool/__init__.py b/icu4c/source/data/buildtool/__init__.py new file mode 100644 index 00000000000..60855c07a08 --- /dev/null +++ b/icu4c/source/data/buildtool/__init__.py @@ -0,0 +1,82 @@ +# Copyright (C) 2018 and later: Unicode, Inc. and others. +# License & terms of use: http://www.unicode.org/copyright.html + +from collections import namedtuple + +AVAILABLE_FEATURES = [ + "confusables", + "cnvalias", + "uconv", + "brkitr", + "stringprep", + "dictionaries", + "normalization", + "coll", + "unames", + "misc", + "locales", + "curr", + "lang", + "region", + "zone", + "unit", + "rbnf", + "translit" +] + +InFile = namedtuple("InFile", ["filename"]) +TmpFile = namedtuple("TmpFile", ["filename"]) +OutFile = namedtuple("OutFile", ["filename"]) +PkgFile = namedtuple("PkgFile", ["filename"]) + +IcuTool = namedtuple("IcuTool", ["name"]) +SystemTool = namedtuple("SystemTool", ["name"]) + +SingleExecutionRequest = namedtuple("SingleExecutionRequest", [ + "name", + "input_files", + "output_files", + "tool", + "args", + "format_with" +]) + +RepeatedExecutionRequest = namedtuple("RepeatedExecutionRequest", [ + "name", + "dep_files", + "input_files", + "output_files", + "tool", + "args", + "format_with", + "repeat_with" +]) + +RepeatedOrSingleExecutionRequest = namedtuple("RepeatedOrSingleExecutionRequest", [ + "name", + "dep_files", + "input_files", + "output_files", + "tool", + "args", + "format_with", + "repeat_with", + "flatten_with" +]) + +PrintFileRequest = namedtuple("PrintFileRequest", [ + "name", + "output_file", + "content" +]) + +CopyRequest = namedtuple("CopyRequest", [ + "name", + "input_file", + "output_file" +]) + +VariableRequest = namedtuple("VariableRequest", [ + "name", + "input_files" +]) diff --git a/icu4c/source/data/buildtool/__main__.py b/icu4c/source/data/buildtool/__main__.py new file mode 100644 index 00000000000..95b7a0b3331 --- /dev/null +++ b/icu4c/source/data/buildtool/__main__.py @@ -0,0 +1,176 @@ +# Copyright (C) 2018 and later: Unicode, Inc. and others. +# License & terms of use: http://www.unicode.org/copyright.html + +import argparse +import glob as pyglob +import sys + +from . import * +from .renderers import makefile, windirect +from . import utils +import BUILDRULES + +flag_parser = argparse.ArgumentParser( + description = """Generates rules for building ICU binary data files from text +and other input files in source control. + +You can select features using either the --whitelist or --blacklist option. +Available features include: + +{AVAILABLE_FEATURES} +""".format(AVAILABLE_FEATURES = "\n".join(" %s" % v for v in AVAILABLE_FEATURES)), + formatter_class = argparse.RawDescriptionHelpFormatter +) +flag_parser.add_argument( + "--format", + help = "How to output the rules to run to build ICU data.", + choices = ["gnumake", "windirect"], + required = True +) +flag_parser.add_argument( + "--glob_dir", + help = "Path to data input folder (icu4c/source/data) when this script is being run.", + default = "." +) +flag_parser.add_argument( + "--in_dir", + help = "Path to data input folder (icu4c/source/data) for file processing. Not used in gnumake format.", + default = "." +) +flag_parser.add_argument( + "--out_dir", + help = "Path to where to save output data files. Not used in gnumake format.", + default = "icudata" +) +flag_parser.add_argument( + "--tmp_dir", + help = "Path to where to save temporary files. Not used in gnumake format.", + default = "icutmp" +) +flag_parser.add_argument( + "--tool_dir", + help = "Path to where to find binary tools (genrb, genbrk, etc). Used for 'windirect' format only.", + default = "../tool" +) +flag_parser.add_argument( + "--tool_cfg", + help = "The build configuration of the tools. Used for 'windirect' format only.", + default = "x86/Debug" +) +flag_parser.add_argument( + "--seqmode", + help = "Whether to optimize rules to be run sequentially (fewer threads) or in parallel (many threads).", + choices = ["sequential", "parallel"], + default = "parallel" +) +flag_parser.add_argument( + "--collation_ucadata", + help = "Which data set to use for ucadata in collation.", + choices = ["unihan", "implicithan"], + default = "unihan" +) +features_group = flag_parser.add_mutually_exclusive_group() +features_group.add_argument( + "--blacklist", + metavar = "FEATURE", + help = "A list of one or more features to disable; all others will be enabled by default. New users should favor a blacklist to ensure important data is not left out.", + nargs = "+", + choices = AVAILABLE_FEATURES +) +features_group.add_argument( + "--whitelist", + metavar = "FEATURE", + help = "A list of one or more features to enable; all others will be disabled by default.", + nargs = "+", + choices = AVAILABLE_FEATURES +) + + +class Config(object): + + def __init__(self, args): + if args.whitelist: + self._feature_set = set(args.whitelist) + elif args.blacklist: + self._feature_set = set(AVAILABLE_FEATURES) - set(args.blacklist) + else: + self._feature_set = set(AVAILABLE_FEATURES) + self._max_parallel = (args.seqmode == "parallel") + self._coll_han_type = args.collation_ucadata + + def has_feature(self, feature_name): + assert feature_name in AVAILABLE_FEATURES + return feature_name in self._feature_set + + def max_parallel(self): + return self._max_parallel + + def coll_han_type(self): + # Either "unihan" or "implicithan" + return self._coll_han_type + + +def main(): + args = flag_parser.parse_args() + config = Config(args) + + if args.format == "gnumake": + makefile_vars = { + "IN_DIR": "$(srcdir)", + "INDEX_NAME": "res_index" + } + makefile_env = ["ICUDATA_CHAR", "OUT_DIR", "TMP_DIR"] + common = { + key: "$(%s)" % key + for key in list(makefile_vars.keys()) + makefile_env + } + common["GLOB_DIR"] = args.glob_dir + else: + common = { + "GLOB_DIR": args.glob_dir, + "IN_DIR": args.in_dir, + "OUT_DIR": args.out_dir, + "TMP_DIR": args.tmp_dir, + "INDEX_NAME": "res_index", + # TODO: Pull this from configure script: + "ICUDATA_CHAR": "l" + } + + def glob(pattern): + result_paths = pyglob.glob("{IN_DIR}/{PATTERN}".format( + IN_DIR = args.glob_dir, + PATTERN = pattern + )) + # For the purposes of buildtool, force Unix-style directory separators. + return [v.replace("\\", "/")[len(args.glob_dir)+1:] for v in sorted(result_paths)] + + build_dirs, raw_requests = BUILDRULES.generate(config, glob, common) + requests = [] + for req in raw_requests: + if isinstance(req, RepeatedOrSingleExecutionRequest): + requests.append(utils.flatten(req, config.max_parallel())) + else: + requests.append(req) + + if args.format == "gnumake": + print(makefile.get_gnumake_rules( + build_dirs, + requests, + makefile_vars, + common_vars = common + )) + elif args.format == "windirect": + return windirect.run( + build_dirs, + requests, + common_vars = common, + tool_dir = args.tool_dir, + tool_cfg = args.tool_cfg + ) + else: + print("Format not supported: %s" % args.format) + return 1 + return 0 + +if __name__ == "__main__": + exit(main()) diff --git a/icu4c/source/data/buildtool/renderers/__init__.py b/icu4c/source/data/buildtool/renderers/__init__.py new file mode 100644 index 00000000000..7c402c2b78b --- /dev/null +++ b/icu4c/source/data/buildtool/renderers/__init__.py @@ -0,0 +1,10 @@ +# Copyright (C) 2018 and later: Unicode, Inc. and others. +# License & terms of use: http://www.unicode.org/copyright.html + +from collections import namedtuple + +MakeRule = namedtuple("MakeRule", ["name", "dep_literals", "dep_files", "output_file", "cmds"]) + +MakeFilesVar = namedtuple("MakeFilesVar", ["name", "files"]) + +MakeStringVar = namedtuple("MakeStringVar", ["name", "content"]) diff --git a/icu4c/source/data/buildtool/renderers/makefile.py b/icu4c/source/data/buildtool/renderers/makefile.py new file mode 100644 index 00000000000..6d028b58a2b --- /dev/null +++ b/icu4c/source/data/buildtool/renderers/makefile.py @@ -0,0 +1,239 @@ +# Copyright (C) 2018 and later: Unicode, Inc. and others. +# License & terms of use: http://www.unicode.org/copyright.html + +from . import * +from .. import * +from .. import utils + +def get_gnumake_rules(build_dirs, requests, makefile_vars, **kwargs): + makefile_string = "" + + # Common Variables + common_vars = kwargs["common_vars"] + for key, value in makefile_vars.items(): + makefile_string += "{KEY} = {VALUE}\n".format( + KEY = key, + VALUE = value + ) + makefile_string += "\n" + + # Directories + dirs_timestamp_file = "{TMP_DIR}/dirs.timestamp".format(**common_vars) + makefile_string += "DIRS = {TIMESTAMP_FILE}\n\n".format( + TIMESTAMP_FILE = dirs_timestamp_file + ) + makefile_string += "{TIMESTAMP_FILE}:\n\t$(MKINSTALLDIRS) {ALL_DIRS}\n\techo timestamp > {TIMESTAMP_FILE}\n\n".format( + TIMESTAMP_FILE = dirs_timestamp_file, + ALL_DIRS = " ".join(build_dirs).format(**common_vars) + ) + + # Generate Rules + make_rules = [] + for request in requests: + make_rules += get_gnumake_rules_helper(request, **kwargs) + + # Main Commands + for rule in make_rules: + if isinstance(rule, MakeFilesVar): + makefile_string += "{NAME} = {FILE_LIST}\n\n".format( + NAME = rule.name, + FILE_LIST = files_to_makefile(rule.files, wrap = True, **kwargs), + ) + continue + + if isinstance(rule, MakeStringVar): + makefile_string += "define {NAME}\n{CONTENT}\nendef\nexport {NAME}\n\n".format( + NAME = rule.name, + CONTENT = rule.content + ) + continue + + assert isinstance(rule, MakeRule) + header_line = "{OUT_FILE}: {DEP_FILES} {DEP_LITERALS} | $(DIRS)".format( + OUT_FILE = files_to_makefile([rule.output_file], **kwargs), + DEP_FILES = files_to_makefile(rule.dep_files, wrap = True, **kwargs), + DEP_LITERALS = " ".join(rule.dep_literals) + ) + + if len(rule.cmds) == 0: + makefile_string += "%s\n\n" % header_line + continue + + makefile_string += "{HEADER_LINE}\n{RULE_LINES}\n\n".format( + HEADER_LINE = header_line, + RULE_LINES = "\n".join("\t%s" % cmd for cmd in rule.cmds) + ) + + return makefile_string + +def files_to_makefile(files, common_vars, wrap = False, **kwargs): + if len(files) == 0: + return "" + dirnames = [utils.dir_for(file).format(**common_vars) for file in files] + join_str = " \\\n\t\t" if wrap and len(files) > 2 else " " + if len(files) == 1: + return "%s/%s" % (dirnames[0], files[0].filename) + elif len(set(dirnames)) == 1: + return "$(addprefix %s/,%s)" % (dirnames[0], join_str.join(file.filename for file in files)) + else: + return join_str.join("%s/%s" % (d, f.filename) for d,f in zip(dirnames, files)) + +def get_gnumake_rules_helper(request, common_vars, **kwargs): + + if isinstance(request, PrintFileRequest): + var_name = "%s_CONTENT" % request.name.upper() + return [ + MakeStringVar( + name = var_name, + content = request.content + ), + MakeRule( + name = request.name, + dep_literals = [], + dep_files = [], + output_file = request.output_file, + cmds = [ + "echo \"$${VAR_NAME}\" > {MAKEFILENAME}".format( + VAR_NAME = var_name, + MAKEFILENAME = files_to_makefile([request.output_file], common_vars), + **common_vars + ) + ] + ) + ] + + + if isinstance(request, CopyRequest): + return [ + MakeRule( + name = request.name, + dep_literals = [], + dep_files = [request.input_file], + output_file = request.output_file, + cmds = ["cp %s %s" % ( + files_to_makefile([request.input_file], common_vars), + files_to_makefile([request.output_file], common_vars)) + ] + ) + ] + + if isinstance(request, VariableRequest): + return [ + MakeFilesVar( + name = request.name.upper(), + files = request.input_files + ) + ] + + if request.tool.name == "make": + cmd_template = "$(MAKE) {ARGS}" + elif request.tool.name == "gentest": + cmd_template = "$(INVOKE) $(GENTEST) {ARGS}" + else: + assert isinstance(request.tool, IcuTool) + cmd_template = "$(INVOKE) $(TOOLBINDIR)/{TOOL} {{ARGS}}".format( + TOOL = request.tool.name + ) + + if isinstance(request, SingleExecutionRequest): + cmd = utils.format_single_request_command(request, cmd_template, common_vars) + + if len(request.output_files) > 1: + # Special case for multiple output files: Makefile rules should have only one + # output file apiece. More information: + # https://www.gnu.org/software/automake/manual/html_node/Multiple-Outputs.html + timestamp_var_name = "%s_ALL" % request.name.upper() + timestamp_file = TmpFile("%s.timestamp" % request.name) + rules = [ + MakeFilesVar( + name = timestamp_var_name, + files = [timestamp_file] + ), + MakeRule( + name = "%s_all" % request.name, + dep_literals = [], + dep_files = request.input_files, + output_file = timestamp_file, + cmds = [ + cmd, + "echo timestamp > {MAKEFILENAME}".format( + MAKEFILENAME = files_to_makefile([timestamp_file], common_vars) + ) + ] + ) + ] + for i, file in enumerate(request.output_files): + rules += [ + MakeRule( + name = "%s_%d" % (request.name, i), + dep_literals = ["$(%s)" % timestamp_var_name], + dep_files = [], + output_file = file, + cmds = [] + ) + ] + return rules + + elif len(request.input_files) > 5: + # For nicer printing, for long input lists, use a helper variable. + dep_var_name = "%s_DEPS" % request.name.upper() + return [ + MakeFilesVar( + name = dep_var_name, + files = request.input_files + ), + MakeRule( + name = request.name, + dep_literals = ["$(%s)" % dep_var_name], + dep_files = [], + output_file = request.output_files[0], + cmds = [cmd] + ) + ] + + else: + return [ + MakeRule( + name = request.name, + dep_literals = [], + dep_files = request.input_files, + output_file = request.output_files[0], + cmds = [cmd] + ) + ] + + if isinstance(request, RepeatedExecutionRequest): + rules = [] + dep_literals = [] + # To keep from repeating the same dep files many times, make a variable. + if len(request.dep_files) > 0: + dep_var_name = "%s_DEPS" % request.name.upper() + dep_literals = ["$(%s)" % dep_var_name] + rules += [ + MakeFilesVar( + name = dep_var_name, + files = request.dep_files + ) + ] + # Add a rule for each individual file. + for loop_vars in utils.repeated_execution_request_looper(request): + (_, input_file, output_file) = loop_vars + name_suffix = input_file[input_file.filename.rfind("/")+1:input_file.filename.rfind(".")] + cmd = utils.format_repeated_request_command( + request, + cmd_template, + loop_vars, + common_vars + ) + rules += [ + MakeRule( + name = "%s_%s" % (request.name, name_suffix), + dep_literals = dep_literals, + dep_files = [input_file], + output_file = output_file, + cmds = [cmd] + ) + ] + return rules + + assert False diff --git a/icu4c/source/data/buildtool/renderers/windirect.py b/icu4c/source/data/buildtool/renderers/windirect.py new file mode 100644 index 00000000000..b85243ef202 --- /dev/null +++ b/icu4c/source/data/buildtool/renderers/windirect.py @@ -0,0 +1,78 @@ +# Copyright (C) 2018 and later: Unicode, Inc. and others. +# License & terms of use: http://www.unicode.org/copyright.html + +from . import * +from .. import * +from .. import utils + +import os +import shutil +import subprocess + +def run(build_dirs, requests, common_vars, **kwargs): + for bd in build_dirs: + os.makedirs(bd.format(**common_vars), exist_ok=True) + for request in requests: + status = run_helper(request, common_vars, **kwargs) + if status != 0: + print("!!! ERROR executing above command line: exit code %d" % status) + return 1 + print("windirect: All data build commands executed") + return 0 + +def run_helper(request, common_vars, tool_dir, tool_cfg, **kwargs): + if isinstance(request, PrintFileRequest): + output_path = "{DIRNAME}/{FILENAME}".format( + DIRNAME = utils.dir_for(request.output_file).format(**common_vars), + FILENAME = request.output_file.filename, + ) + print("Printing to file: %s" % output_path) + with open(output_path, "w") as f: + f.write(request.content) + return 0 + if isinstance(request, CopyRequest): + input_path = "{DIRNAME}/{FILENAME}".format( + DIRNAME = utils.dir_for(request.input_file).format(**common_vars), + FILENAME = request.input_file.filename, + ) + output_path = "{DIRNAME}/{FILENAME}".format( + DIRNAME = utils.dir_for(request.output_file).format(**common_vars), + FILENAME = request.output_file.filename, + ) + print("Copying file to: %s" % output_path) + shutil.copyfile(input_path, output_path) + return 0 + if isinstance(request, VariableRequest): + # No-op + return 0 + + assert isinstance(request.tool, IcuTool) + cmd_template = "{TOOL_DIR}/{TOOL}/{TOOL_CFG}/{TOOL}.exe {{ARGS}}".format( + TOOL_DIR = tool_dir, + TOOL_CFG = tool_cfg, + TOOL = request.tool.name, + **common_vars + ) + if isinstance(request, RepeatedExecutionRequest): + for loop_vars in utils.repeated_execution_request_looper(request): + command_line = utils.format_repeated_request_command( + request, + cmd_template, + loop_vars, + common_vars + ) + # TODO: Is this / to \ substitution too aggressive? + command_line = command_line.replace("/", "\\") + print("Running: %s" % command_line) + res = subprocess.run(command_line) + if res.returncode != 0: + return res.returncode + return 0 + if isinstance(request, SingleExecutionRequest): + command_line = utils.format_single_request_command(request, cmd_template, common_vars) + # TODO: Is this / to \ substitution too aggressive? + command_line = command_line.replace("/", "\\") + print("Running: %s" % command_line) + res = subprocess.run(command_line) + return res.returncode + assert False diff --git a/icu4c/source/data/buildtool/utils.py b/icu4c/source/data/buildtool/utils.py new file mode 100644 index 00000000000..0ca3b811604 --- /dev/null +++ b/icu4c/source/data/buildtool/utils.py @@ -0,0 +1,130 @@ +# Copyright (C) 2018 and later: Unicode, Inc. and others. +# License & terms of use: http://www.unicode.org/copyright.html + +from . import * + + +def dir_for(file): + if isinstance(file, InFile): + return "{IN_DIR}" + if isinstance(file, TmpFile): + return "{TMP_DIR}" + if isinstance(file, OutFile): + return "{OUT_DIR}" + if isinstance(file, PkgFile): + return "{PKG_DIR}" + assert False + + +def concat_dicts(*dicts): + # There is not a super great way to do this in Python: + new_dict = {} + for dict in dicts: + new_dict.update(dict) + return new_dict + + +def repeated_execution_request_looper(request): + # dictionary of lists to list of dictionaries: + ld = [ + dict(zip(request.repeat_with, t)) + for t in zip(*request.repeat_with.values()) + ] + if not ld: + # No special options given in repeat_with + ld = [{} for _ in range(len(request.input_files))] + return zip(ld, request.input_files, request.output_files) + + +def format_single_request_command(request, cmd_template, common_vars): + return cmd_template.format( + ARGS = request.args.format( + INPUT_FILES = [file.filename for file in request.input_files], + OUTPUT_FILES = [file.filename for file in request.output_files], + **concat_dicts(common_vars, request.format_with) + ) + ) + + +def format_repeated_request_command(request, cmd_template, loop_vars, common_vars): + (iter_vars, input_file, output_file) = loop_vars + return cmd_template.format( + ARGS = request.args.format( + INPUT_FILE = input_file.filename, + OUTPUT_FILE = output_file.filename, + **concat_dicts(common_vars, request.format_with, iter_vars) + ) + ) + + +def flatten(request, max_parallel): + """Flatten a RepeatedOrSingleExecutionRequest + + Becomes either a SingleExecutionRequest or a RepeatedExecutionRequest. + """ + if max_parallel: + return RepeatedExecutionRequest( + name = request.name, + dep_files = request.dep_files, + input_files = request.input_files, + output_files = request.output_files, + tool = request.tool, + args = request.args, + format_with = request.format_with, + repeat_with = request.repeat_with + ) + else: + return SingleExecutionRequest( + name = request.name, + input_files = request.dep_files + request.input_files, + output_files = request.output_files, + tool = request.tool, + args = request.args, + format_with = concat_dicts(request.format_with, request.flatten_with) + ) + + +def generate_index_file(locales, cldr_version, common_vars): + formatted_version = " CLDRVersion { \"%s\" }\n" % cldr_version if cldr_version else "" + formatted_locales = "\n".join([" %s {\"\"}" % v for v in locales]) + # TODO: CLDRVersion is required only in the base file + return ("// Warning this file is automatically generated\n" + "{INDEX_NAME}:table(nofallback) {{\n" + "{FORMATTED_VERSION}" + " InstalledLocales {{\n" + "{FORMATTED_LOCALES}\n" + " }}\n" + "}}").format( + FORMATTED_VERSION = formatted_version, + FORMATTED_LOCALES = formatted_locales, + **common_vars + ) + + +def get_all_output_files(requests, include_tmp=False): + files = [] + for request in requests: + if isinstance(request, SingleExecutionRequest): + files += request.output_files + elif isinstance(request, RepeatedExecutionRequest): + files += request.output_files + elif isinstance(request, RepeatedOrSingleExecutionRequest): + files += request.output_files + elif isinstance(request, PrintFileRequest): + files += [request.output_file] + elif isinstance(request, CopyRequest): + files += [request.output_file] + elif isinstance(request, VariableRequest): + pass + else: + assert False + + # Filter out all files but those in OUT_DIR if necessary. + # It is also easy to filter for uniqueness; do it right now and return. + if not include_tmp: + files = (file for file in files if isinstance(file, OutFile)) + return list(set(files)) + + # Filter for unique values. NOTE: Cannot use set() because we need to accept same filename as + # OutFile and TmpFile as different, and by default they evaluate as equal. + return [f for _, f in set((type(f), f) for f in files)] diff --git a/icu4c/source/data/curr/pool.res b/icu4c/source/data/curr/pool.res deleted file mode 100644 index 20658a7a453..00000000000 Binary files a/icu4c/source/data/curr/pool.res and /dev/null differ diff --git a/icu4c/source/data/lang/pool.res b/icu4c/source/data/lang/pool.res deleted file mode 100644 index 00b42ad7e1e..00000000000 Binary files a/icu4c/source/data/lang/pool.res and /dev/null differ diff --git a/icu4c/source/data/locales/pool.res b/icu4c/source/data/locales/pool.res deleted file mode 100644 index 31e94c9904f..00000000000 Binary files a/icu4c/source/data/locales/pool.res and /dev/null differ diff --git a/icu4c/source/data/makedata.mak b/icu4c/source/data/makedata.mak index 6a54d0acc8e..d1f13d06a94 100644 --- a/icu4c/source/data/makedata.mak +++ b/icu4c/source/data/makedata.mak @@ -33,12 +33,8 @@ ICU_LIB_TARGET=$(DLL_OUTPUT)\$(U_ICUDATA_NAME).dll !ENDIF !MESSAGE ICU data make path is $(ICUMAKE) -!IF [py -3]!=0 -!MESSAGE Information: Unable to find Python 3. ICU versions 64 and later will require Python 3 to build. -!MESSAGE Information: See ICU-10923 for more information: https://unicode-org.atlassian.net/browse/ICU-10923 -!ELSE -!MESSAGE Information: Found Python 3. You are all set for ICU 64, which will require Python 3 to build. -!MESSAGE Information: For more info on Python 3 requirement, see: https://unicode-org.atlassian.net/browse/ICU-10923 +!IF [py -3 -c "exit(0)"]!=0 +!MESSAGE Information: Unable to find Python 3. Data will fail to build from source. !ENDIF # Suffixes for data files @@ -72,15 +68,9 @@ ICUP=$(ICUP:\source\data\\..\..=) ICUSRCDATA=$(ICUP)\source\data ICUSRCDATA_RELATIVE_PATH=..\..\.. -# ICUUCM -# The directory that contains ucmcore.mk files along with *.ucm files -# -ICUUCM=mappings - -# ICULOC -# The directory that contains resfiles.mk files along with *.txt locale data files -# -ICULOC=locales +# Timestamp files to keep track of current build state +TOOLS_TS=$(ICUTMP)\tools.timestamp +COREDATA_TS=$(ICUTMP)\coredata.timestamp # ICUCOL # The directory that contains colfiles.mk files along with *.txt collation data files @@ -102,23 +92,6 @@ ICUTRNS=translit # ICUBRK=brkitr -# ICUUNIDATA -# The directory that contains Unicode data files -# -ICUUNIDATA=$(ICUP)\source\data\unidata - - -# ICUMISC -# The directory that contains miscfiles.mk along with files that are miscelleneous data -# -ICUMISC=$(ICUP)\source\data\misc -ICUMISC2=misc - -# ICUSPREP -# The directory that contains sprepfiles.mk files along with *.txt stringprep files -# -ICUSPREP=sprep - # # ICUDATA # The source directory. Contains the source files for the common data to be built. @@ -210,345 +183,8 @@ ICUDATA_SOURCE_ARCHIVE=$(ICUTMP)\$(ICUPKG).dat !IFDEF ICUDATA_SOURCE_ARCHIVE !MESSAGE ICU data source archive is $(ICUDATA_SOURCE_ARCHIVE) -!ELSE -# We're including a list of .ucm files. -# There are several lists, they are all optional. - -# Always build the mapping files for the EBCDIC fallback codepages -# They are necessary on EBCDIC machines, and -# the following logic is much easier if UCM_SOURCE is never empty. -# (They are small.) -UCM_SOURCE=ibm-37_P100-1995.ucm ibm-1047_P100-1995.ucm - -!IF EXISTS("$(ICUSRCDATA)\$(ICUUCM)\ucmcore.mk") -!INCLUDE "$(ICUSRCDATA)\$(ICUUCM)\ucmcore.mk" -UCM_SOURCE=$(UCM_SOURCE) $(UCM_SOURCE_CORE) -!ELSE -!MESSAGE Warning: cannot find "ucmcore.mk". Not building core MIME/Unix/Windows converter files. !ENDIF -!IF EXISTS("$(ICUSRCDATA)\$(ICUUCM)\ucmfiles.mk") -!INCLUDE "$(ICUSRCDATA)\$(ICUUCM)\ucmfiles.mk" -UCM_SOURCE=$(UCM_SOURCE) $(UCM_SOURCE_FILES) -!ELSE -!MESSAGE Warning: cannot find "ucmfiles.mk". Not building many converter files. -!ENDIF - -!IF EXISTS("$(ICUSRCDATA)\$(ICUUCM)\ucmebcdic.mk") -!INCLUDE "$(ICUSRCDATA)\$(ICUUCM)\ucmebcdic.mk" -UCM_SOURCE=$(UCM_SOURCE) $(UCM_SOURCE_EBCDIC) -!IFDEF UCM_SOURCE_EBCDIC_IGNORE_SISO -BUILD_SPECIAL_CNV_FILES=YES -UCM_SOURCE_SPECIAL=$(UCM_SOURCE_EBCDIC_IGNORE_SISO) -!ELSE -!UNDEF BUILD_SPECIAL_CNV_FILES -!ENDIF -!ELSE -!MESSAGE Warning: cannot find "ucmebcdic.mk". Not building EBCDIC converter files. -!ENDIF - -!IF EXISTS("$(ICUSRCDATA)\$(ICUUCM)\ucmlocal.mk") -!INCLUDE "$(ICUSRCDATA)\$(ICUUCM)\ucmlocal.mk" -!IFDEF UCM_SOURCE_LOCAL -UCM_SOURCE=$(UCM_SOURCE) $(UCM_SOURCE_LOCAL) -!ENDIF -!IFDEF UCM_SOURCE_EBCDIC_IGNORE_SISO_LOCAL -UCM_SOURCE_SPECIAL=$(UCM_SOURCE_SPECIAL) $(UCM_SOURCE_EBCDIC_IGNORE_SISO_LOCAL) -BUILD_SPECIAL_CNV_FILES=YES -!ENDIF -!ELSE -!MESSAGE Information: cannot find "ucmlocal.mk". Not building user-additional converter files. -!ENDIF - -CNV_FILES=$(UCM_SOURCE:.ucm=.cnv) -!IFDEF BUILD_SPECIAL_CNV_FILES -CNV_FILES_SPECIAL=$(UCM_SOURCE_SPECIAL:.ucm=.cnv) -!ENDIF - -!IF EXISTS("$(ICUSRCDATA)\$(ICUBRK)\brkfiles.mk") -!INCLUDE "$(ICUSRCDATA)\$(ICUBRK)\brkfiles.mk" -!IF EXISTS("$(ICUSRCDATA)\$(ICUBRK)\brklocal.mk") -!INCLUDE "$(ICUSRCDATA)\$(ICUBRK)\brklocal.mk" -!IFDEF BRK_SOURCE_LOCAL -BRK_SOURCE=$(BRK_SOURCE) $(BRK_SOURCE_LOCAL) -!ENDIF -!IFDEF BRK_DICT_SOURCE_LOCAL -BRK_DICT_SOURCE=$(BRK_DICT_SOURCE) $(BRK_DICT_SOURCE_LOCAL) -!ENDIF -!IFDEF BRK_RES_SOURCE_LOCAL -BRK_RES_SOURCE=$(BRK_RES_SOURCE) $(BRK_RES_SOURCE_LOCAL) -!ENDIF -!ELSE -!MESSAGE Information: cannot find "brklocal.mk". Not building user-additional break iterator files. -!ENDIF -!ELSE -!MESSAGE Warning: cannot find "brkfiles.mk" -!ENDIF - -# -# Break iterator data files. -# -BRK_FILES=$(ICUBRK)\$(BRK_SOURCE:.txt =.brk brkitr\) -BRK_FILES=$(BRK_FILES:.txt=.brk) -BRK_FILES=$(BRK_FILES:brkitr\ =brkitr\) - -!IFDEF BRK_DICT_SOURCE -BRK_DICT_FILES = $(ICUBRK)\$(BRK_DICT_SOURCE:.txt =.dict brkitr\) -BRK_DICT_FILES = $(BRK_DICT_FILES:.txt=.dict) -BRK_DICT_FILES = $(BRK_DICT_FILES:brkitr\ =brkitr\) -!ENDIF - -!IFDEF BRK_RES_SOURCE -BRK_RES_FILES = $(BRK_RES_SOURCE:.txt =.res brkitr\) -BRK_RES_FILES = $(BRK_RES_FILES:.txt=.res) -BRK_RES_FILES = $(ICUBRK)\root.res $(ICUBRK)\$(BRK_RES_FILES:brkitr\ =) -ALL_RES = $(ALL_RES) $(ICUBRK)\res_index.res -!ENDIF - -# Read list of locale resource bundle files -!IF EXISTS("$(ICUSRCDATA)\$(ICULOC)\resfiles.mk") -!INCLUDE "$(ICUSRCDATA)\$(ICULOC)\resfiles.mk" -!IF EXISTS("$(ICUSRCDATA)\$(ICULOC)\reslocal.mk") -!INCLUDE "$(ICUSRCDATA)\$(ICULOC)\reslocal.mk" -!IFDEF GENRB_SOURCE_LOCAL -GENRB_SOURCE=$(GENRB_SOURCE) $(GENRB_SOURCE_LOCAL) -!ENDIF -!ELSE -!MESSAGE Information: cannot find "reslocal.mk". Not building user-additional resource bundle files. -!ENDIF -!ELSE -!MESSAGE Warning: cannot find "resfiles.mk" -!ENDIF - -!IFDEF GENRB_SOURCE -RB_FILES = root.res pool.res $(GENRB_ALIAS_SOURCE:.txt=.res) $(GENRB_ALIAS_SOURCE_LOCAL:.txt=.res) $(GENRB_SOURCE:.txt=.res) -ALL_RES = $(ALL_RES) res_index.res -!ENDIF - - -# Read the list of currency display name resource bundle files -!IF EXISTS("$(ICUSRCDATA)\curr\resfiles.mk") -!INCLUDE "$(ICUSRCDATA)\curr\resfiles.mk" -!IF EXISTS("$(ICUSRCDATA)\curr\reslocal.mk") -!INCLUDE "$(ICUSRCDATA)\curr\reslocal.mk" -!IFDEF CURR_SOURCE_LOCAL -CURR_SOURCE=$(CURR_SOURCE) $(CURR_SOURCE_LOCAL) -!ENDIF -!ELSE -!MESSAGE Information: cannot find "curr\reslocal.mk". Not building user-additional resource bundle files. -!ENDIF -!ELSE -!MESSAGE Warning: cannot find "curr\resfiles.mk" -!ENDIF - -!IFDEF CURR_SOURCE -CURR_FILES = curr\root.txt supplementalData.txt $(CURR_ALIAS_SOURCE) $(CURR_SOURCE) -CURR_RES_FILES = $(CURR_FILES:.txt =.res curr\) -CURR_RES_FILES = $(CURR_RES_FILES:.txt=.res) -CURR_RES_FILES = curr\pool.res $(CURR_RES_FILES:curr\ =curr\) -ALL_RES = $(ALL_RES) curr\res_index.res -!ENDIF - -# Read the list of language/script display name resource bundle files -!IF EXISTS("$(ICUSRCDATA)\lang\resfiles.mk") -!INCLUDE "$(ICUSRCDATA)\lang\resfiles.mk" -!IF EXISTS("$(ICUSRCDATA)\lang\reslocal.mk") -!INCLUDE "$(ICUSRCDATA)\lang\reslocal.mk" -!IFDEF LANG_SOURCE_LOCAL -LANG_SOURCE=$(LANG_SOURCE) $(LANG_SOURCE_LOCAL) -!ENDIF -!ELSE -!MESSAGE Information: cannot find "lang\reslocal.mk". Not building user-additional resource bundle files. -!ENDIF -!ELSE -!MESSAGE Warning: cannot find "lang\resfiles.mk" -!ENDIF - -!IFDEF LANG_SOURCE -LANG_FILES = lang\root.txt $(LANG_ALIAS_SOURCE) $(LANG_SOURCE) -LANG_RES_FILES = $(LANG_FILES:.txt =.res lang\) -LANG_RES_FILES = $(LANG_RES_FILES:.txt=.res) -LANG_RES_FILES = lang\pool.res $(LANG_RES_FILES:lang\ =lang\) -ALL_RES = $(ALL_RES) lang\res_index.res -!ENDIF - -# Read the list of region display name resource bundle files -!IF EXISTS("$(ICUSRCDATA)\region\resfiles.mk") -!INCLUDE "$(ICUSRCDATA)\region\resfiles.mk" -!IF EXISTS("$(ICUSRCDATA)\region\reslocal.mk") -!INCLUDE "$(ICUSRCDATA)\region\reslocal.mk" -!IFDEF REGION_SOURCE_LOCAL -REGION_SOURCE=$(REGION_SOURCE) $(REGION_SOURCE_LOCAL) -!ENDIF -!ELSE -!MESSAGE Information: cannot find "region\reslocal.mk". Not building user-additional resource bundle files. -!ENDIF -!ELSE -!MESSAGE Warning: cannot find "region\resfiles.mk" -!ENDIF - -!IFDEF REGION_SOURCE -REGION_FILES = region\root.txt $(REGION_ALIAS_SOURCE) $(REGION_SOURCE) -REGION_RES_FILES = $(REGION_FILES:.txt =.res region\) -REGION_RES_FILES = $(REGION_RES_FILES:.txt=.res) -REGION_RES_FILES = region\pool.res $(REGION_RES_FILES:region\ =region\) -ALL_RES = $(ALL_RES) region\res_index.res -!ENDIF - -# Read the list of time zone display name resource bundle files -!IF EXISTS("$(ICUSRCDATA)\zone\resfiles.mk") -!INCLUDE "$(ICUSRCDATA)\zone\resfiles.mk" -!IF EXISTS("$(ICUSRCDATA)\zone\reslocal.mk") -!INCLUDE "$(ICUSRCDATA)\zone\reslocal.mk" -!IFDEF ZONE_SOURCE_LOCAL -ZONE_SOURCE=$(ZONE_SOURCE) $(ZONE_SOURCE_LOCAL) -!ENDIF -!ELSE -!MESSAGE Information: cannot find "zone\reslocal.mk". Not building user-additional resource bundle files. -!ENDIF -ZONE_SOURCE=$(ZONE_SOURCE) tzdbNames.txt -!ELSE -!MESSAGE Warning: cannot find "zone\resfiles.mk" -!ENDIF - -!IFDEF ZONE_SOURCE -ZONE_FILES = zone\root.txt $(ZONE_ALIAS_SOURCE) $(ZONE_SOURCE) -ZONE_RES_FILES = $(ZONE_FILES:.txt =.res zone\) -ZONE_RES_FILES = $(ZONE_RES_FILES:.txt=.res) -ZONE_RES_FILES = zone\pool.res $(ZONE_RES_FILES:zone\ =zone\) -ALL_RES = $(ALL_RES) zone\res_index.res -!ENDIF - -# Read the list of units display name resource bundle files -!IF EXISTS("$(ICUSRCDATA)\unit\resfiles.mk") -!INCLUDE "$(ICUSRCDATA)\unit\resfiles.mk" -!IF EXISTS("$(ICUSRCDATA)\unit\reslocal.mk") -!INCLUDE "$(ICUSRCDATA)\unit\reslocal.mk" -!IFDEF UNIT_SOURCE_LOCAL -UNIT_SOURCE=$(UNIT_SOURCE) $(UNIT_SOURCE_LOCAL) -!ENDIF -!ELSE -!MESSAGE Information: cannot find "unit\reslocal.mk". Not building user-additional resource bundle files. -!ENDIF -!ELSE -!MESSAGE Warning: cannot find "unit\resfiles.mk" -!ENDIF - -!IFDEF UNIT_SOURCE -UNIT_FILES = unit\root.txt $(UNIT_ALIAS_SOURCE) $(UNIT_SOURCE) -UNIT_RES_FILES = $(UNIT_FILES:.txt =.res unit\) -UNIT_RES_FILES = $(UNIT_RES_FILES:.txt=.res) -UNIT_RES_FILES = unit\pool.res $(UNIT_RES_FILES:unit\ =unit\) -ALL_RES = $(ALL_RES) unit\res_index.res -!ENDIF - -# Read the list of collation resource bundle files -!IF EXISTS("$(ICUSRCDATA)\$(ICUCOL)\colfiles.mk") -!INCLUDE "$(ICUSRCDATA)\$(ICUCOL)\colfiles.mk" -!IF EXISTS("$(ICUSRCDATA)\$(ICUCOL)\collocal.mk") -!INCLUDE "$(ICUSRCDATA)\$(ICUCOL)\collocal.mk" -!IFDEF COLLATION_SOURCE_LOCAL -COLLATION_SOURCE=$(COLLATION_SOURCE) $(COLLATION_SOURCE_LOCAL) -!ENDIF -!ELSE -!MESSAGE Information: cannot find "collocal.mk". Not building user-additional resource bundle files. -!ENDIF -!ELSE -!MESSAGE Warning: cannot find "colfiles.mk" -!ENDIF - -!IFDEF COLLATION_SOURCE -COL_FILES = $(ICUCOL)\root.txt $(COLLATION_ALIAS_SOURCE) $(COLLATION_SOURCE) -COL_COL_FILES = $(COL_FILES:.txt =.res coll\) -COL_COL_FILES = $(COL_COL_FILES:.txt=.res) -COL_COL_FILES = $(COL_COL_FILES:coll\ =) -ALL_RES = $(ALL_RES) $(ICUCOL)\res_index.res -!ENDIF - -# Read the list of RBNF resource bundle files -!IF EXISTS("$(ICUSRCDATA)\$(ICURBNF)\rbnffiles.mk") -!INCLUDE "$(ICUSRCDATA)\$(ICURBNF)\rbnffiles.mk" -!IF EXISTS("$(ICUSRCDATA)\$(ICURBNF)\rbnflocal.mk") -!INCLUDE "$(ICUSRCDATA)\$(ICURBNF)\rbnflocal.mk" -!IFDEF RBNF_SOURCE_LOCAL -RBNF_SOURCE=$(RBNF_SOURCE) $(RBNF_SOURCE_LOCAL) -!ENDIF -!ELSE -!MESSAGE Information: cannot find "rbnflocal.mk". Not building user-additional resource bundle files. -!ENDIF -!ELSE -!MESSAGE Warning: cannot find "rbnffiles.mk" -!ENDIF - -!IFDEF RBNF_SOURCE -RBNF_FILES = $(ICURBNF)\root.txt $(RBNF_ALIAS_SOURCE) $(RBNF_SOURCE) -RBNF_RES_FILES = $(RBNF_FILES:.txt =.res rbnf\) -RBNF_RES_FILES = $(RBNF_RES_FILES:.txt=.res) -RBNF_RES_FILES = $(RBNF_RES_FILES:rbnf\ =rbnf\) -ALL_RES = $(ALL_RES) $(ICURBNF)\res_index.res -!ENDIF - -# Read the list of transliterator resource bundle files -!IF EXISTS("$(ICUSRCDATA)\$(ICUTRNS)\trnsfiles.mk") -!INCLUDE "$(ICUSRCDATA)\$(ICUTRNS)\trnsfiles.mk" -!IF EXISTS("$(ICUSRCDATA)\$(ICUTRNS)\trnslocal.mk") -!INCLUDE "$(ICUSRCDATA)\$(ICUTRNS)\trnslocal.mk" -!IFDEF TRANSLIT_SOURCE_LOCAL -TRANSLIT_SOURCE=$(TRANSLIT_SOURCE) $(TRANSLIT_SOURCE_LOCAL) -!ENDIF -!ELSE -!MESSAGE Information: cannot find "trnslocal.mk". Not building user-additional transliterator files. -!ENDIF -!ELSE -!MESSAGE Warning: cannot find "trnsfiles.mk" -!ENDIF - -!IFDEF TRANSLIT_SOURCE -TRANSLIT_FILES = $(ICUTRNS)\$(TRANSLIT_ALIAS_SOURCE) $(TRANSLIT_SOURCE) -TRANSLIT_RES_FILES = $(TRANSLIT_FILES:.txt =.res translit\) -TRANSLIT_RES_FILES = $(TRANSLIT_RES_FILES:.txt=.res) -TRANSLIT_RES_FILES = $(TRANSLIT_RES_FILES:translit\ =translit\) -#ALL_RES = $(ALL_RES) $(ICUTRNS)\res_index.res -!ENDIF - -# Read the list of miscellaneous resource bundle files -!IF EXISTS("$(ICUSRCDATA)\$(ICUMISC2)\miscfiles.mk") -!INCLUDE "$(ICUSRCDATA)\$(ICUMISC2)\miscfiles.mk" -!IF EXISTS("$(ICUSRCDATA)\$(ICUMISC2)\misclocal.mk") -!INCLUDE "$(ICUSRCDATA)\$(ICUMISC2)\misclocal.mk" -!IFDEF MISC_SOURCE_LOCAL -MISC_SOURCE=$(MISC_SOURCE) $(MISC_SOURCE_LOCAL) -!ENDIF -!ELSE -!MESSAGE Information: cannot find "misclocal.mk". Not building user-additional miscellaenous files. -!ENDIF -!ELSE -!MESSAGE Warning: cannot find "miscfiles.mk" -!ENDIF - -MISC_FILES = $(MISC_SOURCE:.txt=.res) - -# don't include COL_FILES -ALL_RES = $(ALL_RES) $(RB_FILES) $(MISC_FILES) -!ENDIF - -# Read the list of stringprep profile files -!IF EXISTS("$(ICUSRCDATA)\$(ICUSPREP)\sprepfiles.mk") -!INCLUDE "$(ICUSRCDATA)\$(ICUSPREP)\sprepfiles.mk" -!IF EXISTS("$(ICUSRCDATA)\$(ICUSPREP)\spreplocal.mk") -!INCLUDE "$(ICUSRCDATA)\$(ICUSPREP)\spreplocal.mk" -!IFDEF SPREP_SOURCE_LOCAL -SPREP_SOURCE=$(SPREP_SOURCE) $(SPREP_SOURCE_LOCAL) -!ENDIF -!ELSE -!MESSAGE Information: cannot find "spreplocal.mk". Not building user-additional stringprep files. -!ENDIF -!ELSE -!MESSAGE Warning: cannot find "sprepfiles.mk" -!ENDIF - -SPREP_FILES = $(SPREP_SOURCE:.txt=.spp) - # Common defines for both ways of building ICU's data library. COMMON_ICUDATA_DEPENDENCIES="$(ICUPBIN)\pkgdata.exe" "$(ICUTMP)\icudata.res" "$(ICUP)\source\stubdata\stubdatabuilt.txt" COMMON_ICUDATA_ARGUMENTS=-f -e $(U_ICUDATA_NAME) -v $(ICU_PACKAGE_MODE) -c -p $(ICUPKG) -T "$(ICUTMP)" -L $(U_ICUDATA_NAME) -d "$(ICUBLD_PKG)" -s . @@ -576,6 +212,37 @@ ALL : GODATA "$(ICU_LIB_TARGET)" "$(TESTDATAOUT)\testdata.dat" copy "$(ICUOUT)\$(U_ICUDATA_NAME)$(U_ICUDATA_ENDIAN_SUFFIX).dat" "$(ICUMAKE)\..\..\commondata\" !ENDIF + +# Three main targets: tools, core data, and test data. +# Keep track of whether they are built via timestamp files. + +$(TOOLS_TS): "$(ICUTOOLS)\genrb\$(CFGTOOLS)\genrb.exe" "$(ICUTOOLS)\gencnval\$(CFGTOOLS)\gencnval.exe" "$(ICUTOOLS)\gencfu\$(CFGTOOLS)\gencfu.exe" "$(ICUTOOLS)\icupkg\$(CFGTOOLS)\icupkg.exe" "$(ICUTOOLS)\makeconv\$(CFGTOOLS)\makeconv.exe" "$(ICUPBIN)\pkgdata.exe" + @echo "timestamp" > $(TOOLS_TS) + +# On Unix, Python generates at configure time a list of Makefile rules. +# On Windows, however, we run the Python data build script at build time instead. +# The alternative would be to use a preprocessor macro to generate rules for nmake. +# However, this approach was abandoned for reasons including: +# +# - nmake imposes more stringent restrictions on command line length. +# - The lack of gnumake features makes nmake file construction more complex. +# - Windows builds are single-threaded, giving less advantage to a Makefile approach. +# +# Currently, the entire script needs to run even for small changes to data. Maybe consider +# checking file-changed timestamps in Python to build only the required subset of data. + +$(COREDATA_TS): + @cd "$(ICUSRCDATA)" + py -3 -m buildtool \ + --format windirect \ + --in_dir "$(ICUSRCDATA)" \ + --tool_dir "$(ICUTOOLS)" \ + --tool_cfg "$(CFG)" \ + --out_dir "$(ICUBLD_PKG)" \ + --tmp_dir "$(ICUTMP)" + @echo "timestamp" > $(COREDATA_TS) + + # The core Unicode properties files (uprops.icu, ucase.icu, ubidi.icu) # are hardcoded in the common DLL and therefore not included in the data package any more. # They are not built by default but need to be built for ICU4J data and for getting the .c source files @@ -673,10 +340,10 @@ icu4j-data-install : # # testdata - nmake will invoke pkgdata, which will create testdata.dat # -"$(TESTDATAOUT)\testdata.dat": "$(TESTDATA)\*" "$(ICUBLD_PKG)\$(ICUCOL)\ucadata.icu" $(TRANSLIT_RES_FILES) $(MISC_FILES) $(RB_FILES) {"$(ICUTOOLS)\genrb\$(CFGTOOLS)"}genrb.exe +"$(TESTDATAOUT)\testdata.dat": "$(TESTDATA)\*" $(TOOLS_TS) $(COREDATA_TS) @cd "$(TESTDATA)" @echo building testdata... - nmake /nologo /f "$(TESTDATA)\testdata.mak" TESTDATA=. ICUTOOLS="$(ICUTOOLS)" ICUPBIN="$(ICUPBIN)" ICUP="$(ICUP)" CFG=$(CFGTOOLS) TESTDATAOUT="$(TESTDATAOUT)" TESTDATABLD="$(TESTDATABLD)" + nmake /nologo /f "$(TESTDATA)\testdata.mak" TESTDATA=. ICUTOOLS="$(ICUTOOLS)" ICUPBIN="$(ICUPBIN)" ICUP="$(ICUP)" CFG=$(CFGTOOLS) TESTDATAOUT="$(TESTDATAOUT)" TESTDATABLD="$(TESTDATABLD)" ICUSRCDATA="$(ICUSRCDATA)" DLL_OUTPUT="$(DLL_OUTPUT)" #invoke pkgdata for ICU common data # pkgdata will drop all output files (.dat, .dll, .lib) into the target (ICUBLD_PKG) directory. @@ -697,50 +364,12 @@ icu4j-data-install : copy "$(ICUTMP)\$(ICUPKG).dat" "$(ICUOUT)\$(U_ICUDATA_NAME)$(U_ICUDATA_ENDIAN_SUFFIX).dat" -@erase "$(ICUTMP)\$(ICUPKG).dat" !ELSE -"$(ICU_LIB_TARGET)" : $(COMMON_ICUDATA_DEPENDENCIES) $(CNV_FILES) $(CNV_FILES_SPECIAL) "$(ICUBLD_PKG)\unames.icu" "$(ICUBLD_PKG)\cnvalias.icu" "$(ICUBLD_PKG)\nfkc.nrm" "$(ICUBLD_PKG)\nfkc_cf.nrm" "$(ICUBLD_PKG)\uts46.nrm" "$(ICUBLD_PKG)\$(ICUCOL)\ucadata.icu" $(CURR_RES_FILES) $(LANG_RES_FILES) $(REGION_RES_FILES) $(ZONE_RES_FILES) $(UNIT_RES_FILES) $(BRK_FILES) $(BRK_DICT_FILES) $(BRK_RES_FILES) $(ALL_RES) $(COL_COL_FILES) $(RBNF_RES_FILES) $(TRANSLIT_RES_FILES) $(SPREP_FILES) "$(ICUBLD_PKG)\confusables.cfu" - @echo Building icu data +"$(ICU_LIB_TARGET)" : $(COMMON_ICUDATA_DEPENDENCIES) $(COREDATA_TS) + @echo Building ICU data from scratch cd "$(ICUBLD_PKG)" - "$(ICUPBIN)\pkgdata" $(COMMON_ICUDATA_ARGUMENTS) <<"$(ICUTMP)\icudata.lst" -unames.icu -confusables.cfu -$(ICUCOL)\ucadata.icu -cnvalias.icu -nfkc.nrm -nfkc_cf.nrm -uts46.nrm -$(CNV_FILES:.cnv =.cnv -) -$(CNV_FILES_SPECIAL:.cnv =.cnv -) -$(ALL_RES:.res =.res -) -$(CURR_RES_FILES:.res =.res -) -$(LANG_RES_FILES:.res =.res -) -$(REGION_RES_FILES:.res =.res -) -$(ZONE_RES_FILES:.res =.res -) -$(UNIT_RES_FILES:.res =.res -) -$(COL_COL_FILES:.res =.res -) -$(RBNF_RES_FILES:.res =.res -) -$(TRANSLIT_RES_FILES:.res =.res -) -$(BRK_FILES:.brk =.brk -) -$(BRK_DICT_FILES:.dict =.dict -) -$(BRK_RES_FILES:.res =.res -) -$(SPREP_FILES:.spp=.spp -) -<> $@; \ - done; +include $(top_builddir)/$(subdir)/rules.mk -build-testdata: build-dir $(ALL_TEST_FILES) $(TESTBUILDDIR)/testdata.lst $(TESTBUILDDIR)/encoded.res +build-testdata: build-dir $(TESTDATA_ALL_OUTPUT_FILES) -# test.icu -$(TESTBUILDDIR)/test.icu: $(GENTEST) - $(INVOKE) $(GENTEST) -d $(TESTBUILDDIR) +testdata: build-testdata -$(TESTBUILDDIR)/testtable32.txt: $(GENTEST) - $(INVOKE) $(GENTEST) -r -d $(TESTBUILDDIR) -$(TESTBUILDDIR)/testtable32.res: $(TESTBUILDDIR)/testtable32.txt $(TOOLBINDIR)/genrb$(EXEEXT) - $(INVOKE) $(TOOLBINDIR)/genrb $(GENRBOPTS) -s $(TESTBUILDDIR) $(ICU_DATA_OPT) -d $(TESTBUILDDIR) $(=2) { - path=argv[1]; + if (argc >= 2) { + path = argv[1]; } else { - path=options[SOURCEDIR].value; - if(path!=NULL && *path!=0) { - char *end; + path = "convrtrs.txt"; + } - uprv_strcpy(pathBuf, path); - end = uprv_strchr(pathBuf, 0); - if(*(end-1)!=U_FILE_SEP_CHAR) { - *(end++)=U_FILE_SEP_CHAR; - } - uprv_strcpy(end, "convrtrs.txt"); - path=pathBuf; - } else { - path = "convrtrs.txt"; + const char* sourcedir = options[SOURCEDIR].value; + if (sourcedir != NULL && *sourcedir != 0) { + char *end; + uprv_strcpy(pathBuf, sourcedir); + end = uprv_strchr(pathBuf, 0); + if(*(end-1)!=U_FILE_SEP_CHAR) { + *(end++)=U_FILE_SEP_CHAR; } + uprv_strcpy(end, path); + path = pathBuf; } uprv_memset(stringStore, 0, sizeof(stringStore)); diff --git a/icu4c/source/tools/genrb/genrb.cpp b/icu4c/source/tools/genrb/genrb.cpp index c4fc462066a..1135254ca92 100644 --- a/icu4c/source/tools/genrb/genrb.cpp +++ b/icu4c/source/tools/genrb/genrb.cpp @@ -99,7 +99,7 @@ UOption options[]={ UOPTION_DEF("language", 'l', UOPT_REQUIRES_ARG), /* 16 */ UOPTION_DEF("omitCollationRules", 'R', UOPT_NO_ARG),/* 17 */ UOPTION_DEF("formatVersion", '\x01', UOPT_REQUIRES_ARG),/* 18 */ - UOPTION_DEF("writePoolBundle", '\x01', UOPT_NO_ARG),/* 19 */ + UOPTION_DEF("writePoolBundle", '\x01', UOPT_OPTIONAL_ARG),/* 19 */ UOPTION_DEF("usePoolBundle", '\x01', UOPT_OPTIONAL_ARG),/* 20 */ UOPTION_DEF("includeUnihanColl", '\x01', UOPT_NO_ARG),/* 21 */ /* temporary, don't display in usage info */ }; @@ -224,8 +224,8 @@ main(int argc, "\t --formatVersion write a .res file compatible with the requested formatVersion (single digit);\n" "\t for example, --formatVersion 1\n"); fprintf(stderr, - "\t --writePoolBundle write a pool.res file with all of the keys of all input bundles\n" - "\t --usePoolBundle [path-to-pool.res] point to keys from the pool.res keys pool bundle if they are available there;\n" + "\t --writePoolBundle [directory] write a pool.res file with all of the keys of all input bundles\n" + "\t --usePoolBundle [directory] point to keys from the pool.res keys pool bundle if they are available there;\n" "\t makes .res files smaller but dependent on the pool bundle\n" "\t (--writePoolBundle and --usePoolBundle cannot be combined)\n"); @@ -532,8 +532,14 @@ main(int argc, poolBundle.close(); if(U_SUCCESS(status) && options[WRITE_POOL_BUNDLE].doesOccur) { + const char* writePoolDir; + if (options[WRITE_POOL_BUNDLE].value!=NULL) { + writePoolDir = options[WRITE_POOL_BUNDLE].value; + } else { + writePoolDir = outputDir; + } char outputFileName[256]; - newPoolBundle->write(outputDir, NULL, outputFileName, sizeof(outputFileName), status); + newPoolBundle->write(writePoolDir, NULL, outputFileName, sizeof(outputFileName), status); if(U_FAILURE(status)) { fprintf(stderr, "unable to write the pool bundle: %s\n", u_errorName(status)); } diff --git a/icu4c/source/tools/genrb/reslist.cpp b/icu4c/source/tools/genrb/reslist.cpp index 0493347ebe5..5fb75fbd74c 100644 --- a/icu4c/source/tools/genrb/reslist.cpp +++ b/icu4c/source/tools/genrb/reslist.cpp @@ -1037,14 +1037,15 @@ void SRBRoot::write(const char *outputDir, const char *outputPkg, // Swap to big-endian so we get the same checksum on all platforms // (except for charset family, due to the key strings). UnicodeString s(f16BitUnits); - s.append((UChar)1); // Ensure that we own this buffer. assert(!s.isBogus()); - uint16_t *p = const_cast(reinterpret_cast(s.getBuffer())); + // .getBuffer(capacity) returns a mutable buffer + char16_t* p = s.getBuffer(f16BitUnits.length()); for (int32_t count = f16BitUnits.length(); count > 0; --count) { uint16_t x = *p; *p++ = (uint16_t)((x << 8) | (x >> 8)); } - checksum = computeCRC((const char *)p, + s.releaseBuffer(f16BitUnits.length()); + checksum = computeCRC((const char *)s.getBuffer(), (uint32_t)f16BitUnits.length() * 2, checksum); } indexes[URES_INDEX_POOL_CHECKSUM] = (int32_t)checksum; diff --git a/icu4c/source/tools/makeconv/makeconv.cpp b/icu4c/source/tools/makeconv/makeconv.cpp index 5a01e209152..12db2860df2 100644 --- a/icu4c/source/tools/makeconv/makeconv.cpp +++ b/icu4c/source/tools/makeconv/makeconv.cpp @@ -179,6 +179,7 @@ enum { OPT_SMALL, OPT_IGNORE_SISO_CHECK, OPT_QUIET, + OPT_SOURCEDIR, OPT_COUNT }; @@ -193,6 +194,7 @@ static UOption options[]={ { "small", NULL, NULL, NULL, '\1', UOPT_NO_ARG, 0 }, { "ignore-siso-check", NULL, NULL, NULL, '\1', UOPT_NO_ARG, 0 }, UOPTION_QUIET, + UOPTION_SOURCEDIR, }; int main(int argc, char* argv[]) @@ -230,7 +232,8 @@ int main(int argc, char* argv[]) "\t-c or --copyright include a copyright notice\n" "\t-d or --destdir destination directory, followed by the path\n" "\t-v or --verbose Turn on verbose output\n" - "\t-q or --quiet do not display warnings and progress\n", + "\t-q or --quiet do not display warnings and progress\n" + "\t-s or --sourcedir source directory, followed by the path\n", argv[0]); fprintf(stdfile, "\t --small Generate smaller .cnv files. They will be\n" @@ -282,11 +285,20 @@ int main(int argc, char* argv[]) #endif UBool printFilename = (UBool) (argc > 2 || VERBOSE); + icu::CharString pathBuf; for (++argv; --argc; ++argv) { UErrorCode localError = U_ZERO_ERROR; const char *arg = getLongPathname(*argv); + const char* sourcedir = options[OPT_SOURCEDIR].value; + if (sourcedir != NULL && *sourcedir != 0) { + pathBuf.clear(); + pathBuf.appendPathPart(sourcedir, localError); + pathBuf.appendPathPart(arg, localError); + arg = pathBuf.data(); + } + /*produces the right destination path for display*/ outFileName.truncate(outBasenameStart); if (outBasenameStart != 0)