organicmaps/icu
13
Fork 0
mirror of https://github.com/unicode-org/icu.git synced 2025-04-05 13:35:32 +00:00
Code Issues 1 Projects Releases Wiki Activity Actions 12

ICU-10923 Rewriting data build orchestration in Python.

Browse source 
- Creates new Python package in icu4c/data/buildtool
- Creates BUILRDULES.py in icu4c/data and icu4c/test/testdata, unified between Unix/Windows
- Removes most data build orchestration rules from makedata.mak, testdata.mak, data/Makefile.in, and test/testdata/Makefile.in
- Removes pool.res files and builds them on the fly instead
This commit is contained in:
Shane Carr 2018-08-24 23:39:10 -07:00 • committed by Shane F. Carr
parent 2d3901ae5d
commit ee9c0b944c
25 changed files with 1528 additions and 1458 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
.gitignore vendored
View file

@ -1,3 +1,8 @@
# Global ignores
__pycache__/
*.pyc
# Specific file name ignores
/.git
icu4c/bin
icu4c/bin32uwp
@ -77,6 +82,7 @@ icu4c/source/data/obj
icu4c/source/data/out
icu4c/source/data/packagedata
icu4c/source/data/pkgdataMakefile
icu4c/source/data/rules.mk
icu4c/source/data/uni-core-data
icu4c/source/data/x64
icu4c/source/data/x86
@ -621,6 +627,7 @@ icu4c/source/test/testdata/Makefile
icu4c/source/test/testdata/out
icu4c/source/test/testdata/pkgdata.inc
icu4c/source/test/testdata/pkgdataMakefile
icu4c/source/test/testdata/rules.mk
icu4c/source/test/testmap/*.d
icu4c/source/test/testmap/Debug
icu4c/source/test/testmap/Makefile

30
icu4c/source/configure vendored
View file

@ -4200,7 +4200,7 @@ fi
#AC_CHECK_PROG(AUTOCONF, autoconf, autoconf, true)
#AC_CHECK_PROG(STRIP, strip, strip, true)
for ac_prog in python3
for ac_prog in python3 "py -3"
do
# Extract the first word of "$ac_prog", so it can be a program name with args.
set dummy $ac_prog; ac_word=$2
@ -9121,6 +9121,24 @@ $as_echo "$as_me: WARNING: unrecognized options: $ac_unrecognized_opts" >&2;}
fi
if test -z "$PYTHON3";
then
echo "" > data/rules.mk
else
echo "Spawning Python to generate data/rules.mk..."
PYTHONPATH="$srcdir/data" $PYTHON3 -m buildtool \
--format gnumake \
--seqmode parallel \
--glob_dir "$srcdir/data" \
> data/rules.mk
echo "Spawning Python to generate test/testdata/rules.mk..."
PYTHONPATH="$srcdir/test/testdata:$srcdir/data" $PYTHON3 -m buildtool \
--format gnumake \
--seqmode parallel \
--glob_dir "$srcdir/test/testdata" \
> test/testdata/rules.mk
fi
echo
echo "ICU for C/C++ $VERSION is ready to be built."
echo "=== Important Notes: ==="
@ -9205,15 +9223,7 @@ fi
if test -z "$PYTHON3";
then
echo ""
echo "NOTICE: Unable to find Python 3. ICU versions 64 and later will require Python 3 to build."
echo "See ICU-10923 for more information: https://unicode-org.atlassian.net/browse/ICU-10923"
echo ""
else
echo ""
echo "Found Python 3. You are all set for ICU 64, which will require Python 3 to build."
echo "For more info on Python 3 requirement, see: https://unicode-org.atlassian.net/browse/ICU-10923"
echo ""
echo "** Note: Python 3 not found. You will not be able to build data from source."
fi
$as_unset _CXX_CXXSUFFIX

30
icu4c/source/configure.ac
View file

@ -195,7 +195,7 @@ fi
#AC_CHECK_PROG(AUTOCONF, autoconf, autoconf, true)
#AC_CHECK_PROG(STRIP, strip, strip, true)
AC_CHECK_PROGS(PYTHON3, python3)
AC_CHECK_PROGS(PYTHON3, python3 "py -3")
# Check for the platform make
AC_PATH_PROGS(U_MAKE, gmake gnumake, make)
@ -1385,6 +1385,24 @@ AC_CONFIG_FILES([icudefs.mk \
samples/cal/Makefile samples/layout/Makefile])
AC_OUTPUT
if test -z "$PYTHON3";
then
echo "" > data/rules.mk
else
echo "Spawning Python to generate data/rules.mk..."
PYTHONPATH="$srcdir/data" $PYTHON3 -m buildtool \
--format gnumake \
--seqmode parallel \
--glob_dir "$srcdir/data" \
> data/rules.mk
echo "Spawning Python to generate test/testdata/rules.mk..."
PYTHONPATH="$srcdir/test/testdata:$srcdir/data" $PYTHON3 -m buildtool \
--format gnumake \
--seqmode parallel \
--glob_dir "$srcdir/test/testdata" \
> test/testdata/rules.mk
fi
echo
echo "ICU for C/C++ $VERSION is ready to be built."
echo "=== Important Notes: ==="
@ -1466,15 +1484,7 @@ fi
if test -z "$PYTHON3";
then
echo ""
echo "NOTICE: Unable to find Python 3. ICU versions 64 and later will require Python 3 to build."
echo "See ICU-10923 for more information: https://unicode-org.atlassian.net/browse/ICU-10923"
echo ""
else
echo ""
echo "Found Python 3. You are all set for ICU 64, which will require Python 3 to build."
echo "For more info on Python 3 requirement, see: https://unicode-org.atlassian.net/browse/ICU-10923"
echo ""
echo "** Note: Python 3 not found. You will not be able to build data from source."
fi
$as_unset _CXX_CXXSUFFIX

399
icu4c/source/data/BUILDRULES.py Normal file
View file

@ -0,0 +1,399 @@
# Copyright (C) 2018 and later: Unicode, Inc. and others.
# License & terms of use: http://www.unicode.org/copyright.html
from distutils.sysconfig import parse_makefile
from buildtool import *
from buildtool import utils
import sys
def generate(config, glob, common_vars):
requests = []
pkg_exclusions = set()
if len(glob("misc/*")) == 0:
print("Error: Cannot find data directory; please specify --glob_dir", file=sys.stderr)
exit(1)
# DIRECTORIES
build_dirs = [
"{OUT_DIR}",
"{OUT_DIR}/curr",
"{OUT_DIR}/lang",
"{OUT_DIR}/region",
"{OUT_DIR}/zone",
"{OUT_DIR}/unit",
"{OUT_DIR}/brkitr",
"{OUT_DIR}/coll",
"{OUT_DIR}/rbnf",
"{OUT_DIR}/translit",
"{TMP_DIR}",
"{TMP_DIR}/curr",
"{TMP_DIR}/lang",
"{TMP_DIR}/locales",
"{TMP_DIR}/region",
"{TMP_DIR}/zone",
"{TMP_DIR}/unit",
"{TMP_DIR}/coll",
"{TMP_DIR}/rbnf",
"{TMP_DIR}/translit",
"{TMP_DIR}/brkitr"
]
# UConv Name Aliases
if config.has_feature("cnvalias"):
input_file = InFile("mappings/convrtrs.txt")
output_file = OutFile("cnvalias.icu")
requests += [
SingleExecutionRequest(
name = "cnvalias",
input_files = [input_file],
output_files = [output_file],
tool = IcuTool("gencnval"),
args = "-s {IN_DIR} -d {OUT_DIR} "
"{INPUT_FILES[0]}",
format_with = {}
)
]
# CONFUSABLES
if config.has_feature("confusables"):
txt1 = InFile("unidata/confusables.txt")
txt2 = InFile("unidata/confusablesWholeScript.txt")
cfu = OutFile("confusables.cfu")
requests += [
SingleExecutionRequest(
name = "confusables",
input_files = [txt1, txt2, OutFile("cnvalias.icu")],
output_files = [cfu],
tool = IcuTool("gencfu"),
args = "-d {OUT_DIR} -i {OUT_DIR} "
"-c -r {IN_DIR}/{INPUT_FILES[0]} -w {IN_DIR}/{INPUT_FILES[1]} "
"-o {OUTPUT_FILES[0]}",
format_with = {}
)
]
# UConv Conversion Table Files
if config.has_feature("uconv"):
input_files = [InFile(filename) for filename in glob("mappings/*.ucm")]
output_files = [OutFile("%s.cnv" % v.filename[9:-4]) for v in input_files]
# TODO: handle BUILD_SPECIAL_CNV_FILES? Means to add --ignore-siso-check flag to makeconv
requests += [
RepeatedOrSingleExecutionRequest(
name = "uconv",
dep_files = [],
input_files = input_files,
output_files = output_files,
tool = IcuTool("makeconv"),
args = "-s {IN_DIR} -d {OUT_DIR} -c {INPUT_FILE_PLACEHOLDER}",
format_with = {},
repeat_with = {
"INPUT_FILE_PLACEHOLDER": [file.filename for file in input_files]
},
flatten_with = {
"INPUT_FILE_PLACEHOLDER": " ".join(file.filename for file in input_files)
}
)
]
# BRK Files
brkitr_brk_files = []
if config.has_feature("brkitr"):
input_files = [InFile(filename) for filename in glob("brkitr/rules/*.txt")]
output_files = [OutFile("brkitr/%s.brk" % v.filename[13:-4]) for v in input_files]
brkitr_brk_files += output_files
requests += [
RepeatedExecutionRequest(
name = "brkitr_brk",
dep_files = [OutFile("cnvalias.icu")],
input_files = input_files,
output_files = output_files,
tool = IcuTool("genbrk"),
args = "-d {OUT_DIR} -i {OUT_DIR} "
"-c -r {IN_DIR}/{INPUT_FILE} "
"-o {OUTPUT_FILE}",
format_with = {},
repeat_with = {}
)
]
# SPP FILES
if config.has_feature("stringprep"):
input_files = [InFile(filename) for filename in glob("sprep/*.txt")]
output_files = [OutFile("%s.spp" % v.filename[6:-4]) for v in input_files]
bundle_names = [v.filename[6:-4] for v in input_files]
requests += [
RepeatedExecutionRequest(
name = "stringprep",
dep_files = [],
input_files = input_files,
output_files = output_files,
tool = IcuTool("gensprep"),
args = "-s {IN_DIR}/sprep -d {OUT_DIR} -i {OUT_DIR} "
"-b {BUNDLE_NAME} -m {IN_DIR}/unidata -u 3.2.0 {BUNDLE_NAME}.txt",
format_with = {},
repeat_with = {
"BUNDLE_NAME": bundle_names
}
)
]
# Dict Files
dict_files = []
if config.has_feature("dictionaries"):
input_files = [InFile(filename) for filename in glob("brkitr/dictionaries/*.txt")]
output_files = [OutFile("brkitr/%s.dict" % v.filename[20:-4]) for v in input_files]
dict_files += output_files
extra_options_map = {
"brkitr/dictionaries/burmesedict.txt": "--bytes --transform offset-0x1000",
"brkitr/dictionaries/cjdict.txt": "--uchars",
"brkitr/dictionaries/khmerdict.txt": "--bytes --transform offset-0x1780",
"brkitr/dictionaries/laodict.txt": "--bytes --transform offset-0x0e80",
"brkitr/dictionaries/thaidict.txt": "--bytes --transform offset-0x0e00"
}
extra_optionses = [extra_options_map[v.filename] for v in input_files]
requests += [
RepeatedExecutionRequest(
name = "dictionaries",
dep_files = [],
input_files = input_files,
output_files = output_files,
tool = IcuTool("gendict"),
args = "-i {OUT_DIR} "
"-c {EXTRA_OPTIONS} "
"{IN_DIR}/{INPUT_FILE} {OUT_DIR}/{OUTPUT_FILE}",
format_with = {},
repeat_with = {
"EXTRA_OPTIONS": extra_optionses
}
)
]
# NRM Files
if config.has_feature("normalization"):
input_files = [InFile(filename) for filename in glob("in/*.nrm")]
input_files.remove(InFile("in/nfc.nrm")) # nfc.nrm is pre-compiled into C++
output_files = [OutFile(v.filename[3:]) for v in input_files]
requests += [
RepeatedExecutionRequest(
name = "normalization",
dep_files = [],
input_files = input_files,
output_files = output_files,
tool = IcuTool("icupkg"),
args = "-t{ICUDATA_CHAR} {IN_DIR}/{INPUT_FILE} {OUT_DIR}/{OUTPUT_FILE}",
format_with = {},
repeat_with = {}
)
]
# Collation Dependency File (ucadata.icu)
if config.has_feature("coll"):
input_file = InFile("in/coll/ucadata-%s.icu" % config.coll_han_type())
output_file = OutFile("coll/ucadata.icu")
requests += [
SingleExecutionRequest(
name = "coll_ucadata",
input_files = [input_file],
output_files = [output_file],
tool = IcuTool("icupkg"),
args = "-t{ICUDATA_CHAR} {IN_DIR}/{INPUT_FILES[0]} {OUT_DIR}/{OUTPUT_FILES[0]}",
format_with = {}
)
]
# Unicode Character Names
if config.has_feature("unames"):
input_file = InFile("in/unames.icu")
output_file = OutFile("unames.icu")
requests += [
SingleExecutionRequest(
name = "unames",
input_files = [input_file],
output_files = [output_file],
tool = IcuTool("icupkg"),
args = "-t{ICUDATA_CHAR} {IN_DIR}/{INPUT_FILES[0]} {OUT_DIR}/{OUTPUT_FILES[0]}",
format_with = {}
)
]
# Misc Data Res Files
if config.has_feature("misc"):
# TODO: Treat each misc file separately
input_files = [InFile(filename) for filename in glob("misc/*.txt")]
input_basenames = [v.filename[5:] for v in input_files]
output_files = [OutFile("%s.res" % v[:-4]) for v in input_basenames]
requests += [
RepeatedExecutionRequest(
name = "misc",
dep_files = [],
input_files = input_files,
output_files = output_files,
tool = IcuTool("genrb"),
args = "-s {IN_DIR}/misc -d {OUT_DIR} -i {OUT_DIR} "
"-k -q "
"{INPUT_BASENAME}",
format_with = {},
repeat_with = {
"INPUT_BASENAME": input_basenames
}
)
]
# Specialized Locale Data Res Files
specialized_sub_dirs = [
# (input dirname, output dirname, resfiles.mk path, mk version var, mk source var, use pool file, dep files)
("locales", None, "resfiles.mk", "GENRB_CLDR_VERSION", "GENRB_SOURCE", True,
[]),
("curr", "curr", "resfiles.mk", "CURR_CLDR_VERSION", "CURR_SOURCE", True,
[]),
("lang", "lang", "resfiles.mk", "LANG_CLDR_VERSION", "LANG_SOURCE", True,
[]),
("region", "region", "resfiles.mk", "REGION_CLDR_VERSION", "REGION_SOURCE", True,
[]),
("zone", "zone", "resfiles.mk", "ZONE_CLDR_VERSION", "ZONE_SOURCE", True,
[]),
("unit", "unit", "resfiles.mk", "UNIT_CLDR_VERSION", "UNIT_SOURCE", True,
[]),
# TODO: We should not need timezoneTypes.res to build collation resource bundles.
# TODO: Maybe keyTypeData.res should be baked into the common library.
("coll", "coll", "colfiles.mk", "COLLATION_CLDR_VERSION", "COLLATION_SOURCE", False,
[OutFile("coll/ucadata.icu"), OutFile("timezoneTypes.res"), OutFile("keyTypeData.res")]),
("brkitr", "brkitr", "brkfiles.mk", "BRK_RES_CLDR_VERSION", "BRK_RES_SOURCE", False,
brkitr_brk_files + dict_files),
("rbnf", "rbnf", "rbnffiles.mk", "RBNF_CLDR_VERSION", "RBNF_SOURCE", False,
[]),
("translit", "translit", "trnsfiles.mk", None, "TRANSLIT_SOURCE", False,
[])
]
for sub_dir, out_sub_dir, resfile_name, version_var, source_var, use_pool_bundle, dep_files in specialized_sub_dirs:
out_prefix = "%s/" % out_sub_dir if out_sub_dir else ""
if config.has_feature(sub_dir):
# TODO: Clean this up for translit
if sub_dir == "translit":
input_files = [
InFile("translit/root.txt"),
InFile("translit/en.txt"),
InFile("translit/el.txt")
]
else:
input_files = [InFile(filename) for filename in glob("%s/*.txt" % sub_dir)]
input_basenames = [v.filename[len(sub_dir)+1:] for v in input_files]
output_files = [
OutFile("%s%s.res" % (out_prefix, v[:-4]))
for v in input_basenames
]
if use_pool_bundle:
input_pool_files = [OutFile("%spool.res" % out_prefix)]
use_pool_bundle_option = "--usePoolBundle {OUT_DIR}/{OUT_PREFIX}".format(
OUT_PREFIX = out_prefix,
**common_vars
)
requests += [
SingleExecutionRequest(
name = "%s_pool_write" % sub_dir,
input_files = dep_files + input_files,
output_files = input_pool_files,
tool = IcuTool("genrb"),
args = "-s {IN_DIR}/{IN_SUB_DIR} -d {OUT_DIR}/{OUT_PREFIX} -i {OUT_DIR} "
"--writePoolBundle -k "
"{INPUT_BASENAMES_SPACED}",
format_with = {
"IN_SUB_DIR": sub_dir,
"OUT_PREFIX": out_prefix,
"INPUT_BASENAMES_SPACED": " ".join(input_basenames)
}
),
]
else:
input_pool_files = []
use_pool_bundle_option = ""
requests += [
RepeatedOrSingleExecutionRequest(
name = "%s_res" % sub_dir,
dep_files = dep_files + input_pool_files,
input_files = input_files,
output_files = output_files,
tool = IcuTool("genrb"),
args = "-s {IN_DIR}/{IN_SUB_DIR} -d {OUT_DIR}/{OUT_PREFIX} -i {OUT_DIR} "
"{EXTRA_OPTION} -k "
"{INPUT_BASENAME}",
format_with = {
"IN_SUB_DIR": sub_dir,
"OUT_PREFIX": out_prefix,
"EXTRA_OPTION": use_pool_bundle_option
},
repeat_with = {
"INPUT_BASENAME": input_basenames,
},
flatten_with = {
"INPUT_BASENAME": " ".join(input_basenames)
}
)
]
# Generate index txt file
if sub_dir != "translit":
# TODO: Change .mk files to .py files so they can be loaded directly.
# Alternatively, figure out a way to require reading this file altogether.
# Right now, it is required for the index list file.
# Reading these files as .py will be required for Bazel.
mk_values = parse_makefile("{GLOB_DIR}/{IN_SUB_DIR}/{RESFILE_NAME}".format(
IN_SUB_DIR = sub_dir,
RESFILE_NAME = resfile_name,
**common_vars
))
cldr_version = mk_values[version_var] if version_var and sub_dir == "locales" else None
locales = [v[:-4] for v in mk_values[source_var].split()]
pkg_exclusions |= set(output_files) - set(OutFile("%s%s.res" % (out_prefix, locale)) for locale in locales)
index_file_txt = TmpFile("{IN_SUB_DIR}/{INDEX_NAME}.txt".format(
IN_SUB_DIR = sub_dir,
**common_vars
))
requests += [
PrintFileRequest(
name = "%s_index_txt" % sub_dir,
output_file = index_file_txt,
content = utils.generate_index_file(locales, cldr_version, common_vars)
)
]
# Generate index res file
index_res_file = OutFile("{OUT_PREFIX}{INDEX_NAME}.res".format(
OUT_PREFIX = out_prefix,
**common_vars
))
requests += [
SingleExecutionRequest(
name = "%s_index_res" % sub_dir,
input_files = [index_file_txt],
output_files = [index_res_file],
tool = IcuTool("genrb"),
args = "-s {TMP_DIR}/{IN_SUB_DIR} -d {OUT_DIR}/{OUT_PREFIX} -i {OUT_DIR} "
"-k "
"{INDEX_NAME}.txt",
format_with = {
"IN_SUB_DIR": sub_dir,
"OUT_PREFIX": out_prefix
}
)
]
# Finally, make the package.
all_output_files = list(sorted(utils.get_all_output_files(requests)))
icudata_list_file = TmpFile("icudata.lst")
requests += [
PrintFileRequest(
name = "icudata_list",
output_file = icudata_list_file,
content = "\n".join(file.filename for file in all_output_files)
),
VariableRequest(
name = "icudata_all_output_files",
input_files = all_output_files + [icudata_list_file]
)
]
return (build_dirs, requests)

520
icu4c/source/data/Makefile.in
View file

@ -112,6 +112,10 @@ COMINCDIR=$(top_srcdir)/common/unicode
SRCLISTDEPS=Makefile $(srcdir)/Makefile.in
BUILD_DIRS=$(OUTDIR) $(MAINBUILDDIR) $(BUILDDIR) $(CURRBLDDIR) $(LANGBLDDIR) $(REGIONBLDDIR) $(ZONEBLDDIR) $(UNITBLDDIR) $(BRKBLDDIR) $(COLBLDDIR) $(RBNFBLDDIR) $(TRANSLITBLDDIR) $(OUTTMPDIR) $(OUTTMPDIR_390STUB) $(OUTTMPDIR)/$(CURR_TREE) $(OUTTMPDIR)/$(LANG_TREE) $(OUTTMPDIR)/$(REGION_TREE) $(OUTTMPDIR)/$(ZONE_TREE) $(OUTTMPDIR)/$(UNIT_TREE) $(OUTTMPDIR)/$(COLLATION_TREE) $(OUTTMPDIR)/$(RBNF_TREE) $(OUTTMPDIR)/$(TRANSLIT_TREE) $(OUTTMPDIR)/$(BREAK_TREE)
# Variable names for rules.mk
OUT_DIR=$(BUILDDIR)
TMP_DIR=$(OUTTMPDIR)
# relative lib links from pkgdata are the same as for tmp
TOOLDIR=$(top_builddir)/tools
@ -240,128 +244,9 @@ package390: $(OUTTMPDIR)/icudata390.lst $(PKGDATA_LIST) ./icupkg.inc packagedata
cp $(ICUPKGDATA_OUTDIR)/$(LIB_ICUDATA_NAME)$(STUB_SUFFIX).$(SO) $(top_builddir)/stubdata/$(LIB_ICUDATA_NAME)$(STUB_SUFFIX).$(SO)
##### Define all the data files. the build rule that depends on them is below.
# X_FILES_SHORT = just the base names (for lists)
# X_FILES = full paths (for dependency)
## Include the main build rules for data files
include $(top_builddir)/$(subdir)/rules.mk
## DAT files - Misc. data files.
# 2005-may-05 Removed Unicode properties files (unorm.icu, uprops.icu, ucase.icu, ubidi.icu)
# from data build. See Jitterbug 4497. (makedata.mak revision 1.117)
# 2010-dec Removed pnames.icu.
# These are now hardcoded in ICU4C and only loaded in ICU4J.
#
DAT_FILES_SHORT=unames.icu cnvalias.icu coll/ucadata.icu nfkc.nrm nfkc_cf.nrm uts46.nrm
DAT_FILES=$(DAT_FILES_SHORT:%=$(BUILDDIR)/%)
## BRK files
BREAK_TREE=brkitr
-include $(BRKSRCDIR)/brkfiles.mk
-include $(BRKSRCDIR)/brklocal.mk
ALL_BRK_SOURCE= $(BRK_SOURCE) $(BRK_SOURCE_LOCAL)
BRK_FILES_SHORT=$(ALL_BRK_SOURCE:%.txt=$(BREAK_TREE)/%.brk)
BRK_FILES=$(ALL_BRK_SOURCE:%.txt=$(BRKBLDDIR)/%.brk)
ifdef BRK_DICT_SOURCE
ALL_DICT_SOURCE=$(BRK_DICT_SOURCE) $(BRK_DICT_SOURCE_LOCAL)
DICT_FILES_SHORT=$(ALL_DICT_SOURCE:%.txt=$(BREAK_TREE)/%.dict)
DICT_FILES=$(ALL_DICT_SOURCE:%.txt=$(BRKBLDDIR)/%.dict)
endif
ifdef BRK_RES_SOURCE
BRS_SRC= root.txt $(BRK_RES_SOURCE) $(BRK_RES_SOURCE_LOCAL)
BRS_SRC_FILES = $(BRS_SRC:%=$(BRKSRCDIR)/%)
INSTALLED_BRS_FILES = $(BRK_RES_SOURCE:%.txt=%) $(BRK_RES_SOURCE_LOCAL:%.txt=%)
endif
## Confusables (Spoofing) files
ALL_CFU_SOURCE=$(UNICODEDATADIR)/confusables.txt $(UNICODEDATADIR)/confusablesWholeScript.txt
CFU_FILES_SHORT=confusables.cfu
CFU_FILES=$(BUILDDIR)/$(CFU_FILES_SHORT)
## UCM files
-include $(UCMSRCDIR)/ucmcore.mk
-include $(UCMSRCDIR)/ucmfiles.mk
-include $(UCMSRCDIR)/ucmebcdic.mk
-include $(UCMSRCDIR)/ucmlocal.mk
ALL_UCM_SOURCE=ibm-37_P100-1995.ucm ibm-1047_P100-1995.ucm $(UCM_SOURCE_CORE) $(UCM_SOURCE_FILES) $(UCM_SOURCE_EBCDIC) $(UCM_SOURCE_LOCAL)
UCM_FILES = $(ALL_UCM_SOURCE:%=$(SRCDATADIR)/%)
CNV_FILES = $(ALL_UCM_SOURCE:%.ucm=$(BUILDDIR)/%.cnv)
CNV_FILES_SHORT = $(ALL_UCM_SOURCE:%.ucm=%.cnv)
UCM_SOURCE_SPECIAL=$(UCM_SOURCE_EBCDIC_IGNORE_SISO) $(UCM_SOURCE_EBCDIC_IGNORE_SISO_LOCAL)
UCM_FILES_SPECIAL=$(UCM_SOURCE_SPECIAL:%=$(UCMSRCDIR)/%)
CNV_FILES_SPECIAL=$(UCM_SOURCE_SPECIAL:%.ucm=$(BUILDDIR)/%.cnv)
CNV_FILES_SHORT_SPECIAL=$(UCM_SOURCE_SPECIAL:%.ucm=%.cnv)
## RES files
-include $(LOCSRCDIR)/resfiles.mk
-include $(CURRSRCDIR)/resfiles.mk
-include $(LANGSRCDIR)/resfiles.mk
-include $(REGIONSRCDIR)/resfiles.mk
-include $(ZONESRCDIR)/resfiles.mk
-include $(UNITSRCDIR)/resfiles.mk
-include $(COLSRCDIR)/colfiles.mk
-include $(RBNFSRCDIR)/rbnffiles.mk
-include $(TRANSLITSRCDIR)/trnsfiles.mk
-include $(LOCSRCDIR)/reslocal.mk
-include $(CURRSRCDIR)/reslocal.mk
-include $(LANGSRCDIR)/reslocal.mk
-include $(REGIONSRCDIR)/reslocal.mk
-include $(ZONESRCDIR)/reslocal.mk
-include $(UNITSRCDIR)/reslocal.mk
-include $(COLSRCDIR)/collocal.mk
-include $(BRKSRCDIR)/brslocal.mk
-include $(RBNFSRCDIR)/rbnflocal.mk
-include $(TRANSLITSRCDIR)/trnslocal.mk
ifdef GENRB_SOURCE
RES_SRC= root.txt $(GENRB_SOURCE) $(GENRB_ALIAS_SOURCE) $(GENRB_SOURCE_LOCAL) $(GENRB_ALIAS_SOURCE_LOCAL)
RES_SRC_FILES = $(RES_SRC:%=$(LOCSRCDIR)/%)
INSTALLED_RB_FILES = $(GENRB_SOURCE:%.txt=%) $(GENRB_SOURCE_LOCAL:%.txt=%)
endif
ifdef CURR_SOURCE
CURR_SRC= root.txt supplementalData.txt $(CURR_SOURCE) $(CURR_ALIAS_SOURCE) $(CURR_SOURCE_LOCAL)
CURR_SRC_FILES = $(CURR_SRC:%=$(CURRSRCDIR)/%)
INSTALLED_CURR_FILES = $(CURR_SOURCE:%.txt=%) $(CURR_SOURCE_LOCAL:%.txt=%)
endif
ifdef LANG_SOURCE
LANG_SRC= root.txt $(LANG_SOURCE) $(LANG_ALIAS_SOURCE) $(LANG_SOURCE_LOCAL)
LANG_SRC_FILES = $(LANG_SRC:%=$(LANGSRCDIR)/%)
INSTALLED_LANG_FILES = $(LANG_SOURCE:%.txt=%) $(LANG_SOURCE_LOCAL:%.txt=%)
endif
ifdef REGION_SOURCE
REGION_SRC= root.txt $(REGION_SOURCE) $(REGION_ALIAS_SOURCE) $(REGION_SOURCE_LOCAL)
REGION_SRC_FILES = $(REGION_SRC:%=$(REGIONSRCDIR)/%)
INSTALLED_REGION_FILES = $(REGION_SOURCE:%.txt=%) $(REGION_SOURCE_LOCAL:%.txt=%)
endif
ifdef ZONE_SOURCE
ZONE_SRC= root.txt $(ZONE_SOURCE) $(ZONE_ALIAS_SOURCE) $(ZONE_SOURCE_LOCAL) tzdbNames.txt
ZONE_SRC_FILES = $(ZONE_SRC:%=$(ZONESRCDIR)/%)
INSTALLED_ZONE_FILES = $(ZONE_SOURCE:%.txt=%) $(ZONE_SOURCE_LOCAL:%.txt=%)
endif
ifdef UNIT_SOURCE
UNIT_SRC= root.txt $(UNIT_SOURCE) $(UNIT_ALIAS_SOURCE) $(UNIT_SOURCE_LOCAL)
UNIT_SRC_FILES = $(UNIT_SRC:%=$(UNITSRCDIR)/%)
INSTALLED_UNIT_FILES = $(UNIT_SOURCE:%.txt=%) $(UNIT_SOURCE_LOCAL:%.txt=%)
endif
ifdef COLLATION_SOURCE
COL_SRC= root.txt $(COLLATION_SOURCE) $(COLLATION_ALIAS_SOURCE) $(COLLATION_SOURCE_LOCAL)
COL_SRC_FILES = $(COL_SRC:%=$(COLSRCDIR)/%)
INSTALLED_COL_FILES = $(COLLATION_SOURCE:%.txt=%) $(COLLATION_SOURCE_LOCAL:%.txt=%)
endif
ifdef RBNF_SOURCE
RBNF_SRC= root.txt $(RBNF_SOURCE) $(RBNF_ALIAS_SOURCE) $(RBNF_SOURCE_LOCAL)
RBNF_SRC_FILES = $(RBNF_SRC:%=$(RBNFSRCDIR)/%)
INSTALLED_RBNF_FILES = $(RBNF_SOURCE:%.txt=%) $(RBNF_SOURCE_LOCAL:%.txt=%)
endif
ifdef TRANSLIT_SOURCE
TRANSLIT_SRC= $(TRANSLIT_SOURCE) $(TRANSLIT_ALIAS_SOURCE) $(TRANSLIT_SOURCE_LOCAL)
TRANSLIT_SRC_FILES = $(TRANSLIT_SRC:%=$(TRANSLITSRCDIR)/%)
INSTALLED_TRANSLIT_FILES = $(TRANSLIT_SOURCE:%.txt=%) $(TRANSLIT_SOURCE_LOCAL:%.txt=%)
endif
GENRBOPTS=-k
## MISC files
-include $(MISCSRCDIR)/miscfiles.mk
-include $(MISCSRCDIR)/misclocal.mk
MSC_SOURCE= $(MISC_SOURCE) $(MISC_SOURCE_LOCAL)
MSC_SRC_FILES=$(MSC_SOURCE:%=$(MISCSRCDIR)/%)
ifeq ($(ENABLE_SO_VERSION_DATA),1)
ifeq ($(PKGDATA_MODE),dll)
@ -375,72 +260,21 @@ endif
endif
endif
INDEX_NAME=res_index
INDEX_FILE=$(OUTTMPDIR)/$(INDEX_NAME).txt
PKGDATA_LIST = $(TMP_DIR)/icudata.lst
ALL_RES_SRC= $(RES_SRC) $(TRNS_SOURCE) $(MSC_SOURCE)
RES_FILES = $(ALL_RES_SRC:%.txt=$(BUILDDIR)/%.res) $(BUILDDIR)/$(INDEX_NAME).res $(BUILDDIR)/pool.res
RES_FILES_SHORT = $(ALL_RES_SRC:%.txt=%.res) $(INDEX_NAME).res pool.res
PKGDATA_LIST = $(OUTTMPDIR)/icudata.lst
CURR_TREE=curr
CURR_INDEX_FILE=$(OUTTMPDIR)/$(CURR_TREE)/$(INDEX_NAME).txt
CURR_FILES = $(CURR_SRC:%.txt=$(CURRBLDDIR)/%.res) $(CURRBLDDIR)/$(INDEX_NAME).res $(CURRBLDDIR)/pool.res
CURR_FILES_SHORT = $(CURR_SRC:%.txt=$(CURR_TREE)/%.res) $(CURR_TREE)/$(INDEX_NAME).res $(CURR_TREE)/pool.res
##### Define all the data files. the build rule that depends on them is below.
# X_FILES_SHORT = just the base names (for lists)
# X_FILES = full paths (for dependency)
LANG_TREE=lang
LANG_INDEX_FILE=$(OUTTMPDIR)/$(LANG_TREE)/$(INDEX_NAME).txt
LANG_FILES = $(LANG_SRC:%.txt=$(LANGBLDDIR)/%.res) $(LANGBLDDIR)/$(INDEX_NAME).res $(LANGBLDDIR)/pool.res
LANG_FILES_SHORT = $(LANG_SRC:%.txt=$(LANG_TREE)/%.res) $(LANG_TREE)/$(INDEX_NAME).res $(LANG_TREE)/pool.res
REGION_TREE=region
REGION_INDEX_FILE=$(OUTTMPDIR)/$(REGION_TREE)/$(INDEX_NAME).txt
REGION_FILES = $(REGION_SRC:%.txt=$(REGIONBLDDIR)/%.res) $(REGIONBLDDIR)/$(INDEX_NAME).res $(REGIONBLDDIR)/pool.res
REGION_FILES_SHORT = $(REGION_SRC:%.txt=$(REGION_TREE)/%.res) $(REGION_TREE)/$(INDEX_NAME).res $(REGION_TREE)/pool.res
ZONE_TREE=zone
ZONE_INDEX_FILE=$(OUTTMPDIR)/$(ZONE_TREE)/$(INDEX_NAME).txt
ZONE_FILES = $(ZONE_SRC:%.txt=$(ZONEBLDDIR)/%.res) $(ZONEBLDDIR)/$(INDEX_NAME).res $(ZONEBLDDIR)/pool.res
ZONE_FILES_SHORT = $(ZONE_SRC:%.txt=$(ZONE_TREE)/%.res) $(ZONE_TREE)/$(INDEX_NAME).res $(ZONE_TREE)/pool.res
UNIT_TREE=unit
UNIT_INDEX_FILE=$(OUTTMPDIR)/$(UNIT_TREE)/$(INDEX_NAME).txt
UNIT_FILES = $(UNIT_SRC:%.txt=$(UNITBLDDIR)/%.res) $(UNITBLDDIR)/$(INDEX_NAME).res $(UNITBLDDIR)/pool.res
UNIT_FILES_SHORT = $(UNIT_SRC:%.txt=$(UNIT_TREE)/%.res) $(UNIT_TREE)/$(INDEX_NAME).res $(UNIT_TREE)/pool.res
COLLATION_TREE=coll
COLLATION_INDEX_FILE=$(OUTTMPDIR)/$(COLLATION_TREE)/$(INDEX_NAME).txt
COLLATION_INDEX_RES=$(COLBLDDIR)/$(INDEX_NAME).res
COLLATION_INDEX_RES_SHORT=$(COLLATION_TREE)/$(INDEX_NAME).res
COLLATION_FILES = $(COL_SRC:%.txt=$(COLBLDDIR)/%.res) $(COLLATION_INDEX_RES)
COLLATION_FILES_SHORT = $(COL_SRC:%.txt=$(COLLATION_TREE)/%.res)
BRK_RES_INDEX_FILE=$(OUTTMPDIR)/$(BREAK_TREE)/$(INDEX_NAME).txt
BRK_RES_INDEX_RES=$(BRKBLDDIR)/$(INDEX_NAME).res
BRK_RES_INDEX_RES_SHORT=$(BREAK_TREE)/$(INDEX_NAME).res
BRK_RES_FILES = $(BRS_SRC:%.txt=$(BRKBLDDIR)/%.res) $(BRK_RES_INDEX_RES)
BRK_RES_FILES_SHORT = $(BRS_SRC:%.txt=$(BREAK_TREE)/%.res)
RBNF_TREE=rbnf
RBNF_INDEX_FILE=$(OUTTMPDIR)/$(RBNF_TREE)/$(INDEX_NAME).txt
RBNF_INDEX_RES=$(RBNFBLDDIR)/$(INDEX_NAME).res
RBNF_INDEX_RES_SHORT=$(RBNF_TREE)/$(INDEX_NAME).res
RBNF_FILES = $(RBNF_SRC:%.txt=$(RBNFBLDDIR)/%.res) $(RBNF_INDEX_RES)
RBNF_FILES_SHORT = $(RBNF_SRC:%.txt=$(RBNF_TREE)/%.res)
TRANSLIT_TREE=translit
#TRANSLIT_INDEX_FILE=$(OUTTMPDIR)/$(TRANSLIT_TREE)/$(INDEX_NAME).txt
#TRANSLIT_INDEX_RES=$(TRANSLITBLDDIR)/$(INDEX_NAME).res
#TRANSLIT_INDEX_RES_SHORT=$(TRANSLIT_TREE)/$(INDEX_NAME).res
TRANSLIT_FILES = $(TRANSLIT_SRC:%.txt=$(TRANSLITBLDDIR)/%.res)
TRANSLIT_FILES_SHORT = $(TRANSLIT_SRC:%.txt=$(TRANSLIT_TREE)/%.res)
## SPP files
-include $(SPREPSRCDIR)/sprepfiles.mk
-include $(SPREPSRCDIR)/spreplocal.mk
ALL_SPREP_SOURCE=$(SPREP_SOURCE) $(SPREP_SOURCE_LOCAL)
SPREP_FILES = $(ALL_SPREP_SOURCE:%.txt=$(BUILDDIR)/%.spp)
SPREP_FILES_SHORT = $(ALL_SPREP_SOURCE:%.txt=%.spp)
## DAT files - Misc. data files.
# 2005-may-05 Removed Unicode properties files (unorm.icu, uprops.icu, ucase.icu, ubidi.icu)
# from data build. See Jitterbug 4497. (makedata.mak revision 1.117)
# 2010-dec Removed pnames.icu.
# These are now hardcoded in ICU4C and only loaded in ICU4J.
#
DAT_FILES_SHORT=unames.icu cnvalias.icu coll/ucadata.icu nfkc.nrm nfkc_cf.nrm uts46.nrm
DAT_FILES=$(DAT_FILES_SHORT:%=$(BUILDDIR)/%)
## All generated files
ALL_FILES = $(DAT_FILES) $(CNV_FILES) $(CNV_FILES_SPECIAL) $(BRK_FILES) $(DICT_FILES) $(RES_FILES) $(INDEX_RES_FILE) $(CURR_FILES) $(LANG_FILES) $(REGION_FILES) $(ZONE_FILES) $(UNIT_FILES) $(COLLATION_FILES) $(BRK_RES_FILES) $(RBNF_FILES) $(TRANSLIT_FILES) $(SPREP_FILES) $(CFU_FILES)
@ -466,23 +300,8 @@ endif
CLEANFILES = *~ icupkg.inc *.x
ifeq ($(ICUDATA_SOURCE_ARCHIVE),)
build-local: build-dir $(SO_VERSION_DATA) $(ALL_FILES) $(PKGDATA_LIST) $(OS390LIST)
build-local: build-dir $(SO_VERSION_DATA) $(ICUDATA_ALL_OUTPUT_FILES) $(PKGDATA_LIST) $(OS390LIST)
echo timestamp > $@
$(PKGDATA_LIST): $(SRCLISTDEPS)
@echo "generating $@ (list of data files)"
@-$(RMV) $@
@for file in $(COLL_FILES_LIST); do \
echo $$file >> $@; \
done;
@for file in $(BRK_FILES_LIST); do \
echo $$file >> $@; \
done;
@for file in $(LOCALE_FILES_LIST); do \
echo $$file >> $@; \
done;
@for file in $(MISC_FILES_LIST); do \
echo $$file >> $@; \
done;
else
build-local: build-dir $(SO_VERSION_DATA) $(PKGDATA_LIST) $(OS390LIST)
echo timestamp > $@
@ -517,279 +336,12 @@ ifneq ($(filter order-only,$(.FEATURES)),)
$(ALL_FILES) $(ALL_INDEX_SRC_FILES): | build-dir
endif
# Now, sections for building each kind of data.
#################################################### DAT
# DAT FILES
# cnvalias.icu
$(BUILDDIR)/cnvalias.icu: $(UCMSRCDIR)/convrtrs.txt $(TOOLBINDIR)/gencnval$(TOOLEXEEXT)
$(INVOKE) $(TOOLBINDIR)/gencnval -d $(BUILDDIR) $(UCMSRCDIR)/convrtrs.txt
# Targets for prebuilt Unicode data
$(BUILDDIR)/%.icu: $(SRCDATADIR)/in/%.icu
$(INVOKE) $(TOOLBINDIR)/icupkg -t$(ICUDATA_CHAR) $< $@
$(BUILDDIR)/%.nrm: $(SRCDATADIR)/in/%.nrm
$(INVOKE) $(TOOLBINDIR)/icupkg -t$(ICUDATA_CHAR) $< $@
$(BUILDDIR)/coll/ucadata.icu: $(SRCDATADIR)/in/coll/ucadata-unihan.icu
$(INVOKE) $(TOOLBINDIR)/icupkg -t$(ICUDATA_CHAR) $< $@
#################################################### SPP
# SPP FILES
$(BUILDDIR)/%.spp: $(SPREPSRCDIR)/%.txt $(TOOLBINDIR)/gensprep$(TOOLEXEEXT) $(BUILDDIR)/unames.icu
$(INVOKE) $(TOOLBINDIR)/gensprep -d $(BUILDDIR) -i $(BUILDDIR) -s $(SPREPSRCDIR) -b $(@F:%.spp=%) -m $(UNICODEDATADIR) -u 3.2.0 $(<F)
#################################################### BRK
# BRK FILES
$(BRKBLDDIR)/%.brk: $(BRKRULESRCDIR)/%.txt $(TOOLBINDIR)/genbrk$(TOOLEXEEXT) $(DAT_FILES)
$(INVOKE) $(TOOLBINDIR)/genbrk -c -i $(BUILDDIR) -r $< -o $@
#################################################### DICT
# DICT FILES
# .dict file generated regardless of whether dictionary file exists
$(BRKBLDDIR)/%.dict: $(TOOLBINDIR)/gendict$(TOOLEXEEXT) $(DAT_FILES)
$(INVOKE) $(TOOLBINDIR)/gendict --uchars -c -i $(BUILDDIR) $(DICTSRCDIR)/$(*F).txt $@
$(BRKBLDDIR)/thaidict.dict: $(TOOLBINDIR)/gendict$(TOOLEXEEXT) $(DAT_FILES)
$(INVOKE) $(TOOLBINDIR)/gendict --bytes --transform offset-0x0e00 -c -i $(BUILDDIR) $(DICTSRCDIR)/thaidict.txt $(BRKBLDDIR)/thaidict.dict
$(BRKBLDDIR)/laodict.dict: $(TOOLBINDIR)/gendict$(TOOLEXEEXT) $(DAT_FILES)
$(INVOKE) $(TOOLBINDIR)/gendict --bytes --transform offset-0x0e80 -c -i $(BUILDDIR) $(DICTSRCDIR)/laodict.txt $(BRKBLDDIR)/laodict.dict
$(BRKBLDDIR)/burmesedict.dict: $(TOOLBINDIR)/gendict$(TOOLEXEEXT) $(DAT_FILES)
$(INVOKE) $(TOOLBINDIR)/gendict --bytes --transform offset-0x1000 -c -i $(BUILDDIR) $(DICTSRCDIR)/burmesedict.txt $(BRKBLDDIR)/burmesedict.dict
# TODO: figure out why combining characters are here?
$(BRKBLDDIR)/khmerdict.dict: $(TOOLBINDIR)/gendict$(TOOLEXEEXT) $(DAT_FILES)
$(INVOKE) $(TOOLBINDIR)/gendict --bytes --transform offset-0x1780 -c -i $(BUILDDIR) $(DICTSRCDIR)/khmerdict.txt $(BRKBLDDIR)/khmerdict.dict
#################################################### CFU
# CFU FILES
# Note: gencfu requires two input files to produce a single output file.
# There will be exactly one target file and two source files.
# The $(word n, ...) selects the nth word from the following stuff.
# There must be a nicer way to do this.
$(CFU_FILES): $(ALL_CFU_SOURCE) $(TOOLBINDIR)/gencfu$(TOOLEXEEXT) $(DAT_FILES)
$(INVOKE) echo ALL_CFU_SOURCE: $(ALL_CFU_SOURCE)
$(INVOKE) echo CFU_FILES: $(CFU_FILES)
$(INVOKE) echo CFU_FILES_SHORT: $(CFU_FILES_SHORT)
$(INVOKE) $(TOOLBINDIR)/gencfu -c -i $(BUILDDIR) -r $(word 1,$(ALL_CFU_SOURCE)) -w $(word 2,$(ALL_CFU_SOURCE)) -o $@
#################################################### CNV
# CNV FILES
$(CNV_FILES_SPECIAL) : $(UCM_FILES_SPECIAL) $(TOOLBINDIR)/makeconv$(TOOLEXEEXT)
$(INVOKE) $(TOOLBINDIR)/makeconv --ignore-siso-check -c -d $(BUILDDIR) $(UCMSRCDIR)/$(@F:%.cnv=%.ucm)
$(BUILDDIR)/%.cnv: $(UCMSRCDIR)/%.ucm $(TOOLBINDIR)/makeconv$(TOOLEXEEXT)
$(INVOKE) $(TOOLBINDIR)/makeconv -c -d $(BUILDDIR) $(UCMSRCDIR)/$(<F)
#################################################### RES
# RES FILES
### curr res
$(CURRBLDDIR)/%.res: $(CURRSRCDIR)/%.txt $(TOOLBINDIR)/genrb$(TOOLEXEEXT) $(DAT_FILES) | $(BUILDDIR)/cnvalias.icu
$(INVOKE) $(TOOLBINDIR)/genrb --usePoolBundle $(GENRBOPTS) -i $(BUILDDIR) -s $(CURRSRCDIR) -d $(CURRBLDDIR) $(<F)
# copy the curr/pool.res file from the source folder to the build output folder
# and swap it to native endianness
$(CURRBLDDIR)/pool.res: $(CURRSRCDIR)/pool.res
$(INVOKE) $(TOOLBINDIR)/icupkg -t$(ICUDATA_CHAR) $(CURRSRCDIR)/pool.res $(CURRBLDDIR)/pool.res
$(CURRBLDDIR)/$(INDEX_NAME).res: $(OUTTMPDIR)/$(CURR_TREE)/$(INDEX_NAME).txt $(TOOLBINDIR)/genrb$(TOOLEXEEXT) | $(BUILDDIR)/cnvalias.icu
$(INVOKE) $(TOOLBINDIR)/genrb $(GENRBOPTS) -i $(BUILDDIR) -s $(OUTTMPDIR)/$(CURR_TREE) -d $(CURRBLDDIR) $(INDEX_NAME).txt
$(CURR_INDEX_FILE): $(SRCLISTDEPS)
@echo "generating $@ (list of installed currency name locales)"; \
$(RMV) $@; \
echo "// Warning this file is automatically generated" > $@; \
echo "$(INDEX_NAME):table(nofallback) {" >> $@; \
echo " InstalledLocales {" >> $@; \
for file in $(INSTALLED_CURR_FILES); do \
echo " $$file {\"\"}" >> $@; \
done; \
echo " }" >> $@; \
echo "}" >> $@;
### lang res
$(LANGBLDDIR)/%.res: $(LANGSRCDIR)/%.txt $(TOOLBINDIR)/genrb$(TOOLEXEEXT) $(DAT_FILES) | $(BUILDDIR)/cnvalias.icu
$(INVOKE) $(TOOLBINDIR)/genrb --usePoolBundle $(GENRBOPTS) -i $(BUILDDIR) -s $(LANGSRCDIR) -d $(LANGBLDDIR) $(<F)
# copy the lang/pool.res file from the source folder to the build output folder
# and swap it to native endianness
$(LANGBLDDIR)/pool.res: $(LANGSRCDIR)/pool.res
$(INVOKE) $(TOOLBINDIR)/icupkg -t$(ICUDATA_CHAR) $(LANGSRCDIR)/pool.res $(LANGBLDDIR)/pool.res
$(LANGBLDDIR)/$(INDEX_NAME).res: $(OUTTMPDIR)/$(LANG_TREE)/$(INDEX_NAME).txt $(TOOLBINDIR)/genrb$(TOOLEXEEXT) | $(BUILDDIR)/cnvalias.icu
$(INVOKE) $(TOOLBINDIR)/genrb $(GENRBOPTS) -i $(BUILDDIR) -s $(OUTTMPDIR)/$(LANG_TREE) -d $(LANGBLDDIR) $(INDEX_NAME).txt
$(LANG_INDEX_FILE): $(SRCLISTDEPS)
@echo "generating $@ (list of installed language name locales)"; \
$(RMV) $@; \
echo "// Warning this file is automatically generated" > $@; \
echo "$(INDEX_NAME):table(nofallback) {" >> $@; \
echo " InstalledLocales {" >> $@; \
for file in $(INSTALLED_LANG_FILES); do \
echo " $$file {\"\"}" >> $@; \
done; \
echo " }" >> $@; \
echo "}" >> $@;
### region res
$(REGIONBLDDIR)/%.res: $(REGIONSRCDIR)/%.txt $(TOOLBINDIR)/genrb$(TOOLEXEEXT) $(DAT_FILES) | $(BUILDDIR)/cnvalias.icu
$(INVOKE) $(TOOLBINDIR)/genrb --usePoolBundle $(GENRBOPTS) -i $(BUILDDIR) -s $(REGIONSRCDIR) -d $(REGIONBLDDIR) $(<F)
# copy the region/pool.res file from the source folder to the build output folder
# and swap it to native endianness
$(REGIONBLDDIR)/pool.res: $(REGIONSRCDIR)/pool.res
$(INVOKE) $(TOOLBINDIR)/icupkg -t$(ICUDATA_CHAR) $(REGIONSRCDIR)/pool.res $(REGIONBLDDIR)/pool.res
$(REGIONBLDDIR)/$(INDEX_NAME).res: $(OUTTMPDIR)/$(REGION_TREE)/$(INDEX_NAME).txt $(TOOLBINDIR)/genrb$(TOOLEXEEXT) | $(BUILDDIR)/cnvalias.icu
$(INVOKE) $(TOOLBINDIR)/genrb $(GENRBOPTS) -i $(BUILDDIR) -s $(OUTTMPDIR)/$(REGION_TREE) -d $(REGIONBLDDIR) $(INDEX_NAME).txt
$(REGION_INDEX_FILE): $(SRCLISTDEPS)
@echo "generating $@ (list of installed region name locales)"; \
$(RMV) $@; \
echo "// Warning this file is automatically generated" > $@; \
echo "$(INDEX_NAME):table(nofallback) {" >> $@; \
echo " InstalledLocales {" >> $@; \
for file in $(INSTALLED_REGION_FILES); do \
echo " $$file {\"\"}" >> $@; \
done; \
echo " }" >> $@; \
echo "}" >> $@;
### zone res
$(ZONEBLDDIR)/%.res: $(ZONESRCDIR)/%.txt $(TOOLBINDIR)/genrb$(TOOLEXEEXT) $(DAT_FILES) | $(BUILDDIR)/cnvalias.icu
$(INVOKE) $(TOOLBINDIR)/genrb --usePoolBundle $(GENRBOPTS) -i $(BUILDDIR) -s $(ZONESRCDIR) -d $(ZONEBLDDIR) $(<F)
# copy the zone/pool.res file from the source folder to the build output folder
# and swap it to native endianness
$(ZONEBLDDIR)/pool.res: $(ZONESRCDIR)/pool.res
$(INVOKE) $(TOOLBINDIR)/icupkg -t$(ICUDATA_CHAR) $(ZONESRCDIR)/pool.res $(ZONEBLDDIR)/pool.res
$(ZONEBLDDIR)/$(INDEX_NAME).res: $(OUTTMPDIR)/$(ZONE_TREE)/$(INDEX_NAME).txt $(TOOLBINDIR)/genrb$(TOOLEXEEXT) | $(BUILDDIR)/cnvalias.icu
$(INVOKE) $(TOOLBINDIR)/genrb $(GENRBOPTS) -i $(BUILDDIR) -s $(OUTTMPDIR)/$(ZONE_TREE) -d $(ZONEBLDDIR) $(INDEX_NAME).txt
$(ZONE_INDEX_FILE): $(SRCLISTDEPS)
@echo "generating $@ (list of installed time zone name locales)"; \
$(RMV) $@; \
echo "// Warning this file is automatically generated" > $@; \
echo "$(INDEX_NAME):table(nofallback) {" >> $@; \
echo " InstalledLocales {" >> $@; \
for file in $(INSTALLED_ZONE_FILES); do \
echo " $$file {\"\"}" >> $@; \
done; \
echo " }" >> $@; \
echo "}" >> $@;
### unit res
$(UNITBLDDIR)/%.res: $(UNITSRCDIR)/%.txt $(TOOLBINDIR)/genrb$(TOOLEXEEXT) $(DAT_FILES)
$(INVOKE) $(TOOLBINDIR)/genrb --usePoolBundle $(GENRBOPTS) -i $(BUILDDIR) -s $(UNITSRCDIR) -d $(UNITBLDDIR) $(<F)
# copy the unit/pool.res file from the source folder to the build output folder
# and swap it to native endianness
$(UNITBLDDIR)/pool.res: $(UNITSRCDIR)/pool.res
$(INVOKE) $(TOOLBINDIR)/icupkg -t$(ICUDATA_CHAR) $(UNITSRCDIR)/pool.res $(UNITBLDDIR)/pool.res
$(UNITBLDDIR)/$(INDEX_NAME).res: $(OUTTMPDIR)/$(UNIT_TREE)/$(INDEX_NAME).txt $(TOOLBINDIR)/genrb$(TOOLEXEEXT)
$(INVOKE) $(TOOLBINDIR)/genrb $(GENRBOPTS) -i $(BUILDDIR) -s $(OUTTMPDIR)/$(UNIT_TREE) -d $(UNITBLDDIR) $(INDEX_NAME).txt
$(UNIT_INDEX_FILE): $(SRCLISTDEPS)
@echo "generating $@ (list of installed time zone name locales)"; \
$(RMV) $@; \
echo "// Warning this file is automatically generated" > $@; \
echo "$(INDEX_NAME):table(nofallback) {" >> $@; \
echo " InstalledLocales {" >> $@; \
for file in $(INSTALLED_UNIT_FILES); do \
echo " $$file {\"\"}" >> $@; \
done; \
echo " }" >> $@; \
echo "}" >> $@;
### collation res
$(COLBLDDIR)/%.res: $(COLSRCDIR)/%.txt $(TOOLBINDIR)/genrb$(TOOLEXEEXT) $(DAT_FILES) | $(BUILDDIR)/cnvalias.icu
$(INVOKE) $(TOOLBINDIR)/genrb $(GENRBOPTS) -i $(BUILDDIR) -s $(COLSRCDIR) -d $(COLBLDDIR) $(<F)
$(COLBLDDIR)/$(INDEX_NAME).res: $(OUTTMPDIR)/$(COLLATION_TREE)/$(INDEX_NAME).txt $(TOOLBINDIR)/genrb$(TOOLEXEEXT) | $(BUILDDIR)/cnvalias.icu
$(INVOKE) $(TOOLBINDIR)/genrb $(GENRBOPTS) -i $(BUILDDIR) -s $(OUTTMPDIR)/$(COLLATION_TREE) -d $(COLBLDDIR) $(INDEX_NAME).txt
$(COLLATION_INDEX_FILE): $(SRCLISTDEPS)
@echo "generating $@ (list of installed collation locales)"; \
$(RMV) $@; \
echo "// Warning this file is automatically generated" > $@; \
echo "$(INDEX_NAME):table(nofallback) {" >> $@; \
echo " InstalledLocales {" >> $@; \
for file in $(INSTALLED_COL_FILES); do \
echo " $$file {\"\"}" >> $@; \
done; \
echo " }" >> $@; \
echo "}" >> $@;
### brk res
$(BRKBLDDIR)/%.res: $(BRKSRCDIR)/%.txt $(TOOLBINDIR)/genrb$(TOOLEXEEXT) $(BRK_FILES) $(DICT_FILES) $(DAT_FILES) | $(BUILDDIR)/cnvalias.icu
$(INVOKE) $(TOOLBINDIR)/genrb $(GENRBOPTS) -i $(BUILDDIR) -s $(BRKSRCDIR) -d $(BRKBLDDIR) $(<F)
$(BRKBLDDIR)/$(INDEX_NAME).res: $(OUTTMPDIR)/$(BREAK_TREE)/$(INDEX_NAME).txt $(TOOLBINDIR)/genrb$(TOOLEXEEXT) | $(BUILDDIR)/cnvalias.icu
$(INVOKE) $(TOOLBINDIR)/genrb $(GENRBOPTS) -i $(BUILDDIR) -s $(OUTTMPDIR)/$(BREAK_TREE) -d $(BRKBLDDIR) $(INDEX_NAME).txt
$(BRK_RES_INDEX_FILE): $(SRCLISTDEPS)
@echo "generating $@ (list of installed break locales)"; \
$(RMV) $@; \
echo "// Warning this file is automatically generated" > $@; \
echo "$(INDEX_NAME):table(nofallback) {" >> $@; \
echo " InstalledLocales {" >> $@; \
for file in $(INSTALLED_BRS_FILES); do \
echo " $$file {\"\"}" >> $@; \
done; \
echo " }" >> $@; \
echo "}" >> $@;
### RBNF res
$(RBNFBLDDIR)/%.res: $(RBNFSRCDIR)/%.txt $(TOOLBINDIR)/genrb$(TOOLEXEEXT) $(DAT_FILES) | $(BUILDDIR)/cnvalias.icu
$(INVOKE) $(TOOLBINDIR)/genrb $(GENRBOPTS) -i $(BUILDDIR) -s $(RBNFSRCDIR) -d $(RBNFBLDDIR) $(<F)
$(RBNFBLDDIR)/$(INDEX_NAME).res: $(OUTTMPDIR)/$(RBNF_TREE)/$(INDEX_NAME).txt $(TOOLBINDIR)/genrb$(TOOLEXEEXT) | $(BUILDDIR)/cnvalias.icu
$(INVOKE) $(TOOLBINDIR)/genrb $(GENRBOPTS) -i $(BUILDDIR) -s $(OUTTMPDIR)/$(RBNF_TREE) -d $(RBNFBLDDIR) $(INDEX_NAME).txt
$(RBNF_INDEX_FILE): $(SRCLISTDEPS)
@echo "generating $@ (list of installed RBNF locales)"; \
$(RMV) $@; \
echo "// Warning this file is automatically generated" > $@; \
echo "$(INDEX_NAME):table(nofallback) {" >> $@; \
echo " InstalledLocales {" >> $@; \
for file in $(INSTALLED_RBNF_FILES); do \
echo " $$file {\"\"}" >> $@; \
done; \
echo " }" >> $@; \
echo "}" >> $@;
### TRANSLIT res
$(TRANSLITBLDDIR)/%.res: $(TRANSLITSRCDIR)/%.txt $(TOOLBINDIR)/genrb$(TOOLEXEEXT) $(DAT_FILES) | $(BUILDDIR)/cnvalias.icu
$(INVOKE) $(TOOLBINDIR)/genrb $(GENRBOPTS) -i $(BUILDDIR) -s $(TRANSLITSRCDIR) -d $(TRANSLITBLDDIR) $(<F)
### normal (locale) res
all-RES: $(RES_FILES)
$(BUILDDIR)/%.res: $(LOCSRCDIR)/%.txt $(TOOLBINDIR)/genrb$(TOOLEXEEXT) $(DAT_FILES) | $(BUILDDIR)/cnvalias.icu
$(INVOKE) $(TOOLBINDIR)/genrb --usePoolBundle $(GENRBOPTS) -i $(BUILDDIR) -s $(LOCSRCDIR) -d $(BUILDDIR) $(<F)
# copy the locales/pool.res file from the source folder to the build output folder
# and swap it to native endianness
$(BUILDDIR)/pool.res: $(LOCSRCDIR)/pool.res
$(INVOKE) $(TOOLBINDIR)/icupkg -t$(ICUDATA_CHAR) $(LOCSRCDIR)/pool.res $(BUILDDIR)/pool.res
# if the tzcode directory contains a new tzdata*.tar.gz file, use it for zoneinfo
ifeq ($(TZDATA),)
TZDATA = $(firstword $(wildcard $(top_builddir)/tools/tzcode/tzdata*.tar.gz) $(wildcard $(top_srcdir)/tools/tzcode/tzdata*.tar.gz))
endif
# TODO: Make the TZDATA override part of Python buildtool
ifneq ($(TZDATA),)
TZCODE_DIR=$(top_builddir)/tools/tzcode
@ -809,31 +361,6 @@ $(ZONEINFO): $(TZDATA)
# end of zoneinfo-generation
endif
# zoneinfo has some issues. Ignore some warnings with -q
$(BUILDDIR)/%.res: $(MISCSRCDIR)/%.txt $(TOOLBINDIR)/genrb$(TOOLEXEEXT) | $(BUILDDIR)/cnvalias.icu
$(INVOKE) $(TOOLBINDIR)/genrb $(GENRBOPTS) -q -i $(BUILDDIR) -s $(MISCSRCDIR) -d $(BUILDDIR) $(<F)
$(INDEX_FILE): $(SRCLISTDEPS)
@echo "generating $@ (list of installed locales)"; \
$(RMV) $@; \
echo "// Warning this file is automatically generated" > $@; \
echo "$(INDEX_NAME):table(nofallback) {" >> $@; \
echo " CLDRVersion { \"$(GENRB_CLDR_VERSION)\" }" >> $@; \
echo " InstalledLocales {" >> $@; \
for file in $(INSTALLED_RB_FILES); do \
echo " $$file {\"\"}" >> $@; \
done; \
echo " }" >> $@; \
echo "}" >> $@;
clean-resindex:
-$(RMV) $(BUILDDIR)/$(INDEX_NAME).txt $(PKGDATA_LIST)
$(BUILDDIR)/$(INDEX_NAME).res: $(INDEX_FILE) $(TOOLBINDIR)/genrb$(TOOLEXEEXT) | $(BUILDDIR)/cnvalias.icu
$(INVOKE) $(TOOLBINDIR)/genrb $(GENRBOPTS) -i $(BUILDDIR) -d $(BUILDDIR) $(INDEX_FILE)
# The core Unicode properties files (pnames.icu, uprops.icu, ucase.icu, ubidi.icu)
# are hardcoded in the common DLL and therefore not included in the data package any more.
# They are not built by default but need to be built for ICU4J data and for getting the .c source files
@ -857,6 +384,13 @@ ICU4J_TZDATA_FILES=zoneinfo64 metaZones timezoneTypes windowsZones
ICU4J_DATA_DIRNAME=com/ibm/icu/impl/data/$(ICUDATA_BASENAME_VERSION)b
ICU4J_TZDATA_PATHS=$(ICU4J_TZDATA_FILES:%="$(ICU4J_DATA_DIRNAME)/%.res")
# Targets for prebuilt Unicode data
$(BUILDDIR)/%.icu: $(SRCDATADIR)/in/%.icu | $(DIRS)
$(INVOKE) $(TOOLBINDIR)/icupkg -t$(ICUDATA_CHAR) $< $@
$(BUILDDIR)/nfc.nrm: $(SRCDATADIR)/in/nfc.nrm | $(DIRS)
$(INVOKE) $(TOOLBINDIR)/icupkg -t$(ICUDATA_CHAR) $< $@
# generate icu4j-related data to $(OUTDIR)/icu4j/com/ibm/icu/impl/data/...
generate-data: build-dir packagedata $(OUTTMPDIR)/$(ICUDATA_PLATFORM_NAME).dat uni-core-data
mkdir -p $(OUTDIR)/icu4j/$(ICU4J_DATA_DIRNAME)

82
icu4c/source/data/buildtool/__init__.py Normal file
View file

@ -0,0 +1,82 @@
# Copyright (C) 2018 and later: Unicode, Inc. and others.
# License & terms of use: http://www.unicode.org/copyright.html
from collections import namedtuple
AVAILABLE_FEATURES = [
"confusables",
"cnvalias",
"uconv",
"brkitr",
"stringprep",
"dictionaries",
"normalization",
"coll",
"unames",
"misc",
"locales",
"curr",
"lang",
"region",
"zone",
"unit",
"rbnf",
"translit"
]
InFile = namedtuple("InFile", ["filename"])
TmpFile = namedtuple("TmpFile", ["filename"])
OutFile = namedtuple("OutFile", ["filename"])
PkgFile = namedtuple("PkgFile", ["filename"])
IcuTool = namedtuple("IcuTool", ["name"])
SystemTool = namedtuple("SystemTool", ["name"])
SingleExecutionRequest = namedtuple("SingleExecutionRequest", [
"name",
"input_files",
"output_files",
"tool",
"args",
"format_with"
])
RepeatedExecutionRequest = namedtuple("RepeatedExecutionRequest", [
"name",
"dep_files",
"input_files",
"output_files",
"tool",
"args",
"format_with",
"repeat_with"
])
RepeatedOrSingleExecutionRequest = namedtuple("RepeatedOrSingleExecutionRequest", [
"name",
"dep_files",
"input_files",
"output_files",
"tool",
"args",
"format_with",
"repeat_with",
"flatten_with"
])
PrintFileRequest = namedtuple("PrintFileRequest", [
"name",
"output_file",
"content"
])
CopyRequest = namedtuple("CopyRequest", [
"name",
"input_file",
"output_file"
])
VariableRequest = namedtuple("VariableRequest", [
"name",
"input_files"
])

176
icu4c/source/data/buildtool/__main__.py Normal file
View file

@ -0,0 +1,176 @@
# Copyright (C) 2018 and later: Unicode, Inc. and others.
# License & terms of use: http://www.unicode.org/copyright.html
import argparse
import glob as pyglob
import sys
from . import *
from .renderers import makefile, windirect
from . import utils
import BUILDRULES
flag_parser = argparse.ArgumentParser(
description = """Generates rules for building ICU binary data files from text
and other input files in source control.
You can select features using either the --whitelist or --blacklist option.
Available features include:
{AVAILABLE_FEATURES}
""".format(AVAILABLE_FEATURES = "\n".join(" %s" % v for v in AVAILABLE_FEATURES)),
formatter_class = argparse.RawDescriptionHelpFormatter
)
flag_parser.add_argument(
"--format",
help = "How to output the rules to run to build ICU data.",
choices = ["gnumake", "windirect"],
required = True
)
flag_parser.add_argument(
"--glob_dir",
help = "Path to data input folder (icu4c/source/data) when this script is being run.",
default = "."
)
flag_parser.add_argument(
"--in_dir",
help = "Path to data input folder (icu4c/source/data) for file processing. Not used in gnumake format.",
default = "."
)
flag_parser.add_argument(
"--out_dir",
help = "Path to where to save output data files. Not used in gnumake format.",
default = "icudata"
)
flag_parser.add_argument(
"--tmp_dir",
help = "Path to where to save temporary files. Not used in gnumake format.",
default = "icutmp"
)
flag_parser.add_argument(
"--tool_dir",
help = "Path to where to find binary tools (genrb, genbrk, etc). Used for 'windirect' format only.",
default = "../tool"
)
flag_parser.add_argument(
"--tool_cfg",
help = "The build configuration of the tools. Used for 'windirect' format only.",
default = "x86/Debug"
)
flag_parser.add_argument(
"--seqmode",
help = "Whether to optimize rules to be run sequentially (fewer threads) or in parallel (many threads).",
choices = ["sequential", "parallel"],
default = "parallel"
)
flag_parser.add_argument(
"--collation_ucadata",
help = "Which data set to use for ucadata in collation.",
choices = ["unihan", "implicithan"],
default = "unihan"
)
features_group = flag_parser.add_mutually_exclusive_group()
features_group.add_argument(
"--blacklist",
metavar = "FEATURE",
help = "A list of one or more features to disable; all others will be enabled by default. New users should favor a blacklist to ensure important data is not left out.",
nargs = "+",
choices = AVAILABLE_FEATURES
)
features_group.add_argument(
"--whitelist",
metavar = "FEATURE",
help = "A list of one or more features to enable; all others will be disabled by default.",
nargs = "+",
choices = AVAILABLE_FEATURES
)
class Config(object):
def __init__(self, args):
if args.whitelist:
self._feature_set = set(args.whitelist)
elif args.blacklist:
self._feature_set = set(AVAILABLE_FEATURES) - set(args.blacklist)
else:
self._feature_set = set(AVAILABLE_FEATURES)
self._max_parallel = (args.seqmode == "parallel")
self._coll_han_type = args.collation_ucadata
def has_feature(self, feature_name):
assert feature_name in AVAILABLE_FEATURES
return feature_name in self._feature_set
def max_parallel(self):
return self._max_parallel
def coll_han_type(self):
# Either "unihan" or "implicithan"
return self._coll_han_type
def main():
args = flag_parser.parse_args()
config = Config(args)
if args.format == "gnumake":
makefile_vars = {
"IN_DIR": "$(srcdir)",
"INDEX_NAME": "res_index"
}
makefile_env = ["ICUDATA_CHAR", "OUT_DIR", "TMP_DIR"]
common = {
key: "$(%s)" % key
for key in list(makefile_vars.keys()) + makefile_env
}
common["GLOB_DIR"] = args.glob_dir
else:
common = {
"GLOB_DIR": args.glob_dir,
"IN_DIR": args.in_dir,
"OUT_DIR": args.out_dir,
"TMP_DIR": args.tmp_dir,
"INDEX_NAME": "res_index",
# TODO: Pull this from configure script:
"ICUDATA_CHAR": "l"
}
def glob(pattern):
result_paths = pyglob.glob("{IN_DIR}/{PATTERN}".format(
IN_DIR = args.glob_dir,
PATTERN = pattern
))
# For the purposes of buildtool, force Unix-style directory separators.
return [v.replace("\\", "/")[len(args.glob_dir)+1:] for v in sorted(result_paths)]
build_dirs, raw_requests = BUILDRULES.generate(config, glob, common)
requests = []
for req in raw_requests:
if isinstance(req, RepeatedOrSingleExecutionRequest):
requests.append(utils.flatten(req, config.max_parallel()))
else:
requests.append(req)
if args.format == "gnumake":
print(makefile.get_gnumake_rules(
build_dirs,
requests,
makefile_vars,
common_vars = common
))
elif args.format == "windirect":
return windirect.run(
build_dirs,
requests,
common_vars = common,
tool_dir = args.tool_dir,
tool_cfg = args.tool_cfg
)
else:
print("Format not supported: %s" % args.format)
return 1
return 0
if __name__ == "__main__":
exit(main())

10
icu4c/source/data/buildtool/renderers/__init__.py Normal file
View file

@ -0,0 +1,10 @@
# Copyright (C) 2018 and later: Unicode, Inc. and others.
# License & terms of use: http://www.unicode.org/copyright.html
from collections import namedtuple
MakeRule = namedtuple("MakeRule", ["name", "dep_literals", "dep_files", "output_file", "cmds"])
MakeFilesVar = namedtuple("MakeFilesVar", ["name", "files"])
MakeStringVar = namedtuple("MakeStringVar", ["name", "content"])

239
icu4c/source/data/buildtool/renderers/makefile.py Normal file
View file

@ -0,0 +1,239 @@
# Copyright (C) 2018 and later: Unicode, Inc. and others.
# License & terms of use: http://www.unicode.org/copyright.html
from . import *
from .. import *
from .. import utils
def get_gnumake_rules(build_dirs, requests, makefile_vars, **kwargs):
makefile_string = ""
# Common Variables
common_vars = kwargs["common_vars"]
for key, value in makefile_vars.items():
makefile_string += "{KEY} = {VALUE}\n".format(
KEY = key,
VALUE = value
)
makefile_string += "\n"
# Directories
dirs_timestamp_file = "{TMP_DIR}/dirs.timestamp".format(**common_vars)
makefile_string += "DIRS = {TIMESTAMP_FILE}\n\n".format(
TIMESTAMP_FILE = dirs_timestamp_file
)
makefile_string += "{TIMESTAMP_FILE}:\n\t$(MKINSTALLDIRS) {ALL_DIRS}\n\techo timestamp > {TIMESTAMP_FILE}\n\n".format(
TIMESTAMP_FILE = dirs_timestamp_file,
ALL_DIRS = " ".join(build_dirs).format(**common_vars)
)
# Generate Rules
make_rules = []
for request in requests:
make_rules += get_gnumake_rules_helper(request, **kwargs)
# Main Commands
for rule in make_rules:
if isinstance(rule, MakeFilesVar):
makefile_string += "{NAME} = {FILE_LIST}\n\n".format(
NAME = rule.name,
FILE_LIST = files_to_makefile(rule.files, wrap = True, **kwargs),
)
continue
if isinstance(rule, MakeStringVar):
makefile_string += "define {NAME}\n{CONTENT}\nendef\nexport {NAME}\n\n".format(
NAME = rule.name,
CONTENT = rule.content
)
continue
assert isinstance(rule, MakeRule)
header_line = "{OUT_FILE}: {DEP_FILES} {DEP_LITERALS} | $(DIRS)".format(
OUT_FILE = files_to_makefile([rule.output_file], **kwargs),
DEP_FILES = files_to_makefile(rule.dep_files, wrap = True, **kwargs),
DEP_LITERALS = " ".join(rule.dep_literals)
)
if len(rule.cmds) == 0:
makefile_string += "%s\n\n" % header_line
continue
makefile_string += "{HEADER_LINE}\n{RULE_LINES}\n\n".format(
HEADER_LINE = header_line,
RULE_LINES = "\n".join("\t%s" % cmd for cmd in rule.cmds)
)
return makefile_string
def files_to_makefile(files, common_vars, wrap = False, **kwargs):
if len(files) == 0:
return ""
dirnames = [utils.dir_for(file).format(**common_vars) for file in files]
join_str = " \\\n\t\t" if wrap and len(files) > 2 else " "
if len(files) == 1:
return "%s/%s" % (dirnames[0], files[0].filename)
elif len(set(dirnames)) == 1:
return "$(addprefix %s/,%s)" % (dirnames[0], join_str.join(file.filename for file in files))
else:
return join_str.join("%s/%s" % (d, f.filename) for d,f in zip(dirnames, files))
def get_gnumake_rules_helper(request, common_vars, **kwargs):
if isinstance(request, PrintFileRequest):
var_name = "%s_CONTENT" % request.name.upper()
return [
MakeStringVar(
name = var_name,
content = request.content
),
MakeRule(
name = request.name,
dep_literals = [],
dep_files = [],
output_file = request.output_file,
cmds = [
"echo \"$${VAR_NAME}\" > {MAKEFILENAME}".format(
VAR_NAME = var_name,
MAKEFILENAME = files_to_makefile([request.output_file], common_vars),
**common_vars
)
]
)
]
if isinstance(request, CopyRequest):
return [
MakeRule(
name = request.name,
dep_literals = [],
dep_files = [request.input_file],
output_file = request.output_file,
cmds = ["cp %s %s" % (
files_to_makefile([request.input_file], common_vars),
files_to_makefile([request.output_file], common_vars))
]
)
]
if isinstance(request, VariableRequest):
return [
MakeFilesVar(
name = request.name.upper(),
files = request.input_files
)
]
if request.tool.name == "make":
cmd_template = "$(MAKE) {ARGS}"
elif request.tool.name == "gentest":
cmd_template = "$(INVOKE) $(GENTEST) {ARGS}"
else:
assert isinstance(request.tool, IcuTool)
cmd_template = "$(INVOKE) $(TOOLBINDIR)/{TOOL} {{ARGS}}".format(
TOOL = request.tool.name
)
if isinstance(request, SingleExecutionRequest):
cmd = utils.format_single_request_command(request, cmd_template, common_vars)
if len(request.output_files) > 1:
# Special case for multiple output files: Makefile rules should have only one
# output file apiece. More information:
# https://www.gnu.org/software/automake/manual/html_node/Multiple-Outputs.html
timestamp_var_name = "%s_ALL" % request.name.upper()
timestamp_file = TmpFile("%s.timestamp" % request.name)
rules = [
MakeFilesVar(
name = timestamp_var_name,
files = [timestamp_file]
),
MakeRule(
name = "%s_all" % request.name,
dep_literals = [],
dep_files = request.input_files,
output_file = timestamp_file,
cmds = [
cmd,
"echo timestamp > {MAKEFILENAME}".format(
MAKEFILENAME = files_to_makefile([timestamp_file], common_vars)
)
]
)
]
for i, file in enumerate(request.output_files):
rules += [
MakeRule(
name = "%s_%d" % (request.name, i),
dep_literals = ["$(%s)" % timestamp_var_name],
dep_files = [],
output_file = file,
cmds = []
)
]
return rules
elif len(request.input_files) > 5:
# For nicer printing, for long input lists, use a helper variable.
dep_var_name = "%s_DEPS" % request.name.upper()
return [
MakeFilesVar(
name = dep_var_name,
files = request.input_files
),
MakeRule(
name = request.name,
dep_literals = ["$(%s)" % dep_var_name],
dep_files = [],
output_file = request.output_files[0],
cmds = [cmd]
)
]
else:
return [
MakeRule(
name = request.name,
dep_literals = [],
dep_files = request.input_files,
output_file = request.output_files[0],
cmds = [cmd]
)
]
if isinstance(request, RepeatedExecutionRequest):
rules = []
dep_literals = []
# To keep from repeating the same dep files many times, make a variable.
if len(request.dep_files) > 0:
dep_var_name = "%s_DEPS" % request.name.upper()
dep_literals = ["$(%s)" % dep_var_name]
rules += [
MakeFilesVar(
name = dep_var_name,
files = request.dep_files
)
]
# Add a rule for each individual file.
for loop_vars in utils.repeated_execution_request_looper(request):
(_, input_file, output_file) = loop_vars
name_suffix = input_file[input_file.filename.rfind("/")+1:input_file.filename.rfind(".")]
cmd = utils.format_repeated_request_command(
request,
cmd_template,
loop_vars,
common_vars
)
rules += [
MakeRule(
name = "%s_%s" % (request.name, name_suffix),
dep_literals = dep_literals,
dep_files = [input_file],
output_file = output_file,
cmds = [cmd]
)
]
return rules
assert False

78
icu4c/source/data/buildtool/renderers/windirect.py Normal file
View file

@ -0,0 +1,78 @@
# Copyright (C) 2018 and later: Unicode, Inc. and others.
# License & terms of use: http://www.unicode.org/copyright.html
from . import *
from .. import *
from .. import utils
import os
import shutil
import subprocess
def run(build_dirs, requests, common_vars, **kwargs):
for bd in build_dirs:
os.makedirs(bd.format(**common_vars), exist_ok=True)
for request in requests:
status = run_helper(request, common_vars, **kwargs)
if status != 0:
print("!!! ERROR executing above command line: exit code %d" % status)
return 1
print("windirect: All data build commands executed")
return 0
def run_helper(request, common_vars, tool_dir, tool_cfg, **kwargs):
if isinstance(request, PrintFileRequest):
output_path = "{DIRNAME}/{FILENAME}".format(
DIRNAME = utils.dir_for(request.output_file).format(**common_vars),
FILENAME = request.output_file.filename,
)
print("Printing to file: %s" % output_path)
with open(output_path, "w") as f:
f.write(request.content)
return 0
if isinstance(request, CopyRequest):
input_path = "{DIRNAME}/{FILENAME}".format(
DIRNAME = utils.dir_for(request.input_file).format(**common_vars),
FILENAME = request.input_file.filename,
)
output_path = "{DIRNAME}/{FILENAME}".format(
DIRNAME = utils.dir_for(request.output_file).format(**common_vars),
FILENAME = request.output_file.filename,
)
print("Copying file to: %s" % output_path)
shutil.copyfile(input_path, output_path)
return 0
if isinstance(request, VariableRequest):
# No-op
return 0
assert isinstance(request.tool, IcuTool)
cmd_template = "{TOOL_DIR}/{TOOL}/{TOOL_CFG}/{TOOL}.exe {{ARGS}}".format(
TOOL_DIR = tool_dir,
TOOL_CFG = tool_cfg,
TOOL = request.tool.name,
**common_vars
)
if isinstance(request, RepeatedExecutionRequest):
for loop_vars in utils.repeated_execution_request_looper(request):
command_line = utils.format_repeated_request_command(
request,
cmd_template,
loop_vars,
common_vars
)
# TODO: Is this / to \ substitution too aggressive?
command_line = command_line.replace("/", "\\")
print("Running: %s" % command_line)
res = subprocess.run(command_line)
if res.returncode != 0:
return res.returncode
return 0
if isinstance(request, SingleExecutionRequest):
command_line = utils.format_single_request_command(request, cmd_template, common_vars)
# TODO: Is this / to \ substitution too aggressive?
command_line = command_line.replace("/", "\\")
print("Running: %s" % command_line)
res = subprocess.run(command_line)
return res.returncode
assert False

130
icu4c/source/data/buildtool/utils.py Normal file
View file

@ -0,0 +1,130 @@
# Copyright (C) 2018 and later: Unicode, Inc. and others.
# License & terms of use: http://www.unicode.org/copyright.html
from . import *
def dir_for(file):
if isinstance(file, InFile):
return "{IN_DIR}"
if isinstance(file, TmpFile):
return "{TMP_DIR}"
if isinstance(file, OutFile):
return "{OUT_DIR}"
if isinstance(file, PkgFile):
return "{PKG_DIR}"
assert False
def concat_dicts(*dicts):
# There is not a super great way to do this in Python:
new_dict = {}
for dict in dicts:
new_dict.update(dict)
return new_dict
def repeated_execution_request_looper(request):
# dictionary of lists to list of dictionaries:
ld = [
dict(zip(request.repeat_with, t))
for t in zip(*request.repeat_with.values())
]
if not ld:
# No special options given in repeat_with
ld = [{} for _ in range(len(request.input_files))]
return zip(ld, request.input_files, request.output_files)
def format_single_request_command(request, cmd_template, common_vars):
return cmd_template.format(
ARGS = request.args.format(
INPUT_FILES = [file.filename for file in request.input_files],
OUTPUT_FILES = [file.filename for file in request.output_files],
**concat_dicts(common_vars, request.format_with)
)
)
def format_repeated_request_command(request, cmd_template, loop_vars, common_vars):
(iter_vars, input_file, output_file) = loop_vars
return cmd_template.format(
ARGS = request.args.format(
INPUT_FILE = input_file.filename,
OUTPUT_FILE = output_file.filename,
**concat_dicts(common_vars, request.format_with, iter_vars)
)
)
def flatten(request, max_parallel):
"""Flatten a RepeatedOrSingleExecutionRequest
Becomes either a SingleExecutionRequest or a RepeatedExecutionRequest.
"""
if max_parallel:
return RepeatedExecutionRequest(
name = request.name,
dep_files = request.dep_files,
input_files = request.input_files,
output_files = request.output_files,
tool = request.tool,
args = request.args,
format_with = request.format_with,
repeat_with = request.repeat_with
)
else:
return SingleExecutionRequest(
name = request.name,
input_files = request.dep_files + request.input_files,
output_files = request.output_files,
tool = request.tool,
args = request.args,
format_with = concat_dicts(request.format_with, request.flatten_with)
)
def generate_index_file(locales, cldr_version, common_vars):
formatted_version = " CLDRVersion { \"%s\" }\n" % cldr_version if cldr_version else ""
formatted_locales = "\n".join([" %s {\"\"}" % v for v in locales])
# TODO: CLDRVersion is required only in the base file
return ("// Warning this file is automatically generated\n"
"{INDEX_NAME}:table(nofallback) {{\n"
"{FORMATTED_VERSION}"
" InstalledLocales {{\n"
"{FORMATTED_LOCALES}\n"
" }}\n"
"}}").format(
FORMATTED_VERSION = formatted_version,
FORMATTED_LOCALES = formatted_locales,
**common_vars
)
def get_all_output_files(requests, include_tmp=False):
files = []
for request in requests:
if isinstance(request, SingleExecutionRequest):
files += request.output_files
elif isinstance(request, RepeatedExecutionRequest):
files += request.output_files
elif isinstance(request, RepeatedOrSingleExecutionRequest):
files += request.output_files
elif isinstance(request, PrintFileRequest):
files += [request.output_file]
elif isinstance(request, CopyRequest):
files += [request.output_file]
elif isinstance(request, VariableRequest):
pass
else:
assert False
# Filter out all files but those in OUT_DIR if necessary.
# It is also easy to filter for uniqueness; do it right now and return.
if not include_tmp:
files = (file for file in files if isinstance(file, OutFile))
return list(set(files))
# Filter for unique values. NOTE: Cannot use set() because we need to accept same filename as
# OutFile and TmpFile as different, and by default they evaluate as equal.
return [f for _, f in set((type(f), f) for f in files)]

BIN
icu4c/source/data/curr/pool.res
View file

Binary file not shown.

BIN
icu4c/source/data/lang/pool.res
View file

Binary file not shown.

BIN
icu4c/source/data/locales/pool.res
View file

Binary file not shown.

742
icu4c/source/data/makedata.mak
View file

@ -33,12 +33,8 @@ ICU_LIB_TARGET=$(DLL_OUTPUT)\$(U_ICUDATA_NAME).dll
!ENDIF
!MESSAGE ICU data make path is $(ICUMAKE)
!IF [py -3]!=0
!MESSAGE Information: Unable to find Python 3. ICU versions 64 and later will require Python 3 to build.
!MESSAGE Information: See ICU-10923 for more information: https://unicode-org.atlassian.net/browse/ICU-10923
!ELSE
!MESSAGE Information: Found Python 3. You are all set for ICU 64, which will require Python 3 to build.
!MESSAGE Information: For more info on Python 3 requirement, see: https://unicode-org.atlassian.net/browse/ICU-10923
!IF [py -3 -c "exit(0)"]!=0
!MESSAGE Information: Unable to find Python 3. Data will fail to build from source.
!ENDIF
# Suffixes for data files
@ -72,15 +68,9 @@ ICUP=$(ICUP:\source\data\\..\..=)
ICUSRCDATA=$(ICUP)\source\data
ICUSRCDATA_RELATIVE_PATH=..\..\..
# ICUUCM
# The directory that contains ucmcore.mk files along with *.ucm files
#
ICUUCM=mappings
# ICULOC
# The directory that contains resfiles.mk files along with *.txt locale data files
#
ICULOC=locales
# Timestamp files to keep track of current build state
TOOLS_TS=$(ICUTMP)\tools.timestamp
COREDATA_TS=$(ICUTMP)\coredata.timestamp
# ICUCOL
# The directory that contains colfiles.mk files along with *.txt collation data files
@ -102,23 +92,6 @@ ICUTRNS=translit
#
ICUBRK=brkitr
# ICUUNIDATA
# The directory that contains Unicode data files
#
ICUUNIDATA=$(ICUP)\source\data\unidata
# ICUMISC
# The directory that contains miscfiles.mk along with files that are miscelleneous data
#
ICUMISC=$(ICUP)\source\data\misc
ICUMISC2=misc
# ICUSPREP
# The directory that contains sprepfiles.mk files along with *.txt stringprep files
#
ICUSPREP=sprep
#
# ICUDATA
# The source directory. Contains the source files for the common data to be built.
@ -210,345 +183,8 @@ ICUDATA_SOURCE_ARCHIVE=$(ICUTMP)\$(ICUPKG).dat
!IFDEF ICUDATA_SOURCE_ARCHIVE
!MESSAGE ICU data source archive is $(ICUDATA_SOURCE_ARCHIVE)
!ELSE
# We're including a list of .ucm files.
# There are several lists, they are all optional.
# Always build the mapping files for the EBCDIC fallback codepages
# They are necessary on EBCDIC machines, and
# the following logic is much easier if UCM_SOURCE is never empty.
# (They are small.)
UCM_SOURCE=ibm-37_P100-1995.ucm ibm-1047_P100-1995.ucm
!IF EXISTS("$(ICUSRCDATA)\$(ICUUCM)\ucmcore.mk")
!INCLUDE "$(ICUSRCDATA)\$(ICUUCM)\ucmcore.mk"
UCM_SOURCE=$(UCM_SOURCE) $(UCM_SOURCE_CORE)
!ELSE
!MESSAGE Warning: cannot find "ucmcore.mk". Not building core MIME/Unix/Windows converter files.
!ENDIF
!IF EXISTS("$(ICUSRCDATA)\$(ICUUCM)\ucmfiles.mk")
!INCLUDE "$(ICUSRCDATA)\$(ICUUCM)\ucmfiles.mk"
UCM_SOURCE=$(UCM_SOURCE) $(UCM_SOURCE_FILES)
!ELSE
!MESSAGE Warning: cannot find "ucmfiles.mk". Not building many converter files.
!ENDIF
!IF EXISTS("$(ICUSRCDATA)\$(ICUUCM)\ucmebcdic.mk")
!INCLUDE "$(ICUSRCDATA)\$(ICUUCM)\ucmebcdic.mk"
UCM_SOURCE=$(UCM_SOURCE) $(UCM_SOURCE_EBCDIC)
!IFDEF UCM_SOURCE_EBCDIC_IGNORE_SISO
BUILD_SPECIAL_CNV_FILES=YES
UCM_SOURCE_SPECIAL=$(UCM_SOURCE_EBCDIC_IGNORE_SISO)
!ELSE
!UNDEF BUILD_SPECIAL_CNV_FILES
!ENDIF
!ELSE
!MESSAGE Warning: cannot find "ucmebcdic.mk". Not building EBCDIC converter files.
!ENDIF
!IF EXISTS("$(ICUSRCDATA)\$(ICUUCM)\ucmlocal.mk")
!INCLUDE "$(ICUSRCDATA)\$(ICUUCM)\ucmlocal.mk"
!IFDEF UCM_SOURCE_LOCAL
UCM_SOURCE=$(UCM_SOURCE) $(UCM_SOURCE_LOCAL)
!ENDIF
!IFDEF UCM_SOURCE_EBCDIC_IGNORE_SISO_LOCAL
UCM_SOURCE_SPECIAL=$(UCM_SOURCE_SPECIAL) $(UCM_SOURCE_EBCDIC_IGNORE_SISO_LOCAL)
BUILD_SPECIAL_CNV_FILES=YES
!ENDIF
!ELSE
!MESSAGE Information: cannot find "ucmlocal.mk". Not building user-additional converter files.
!ENDIF
CNV_FILES=$(UCM_SOURCE:.ucm=.cnv)
!IFDEF BUILD_SPECIAL_CNV_FILES
CNV_FILES_SPECIAL=$(UCM_SOURCE_SPECIAL:.ucm=.cnv)
!ENDIF
!IF EXISTS("$(ICUSRCDATA)\$(ICUBRK)\brkfiles.mk")
!INCLUDE "$(ICUSRCDATA)\$(ICUBRK)\brkfiles.mk"
!IF EXISTS("$(ICUSRCDATA)\$(ICUBRK)\brklocal.mk")
!INCLUDE "$(ICUSRCDATA)\$(ICUBRK)\brklocal.mk"
!IFDEF BRK_SOURCE_LOCAL
BRK_SOURCE=$(BRK_SOURCE) $(BRK_SOURCE_LOCAL)
!ENDIF
!IFDEF BRK_DICT_SOURCE_LOCAL
BRK_DICT_SOURCE=$(BRK_DICT_SOURCE) $(BRK_DICT_SOURCE_LOCAL)
!ENDIF
!IFDEF BRK_RES_SOURCE_LOCAL
BRK_RES_SOURCE=$(BRK_RES_SOURCE) $(BRK_RES_SOURCE_LOCAL)
!ENDIF
!ELSE
!MESSAGE Information: cannot find "brklocal.mk". Not building user-additional break iterator files.
!ENDIF
!ELSE
!MESSAGE Warning: cannot find "brkfiles.mk"
!ENDIF
#
# Break iterator data files.
#
BRK_FILES=$(ICUBRK)\$(BRK_SOURCE:.txt =.brk brkitr\)
BRK_FILES=$(BRK_FILES:.txt=.brk)
BRK_FILES=$(BRK_FILES:brkitr\ =brkitr\)
!IFDEF BRK_DICT_SOURCE
BRK_DICT_FILES = $(ICUBRK)\$(BRK_DICT_SOURCE:.txt =.dict brkitr\)
BRK_DICT_FILES = $(BRK_DICT_FILES:.txt=.dict)
BRK_DICT_FILES = $(BRK_DICT_FILES:brkitr\ =brkitr\)
!ENDIF
!IFDEF BRK_RES_SOURCE
BRK_RES_FILES = $(BRK_RES_SOURCE:.txt =.res brkitr\)
BRK_RES_FILES = $(BRK_RES_FILES:.txt=.res)
BRK_RES_FILES = $(ICUBRK)\root.res $(ICUBRK)\$(BRK_RES_FILES:brkitr\ =)
ALL_RES = $(ALL_RES) $(ICUBRK)\res_index.res
!ENDIF
# Read list of locale resource bundle files
!IF EXISTS("$(ICUSRCDATA)\$(ICULOC)\resfiles.mk")
!INCLUDE "$(ICUSRCDATA)\$(ICULOC)\resfiles.mk"
!IF EXISTS("$(ICUSRCDATA)\$(ICULOC)\reslocal.mk")
!INCLUDE "$(ICUSRCDATA)\$(ICULOC)\reslocal.mk"
!IFDEF GENRB_SOURCE_LOCAL
GENRB_SOURCE=$(GENRB_SOURCE) $(GENRB_SOURCE_LOCAL)
!ENDIF
!ELSE
!MESSAGE Information: cannot find "reslocal.mk". Not building user-additional resource bundle files.
!ENDIF
!ELSE
!MESSAGE Warning: cannot find "resfiles.mk"
!ENDIF
!IFDEF GENRB_SOURCE
RB_FILES = root.res pool.res $(GENRB_ALIAS_SOURCE:.txt=.res) $(GENRB_ALIAS_SOURCE_LOCAL:.txt=.res) $(GENRB_SOURCE:.txt=.res)
ALL_RES = $(ALL_RES) res_index.res
!ENDIF
# Read the list of currency display name resource bundle files
!IF EXISTS("$(ICUSRCDATA)\curr\resfiles.mk")
!INCLUDE "$(ICUSRCDATA)\curr\resfiles.mk"
!IF EXISTS("$(ICUSRCDATA)\curr\reslocal.mk")
!INCLUDE "$(ICUSRCDATA)\curr\reslocal.mk"
!IFDEF CURR_SOURCE_LOCAL
CURR_SOURCE=$(CURR_SOURCE) $(CURR_SOURCE_LOCAL)
!ENDIF
!ELSE
!MESSAGE Information: cannot find "curr\reslocal.mk". Not building user-additional resource bundle files.
!ENDIF
!ELSE
!MESSAGE Warning: cannot find "curr\resfiles.mk"
!ENDIF
!IFDEF CURR_SOURCE
CURR_FILES = curr\root.txt supplementalData.txt $(CURR_ALIAS_SOURCE) $(CURR_SOURCE)
CURR_RES_FILES = $(CURR_FILES:.txt =.res curr\)
CURR_RES_FILES = $(CURR_RES_FILES:.txt=.res)
CURR_RES_FILES = curr\pool.res $(CURR_RES_FILES:curr\ =curr\)
ALL_RES = $(ALL_RES) curr\res_index.res
!ENDIF
# Read the list of language/script display name resource bundle files
!IF EXISTS("$(ICUSRCDATA)\lang\resfiles.mk")
!INCLUDE "$(ICUSRCDATA)\lang\resfiles.mk"
!IF EXISTS("$(ICUSRCDATA)\lang\reslocal.mk")
!INCLUDE "$(ICUSRCDATA)\lang\reslocal.mk"
!IFDEF LANG_SOURCE_LOCAL
LANG_SOURCE=$(LANG_SOURCE) $(LANG_SOURCE_LOCAL)
!ENDIF
!ELSE
!MESSAGE Information: cannot find "lang\reslocal.mk". Not building user-additional resource bundle files.
!ENDIF
!ELSE
!MESSAGE Warning: cannot find "lang\resfiles.mk"
!ENDIF
!IFDEF LANG_SOURCE
LANG_FILES = lang\root.txt $(LANG_ALIAS_SOURCE) $(LANG_SOURCE)
LANG_RES_FILES = $(LANG_FILES:.txt =.res lang\)
LANG_RES_FILES = $(LANG_RES_FILES:.txt=.res)
LANG_RES_FILES = lang\pool.res $(LANG_RES_FILES:lang\ =lang\)
ALL_RES = $(ALL_RES) lang\res_index.res
!ENDIF
# Read the list of region display name resource bundle files
!IF EXISTS("$(ICUSRCDATA)\region\resfiles.mk")
!INCLUDE "$(ICUSRCDATA)\region\resfiles.mk"
!IF EXISTS("$(ICUSRCDATA)\region\reslocal.mk")
!INCLUDE "$(ICUSRCDATA)\region\reslocal.mk"
!IFDEF REGION_SOURCE_LOCAL
REGION_SOURCE=$(REGION_SOURCE) $(REGION_SOURCE_LOCAL)
!ENDIF
!ELSE
!MESSAGE Information: cannot find "region\reslocal.mk". Not building user-additional resource bundle files.
!ENDIF
!ELSE
!MESSAGE Warning: cannot find "region\resfiles.mk"
!ENDIF
!IFDEF REGION_SOURCE
REGION_FILES = region\root.txt $(REGION_ALIAS_SOURCE) $(REGION_SOURCE)
REGION_RES_FILES = $(REGION_FILES:.txt =.res region\)
REGION_RES_FILES = $(REGION_RES_FILES:.txt=.res)
REGION_RES_FILES = region\pool.res $(REGION_RES_FILES:region\ =region\)
ALL_RES = $(ALL_RES) region\res_index.res
!ENDIF
# Read the list of time zone display name resource bundle files
!IF EXISTS("$(ICUSRCDATA)\zone\resfiles.mk")
!INCLUDE "$(ICUSRCDATA)\zone\resfiles.mk"
!IF EXISTS("$(ICUSRCDATA)\zone\reslocal.mk")
!INCLUDE "$(ICUSRCDATA)\zone\reslocal.mk"
!IFDEF ZONE_SOURCE_LOCAL
ZONE_SOURCE=$(ZONE_SOURCE) $(ZONE_SOURCE_LOCAL)
!ENDIF
!ELSE
!MESSAGE Information: cannot find "zone\reslocal.mk". Not building user-additional resource bundle files.
!ENDIF
ZONE_SOURCE=$(ZONE_SOURCE) tzdbNames.txt
!ELSE
!MESSAGE Warning: cannot find "zone\resfiles.mk"
!ENDIF
!IFDEF ZONE_SOURCE
ZONE_FILES = zone\root.txt $(ZONE_ALIAS_SOURCE) $(ZONE_SOURCE)
ZONE_RES_FILES = $(ZONE_FILES:.txt =.res zone\)
ZONE_RES_FILES = $(ZONE_RES_FILES:.txt=.res)
ZONE_RES_FILES = zone\pool.res $(ZONE_RES_FILES:zone\ =zone\)
ALL_RES = $(ALL_RES) zone\res_index.res
!ENDIF
# Read the list of units display name resource bundle files
!IF EXISTS("$(ICUSRCDATA)\unit\resfiles.mk")
!INCLUDE "$(ICUSRCDATA)\unit\resfiles.mk"
!IF EXISTS("$(ICUSRCDATA)\unit\reslocal.mk")
!INCLUDE "$(ICUSRCDATA)\unit\reslocal.mk"
!IFDEF UNIT_SOURCE_LOCAL
UNIT_SOURCE=$(UNIT_SOURCE) $(UNIT_SOURCE_LOCAL)
!ENDIF
!ELSE
!MESSAGE Information: cannot find "unit\reslocal.mk". Not building user-additional resource bundle files.
!ENDIF
!ELSE
!MESSAGE Warning: cannot find "unit\resfiles.mk"
!ENDIF
!IFDEF UNIT_SOURCE
UNIT_FILES = unit\root.txt $(UNIT_ALIAS_SOURCE) $(UNIT_SOURCE)
UNIT_RES_FILES = $(UNIT_FILES:.txt =.res unit\)
UNIT_RES_FILES = $(UNIT_RES_FILES:.txt=.res)
UNIT_RES_FILES = unit\pool.res $(UNIT_RES_FILES:unit\ =unit\)
ALL_RES = $(ALL_RES) unit\res_index.res
!ENDIF
# Read the list of collation resource bundle files
!IF EXISTS("$(ICUSRCDATA)\$(ICUCOL)\colfiles.mk")
!INCLUDE "$(ICUSRCDATA)\$(ICUCOL)\colfiles.mk"
!IF EXISTS("$(ICUSRCDATA)\$(ICUCOL)\collocal.mk")
!INCLUDE "$(ICUSRCDATA)\$(ICUCOL)\collocal.mk"
!IFDEF COLLATION_SOURCE_LOCAL
COLLATION_SOURCE=$(COLLATION_SOURCE) $(COLLATION_SOURCE_LOCAL)
!ENDIF
!ELSE
!MESSAGE Information: cannot find "collocal.mk". Not building user-additional resource bundle files.
!ENDIF
!ELSE
!MESSAGE Warning: cannot find "colfiles.mk"
!ENDIF
!IFDEF COLLATION_SOURCE
COL_FILES = $(ICUCOL)\root.txt $(COLLATION_ALIAS_SOURCE) $(COLLATION_SOURCE)
COL_COL_FILES = $(COL_FILES:.txt =.res coll\)
COL_COL_FILES = $(COL_COL_FILES:.txt=.res)
COL_COL_FILES = $(COL_COL_FILES:coll\ =)
ALL_RES = $(ALL_RES) $(ICUCOL)\res_index.res
!ENDIF
# Read the list of RBNF resource bundle files
!IF EXISTS("$(ICUSRCDATA)\$(ICURBNF)\rbnffiles.mk")
!INCLUDE "$(ICUSRCDATA)\$(ICURBNF)\rbnffiles.mk"
!IF EXISTS("$(ICUSRCDATA)\$(ICURBNF)\rbnflocal.mk")
!INCLUDE "$(ICUSRCDATA)\$(ICURBNF)\rbnflocal.mk"
!IFDEF RBNF_SOURCE_LOCAL
RBNF_SOURCE=$(RBNF_SOURCE) $(RBNF_SOURCE_LOCAL)
!ENDIF
!ELSE
!MESSAGE Information: cannot find "rbnflocal.mk". Not building user-additional resource bundle files.
!ENDIF
!ELSE
!MESSAGE Warning: cannot find "rbnffiles.mk"
!ENDIF
!IFDEF RBNF_SOURCE
RBNF_FILES = $(ICURBNF)\root.txt $(RBNF_ALIAS_SOURCE) $(RBNF_SOURCE)
RBNF_RES_FILES = $(RBNF_FILES:.txt =.res rbnf\)
RBNF_RES_FILES = $(RBNF_RES_FILES:.txt=.res)
RBNF_RES_FILES = $(RBNF_RES_FILES:rbnf\ =rbnf\)
ALL_RES = $(ALL_RES) $(ICURBNF)\res_index.res
!ENDIF
# Read the list of transliterator resource bundle files
!IF EXISTS("$(ICUSRCDATA)\$(ICUTRNS)\trnsfiles.mk")
!INCLUDE "$(ICUSRCDATA)\$(ICUTRNS)\trnsfiles.mk"
!IF EXISTS("$(ICUSRCDATA)\$(ICUTRNS)\trnslocal.mk")
!INCLUDE "$(ICUSRCDATA)\$(ICUTRNS)\trnslocal.mk"
!IFDEF TRANSLIT_SOURCE_LOCAL
TRANSLIT_SOURCE=$(TRANSLIT_SOURCE) $(TRANSLIT_SOURCE_LOCAL)
!ENDIF
!ELSE
!MESSAGE Information: cannot find "trnslocal.mk". Not building user-additional transliterator files.
!ENDIF
!ELSE
!MESSAGE Warning: cannot find "trnsfiles.mk"
!ENDIF
!IFDEF TRANSLIT_SOURCE
TRANSLIT_FILES = $(ICUTRNS)\$(TRANSLIT_ALIAS_SOURCE) $(TRANSLIT_SOURCE)
TRANSLIT_RES_FILES = $(TRANSLIT_FILES:.txt =.res translit\)
TRANSLIT_RES_FILES = $(TRANSLIT_RES_FILES:.txt=.res)
TRANSLIT_RES_FILES = $(TRANSLIT_RES_FILES:translit\ =translit\)
#ALL_RES = $(ALL_RES) $(ICUTRNS)\res_index.res
!ENDIF
# Read the list of miscellaneous resource bundle files
!IF EXISTS("$(ICUSRCDATA)\$(ICUMISC2)\miscfiles.mk")
!INCLUDE "$(ICUSRCDATA)\$(ICUMISC2)\miscfiles.mk"
!IF EXISTS("$(ICUSRCDATA)\$(ICUMISC2)\misclocal.mk")
!INCLUDE "$(ICUSRCDATA)\$(ICUMISC2)\misclocal.mk"
!IFDEF MISC_SOURCE_LOCAL
MISC_SOURCE=$(MISC_SOURCE) $(MISC_SOURCE_LOCAL)
!ENDIF
!ELSE
!MESSAGE Information: cannot find "misclocal.mk". Not building user-additional miscellaenous files.
!ENDIF
!ELSE
!MESSAGE Warning: cannot find "miscfiles.mk"
!ENDIF
MISC_FILES = $(MISC_SOURCE:.txt=.res)
# don't include COL_FILES
ALL_RES = $(ALL_RES) $(RB_FILES) $(MISC_FILES)
!ENDIF
# Read the list of stringprep profile files
!IF EXISTS("$(ICUSRCDATA)\$(ICUSPREP)\sprepfiles.mk")
!INCLUDE "$(ICUSRCDATA)\$(ICUSPREP)\sprepfiles.mk"
!IF EXISTS("$(ICUSRCDATA)\$(ICUSPREP)\spreplocal.mk")
!INCLUDE "$(ICUSRCDATA)\$(ICUSPREP)\spreplocal.mk"
!IFDEF SPREP_SOURCE_LOCAL
SPREP_SOURCE=$(SPREP_SOURCE) $(SPREP_SOURCE_LOCAL)
!ENDIF
!ELSE
!MESSAGE Information: cannot find "spreplocal.mk". Not building user-additional stringprep files.
!ENDIF
!ELSE
!MESSAGE Warning: cannot find "sprepfiles.mk"
!ENDIF
SPREP_FILES = $(SPREP_SOURCE:.txt=.spp)
# Common defines for both ways of building ICU's data library.
COMMON_ICUDATA_DEPENDENCIES="$(ICUPBIN)\pkgdata.exe" "$(ICUTMP)\icudata.res" "$(ICUP)\source\stubdata\stubdatabuilt.txt"
COMMON_ICUDATA_ARGUMENTS=-f -e $(U_ICUDATA_NAME) -v $(ICU_PACKAGE_MODE) -c -p $(ICUPKG) -T "$(ICUTMP)" -L $(U_ICUDATA_NAME) -d "$(ICUBLD_PKG)" -s .
@ -576,6 +212,37 @@ ALL : GODATA "$(ICU_LIB_TARGET)" "$(TESTDATAOUT)\testdata.dat"
copy "$(ICUOUT)\$(U_ICUDATA_NAME)$(U_ICUDATA_ENDIAN_SUFFIX).dat" "$(ICUMAKE)\..\..\commondata\"
!ENDIF
# Three main targets: tools, core data, and test data.
# Keep track of whether they are built via timestamp files.
$(TOOLS_TS): "$(ICUTOOLS)\genrb\$(CFGTOOLS)\genrb.exe" "$(ICUTOOLS)\gencnval\$(CFGTOOLS)\gencnval.exe" "$(ICUTOOLS)\gencfu\$(CFGTOOLS)\gencfu.exe" "$(ICUTOOLS)\icupkg\$(CFGTOOLS)\icupkg.exe" "$(ICUTOOLS)\makeconv\$(CFGTOOLS)\makeconv.exe" "$(ICUPBIN)\pkgdata.exe"
@echo "timestamp" > $(TOOLS_TS)
# On Unix, Python generates at configure time a list of Makefile rules.
# On Windows, however, we run the Python data build script at build time instead.
# The alternative would be to use a preprocessor macro to generate rules for nmake.
# However, this approach was abandoned for reasons including:
#
# - nmake imposes more stringent restrictions on command line length.
# - The lack of gnumake features makes nmake file construction more complex.
# - Windows builds are single-threaded, giving less advantage to a Makefile approach.
#
# Currently, the entire script needs to run even for small changes to data. Maybe consider
# checking file-changed timestamps in Python to build only the required subset of data.
$(COREDATA_TS):
@cd "$(ICUSRCDATA)"
py -3 -m buildtool \
--format windirect \
--in_dir "$(ICUSRCDATA)" \
--tool_dir "$(ICUTOOLS)" \
--tool_cfg "$(CFG)" \
--out_dir "$(ICUBLD_PKG)" \
--tmp_dir "$(ICUTMP)"
@echo "timestamp" > $(COREDATA_TS)
# The core Unicode properties files (uprops.icu, ucase.icu, ubidi.icu)
# are hardcoded in the common DLL and therefore not included in the data package any more.
# They are not built by default but need to be built for ICU4J data and for getting the .c source files
@ -673,10 +340,10 @@ icu4j-data-install :
#
# testdata - nmake will invoke pkgdata, which will create testdata.dat
#
"$(TESTDATAOUT)\testdata.dat": "$(TESTDATA)\*" "$(ICUBLD_PKG)\$(ICUCOL)\ucadata.icu" $(TRANSLIT_RES_FILES) $(MISC_FILES) $(RB_FILES) {"$(ICUTOOLS)\genrb\$(CFGTOOLS)"}genrb.exe
"$(TESTDATAOUT)\testdata.dat": "$(TESTDATA)\*" $(TOOLS_TS) $(COREDATA_TS)
@cd "$(TESTDATA)"
@echo building testdata...
nmake /nologo /f "$(TESTDATA)\testdata.mak" TESTDATA=. ICUTOOLS="$(ICUTOOLS)" ICUPBIN="$(ICUPBIN)" ICUP="$(ICUP)" CFG=$(CFGTOOLS) TESTDATAOUT="$(TESTDATAOUT)" TESTDATABLD="$(TESTDATABLD)"
nmake /nologo /f "$(TESTDATA)\testdata.mak" TESTDATA=. ICUTOOLS="$(ICUTOOLS)" ICUPBIN="$(ICUPBIN)" ICUP="$(ICUP)" CFG=$(CFGTOOLS) TESTDATAOUT="$(TESTDATAOUT)" TESTDATABLD="$(TESTDATABLD)" ICUSRCDATA="$(ICUSRCDATA)" DLL_OUTPUT="$(DLL_OUTPUT)"
#invoke pkgdata for ICU common data
# pkgdata will drop all output files (.dat, .dll, .lib) into the target (ICUBLD_PKG) directory.
@ -697,50 +364,12 @@ icu4j-data-install :
copy "$(ICUTMP)\$(ICUPKG).dat" "$(ICUOUT)\$(U_ICUDATA_NAME)$(U_ICUDATA_ENDIAN_SUFFIX).dat"
-@erase "$(ICUTMP)\$(ICUPKG).dat"
!ELSE
"$(ICU_LIB_TARGET)" : $(COMMON_ICUDATA_DEPENDENCIES) $(CNV_FILES) $(CNV_FILES_SPECIAL) "$(ICUBLD_PKG)\unames.icu" "$(ICUBLD_PKG)\cnvalias.icu" "$(ICUBLD_PKG)\nfkc.nrm" "$(ICUBLD_PKG)\nfkc_cf.nrm" "$(ICUBLD_PKG)\uts46.nrm" "$(ICUBLD_PKG)\$(ICUCOL)\ucadata.icu" $(CURR_RES_FILES) $(LANG_RES_FILES) $(REGION_RES_FILES) $(ZONE_RES_FILES) $(UNIT_RES_FILES) $(BRK_FILES) $(BRK_DICT_FILES) $(BRK_RES_FILES) $(ALL_RES) $(COL_COL_FILES) $(RBNF_RES_FILES) $(TRANSLIT_RES_FILES) $(SPREP_FILES) "$(ICUBLD_PKG)\confusables.cfu"
@echo Building icu data
"$(ICU_LIB_TARGET)" : $(COMMON_ICUDATA_DEPENDENCIES) $(COREDATA_TS)
@echo Building ICU data from scratch
cd "$(ICUBLD_PKG)"
"$(ICUPBIN)\pkgdata" $(COMMON_ICUDATA_ARGUMENTS) <<"$(ICUTMP)\icudata.lst"
unames.icu
confusables.cfu
$(ICUCOL)\ucadata.icu
cnvalias.icu
nfkc.nrm
nfkc_cf.nrm
uts46.nrm
$(CNV_FILES:.cnv =.cnv
)
$(CNV_FILES_SPECIAL:.cnv =.cnv
)
$(ALL_RES:.res =.res
)
$(CURR_RES_FILES:.res =.res
)
$(LANG_RES_FILES:.res =.res
)
$(REGION_RES_FILES:.res =.res
)
$(ZONE_RES_FILES:.res =.res
)
$(UNIT_RES_FILES:.res =.res
)
$(COL_COL_FILES:.res =.res
)
$(RBNF_RES_FILES:.res =.res
)
$(TRANSLIT_RES_FILES:.res =.res
)
$(BRK_FILES:.brk =.brk
)
$(BRK_DICT_FILES:.dict =.dict
)
$(BRK_RES_FILES:.res =.res
)
$(SPREP_FILES:.spp=.spp
)
<<KEEP
"$(ICUPBIN)\pkgdata" $(COMMON_ICUDATA_ARGUMENTS) $(ICUTMP)\icudata.lst
-@erase "$(ICU_LIB_TARGET)"
@if not exist "$(DLL_OUTPUT)" mkdir "$(DLL_OUTPUT)"
@if not exist "$(DLL_OUTPUT)" mkdir "$(DLL_OUTPUT)"
copy "$(U_ICUDATA_NAME).dll" "$(ICU_LIB_TARGET)"
-@erase "$(U_ICUDATA_NAME).dll"
copy "$(ICUTMP)\$(ICUPKG).dat" "$(ICUOUT)\$(U_ICUDATA_NAME)$(U_ICUDATA_ENDIAN_SUFFIX).dat"
@ -748,22 +377,13 @@ $(SPREP_FILES:.spp=.spp
!ENDIF
# utility target to create missing directories
# Most directories are made by Python, but still create ICUTMP
# so it works in the source archive
CREATE_DIRS :
@if not exist "$(ICUOUT)\$(NULL)" mkdir "$(ICUOUT)"
@if not exist "$(ICUTMP)\$(NULL)" mkdir "$(ICUTMP)"
@if not exist "$(ICUOUT)\build\$(NULL)" mkdir "$(ICUOUT)\build"
@if not exist "$(ICUBLD_PKG)\$(NULL)" mkdir "$(ICUBLD_PKG)"
@if not exist "$(ICUBLD_PKG)\curr\$(NULL)" mkdir "$(ICUBLD_PKG)\curr"
@if not exist "$(ICUBLD_PKG)\lang\$(NULL)" mkdir "$(ICUBLD_PKG)\lang"
@if not exist "$(ICUBLD_PKG)\region\$(NULL)" mkdir "$(ICUBLD_PKG)\region"
@if not exist "$(ICUBLD_PKG)\zone\$(NULL)" mkdir "$(ICUBLD_PKG)\zone"
@if not exist "$(ICUBLD_PKG)\$(ICUBRK)\$(NULL)" mkdir "$(ICUBLD_PKG)\$(ICUBRK)"
@if not exist "$(ICUBLD_PKG)\$(ICUCOL)\$(NULL)" mkdir "$(ICUBLD_PKG)\$(ICUCOL)"
@if not exist "$(ICUBLD_PKG)\$(ICURBNF)\$(NULL)" mkdir "$(ICUBLD_PKG)\$(ICURBNF)"
@if not exist "$(ICUBLD_PKG)\$(ICUTRNS)\$(NULL)" mkdir "$(ICUBLD_PKG)\$(ICUTRNS)"
@if not exist "$(TESTDATAOUT)\$(NULL)" mkdir "$(TESTDATAOUT)"
@if not exist "$(TESTDATABLD)\$(NULL)" mkdir "$(TESTDATABLD)"
@if not exist "$(TESTDATAOUT)\testdata\$(NULL)" mkdir "$(TESTDATAOUT)\testdata"
# utility target to send us to the right dir
GODATA : CREATE_DIRS
@ -811,6 +431,7 @@ CLEAN : GODATA
-@erase "*.mak"
-@erase "*.obj"
-@erase "*.res"
-@erase "*.timestamp"
@cd "$(TESTDATABLD)"
-@erase "*.cnv"
-@erase "*.icu"
@ -826,252 +447,14 @@ CLEAN : GODATA
@cd "$(ICUBLD_PKG)"
# RBBI .brk file generation.
{$(ICUSRCDATA_RELATIVE_PATH)\$(ICUBRK)\rules}.txt.brk:
@echo Creating $@
@"$(ICUTOOLS)\genbrk\$(CFGTOOLS)\genbrk" -c -r $< -o $@ -d"$(ICUBLD_PKG)" -i "$(ICUBLD_PKG)"
#RBBI .dict file generation.
{$(ICUSRCDATA_RELATIVE_PATH)\$(ICUBRK)\dictionaries}.txt.dict:
@echo Creating $@
@"$(ICUTOOLS)\gendict\$(CFGTOOLS)\gendict" -c --uchars $< "$(ICUBLD_PKG)\$@"
$(ICUBRK)\thaidict.dict:
@echo Creating $(ICUBRK)\thaidict.dict
@"$(ICUTOOLS)\gendict\$(CFGTOOLS)\gendict" -c --bytes --transform offset-0x0e00 $(ICUSRCDATA_RELATIVE_PATH)\$(ICUBRK)\dictionaries\thaidict.txt "$(ICUBLD_PKG)\$(ICUBRK)\thaidict.dict"
$(ICUBRK)\laodict.dict:
@echo Creating $(ICUBRK)\laodict.dict
@"$(ICUTOOLS)\gendict\$(CFGTOOLS)\gendict" -c --bytes --transform offset-0x0e80 $(ICUSRCDATA_RELATIVE_PATH)\$(ICUBRK)\dictionaries\laodict.txt "$(ICUBLD_PKG)\$(ICUBRK)\laodict.dict"
$(ICUBRK)\burmesedict.dict:
@echo Creating $(ICUBRK)\burmesedict.dict
@"$(ICUTOOLS)\gendict\$(CFGTOOLS)\gendict" -c --bytes --transform offset-0x1000 $(ICUSRCDATA_RELATIVE_PATH)\$(ICUBRK)\dictionaries\burmesedict.txt "$(ICUBLD_PKG)\$(ICUBRK)\burmesedict.dict"
$(ICUBRK)\khmerdict.dict:
@echo Creating $(ICUBRK)\khmerdict.dict
@"$(ICUTOOLS)\gendict\$(CFGTOOLS)\gendict" -c --bytes --transform offset-0x1780 $(ICUSRCDATA_RELATIVE_PATH)\$(ICUBRK)\dictionaries\khmerdict.txt "$(ICUBLD_PKG)\$(ICUBRK)\khmerdict.dict"
!IFNDEF ICUDATA_SOURCE_ARCHIVE
# Rule for creating converters
$(CNV_FILES): $(UCM_SOURCE)
@echo Building Charset Conversion table $(@B)
@"$(ICUTOOLS)\makeconv\$(CFGTOOLS)\makeconv" -c -d"$(ICUBLD_PKG)" $(ICUSRCDATA_RELATIVE_PATH)\$(ICUUCM)\$(@B).ucm
!ENDIF
!IFDEF BUILD_SPECIAL_CNV_FILES
$(CNV_FILES_SPECIAL): $(UCM_SOURCE_SPECIAL)
@echo Building Special Charset Conversion table $(@B)
@"$(ICUTOOLS)\makeconv\$(CFGTOOLS)\makeconv" -c --ignore-siso-check -d"$(ICUBLD_PKG)" $(ICUSRCDATA_RELATIVE_PATH)\$(ICUUCM)\$(@B).ucm
!ENDIF
# Batch inference rule for creating miscellaneous resource files
# TODO: -q option is specified to squelch the 120+ warnings about
# empty intvectors and binary elements. Unfortunately, this may
# squelch other legitimate warnings. When there is a better
# way, remove the -q.
{$(ICUSRCDATA_RELATIVE_PATH)\$(ICUMISC2)}.txt.res::
@echo Making Miscellaneous Resource Bundle files
@"$(ICUTOOLS)\genrb\$(CFGTOOLS)\genrb" -k -q -d"$(ICUBLD_PKG)" $<
# Inference rule for creating resource bundle files
{$(ICUSRCDATA_RELATIVE_PATH)\$(ICULOC)}.txt.res::
@echo Making Locale Resource Bundle files
@"$(ICUTOOLS)\genrb\$(CFGTOOLS)\genrb" --usePoolBundle $(ICUSRCDATA_RELATIVE_PATH)\$(ICULOC) -k -d"$(ICUBLD_PKG)" $<
# copy the locales/pool.res file from the source folder to the build output folder
# and swap it to native endianness
pool.res: $(ICUSRCDATA_RELATIVE_PATH)\$(ICULOC)\pool.res
"$(ICUPBIN)\icupkg" -tl "$(ICUSRCDATA_RELATIVE_PATH)\$(ICULOC)\pool.res" pool.res
res_index.res:
@echo Generating <<res_index.txt
// Warning this file is automatically generated
res_index:table(nofallback) {
InstalledLocales {
$(GENRB_SOURCE:.txt= {""}
)
}
}
<<KEEP
@"$(ICUTOOLS)\genrb\$(CFGTOOLS)\genrb" -k -d"$(ICUBLD_PKG)" .\res_index.txt
{$(ICUSRCDATA_RELATIVE_PATH)\curr}.txt{curr}.res::
@echo Making currency display name files
@"$(ICUTOOLS)\genrb\$(CFGTOOLS)\genrb" --usePoolBundle $(ICUSRCDATA_RELATIVE_PATH)\curr -k -i "$(ICUBLD_PKG)" -d"$(ICUBLD_PKG)\curr" $<
# copy the curr/pool.res file from the source folder to the build output folder
# and swap it to native endianness
curr\pool.res: $(ICUSRCDATA_RELATIVE_PATH)\curr\pool.res
"$(ICUPBIN)\icupkg" -tl "$(ICUSRCDATA_RELATIVE_PATH)\curr\pool.res" curr\pool.res
curr\res_index.res:
@echo Generating <<curr\res_index.txt
// Warning this file is automatically generated
res_index:table(nofallback) {
InstalledLocales {
$(CURR_SOURCE:.txt= {""}
)
}
}
<<KEEP
@"$(ICUTOOLS)\genrb\$(CFGTOOLS)\genrb" -k -d"$(ICUBLD_PKG)\curr" .\curr\res_index.txt
{$(ICUSRCDATA_RELATIVE_PATH)\lang}.txt{lang}.res::
@echo Making language/script display name files
@"$(ICUTOOLS)\genrb\$(CFGTOOLS)\genrb" --usePoolBundle $(ICUSRCDATA_RELATIVE_PATH)\lang -k -i "$(ICUBLD_PKG)" -d"$(ICUBLD_PKG)\lang" $<
# copy the lang/pool.res file from the source folder to the build output folder
# and swap it to native endianness
lang\pool.res: $(ICUSRCDATA_RELATIVE_PATH)\lang\pool.res
"$(ICUPBIN)\icupkg" -tl "$(ICUSRCDATA_RELATIVE_PATH)\lang\pool.res" lang\pool.res
lang\res_index.res:
@echo Generating <<lang\res_index.txt
// Warning this file is automatically generated
res_index:table(nofallback) {
InstalledLocales {
$(LANG_SOURCE:.txt= {""}
)
}
}
<<KEEP
@"$(ICUTOOLS)\genrb\$(CFGTOOLS)\genrb" -k -d"$(ICUBLD_PKG)\lang" .\lang\res_index.txt
{$(ICUSRCDATA_RELATIVE_PATH)\region}.txt{region}.res::
@echo Making region display name files
@"$(ICUTOOLS)\genrb\$(CFGTOOLS)\genrb" --usePoolBundle $(ICUSRCDATA_RELATIVE_PATH)\region -k -i "$(ICUBLD_PKG)" -d"$(ICUBLD_PKG)\region" $<
# copy the region/pool.res file from the source folder to the build output folder
# and swap it to native endianness
region\pool.res: $(ICUSRCDATA_RELATIVE_PATH)\region\pool.res
"$(ICUPBIN)\icupkg" -tl "$(ICUSRCDATA_RELATIVE_PATH)\region\pool.res" region\pool.res
region\res_index.res:
@echo Generating <<region\res_index.txt
// Warning this file is automatically generated
res_index:table(nofallback) {
InstalledLocales {
$(REGION_SOURCE:.txt= {""}
)
}
}
<<KEEP
@"$(ICUTOOLS)\genrb\$(CFGTOOLS)\genrb" -k -d"$(ICUBLD_PKG)\region" .\region\res_index.txt
{$(ICUSRCDATA_RELATIVE_PATH)\zone}.txt{zone}.res::
@echo Making time zone display name files
@"$(ICUTOOLS)\genrb\$(CFGTOOLS)\genrb" --usePoolBundle $(ICUSRCDATA_RELATIVE_PATH)\zone -k -i "$(ICUBLD_PKG)" -d"$(ICUBLD_PKG)\zone" $<
# copy the zone/pool.res file from the source folder to the build output folder
# and swap it to native endianness
zone\pool.res: $(ICUSRCDATA_RELATIVE_PATH)\zone\pool.res
"$(ICUPBIN)\icupkg" -tl "$(ICUSRCDATA_RELATIVE_PATH)\zone\pool.res" zone\pool.res
zone\res_index.res:
@echo Generating <<zone\res_index.txt
// Warning this file is automatically generated
res_index:table(nofallback) {
InstalledLocales {
$(ZONE_SOURCE:.txt= {""}
)
}
}
<<KEEP
@"$(ICUTOOLS)\genrb\$(CFGTOOLS)\genrb" -k -d"$(ICUBLD_PKG)\zone" .\zone\res_index.txt
{$(ICUSRCDATA_RELATIVE_PATH)\unit}.txt{unit}.res::
@echo Making unit display name files
@"$(ICUTOOLS)\genrb\$(CFGTOOLS)\genrb" --usePoolBundle $(ICUSRCDATA_RELATIVE_PATH)\unit -k -i "$(ICUBLD_PKG)" -d"$(ICUBLD_PKG)\unit" $<
# copy the unit/pool.res file from the source folder to the build output folder
# and swap it to native endianness
unit\pool.res: $(ICUSRCDATA_RELATIVE_PATH)\unit\pool.res
"$(ICUPBIN)\icupkg" -tl "$(ICUSRCDATA_RELATIVE_PATH)\unit\pool.res" unit\pool.res
unit\res_index.res:
@echo Generating <<unit\res_index.txt
// Warning this file is automatically generated
res_index:table(nofallback) {
InstalledLocales {
$(UNIT_SOURCE:.txt= {""}
)
}
}
<<KEEP
@"$(ICUTOOLS)\genrb\$(CFGTOOLS)\genrb" -k -d"$(ICUBLD_PKG)\unit" .\unit\res_index.txt
{$(ICUSRCDATA_RELATIVE_PATH)\$(ICUCOL)}.txt{$(ICUCOL)}.res::
@echo Making Collation files
@"$(ICUTOOLS)\genrb\$(CFGTOOLS)\genrb" -k -i "$(ICUBLD_PKG)" -d"$(ICUBLD_PKG)\$(ICUCOL)" $<
$(ICUCOL)\res_index.res:
@echo Generating <<$(ICUCOL)\res_index.txt
// Warning this file is automatically generated
res_index:table(nofallback) {
InstalledLocales {
$(COLLATION_SOURCE:.txt= {""}
)
}
}
<<KEEP
@"$(ICUTOOLS)\genrb\$(CFGTOOLS)\genrb" -k -d"$(ICUBLD_PKG)\$(ICUCOL)" .\$(ICUCOL)\res_index.txt
{$(ICUSRCDATA_RELATIVE_PATH)\$(ICURBNF)}.txt{$(ICURBNF)}.res::
@echo Making RBNF files
@"$(ICUTOOLS)\genrb\$(CFGTOOLS)\genrb" -k -i "$(ICUBLD_PKG)" -d"$(ICUBLD_PKG)\$(ICURBNF)" $<
$(ICURBNF)\res_index.res:
@echo Generating <<$(ICURBNF)\res_index.txt
// Warning this file is automatically generated
res_index:table(nofallback) {
InstalledLocales {
$(RBNF_SOURCE:.txt= {""}
)
}
}
<<KEEP
@"$(ICUTOOLS)\genrb\$(CFGTOOLS)\genrb" -k -d"$(ICUBLD_PKG)\$(ICURBNF)" .\$(ICURBNF)\res_index.txt
$(ICUBRK)\res_index.res:
@echo Generating <<$(ICUBRK)\res_index.txt
// Warning this file is automatically generated
res_index:table(nofallback) {
InstalledLocales {
$(BRK_RES_SOURCE:.txt= {""}
)
}
}
<<KEEP
@"$(ICUTOOLS)\genrb\$(CFGTOOLS)\genrb" -k -d"$(ICUBLD_PKG)\$(ICUBRK)" .\$(ICUBRK)\res_index.txt
{$(ICUSRCDATA_RELATIVE_PATH)\$(ICUBRK)}.txt{$(ICUBRK)}.res::
@echo Making Break Iterator Resource files
@"$(ICUTOOLS)\genrb\$(CFGTOOLS)\genrb" -k -i "$(ICUBLD_PKG)" -d"$(ICUBLD_PKG)\$(ICUBRK)" $<
{$(ICUSRCDATA_RELATIVE_PATH)\$(ICUTRNS)}.txt{$(ICUTRNS)}.res::
@echo Making Transliterator files
@"$(ICUTOOLS)\genrb\$(CFGTOOLS)\genrb" -k -i "$(ICUBLD_PKG)" -d"$(ICUBLD_PKG)\$(ICUTRNS)" $<
# DLL version information
# If you modify this, modify winmode.c in pkgdata.
"$(ICUTMP)\icudata.res": "$(ICUMISC)\icudata.rc"
"$(ICUTMP)\icudata.res": "$(ICUSRCDATA)\misc\icudata.rc"
@echo Creating data DLL version information from $**
@rc.exe /i "..\..\..\..\common" /r /fo $@ $**
# Targets for converters
"$(ICUBLD_PKG)\cnvalias.icu" : {"$(ICUSRCDATA)\$(ICUUCM)"}\convrtrs.txt "$(ICUTOOLS)\gencnval\$(CFGTOOLS)\gencnval.exe"
@echo Creating data file for Converter Aliases
@"$(ICUTOOLS)\gencnval\$(CFGTOOLS)\gencnval" -d "$(ICUBLD_PKG)" "$(ICUSRCDATA)\$(ICUUCM)\convrtrs.txt"
# Targets for prebuilt Unicode data
# Needed for ICU4J!
"$(ICUBLD_PKG)\pnames.icu": $(ICUSRCDATA_RELATIVE_PATH)\in\pnames.icu
"$(ICUPBIN)\icupkg" -tl $? $@
@ -1102,37 +485,8 @@ res_index:table(nofallback) {
"$(ICUBLD_PKG)\coll\ucadata.icu": $(ICUSRCDATA_RELATIVE_PATH)\in\coll\ucadata-unihan.icu
"$(ICUPBIN)\icupkg" -tl $? $@
# Stringprep .spp file generation.
{$(ICUSRCDATA_RELATIVE_PATH)\$(ICUSPREP)}.txt.spp:
@echo Creating $@
@"$(ICUTOOLS)\gensprep\$(CFGTOOLS)\gensprep" -s $(<D) -d "$(ICUBLD_PKG)" -b $(@B) -m "$(ICUUNIDATA)" -u 3.2.0 $(<F)
# Confusables .cfu file generation
# Can't use an inference rule because two .txt source files combine to produce a single .cfu output file
"$(ICUBLD_PKG)\confusables.cfu": "$(ICUUNIDATA)\confusables.txt" "$(ICUUNIDATA)\confusablesWholeScript.txt" "$(ICUTOOLS)\gencfu\$(CFGTOOLS)\gencfu.exe"
@echo Creating $@
@"$(ICUTOOLS)\gencfu\$(CFGTOOLS)\gencfu" -c -r "$(ICUUNIDATA)\confusables.txt" -w "$(ICUUNIDATA)\confusablesWholeScript.txt" -o $@ -i "$(ICUBLD_PKG)"
!IFDEF ICUDATA_ARCHIVE
"$(ICUDATA_SOURCE_ARCHIVE)": CREATE_DIRS $(ICUDATA_ARCHIVE) "$(ICUTOOLS)\icupkg\$(CFGTOOLS)\icupkg.exe"
"$(ICUDATA_SOURCE_ARCHIVE)": CREATE_DIRS $(ICUDATA_ARCHIVE) $(TOOLS_TS)
"$(ICUTOOLS)\icupkg\$(CFGTOOLS)\icupkg" -t$(U_ICUDATA_ENDIAN_SUFFIX) "$(ICUDATA_ARCHIVE)" "$(ICUDATA_SOURCE_ARCHIVE)"
!ENDIF
# Dependencies on the tools for the batch inference rules
!IFNDEF ICUDATA_SOURCE_ARCHIVE
$(UCM_SOURCE) : {"$(ICUTOOLS)\makeconv\$(CFGTOOLS)"}makeconv.exe
!IFDEF BUILD_SPECIAL_CNV_FILES
$(UCM_SOURCE_SPECIAL): {"$(ICUTOOLS)\makeconv\$(CFGTOOLS)"}makeconv.exe
!ENDIF
# This used to depend on "$(ICUBLD_PKG)\uprops.icu" "$(ICUBLD_PKG)\ucase.icu" "$(ICUBLD_PKG)\ubidi.icu"
# This data is now hard coded as a part of the library.
# See Jitterbug 4497 for details.
$(MISC_SOURCE) $(RB_FILES) $(CURR_FILES) $(LANG_FILES) $(REGION_FILES) $(ZONE_FILES) $(UNIT_FILES) $(COL_COL_FILES) $(RBNF_RES_FILES) $(BRK_RES_FILES) $(TRANSLIT_RES_FILES): {"$(ICUTOOLS)\genrb\$(CFGTOOLS)"}genrb.exe "$(ICUBLD_PKG)\$(ICUCOL)\ucadata.icu"
# This used to depend on "$(ICUBLD_PKG)\pnames.icu" "$(ICUBLD_PKG)\uprops.icu" "$(ICUBLD_PKG)\ucase.icu" "$(ICUBLD_PKG)\ubidi.icu"
# These are now hardcoded in ICU4C and only loaded in ICU4J.
$(BRK_SOURCE) : "$(ICUBLD_PKG)\unames.icu"
!ENDIF

BIN
icu4c/source/data/region/pool.res
View file

Binary file not shown.

BIN
icu4c/source/data/unit/pool.res
View file

Binary file not shown.

BIN
icu4c/source/data/zone/pool.res
View file

Binary file not shown.

227
icu4c/source/test/testdata/BUILDRULES.py vendored Normal file
View file

@ -0,0 +1,227 @@
# Copyright (C) 2018 and later: Unicode, Inc. and others.
# License & terms of use: http://www.unicode.org/copyright.html
from distutils.sysconfig import parse_makefile
from buildtool import *
def generate(config, glob, common_vars):
build_dirs = ["{OUT_DIR}", "{TMP_DIR}"]
requests = []
requests += generate_rb(config, glob, common_vars)
requests += generate_sprep(config, glob, common_vars)
requests += generate_conv(config, glob, common_vars)
requests += generate_other(config, glob, common_vars)
requests += generate_copy(config, glob, common_vars)
all_output_files = list(sorted(utils.get_all_output_files(requests)))
all_output_files_with_tmp = list(sorted(utils.get_all_output_files(requests, include_tmp=True)))
testdata_list_file = TmpFile("testdata.lst")
requests += [
PrintFileRequest(
name = "testdata_list",
output_file = testdata_list_file,
content = "\n".join(file.filename for file in all_output_files)
),
VariableRequest(
name = "testdata_all_output_files",
input_files = all_output_files_with_tmp + [testdata_list_file]
)
]
return (build_dirs, requests)
def generate_rb(config, glob, common_vars):
mk_vars = parse_makefile("{GLOB_DIR}/tstfiles.mk".format(**common_vars))
basenames = [v[:-4] for v in mk_vars["TEST_RES_SOURCE"].split()]
basenames += [
"casing",
"mc",
"root",
"sh",
"sh_YU",
"te",
"te_IN",
"te_IN_REVISED",
"testtypes",
"testaliases",
"testempty",
"structLocale",
"idna_rules",
"conversion",
"icuio",
# "metaZones",
# "timezoneTypes",
# "windowsZones",
]
return [
# Inference rule for creating resource bundles
# Some test data resource bundles are known to have warnings and bad data.
# The -q option is there on purpose, so we don't see it normally.
# TODO: Use option -k?
RepeatedExecutionRequest(
name = "testrb",
dep_files = [],
input_files = [InFile("%s.txt" % bn) for bn in basenames],
output_files = [OutFile("%s.res" % bn) for bn in basenames],
tool = IcuTool("genrb"),
args = "-q -s {IN_DIR} -d {OUT_DIR} {INPUT_FILE}",
format_with = {},
repeat_with = {}
),
# Other standalone res files
SingleExecutionRequest(
name = "encoded",
input_files = [InFile("encoded.utf16be")],
output_files = [OutFile("encoded.res")],
tool = IcuTool("genrb"),
args = "-s {IN_DIR} -eUTF-16BE -d {OUT_DIR} {INPUT_FILES[0]}",
format_with = {}
),
SingleExecutionRequest(
name = "idna_rules",
input_files = [InFile("idna_rules.txt")],
output_files = [OutFile("idna_rules.res")],
tool = IcuTool("genrb"),
args = "-s {IN_DIR} -d {OUT_DIR} {INPUT_FILES[0]}",
format_with = {}
),
SingleExecutionRequest(
name = "zoneinfo64",
input_files = [InFile("zoneinfo64.txt")],
output_files = [TmpFile("zoneinfo64.res")],
tool = IcuTool("genrb"),
args = "-s {IN_DIR} -d {TMP_DIR} {INPUT_FILES[0]}",
format_with = {}
)
]
def generate_sprep(config, glob, common_vars):
return [
SingleExecutionRequest(
name = "nfscsi",
input_files = [InFile("nfs4_cs_prep_ci.txt")],
output_files = [OutFile("nfscsi.spp")],
tool = IcuTool("gensprep"),
args = "-s {IN_DIR} -d {OUT_DIR} -b nfscsi -u 3.2.0 {INPUT_FILES[0]}",
format_with = {}
),
SingleExecutionRequest(
name = "nfscss",
input_files = [InFile("nfs4_cs_prep_cs.txt")],
output_files = [OutFile("nfscss.spp")],
tool = IcuTool("gensprep"),
args = "-s {IN_DIR} -d {OUT_DIR} -b nfscss -u 3.2.0 {INPUT_FILES[0]}",
format_with = {}
),
SingleExecutionRequest(
name = "nfscis",
input_files = [InFile("nfs4_cis_prep.txt")],
output_files = [OutFile("nfscis.spp")],
tool = IcuTool("gensprep"),
args = "-s {IN_DIR} -d {OUT_DIR} -b nfscis -u 3.2.0 -k -n {IN_DIR}/../../data/unidata {INPUT_FILES[0]}",
format_with = {}
),
SingleExecutionRequest(
name = "nfsmxs",
input_files = [InFile("nfs4_mixed_prep_s.txt")],
output_files = [OutFile("nfsmxs.spp")],
tool = IcuTool("gensprep"),
args = "-s {IN_DIR} -d {OUT_DIR} -b nfsmxs -u 3.2.0 -k -n {IN_DIR}/../../data/unidata {INPUT_FILES[0]}",
format_with = {}
),
SingleExecutionRequest(
name = "nfsmxp",
input_files = [InFile("nfs4_mixed_prep_p.txt")],
output_files = [OutFile("nfsmxp.spp")],
tool = IcuTool("gensprep"),
args = "-s {IN_DIR} -d {OUT_DIR} -b nfsmxp -u 3.2.0 -k -n {IN_DIR}/../../data/unidata {INPUT_FILES[0]}",
format_with = {}
)
]
def generate_conv(config, glob, common_vars):
basenames = [
"test1",
"test1bmp",
"test2",
"test3",
"test4",
"test4x",
"test5",
"ibm9027"
]
return [
RepeatedExecutionRequest(
name = "test_conv",
dep_files = [],
input_files = [InFile("%s.ucm" % bn) for bn in basenames],
output_files = [OutFile("%s.cnv" % bn) for bn in basenames],
tool = IcuTool("makeconv"),
args = "--small -d {OUT_DIR} {IN_DIR}/{INPUT_FILE}",
format_with = {},
repeat_with = {}
)
]
def generate_copy(config, glob, common_vars):
return [
CopyRequest(
name = "nam_typ",
input_file = OutFile("te.res"),
output_file = TmpFile("nam.typ")
),
CopyRequest(
name = "old_l_testtypes",
input_file = InFile("old_l_testtypes.res"),
output_file = OutFile("old_l_testtypes.res")
),
CopyRequest(
name = "old_e_testtypes",
input_file = InFile("old_e_testtypes.res"),
output_file = OutFile("old_e_testtypes.res")
),
]
def generate_other(config, glob, common_vars):
return [
SingleExecutionRequest(
name = "testnorm",
input_files = [InFile("testnorm.txt")],
output_files = [OutFile("testnorm.nrm")],
tool = IcuTool("gennorm2"),
args = "-s {IN_DIR} {INPUT_FILES[0]} -o {OUT_DIR}/{OUTPUT_FILES[0]}",
format_with = {}
),
SingleExecutionRequest(
name = "test_icu",
input_files = [],
output_files = [OutFile("test.icu")],
tool = IcuTool("gentest"),
args = "-d {OUT_DIR}",
format_with = {}
),
SingleExecutionRequest(
name = "testtable32_txt",
input_files = [],
output_files = [TmpFile("testtable32.txt")],
tool = IcuTool("gentest"),
args = "-r -d {TMP_DIR}",
format_with = {}
),
SingleExecutionRequest(
name = "testtable32_res",
input_files = [TmpFile("testtable32.txt")],
output_files = [OutFile("testtable32.res")],
tool = IcuTool("genrb"),
args = "-s {TMP_DIR} -d {OUT_DIR} {INPUT_FILES[0]}",
format_with = {}
)
]

101
icu4c/source/test/testdata/Makefile.in vendored
View file

@ -127,105 +127,22 @@ TESTDATA=testdata
# prefix for files that are testdata
TESTDT=$(TESTDATA)
# File definitions
TEST_DAT_FILES=$(TESTBUILDDIR)/test.icu
TEST_SPP_FILES=$(TESTBUILDDIR)/nfscsi.spp $(TESTBUILDDIR)/nfscss.spp $(TESTBUILDDIR)/nfscis.spp $(TESTBUILDDIR)/nfsmxs.spp $(TESTBUILDDIR)/nfsmxp.spp
# Variable names for rules.mk
OUT_DIR=$(TESTBUILDDIR)
TMP_DIR=$(TESTOUTDIR)/$(TESTDT)
TEST_UCM_SOURCE= test1.ucm test1bmp.ucm test2.ucm test3.ucm test4.ucm test4x.ucm test5.ucm ibm9027.ucm
TEST_UCM_FILES=$(TEST_UCM_SOURCE:%=$(TESTSRCDATADIR)/data/%)
TEST_CNV_FILES=$(TEST_UCM_SOURCE:%.ucm=$(TESTBUILDDIR)/%.cnv)
## Include the Python-generated rules
TEST_NRM_FILES=$(TESTBUILDDIR)/testnorm.nrm
# import the shared .mk file
include $(TESTSRCDATADIR)/tstfiles.mk
-include $(TESTSRCDATADIR)/tstlocal.mk
# TEST_RES_SOURCE comes from tstfiles.mk
TEST_RES = $(TEST_RES_SOURCE) $(TEST_RES_LOCAL) casing.txt mc.txt root.txt sh.txt sh_YU.txt te.txt te_IN.txt te_IN_REVISED.txt testtypes.txt testaliases.txt testempty.txt structLocale.txt idna_rules.txt conversion.txt icuio.txt testtable32.txt metaZones.txt timezoneTypes.txt windowsZones.txt zoneinfo64.txt
TEST_RES_FILES=$(TEST_RES:%.txt=$(TESTBUILDDIR)/%.res) $(TESTBUILDDIR)/encoded.res $(TESTBUILDDIR)/old_l_testtypes.res $(TESTBUILDDIR)/old_e_testtypes.res
ALL_TEST_FILES = $(TEST_DAT_FILES) $(TEST_SPP_FILES) $(TEST_BRK_FILES) $(TEST_CNV_FILES) $(TEST_NRM_FILES) $(TEST_RES_FILES) $(TESTOUTDIR)/$(TESTDT)/nam.typ $(TESTOUTDIR)/$(TESTDT)/zoneinfo64.res
# list
$(TESTBUILDDIR)/testdata.lst: $(SRCLISTDEPS)
@echo "generating $@ (list of data files)"
@-$(RMV) $@
@for file in $(TEST_RES_FILES:$(TESTBUILDDIR)/%.res=%.res) $(TEST_DAT_FILES:$(TESTBUILDDIR)/%.icu=%.icu) $(TEST_SPP_FILES:$(TESTBUILDDIR)/%.spp=%.spp) $(TEST_CNV_FILES:$(TESTBUILDDIR)/%.cnv=%.cnv) $(TEST_NRM_FILES:$(TESTBUILDDIR)/%.nrm=%.nrm); do \
echo $$file >> $@; \
done;
include $(top_builddir)/$(subdir)/rules.mk
build-testdata: build-dir $(ALL_TEST_FILES) $(TESTBUILDDIR)/testdata.lst $(TESTBUILDDIR)/encoded.res
build-testdata: build-dir $(TESTDATA_ALL_OUTPUT_FILES)
# test.icu
$(TESTBUILDDIR)/test.icu: $(GENTEST)
$(INVOKE) $(GENTEST) -d $(TESTBUILDDIR)
testdata: build-testdata
$(TESTBUILDDIR)/testtable32.txt: $(GENTEST)
$(INVOKE) $(GENTEST) -r -d $(TESTBUILDDIR)
$(TESTBUILDDIR)/testtable32.res: $(TESTBUILDDIR)/testtable32.txt $(TOOLBINDIR)/genrb$(EXEEXT)
$(INVOKE) $(TOOLBINDIR)/genrb $(GENRBOPTS) -s $(TESTBUILDDIR) $(ICU_DATA_OPT) -d $(TESTBUILDDIR) $(<F)
# Targets for nfscsi.spp
$(TESTBUILDDIR)/nfscsi.spp: $(TOOLBINDIR)/gensprep$(EXEEXT) $(TESTSRCDATADIR)/nfs4_cs_prep_ci.txt
@echo Building nfscsi.icu
$(INVOKE) $(TOOLBINDIR)/gensprep -s $(TESTSRCDATADIR) $(ICU_DATA_OPT) -d $(TESTBUILDDIR) -b nfscsi -u 3.2.0 nfs4_cs_prep_ci.txt
# Targets for nfscss.spp
$(TESTBUILDDIR)/nfscss.spp: $(TOOLBINDIR)/gensprep$(EXEEXT) $(TESTSRCDATADIR)/nfs4_cs_prep_cs.txt
@echo Building nfscss.icu
$(INVOKE) $(TOOLBINDIR)/gensprep -s $(TESTSRCDATADIR) $(ICU_DATA_OPT) -d $(TESTBUILDDIR) -b nfscss -u 3.2.0 nfs4_cs_prep_cs.txt
# Targets for nfscis.spp
$(TESTBUILDDIR)/nfscis.spp: $(TOOLBINDIR)/gensprep$(EXEEXT) $(TESTSRCDATADIR)/nfs4_cis_prep.txt
@echo Building nfscis.spp
$(INVOKE) $(TOOLBINDIR)/gensprep -s $(TESTSRCDATADIR) $(ICU_DATA_OPT) -d $(TESTBUILDDIR) -b nfscis -k -n $(UNICODEDATADIR) -u 3.2.0 nfs4_cis_prep.txt
# Targets for nfsmxs.spp
$(TESTBUILDDIR)/nfsmxs.spp: $(TOOLBINDIR)/gensprep$(EXEEXT) $(TESTSRCDATADIR)/nfs4_mixed_prep_s.txt
@echo Building nfsmxs.spp
$(INVOKE) $(TOOLBINDIR)/gensprep -s $(TESTSRCDATADIR) $(ICU_DATA_OPT) -d $(TESTBUILDDIR) -b nfsmxs -k -n $(UNICODEDATADIR) -u 3.2.0 nfs4_mixed_prep_s.txt
# Targets for nfsmxp.spp
$(TESTBUILDDIR)/nfsmxp.spp: $(TOOLBINDIR)/gensprep$(EXEEXT) $(TESTSRCDATADIR)/nfs4_mixed_prep_p.txt
@echo Building nfsmxp.spp
$(INVOKE) $(TOOLBINDIR)/gensprep -s $(TESTSRCDATADIR) $(ICU_DATA_OPT) -d $(TESTBUILDDIR) -b nfsmxp -k -n $(UNICODEDATADIR) -u 3.2.0 nfs4_mixed_prep_p.txt
$(TESTBUILDDIR)/%.cnv: $(TESTSRCDATADIR)/%.ucm $(TOOLBINDIR)/makeconv$(EXEEXT)
$(INVOKE) $(TOOLBINDIR)/makeconv --small -c -d $(TESTBUILDDIR) $(TESTSRCDATADIR)/$(<F)
$(TESTBUILDDIR)/%.nrm: $(TESTSRCDATADIR)/%.txt $(TOOLBINDIR)/gennorm2$(EXEEXT)
$(INVOKE) $(TOOLBINDIR)/gennorm2 -s $(TESTSRCDATADIR) $(<F) -o $@
$(TESTBUILDDIR)/%.res: $(TESTSRCDATADIR)/%.txt $(TOOLBINDIR)/genrb$(EXEEXT) $(DAT_FILES)
$(INVOKE) $(TOOLBINDIR)/genrb $(GENRBOPTS) -q -s $(TESTSRCDATADIR) $(ICU_DATA_OPT) -d $(TESTBUILDDIR) $(<F)
$(TESTBUILDDIR)/encoded.res: $(TESTSRCDATADIR)/encoded.utf16be $(TOOLBINDIR)/genrb$(EXEEXT)
@echo Testing genrb -e option
ICU_DATA=$(BUILDDIR) $(INVOKE) $(TOOLBINDIR)/genrb $(GENRBOPTS) -s $(TESTSRCDATADIR) -eUTF-16BE -d $(TESTBUILDDIR) $(<F)
@echo Finished testing genrb -e option
################################################################### TYP
# TYP FILES
$(TESTOUTDIR)/$(TESTDT)/nam.typ: $(TESTBUILDDIR)/te_IN.res
cp $< $@
$(TESTBUILDDIR)/old_l_testtypes.res: $(TESTSRCDATADIR)/old_l_testtypes.res
cp $< $@
$(TESTBUILDDIR)/old_e_testtypes.res: $(TESTSRCDATADIR)/old_e_testtypes.res
cp $< $@
$(TESTOUTDIR)/$(TESTDT)/zoneinfo64.res: $(TESTBUILDDIR)/zoneinfo64.res
cp $< $@
testdata: build-dir $(ALL_TEST_FILES) $(UNPACKAGEDTESTDATA)
#$(ALL_TEST_FILES) $(TESTBUILDDIR)/testtable32.txt $(TESTBUILDDIR)/testdata.lst: build-dir
packagetest: build-dir testdata pkgdata.inc $(TESTBUILDDIR)/testdata.lst
$(PKGDATA_INVOKE) $(PKGDATA) -T $(TESTBUILDDIR) -d $(TESTOUTDIR) -s $(TESTBUILDDIR) -p $(TESTDATA) -m common $(TESTBUILDDIR)/testdata.lst
packagetest: build-dir testdata pkgdata.inc $(TMP_DIR)/testdata.lst
$(PKGDATA_INVOKE) $(PKGDATA) -T $(TESTBUILDDIR) -d $(TESTOUTDIR) -s $(TESTBUILDDIR) -p $(TESTDATA) -m common $(TMP_DIR)/testdata.lst
build-dir: $(BUILD_DIRS)
$(BUILD_DIRS):

153
icu4c/source/test/testdata/testdata.mak vendored
View file

@ -25,143 +25,22 @@ ALL : "$(TESTDATAOUT)\testdata.dat"
# old_e_testtypes.res is the same, but icuswapped to big-endian EBCDIC
# the following file has $(TEST_RES_SOURCE)
!INCLUDE "$(TESTDATA)\tstfiles.mk"
TESTDATATMP="$(TESTDATAOUT)\testdata"
TEST_RES_FILES = $(TEST_RES_SOURCE:.txt=.res)
CREATE_DIRS :
@if not exist "$(TESTDATAOUT)\$(NULL)" mkdir "$(TESTDATAOUT)"
@if not exist "$(TESTDATABLD)\$(NULL)" mkdir "$(TESTDATABLD)"
@if not exist "$(TESTDATATMP)\$(NULL)" mkdir "$(TESTDATATMP)"
"$(TESTDATAOUT)\testdata.dat" : $(TEST_RES_FILES) "$(TESTDATABLD)\casing.res" "$(TESTDATABLD)\conversion.res" "$(TESTDATABLD)\icuio.res" "$(TESTDATABLD)\mc.res" "$(TESTDATABLD)\structLocale.res" "$(TESTDATABLD)\root.res" "$(TESTDATABLD)\sh.res" "$(TESTDATABLD)\sh_YU.res" "$(TESTDATABLD)\te.res" "$(TESTDATABLD)\te_IN.res" "$(TESTDATABLD)\te_IN_REVISED.res" "$(TESTDATABLD)\testaliases.res" "$(TESTDATABLD)\testtypes.res" "$(TESTDATABLD)\testempty.res" "$(TESTDATABLD)\encoded.res" "$(TESTDATABLD)\idna_rules.res" "$(TESTDATABLD)\test.icu" "$(TESTDATABLD)\testtable32.res" "$(TESTDATABLD)\test1.cnv" "$(TESTDATABLD)\test1bmp.cnv" "$(TESTDATABLD)\test2.cnv" "$(TESTDATABLD)\test3.cnv" "$(TESTDATABLD)\test4.cnv" "$(TESTDATABLD)\test4x.cnv" "$(TESTDATABLD)\test5.cnv" "$(TESTDATABLD)\ibm9027.cnv" "$(TESTDATABLD)\nfscsi.spp" "$(TESTDATABLD)\nfscss.spp" "$(TESTDATABLD)\nfscis.spp" "$(TESTDATABLD)\nfsmxs.spp" "$(TESTDATABLD)\nfsmxp.spp" "$(TESTDATABLD)\testnorm.nrm" "$(TESTDATABLD)\zoneinfo64.res"
"$(TESTDATAOUT)\testdata.dat" :
@echo Building test data
@copy "$(TESTDATABLD)\te.res" "$(TESTDATAOUT)\$(TESTDT)\nam.typ"
@copy "$(TESTDATA)\old_l_testtypes.res" "$(TESTDATABLD)"
@copy "$(TESTDATA)\old_e_testtypes.res" "$(TESTDATABLD)"
@copy "$(TESTDATABLD)\zoneinfo64.res" "$(TESTDATAOUT)\$(TESTDT)"
"$(ICUPBIN)\pkgdata" -f -v -m common -c -p"$(TESTPKG)" -d "$(TESTDATAOUT)" -T "$(TESTDATABLD)" -s "$(TESTDATABLD)" <<
casing.res
conversion.res
mc.res
root.res
testtable32.res
sh.res
sh_YU.res
te.res
te_IN.res
te_IN_REVISED.res
testtypes.res
old_l_testtypes.res
old_e_testtypes.res
testempty.res
testaliases.res
structLocale.res
icuio.res
encoded.res
test.icu
test1.cnv
test1bmp.cnv
test2.cnv
test3.cnv
test4.cnv
test4x.cnv
test5.cnv
ibm9027.cnv
idna_rules.res
nfscsi.spp
nfscss.spp
nfscis.spp
nfsmxs.spp
nfsmxp.spp
testnorm.nrm
$(TEST_RES_FILES:.res =.res
)
<<
# Inference rule for creating resource bundles
# Some test data resource bundles are known to have warnings and bad data.
# The -q option is there on purpose, so we don't see it normally.
{$(TESTDATA)}.txt.res::
@echo Making Test Resource Bundle files $<
@"$(ICUTOOLS)\genrb\$(CFG)\genrb" -q -s"$(TESTDATA)" -d"$(TESTDATABLD)" $<
"$(TESTDATABLD)\encoded.res": "$(TESTDATA)\encoded.utf16be"
@echo Making Test Resource Bundle file with encoding
"$(ICUTOOLS)\genrb\$(CFG)\genrb" -s"$(TESTDATA)" -eUTF-16BE -d"$(TESTDATABLD)" encoded.utf16be
"$(TESTDATABLD)\idna_rules.res": "$(TESTDATA)\idna_rules.txt"
@echo Making Test Resource Bundle file for IDNA reference implementation
@"$(ICUTOOLS)\genrb\$(CFG)\genrb" -s"$(TESTDATA)" -d"$(TESTDATABLD)" idna_rules.txt
"$(TESTDATABLD)\test.icu" : {"$(ICUTOOLS)\gentest\$(CFG)"}gentest.exe
"$(ICUTOOLS)\gentest\$(CFG)\gentest" -d"$(TESTDATABLD)"
# testtable32 resource file
"$(TESTDATABLD)\testtable32.txt" : {"$(ICUTOOLS)\gentest\$(CFG)"}gentest.exe
"$(ICUTOOLS)\gentest\$(CFG)\gentest" -r -d"$(TESTDATABLD)"
"$(TESTDATABLD)\testtable32.res": "$(TESTDATABLD)\testtable32.txt"
"$(ICUTOOLS)\genrb\$(CFG)\genrb" -s"$(TESTDATABLD)" -d"$(TESTDATABLD)" testtable32.txt
# Targets for nfscsi.spp
"$(TESTDATABLD)\nfscsi.spp" : {"$(ICUTOOLS)\gensprep\$(CFG)"}gensprep.exe "$(TESTDATA)\nfs4_cs_prep_ci.txt"
@echo Building $@
@"$(ICUTOOLS)\gensprep\$(CFG)\gensprep" -s "$(TESTDATA)" -d "$(TESTDATABLD)\\" -b nfscsi -u 3.2.0 nfs4_cs_prep_ci.txt
# Targets for nfscss.spp
"$(TESTDATABLD)\nfscss.spp" : {"$(ICUTOOLS)\gensprep\$(CFG)"}gensprep.exe "$(TESTDATA)\nfs4_cs_prep_cs.txt"
@echo Building $@
@"$(ICUTOOLS)\gensprep\$(CFG)\gensprep" -s "$(TESTDATA)" -d "$(TESTDATABLD)\\" -b nfscss -u 3.2.0 nfs4_cs_prep_cs.txt
# Targets for nfscis.spp
"$(TESTDATABLD)\nfscis.spp" : {"$(ICUTOOLS)\gensprep\$(CFG)"}gensprep.exe "$(TESTDATA)\nfs4_cis_prep.txt"
@echo Building $@
@"$(ICUTOOLS)\gensprep\$(CFG)\gensprep" -s "$(TESTDATA)" -d "$(TESTDATABLD)\\" -b nfscis -u 3.2.0 -k -n "$(ICUTOOLS)\..\data\unidata" nfs4_cis_prep.txt
# Targets for nfsmxs.spp
"$(TESTDATABLD)\nfsmxs.spp" : {"$(ICUTOOLS)\gensprep\$(CFG)"}gensprep.exe "$(TESTDATA)\nfs4_mixed_prep_s.txt"
@echo Building $@
@"$(ICUTOOLS)\gensprep\$(CFG)\gensprep" -s "$(TESTDATA)" -d "$(TESTDATABLD)\\" -b nfsmxs -u 3.2.0 -k -n "$(ICUTOOLS)\..\data\unidata" nfs4_mixed_prep_s.txt
# Targets for nfsmxp.spp
"$(TESTDATABLD)\nfsmxp.spp" : {"$(ICUTOOLS)\gensprep\$(CFG)"}gensprep.exe "$(TESTDATA)\nfs4_mixed_prep_p.txt"
@echo Building $@
@"$(ICUTOOLS)\gensprep\$(CFG)\gensprep" -s "$(TESTDATA)" -d "$(TESTDATABLD)\\" -b nfsmxp -u 3.2.0 -k -n "$(ICUTOOLS)\..\data\unidata" nfs4_mixed_prep_p.txt
# Targets for test converter data
"$(TESTDATABLD)\test1.cnv": "$(TESTDATA)\test1.ucm"
@echo Building $@
@"$(ICUTOOLS)\makeconv\$(CFG)\makeconv" --small -d"$(TESTDATABLD)" $**
"$(TESTDATABLD)\test1bmp.cnv": "$(TESTDATA)\test1bmp.ucm"
@echo Building $@
@"$(ICUTOOLS)\makeconv\$(CFG)\makeconv" --small -d"$(TESTDATABLD)" $**
"$(TESTDATABLD)\test2.cnv": "$(TESTDATA)\test2.ucm"
@echo Building $@
@"$(ICUTOOLS)\makeconv\$(CFG)\makeconv" --small -d"$(TESTDATABLD)" $**
"$(TESTDATABLD)\test3.cnv": "$(TESTDATA)\test3.ucm"
@echo Building $@
@"$(ICUTOOLS)\makeconv\$(CFG)\makeconv" --small -d"$(TESTDATABLD)" $**
"$(TESTDATABLD)\test4.cnv": "$(TESTDATA)\test4.ucm"
@echo Building $@
@"$(ICUTOOLS)\makeconv\$(CFG)\makeconv" --small -d"$(TESTDATABLD)" $**
"$(TESTDATABLD)\test4x.cnv": "$(TESTDATA)\test4x.ucm"
@echo Building $@
@"$(ICUTOOLS)\makeconv\$(CFG)\makeconv" --small -d"$(TESTDATABLD)" $**
"$(TESTDATABLD)\test5.cnv": "$(TESTDATA)\test5.ucm"
@echo Building $@
@"$(ICUTOOLS)\makeconv\$(CFG)\makeconv" --small -d"$(TESTDATABLD)" $**
"$(TESTDATABLD)\ibm9027.cnv": "$(TESTDATA)\ibm9027.ucm"
@echo Building $@
@"$(ICUTOOLS)\makeconv\$(CFG)\makeconv" --small -d"$(TESTDATABLD)" $**
# Target for test normalization data
"$(TESTDATABLD)\testnorm.nrm": "$(TESTDATA)\testnorm.txt"
@echo Building $@
@"$(ICUTOOLS)\gennorm2\$(CFG)\gennorm2" -s "$(TESTDATA)" testnorm.txt -o $@
set PYTHONPATH=$(ICUSRCDATA);%PYTHONPATH%
py -3 -m buildtool \
--format windirect \
--tool_dir "$(ICUTOOLS)" \
--tool_cfg "$(CFG)" \
--glob_dir "$(TESTDATA)" \
--in_dir "$(TESTDATA)" \
--tmp_dir "$(TESTDATATMP)" \
--out_dir "$(TESTDATABLD)"
"$(ICUPBIN)\pkgdata" -f -v -m common -c -p"$(TESTPKG)" -d "$(TESTDATAOUT)" -T "$(TESTDATABLD)" -s "$(TESTDATABLD)" $(TESTDATATMP)\testdata.lst

27
icu4c/source/tools/gencnval/gencnval.c
View file

@ -263,23 +263,22 @@ main(int argc, char* argv[]) {
quiet = TRUE;
}
if(argc>=2) {
path=argv[1];
if (argc >= 2) {
path = argv[1];
} else {
path=options[SOURCEDIR].value;
if(path!=NULL && *path!=0) {
char *end;
path = "convrtrs.txt";
}
uprv_strcpy(pathBuf, path);
end = uprv_strchr(pathBuf, 0);
if(*(end-1)!=U_FILE_SEP_CHAR) {
*(end++)=U_FILE_SEP_CHAR;
}
uprv_strcpy(end, "convrtrs.txt");
path=pathBuf;
} else {
path = "convrtrs.txt";
const char* sourcedir = options[SOURCEDIR].value;
if (sourcedir != NULL && *sourcedir != 0) {
char *end;
uprv_strcpy(pathBuf, sourcedir);
end = uprv_strchr(pathBuf, 0);
if(*(end-1)!=U_FILE_SEP_CHAR) {
*(end++)=U_FILE_SEP_CHAR;
}
uprv_strcpy(end, path);
path = pathBuf;
}
uprv_memset(stringStore, 0, sizeof(stringStore));

14
icu4c/source/tools/genrb/genrb.cpp
View file

@ -99,7 +99,7 @@ UOption options[]={
UOPTION_DEF("language", 'l', UOPT_REQUIRES_ARG), /* 16 */
UOPTION_DEF("omitCollationRules", 'R', UOPT_NO_ARG),/* 17 */
UOPTION_DEF("formatVersion", '\x01', UOPT_REQUIRES_ARG),/* 18 */
UOPTION_DEF("writePoolBundle", '\x01', UOPT_NO_ARG),/* 19 */
UOPTION_DEF("writePoolBundle", '\x01', UOPT_OPTIONAL_ARG),/* 19 */
UOPTION_DEF("usePoolBundle", '\x01', UOPT_OPTIONAL_ARG),/* 20 */
UOPTION_DEF("includeUnihanColl", '\x01', UOPT_NO_ARG),/* 21 */ /* temporary, don't display in usage info */
};
@ -224,8 +224,8 @@ main(int argc,
"\t --formatVersion write a .res file compatible with the requested formatVersion (single digit);\n"
"\t for example, --formatVersion 1\n");
fprintf(stderr,
"\t --writePoolBundle write a pool.res file with all of the keys of all input bundles\n"
"\t --usePoolBundle [path-to-pool.res] point to keys from the pool.res keys pool bundle if they are available there;\n"
"\t --writePoolBundle [directory] write a pool.res file with all of the keys of all input bundles\n"
"\t --usePoolBundle [directory] point to keys from the pool.res keys pool bundle if they are available there;\n"
"\t makes .res files smaller but dependent on the pool bundle\n"
"\t (--writePoolBundle and --usePoolBundle cannot be combined)\n");
@ -532,8 +532,14 @@ main(int argc,
poolBundle.close();
if(U_SUCCESS(status) && options[WRITE_POOL_BUNDLE].doesOccur) {
const char* writePoolDir;
if (options[WRITE_POOL_BUNDLE].value!=NULL) {
writePoolDir = options[WRITE_POOL_BUNDLE].value;
} else {
writePoolDir = outputDir;
}
char outputFileName[256];
newPoolBundle->write(outputDir, NULL, outputFileName, sizeof(outputFileName), status);
newPoolBundle->write(writePoolDir, NULL, outputFileName, sizeof(outputFileName), status);
if(U_FAILURE(status)) {
fprintf(stderr, "unable to write the pool bundle: %s\n", u_errorName(status));
}

7
icu4c/source/tools/genrb/reslist.cpp
View file

@ -1037,14 +1037,15 @@ void SRBRoot::write(const char *outputDir, const char *outputPkg,
// Swap to big-endian so we get the same checksum on all platforms
// (except for charset family, due to the key strings).
UnicodeString s(f16BitUnits);
s.append((UChar)1); // Ensure that we own this buffer.
assert(!s.isBogus());
uint16_t *p = const_cast<uint16_t *>(reinterpret_cast<const uint16_t *>(s.getBuffer()));
// .getBuffer(capacity) returns a mutable buffer
char16_t* p = s.getBuffer(f16BitUnits.length());
for (int32_t count = f16BitUnits.length(); count > 0; --count) {
uint16_t x = *p;
*p++ = (uint16_t)((x << 8) | (x >> 8));
}
checksum = computeCRC((const char *)p,
s.releaseBuffer(f16BitUnits.length());
checksum = computeCRC((const char *)s.getBuffer(),
(uint32_t)f16BitUnits.length() * 2, checksum);
}
indexes[URES_INDEX_POOL_CHECKSUM] = (int32_t)checksum;

14
icu4c/source/tools/makeconv/makeconv.cpp
View file

@ -179,6 +179,7 @@ enum {
OPT_SMALL,
OPT_IGNORE_SISO_CHECK,
OPT_QUIET,
OPT_SOURCEDIR,
OPT_COUNT
};
@ -193,6 +194,7 @@ static UOption options[]={
{ "small", NULL, NULL, NULL, '\1', UOPT_NO_ARG, 0 },
{ "ignore-siso-check", NULL, NULL, NULL, '\1', UOPT_NO_ARG, 0 },
UOPTION_QUIET,
UOPTION_SOURCEDIR,
};
int main(int argc, char* argv[])
@ -230,7 +232,8 @@ int main(int argc, char* argv[])
"\t-c or --copyright include a copyright notice\n"
"\t-d or --destdir destination directory, followed by the path\n"
"\t-v or --verbose Turn on verbose output\n"
"\t-q or --quiet do not display warnings and progress\n",
"\t-q or --quiet do not display warnings and progress\n"
"\t-s or --sourcedir source directory, followed by the path\n",
argv[0]);
fprintf(stdfile,
"\t --small Generate smaller .cnv files. They will be\n"
@ -282,11 +285,20 @@ int main(int argc, char* argv[])
#endif
UBool printFilename = (UBool) (argc > 2 || VERBOSE);
icu::CharString pathBuf;
for (++argv; --argc; ++argv)
{
UErrorCode localError = U_ZERO_ERROR;
const char *arg = getLongPathname(*argv);
const char* sourcedir = options[OPT_SOURCEDIR].value;
if (sourcedir != NULL && *sourcedir != 0) {
pathBuf.clear();
pathBuf.appendPathPart(sourcedir, localError);
pathBuf.appendPathPart(arg, localError);
arg = pathBuf.data();
}
/*produces the right destination path for display*/
outFileName.truncate(outBasenameStart);
if (outBasenameStart != 0)