From cfa805ad6ce2828932caf8689e5a7afa1c34adb1 Mon Sep 17 00:00:00 2001 From: Timofey Date: Tue, 11 Oct 2016 19:15:29 +0300 Subject: [PATCH 1/4] Added the parser for .po format. This is the format in which we get translated strings from our partners. --- tools/python/find_untranslated_strings.py | 16 +++++- tools/python/po_parser.py | 59 +++++++++++++++-------- 2 files changed, 54 insertions(+), 21 deletions(-) diff --git a/tools/python/find_untranslated_strings.py b/tools/python/find_untranslated_strings.py index a5ec529eb7..89fc5bd141 100755 --- a/tools/python/find_untranslated_strings.py +++ b/tools/python/find_untranslated_strings.py @@ -27,7 +27,7 @@ class StringsTxt: else: self.strings_path = strings_path - self.translations = defaultdict(dict) # dict> + self.translations = defaultdict(lambda: defaultdict(str)) # dict> self.translations_by_language = defaultdict(dict) # dict> self.comments_and_tags = defaultdict(dict) self.with_english = [] @@ -36,6 +36,7 @@ class StringsTxt: self.keys_in_order = [] self._read_file() + def process_file(self): self._populate_translations_by_langs() self._find_duplicates() @@ -45,6 +46,19 @@ class StringsTxt: self._find_most_similar() + def add_translation(self, translation, key=None, lang=None): + if not key or not lang: + raise UnboundLocalError("You must provide the key and language for the translation") + if key not in self.keys_in_order: + self.keys_in_order.append(key) + self.translations[key][lang] = translation + self.all_langs.add(lang) + + + def append_to_translation(self, key, lang, tail): + self.translations[key][lang] = self.translations[key][lang] + tail + + def _read_file(self): with open(self.strings_path) as strings: for line in strings: diff --git a/tools/python/po_parser.py b/tools/python/po_parser.py index 04f1538daf..33a56b26da 100644 --- a/tools/python/po_parser.py +++ b/tools/python/po_parser.py @@ -1,27 +1,30 @@ +#!/usr/bin/env python2.7 + from __future__ import print_function from collections import defaultdict from argparse import ArgumentParser from os import listdir from os.path import isfile, join +from find_untranslated_strings import StringsTxt TRANSFORMATION_TABLE = { - "zh_CN": "zh_Hans", - "zh_TW": "zh_Hant", - "no_NO": "no", - "en_GB": "en_GB" + "zh_CN": "zh-Hans", + "zh_TW": "zh-Hant", + "no_NO": "nb", + "en_GB": "en-GB" } #msgid #msgstr class PoParser: - def __init__(self, folder_path): + def __init__(self): args = self.parse_args() self.folder_path = args.folder - - all_po_files = self.find_all_po_files() + self.all_po_files = self.find_all_po_files() + self.strings_txt = StringsTxt(args.strings_txt) def find_all_po_files(self): @@ -32,16 +35,22 @@ class PoParser: def parse_files(self): - for key, tr in self.translations.iteritems(): - print(" [{}]\n en = {}".format(key, tr)) + for po_file in self.all_po_files: + self._parse_one_file( + join(self.folder_path, po_file), + self.lang_from_filename(po_file) + ) def lang_from_filename(self, filename): - # strings_ru_RU.po + # file names are in this format: strings_ru_RU.po lang = filename[len("strings_"):-len(".po")] + if lang in TRANSFORMATION_TABLE: + return TRANSFORMATION_TABLE[lang] + return lang[:2] - def _parse_one_file(self, filepath): + def _parse_one_file(self, filepath, lang): self.translations = defaultdict(str) current_key = None string_started = False @@ -54,19 +63,22 @@ class PoParser: continue translation = self.clean_line(line, "msgstr") if not translation: - print("No translation for key {}".format(current_key)) + print("No translation for key {} in file {}".format(current_key, filepath)) continue - self.translations[current_key] = translation + self.strings_txt.add_translation( + translation, + key="[{}]".format(current_key), + lang=lang + ) string_started = True + elif not line or line.startswith("#"): string_started = False current_key = None else: if not string_started: continue - self.translations[current_key] = "{}{}".format(self.translations[current_key], self.clean_line(line)) - - + self.strings_txt.append_to_translation(current_key, lang, self.clean_line(line)) def clean_line(self, line, prefix=""): @@ -87,13 +99,20 @@ class PoParser: help="""Path to the folder where the PO files are. Required.""" ) + parser.add_argument( + "-s", "--strings-txt", + dest="strings_txt", required=True, + help="""The path to the strings.txt file. The strings from the po + files will be added to that strings.txt file.""" + ) + + return parser.parse_args() def main(): - # parser = PoParser("en.po", "en") - # for key, tr in parser.translations.iteritems(): - # print(" [{}]\n en = {}".format(key, tr)) - pass + parser = PoParser() + parser.parse_files() + parser.strings_txt.write_formatted() if __name__ == "__main__": main() \ No newline at end of file From ce7a6f82300a8b32c32ee23a7950f970dda3c153 Mon Sep 17 00:00:00 2001 From: Timofey Date: Wed, 12 Oct 2016 14:21:38 +0300 Subject: [PATCH 2/4] PR fixes --- tools/python/po_parser.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/python/po_parser.py b/tools/python/po_parser.py index 33a56b26da..1d609a8605 100644 --- a/tools/python/po_parser.py +++ b/tools/python/po_parser.py @@ -1,5 +1,5 @@ #!/usr/bin/env python2.7 - +#coding: utf8 from __future__ import print_function from collections import defaultdict @@ -114,5 +114,6 @@ def main(): parser.parse_files() parser.strings_txt.write_formatted() + if __name__ == "__main__": main() \ No newline at end of file From 807732d0088a3638b72f2f1ad24be44729fac06e Mon Sep 17 00:00:00 2001 From: Timofey Date: Wed, 12 Oct 2016 17:30:15 +0300 Subject: [PATCH 3/4] PR Fixes --- tools/python/find_untranslated_strings.py | 4 +--- tools/python/po_parser.py | 3 ++- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/tools/python/find_untranslated_strings.py b/tools/python/find_untranslated_strings.py index 89fc5bd141..74051f3580 100755 --- a/tools/python/find_untranslated_strings.py +++ b/tools/python/find_untranslated_strings.py @@ -46,9 +46,7 @@ class StringsTxt: self._find_most_similar() - def add_translation(self, translation, key=None, lang=None): - if not key or not lang: - raise UnboundLocalError("You must provide the key and language for the translation") + def add_translation(self, translation, key, lang): if key not in self.keys_in_order: self.keys_in_order.append(key) self.translations[key][lang] = translation diff --git a/tools/python/po_parser.py b/tools/python/po_parser.py index 1d609a8605..6721b69d2a 100644 --- a/tools/python/po_parser.py +++ b/tools/python/po_parser.py @@ -75,6 +75,7 @@ class PoParser: elif not line or line.startswith("#"): string_started = False current_key = None + else: if not string_started: continue @@ -116,4 +117,4 @@ def main(): if __name__ == "__main__": - main() \ No newline at end of file + main() From 031f41c56df776a458f57dae33e98c6f198e24e4 Mon Sep 17 00:00:00 2001 From: Timofey Date: Wed, 12 Oct 2016 17:43:45 +0300 Subject: [PATCH 4/4] PR fixes --- tools/python/po_parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/python/po_parser.py b/tools/python/po_parser.py index 6721b69d2a..39e268ab0c 100644 --- a/tools/python/po_parser.py +++ b/tools/python/po_parser.py @@ -101,7 +101,7 @@ class PoParser: ) parser.add_argument( - "-s", "--strings-txt", + "-s", "--strings", dest="strings_txt", required=True, help="""The path to the strings.txt file. The strings from the po files will be added to that strings.txt file."""