From bc95533c6901fa1fb16c599901c27034f5ff3099 Mon Sep 17 00:00:00 2001 From: Alex Zolotarev Date: Fri, 26 Jun 2015 20:21:27 +0300 Subject: [PATCH] Updated Twine utility. --- tools/twine/.gitignore | 3 +- tools/twine/README.md | 58 ++++++++++++++-- tools/twine/bin/twine | 3 +- tools/twine/lib/twine.rb | 1 + tools/twine/lib/twine/cli.rb | 13 +++- tools/twine/lib/twine/formatters.rb | 20 +++++- tools/twine/lib/twine/formatters/abstract.rb | 9 ++- tools/twine/lib/twine/formatters/django.rb | 4 +- tools/twine/lib/twine/formatters/jquery.rb | 1 + tools/twine/lib/twine/plugin.rb | 62 +++++++++++++++++ tools/twine/lib/twine/runner.rb | 69 ++++++++++++------- tools/twine/lib/twine/version.rb | 2 +- .../test-json-line-breaks/consumed.txt | 5 ++ .../test-json-line-breaks/generated.json | 3 + .../test-json-line-breaks/line-breaks.json | 3 + .../test-json-line-breaks/line-breaks.txt | 4 ++ tools/twine/test/fixtures/test-output-10.txt | 2 +- tools/twine/test/fixtures/test-output-12.txt | 12 ++++ tools/twine/test/twine_test.rb | 24 +++++++ tools/twine/twine.gemspec | 1 + 20 files changed, 255 insertions(+), 44 deletions(-) create mode 100644 tools/twine/lib/twine/plugin.rb create mode 100644 tools/twine/test/fixtures/test-json-line-breaks/consumed.txt create mode 100644 tools/twine/test/fixtures/test-json-line-breaks/generated.json create mode 100644 tools/twine/test/fixtures/test-json-line-breaks/line-breaks.json create mode 100644 tools/twine/test/fixtures/test-json-line-breaks/line-breaks.txt create mode 100644 tools/twine/test/fixtures/test-output-12.txt diff --git a/tools/twine/.gitignore b/tools/twine/.gitignore index 7f27d7f1ea..0012a3255d 100644 --- a/tools/twine/.gitignore +++ b/tools/twine/.gitignore @@ -1,2 +1,3 @@ -#Ruby gem *.gem +.idea/ +*.lock diff --git a/tools/twine/README.md b/tools/twine/README.md index 80e0ec8976..0c733a6283 100644 --- a/tools/twine/README.md +++ b/tools/twine/README.md @@ -47,7 +47,7 @@ Whitepace in this file is mostly ignored. If you absolutely need to put spaces a en = No fr = Non ja = いいえ - + [[Errors]] [path_not_found_error] en = The file '%@' could not be found. @@ -57,7 +57,7 @@ Whitepace in this file is mostly ignored. If you absolutely need to put spaces a en = The network is currently unavailable. tags = app1 comment = An error describing when the device can not connect to the internet. - + [[Escaping Example]] [list_item_separator] en = `, ` @@ -77,13 +77,14 @@ Twine currently supports the following formats for outputting strings: * [Gettext PO Files][gettextpo] (format: gettext) * [jquery-localize Language Files][jquerylocalize] (format: jquery) * [Django PO Files][djangopo] (format: django) +* [Tizen String Resources][tizen] (format: tizen) If you would like to enable twine to create language files in another format, create an appropriate formatter in `lib/twine/formatters`. ## Usage Usage: twine COMMAND STRINGS_FILE [INPUT_OR_OUTPUT_PATH] [--lang LANG1,LANG2...] [--tags TAG1,TAG2,TAG3...] [--format FORMAT] - + ### Commands #### `generate-string-file` @@ -129,10 +130,16 @@ This command is a convenient way of taking a zip file and executing the `consume #### `generate-report` -This command gives you useful information about your strings. It will tell you how many strings you have, how many have been translated into each language, and whether your master strings data file has any duplicate string keys. +This command gives you useful information about your strings. It will tell you how many strings you have and how many have been translated into each language. $ twine generate-report /path/to/strings.txt +#### `validate-strings-file` + +This command validates that the strings file can be parsed, contains no duplicate keys, and that all strings have at least one tag. It will exit with a non-zero status code if any of those criteria are not met. + + $ twine validate-strings-file /path/to/strings.txt + ## Creating Your First strings.txt File The easiest way to create your first strings.txt file is to run the `consume-all-string-files` command. The one caveat is to first create a blank strings.txt file to use as your starting point. Then, just point the `consume-all-string-files` command at a directory in your project containing all of your iOS, OS X, or Android strings files. @@ -142,6 +149,8 @@ The easiest way to create your first strings.txt file is to run the `consume-all ## Twine and Your Build Process +### Xcode + It is easy to incorporate Twine right into your iOS and OS X app build processes. 1. In your project folder, create all of the `.lproj` directories that you need. It does not really matter where they are. We tend to put them in `Resources/Locales/`. @@ -158,22 +167,62 @@ It is easy to incorporate Twine right into your iOS and OS X app build processes Now, whenever you build your application, Xcode will automatically invoke Twine to make sure that your `.strings` files are up-to-date. +### Android Studio/Gradle + +Add the following task at the top level in app/build.gradle: +``` +task generateStrings { + String script = 'if hash twine 2>/dev/null; then twine generate-string-file strings.txt ./src/main/res/values/generated_strings.xml; fi' + exec { + executable "sh" + args '-c', script + } +} +``` + +Now every time you build your app the strings are generated from the twine file. + + ## User Interface * [Twine TextMate 2 Bundle](https://github.com/mobiata/twine.tmbundle) — This [TextMate 2](https://github.com/textmate/textmate) bundle will make it easier for you to work with Twine strings files. In particular, it lets you use code folding to easily collapse and expand both strings and sections. * [twine_ui](https://github.com/Daij-Djan/twine_ui) — A user interface for Twine written by [Dominik Pich](https://github.com/Daij-Djan/). Consider using this if you would prefer to use Twine without dropping to a command line. +## Plugin Support + +Twine supports a basic plugin infrastructure, allowing third-party code to provide support for additional formatters. Twine will read a yaml config file specifying which plugins to load from three locations. + +0. `./twine.yml` The current working directory +0. `~/.twine` The home directory +0. `/etc/twine.yml` The etc directory + +Plugins are specified as values for the `gems` key. The following is an example config: + +``` +gems: appium_twine +``` + +Multiple gems can also be specfied in the yaml file. + +``` +gems: [appium_twine, some_other_plugin] +``` + +[appium_twine](https://github.com/appium/appium_twine) is a sample plugin used to provide a C# formatter. + ## Contributors Many thanks to all of the contributors to the Twine project, including: * [Blake Watters](https://github.com/blakewatters) +* [bootstraponline](https://github.com/bootstraponline) * [Ishitoya Kentaro](https://github.com/kent013) * [Joseph Earl](https://github.com/JosephEarl) * [Kevin Everets](https://github.com/keverets) * [Kevin Wood](https://github.com/kwood) * [Mohammad Hejazi](https://github.com/MohammadHejazi) * [Robert Guo](http://www.robertguo.me/) +* [Sergey Pisarchik](https://github.com/SergeyPisarchik) * [Shai Shamir](https://github.com/pichirichi) @@ -185,3 +234,4 @@ Many thanks to all of the contributors to the Twine project, including: [gettextpo]: http://www.gnu.org/savannah-checkouts/gnu/gettext/manual/html_node/PO-Files.html [jquerylocalize]: https://github.com/coderifous/jquery-localize [djangopo]: https://docs.djangoproject.com/en/dev/topics/i18n/translation/ +[tizen]: https://developer.tizen.org/documentation/articles/localization diff --git a/tools/twine/bin/twine b/tools/twine/bin/twine index 690f51f28f..da3cf272a8 100755 --- a/tools/twine/bin/twine +++ b/tools/twine/bin/twine @@ -3,6 +3,5 @@ require 'twine' begin Twine::Runner.run(ARGV) rescue Twine::Error => e - STDERR.puts e.message - exit + abort e.message end diff --git a/tools/twine/lib/twine.rb b/tools/twine/lib/twine.rb index 71d3bdce7f..412a6b01c9 100644 --- a/tools/twine/lib/twine.rb +++ b/tools/twine/lib/twine.rb @@ -2,6 +2,7 @@ module Twine class Error < StandardError end + require 'twine/plugin' require 'twine/cli' require 'twine/encoding' require 'twine/formatters' diff --git a/tools/twine/lib/twine/cli.rb b/tools/twine/lib/twine/cli.rb index d824359cdc..f5210bc48f 100644 --- a/tools/twine/lib/twine/cli.rb +++ b/tools/twine/lib/twine/cli.rb @@ -31,7 +31,9 @@ module Twine opts.separator '' opts.separator 'consume-loc-drop -- Consumes an archive of translated files. This archive should be in the same format as the one created by the generate-loc-drop command.' opts.separator '' - opts.separator 'generate-report -- Generates a report containing data about your strings. For example, it will tell you if you have any duplicate strings or if any of your strings are missing tags. In addition, it will tell you how many strings you have and how many of those strings have been translated into each language.' + opts.separator 'generate-report -- Generates a report containing data about your strings. It will tell you how many strings you have and how many of those strings have been translated into each language.' + opts.separator '' + opts.separator 'validate-strings-file -- Validates that the given strings file is parseable, contains no duplicates, and that every string has a tag. Exits with a non-zero exit code if those criteria are not met.' opts.separator '' opts.separator 'General Options:' opts.separator '' @@ -45,7 +47,7 @@ module Twine @options[:untagged] = true end formats = [] - Formatters::FORMATTERS.each do |formatter| + Formatters.formatters.each do |formatter| formats << formatter::FORMAT_NAME end opts.on('-f', '--format FORMAT', "The file format to read or write (#{formats.join(', ')}). Additional formatters can be placed in the formats/ directory.") do |format| @@ -93,10 +95,11 @@ module Twine opts.separator '> twine generate-string-file strings.txt ko.xml --tags FT' opts.separator '> twine generate-all-string-files strings.txt Resources/Locales/ --tags FT,FB' opts.separator '> twine consume-string-file strings.txt ja.strings' - opts.separator '> twine consume-all-string-files strings.txt Resources/Locales/ --developer-language en' + opts.separator '> twine consume-all-string-files strings.txt Resources/Locales/ --developer-language en --tags DefaultTag1,DefaultTag2' opts.separator '> twine generate-loc-drop strings.txt LocDrop5.zip --tags FT,FB --format android --lang de,en,en-GB,ja,ko' opts.separator '> twine consume-loc-drop strings.txt LocDrop5.zip' opts.separator '> twine generate-report strings.txt' + opts.separator '> twine validate-strings-file strings.txt' end parser.parse! @args @@ -180,6 +183,10 @@ module Twine if @args.length > 2 raise Twine::Error.new "Unknown argument: #{@args[2]}" end + when 'validate-strings-file' + if @args.length > 2 + raise Twine::Error.new "Unknown argument: #{@args[2]}" + end end end end diff --git a/tools/twine/lib/twine/formatters.rb b/tools/twine/lib/twine/formatters.rb index a5e9fc844a..7a90ecccb9 100644 --- a/tools/twine/lib/twine/formatters.rb +++ b/tools/twine/lib/twine/formatters.rb @@ -9,6 +9,22 @@ require 'twine/formatters/tizen' module Twine module Formatters - FORMATTERS = [Formatters::Apple, Formatters::Android, Formatters::Gettext, Formatters::JQuery, Formatters::Flash, Formatters::Django, Formatters::Tizen] + @formatters = [Formatters::Apple, Formatters::Android, Formatters::Gettext, Formatters::JQuery, Formatters::Flash, Formatters::Django, Formatters::Tizen] + + class << self + attr_reader :formatters + + ### + # registers a new formatter + # + # formatter_class - the class of the formatter to register + # + # returns array of active formatters + # + def register_formatter formatter_class + raise "#{formatter_class} already registered" if @formatters.include? formatter_class + @formatters << formatter_class + end + end end -end +end \ No newline at end of file diff --git a/tools/twine/lib/twine/formatters/abstract.rb b/tools/twine/lib/twine/formatters/abstract.rb index c9f2df9770..4d6d844efc 100644 --- a/tools/twine/lib/twine/formatters/abstract.rb +++ b/tools/twine/lib/twine/formatters/abstract.rb @@ -66,7 +66,7 @@ module Twine return str end end - + def set_translation_for_key(key, lang, value) if @strings.strings_map.include?(key) @strings.strings_map[key].translations[lang] = value @@ -80,6 +80,11 @@ module Twine end current_row = StringsRow.new(key) current_section.rows << current_row + + if @options[:tags] && @options[:tags].length > 0 + current_row.tags = @options[:tags] + end + @strings.strings_map[key] = current_row @strings.strings_map[key].translations[lang] = value else @@ -133,7 +138,7 @@ module Twine end end if langs_written.empty? - raise Twine::Error.new("Failed to genertate any files: No languages found at #{path}") + raise Twine::Error.new("Failed to generate any files: No languages found at #{path}") end end end diff --git a/tools/twine/lib/twine/formatters/django.rb b/tools/twine/lib/twine/formatters/django.rb index 4a5352b4a1..981f4d7045 100644 --- a/tools/twine/lib/twine/formatters/django.rb +++ b/tools/twine/lib/twine/formatters/django.rb @@ -21,7 +21,7 @@ module Twine return match[1] end end - + return end @@ -106,7 +106,7 @@ module Twine end printed_section = true end - + basetrans = row.translated_string_for_lang(default_lang) key = row.key diff --git a/tools/twine/lib/twine/formatters/jquery.rb b/tools/twine/lib/twine/formatters/jquery.rb index 404b62b83d..792d52bb33 100644 --- a/tools/twine/lib/twine/formatters/jquery.rb +++ b/tools/twine/lib/twine/formatters/jquery.rb @@ -35,6 +35,7 @@ module Twine open(path) do |io| json = JSON.load(io) json.each do |key, value| + value.gsub!("\n","\\n") set_translation_for_key(key, lang, value) end end diff --git a/tools/twine/lib/twine/plugin.rb b/tools/twine/lib/twine/plugin.rb new file mode 100644 index 0000000000..0967e1ad90 --- /dev/null +++ b/tools/twine/lib/twine/plugin.rb @@ -0,0 +1,62 @@ +require 'safe_yaml/load' + +SafeYAML::OPTIONS[:suppress_warnings] = true + +module Twine + class Plugin + attr_reader :debug, :config + + def initialize + @debug = false + require_gems + end + + ### + # require gems from the yaml config. + # + # gems: [twine-plugin1, twine-2] + # + # also works with single gem + # + # gems: twine-plugin1 + # + def require_gems + # ./twine.yml # current working directory + # ~/.twine # home directory + # /etc/twine.yml # etc + cwd_config = join_path Dir.pwd, 'twine.yml' + home_config = join_path Dir.home, '.twine' + etc_config = '/etc/twine.yml' + + config_order = [cwd_config, home_config, etc_config] + + puts "Config order: #{config_order}" if debug + + config_order.each do |config_file| + next unless valid_file config_file + puts "Loading: #{config_file}" if debug + @config = SafeYAML.load_file config_file + puts "Config yaml: #{config}" if debug + break + end + + return unless config + + # wrap gems in an array. if nil then array will be empty + Kernel.Array(config['gems']).each do |gem_path| + puts "Requiring: #{gem_path}" if debug + require gem_path + end + end + + private + + def valid_file path + File.exist?(path) && File.readable?(path) && !File.directory?(path) + end + + def join_path *paths + File.expand_path File.join *paths + end + end +end diff --git a/tools/twine/lib/twine/runner.rb b/tools/twine/lib/twine/runner.rb index 7e6b89e377..9946cbcb76 100644 --- a/tools/twine/lib/twine/runner.rb +++ b/tools/twine/lib/twine/runner.rb @@ -1,7 +1,9 @@ require 'tmpdir' +Twine::Plugin.new # Initialize plugins first in Runner. + module Twine - VALID_COMMANDS = ['generate-string-file', 'generate-all-string-files', 'consume-string-file', 'consume-all-string-files', 'generate-loc-drop', 'consume-loc-drop', 'generate-report'] + VALID_COMMANDS = ['generate-string-file', 'generate-all-string-files', 'consume-string-file', 'consume-all-string-files', 'generate-loc-drop', 'consume-loc-drop', 'generate-report', 'validate-strings-file'] class Runner def initialize(args) @@ -48,6 +50,8 @@ module Twine consume_loc_drop when 'generate-report' generate_report + when 'validate-strings-file' + validate_strings_file end end @@ -208,13 +212,34 @@ module Twine def generate_report total_strings = 0 strings_per_lang = {} - all_keys = Set.new - duplicate_keys = Set.new - keys_without_tags = Set.new @strings.language_codes.each do |code| strings_per_lang[code] = 0 end + @strings.sections.each do |section| + section.rows.each do |row| + total_strings += 1 + + row.translations.each_key do |code| + strings_per_lang[code] += 1 + end + end + end + + # Print the report. + puts "Total number of strings = #{total_strings}" + @strings.language_codes.each do |code| + puts "#{code}: #{strings_per_lang[code]}" + end + end + + def validate_strings_file + total_strings = 0 + all_keys = Set.new + duplicate_keys = Set.new + keys_without_tags = Set.new + errors = [] + @strings.sections.each do |section| section.rows.each do |row| total_strings += 1 @@ -225,37 +250,29 @@ module Twine all_keys.add(row.key) end - row.translations.each_key do |code| - strings_per_lang[code] += 1 - end - if row.tags == nil || row.tags.length == 0 keys_without_tags.add(row.key) end end end - # Print the report. - puts "Total number of strings = #{total_strings}" - @strings.language_codes.each do |code| - puts "#{code}: #{strings_per_lang[code]}" - end - if duplicate_keys.length > 0 - puts "\nDuplicate string keys:" - duplicate_keys.each do |key| - puts key - end + error_body = duplicate_keys.to_a.join("\n ") + errors << "Found duplicate string key(s):\n #{error_body}" end if keys_without_tags.length == total_strings - puts "\nNone of your strings have tags." + errors << "None of your strings have tags." elsif keys_without_tags.length > 0 - puts "\nStrings without tags:" - keys_without_tags.each do |key| - puts key - end + error_body = keys_without_tags.to_a.join("\n ") + errors << "Found strings(s) without tags:\n #{error_body}" end + + if errors.length > 0 + raise Twine::Error.new errors.join("\n\n") + end + + puts "#{@options[:strings_file]} is valid." end def determine_language_given_path(path) @@ -269,7 +286,7 @@ module Twine def determine_format_given_path(path) ext = File.extname(path) - Formatters::FORMATTERS.each do |formatter| + Formatters.formatters.each do |formatter| if formatter::EXTENSION == ext return formatter::FORMAT_NAME end @@ -279,7 +296,7 @@ module Twine end def determine_format_given_directory(directory) - Formatters::FORMATTERS.each do |formatter| + Formatters.formatters.each do |formatter| if formatter.can_handle_directory?(directory) return formatter::FORMAT_NAME end @@ -289,7 +306,7 @@ module Twine end def formatter_for_format(format) - Formatters::FORMATTERS.each do |formatter| + Formatters.formatters.each do |formatter| if formatter::FORMAT_NAME == format return formatter.new(@strings, @options) end diff --git a/tools/twine/lib/twine/version.rb b/tools/twine/lib/twine/version.rb index 8df44ab28b..f34e1f23af 100644 --- a/tools/twine/lib/twine/version.rb +++ b/tools/twine/lib/twine/version.rb @@ -1,3 +1,3 @@ module Twine - VERSION = '0.5.0' + VERSION = '0.6.0' end diff --git a/tools/twine/test/fixtures/test-json-line-breaks/consumed.txt b/tools/twine/test/fixtures/test-json-line-breaks/consumed.txt new file mode 100644 index 0000000000..42d826bab6 --- /dev/null +++ b/tools/twine/test/fixtures/test-json-line-breaks/consumed.txt @@ -0,0 +1,5 @@ +[[Line Break Strings]] + [line_breaking] + en = This\nstring\ncontains\nline\nbreaks + tags = tag1 + fr = This\nstring\nalso\ncontains\nline\nbreaks diff --git a/tools/twine/test/fixtures/test-json-line-breaks/generated.json b/tools/twine/test/fixtures/test-json-line-breaks/generated.json new file mode 100644 index 0000000000..fcb238220a --- /dev/null +++ b/tools/twine/test/fixtures/test-json-line-breaks/generated.json @@ -0,0 +1,3 @@ +{ +"line_breaking":"This\nstring\ncontains\nline\nbreaks" +} diff --git a/tools/twine/test/fixtures/test-json-line-breaks/line-breaks.json b/tools/twine/test/fixtures/test-json-line-breaks/line-breaks.json new file mode 100644 index 0000000000..989b6e0f20 --- /dev/null +++ b/tools/twine/test/fixtures/test-json-line-breaks/line-breaks.json @@ -0,0 +1,3 @@ +{ +"line_breaking":"This\nstring\nalso\ncontains\nline\nbreaks" +} diff --git a/tools/twine/test/fixtures/test-json-line-breaks/line-breaks.txt b/tools/twine/test/fixtures/test-json-line-breaks/line-breaks.txt new file mode 100644 index 0000000000..f82d39e8da --- /dev/null +++ b/tools/twine/test/fixtures/test-json-line-breaks/line-breaks.txt @@ -0,0 +1,4 @@ +[[Line Break Strings]] + [line_breaking] + en = This\nstring\ncontains\nline\nbreaks + tags = tag1 diff --git a/tools/twine/test/fixtures/test-output-10.txt b/tools/twine/test/fixtures/test-output-10.txt index d08351c7a9..bc0e4d459d 100644 --- a/tools/twine/test/fixtures/test-output-10.txt +++ b/tools/twine/test/fixtures/test-output-10.txt @@ -1,6 +1,6 @@ - + diff --git a/tools/twine/test/fixtures/test-output-12.txt b/tools/twine/test/fixtures/test-output-12.txt new file mode 100644 index 0000000000..8449225d22 --- /dev/null +++ b/tools/twine/test/fixtures/test-output-12.txt @@ -0,0 +1,12 @@ + + + + + + + + key1-french + key2-french + key3-english + key4-english + diff --git a/tools/twine/test/twine_test.rb b/tools/twine/test/twine_test.rb index 38ef26e7bb..214b9a87e8 100644 --- a/tools/twine/test/twine_test.rb +++ b/tools/twine/test/twine_test.rb @@ -60,6 +60,14 @@ class TwineTest < Test::Unit::TestCase end end + def test_generate_string_file_8 + Dir.mktmpdir do |dir| + output_path = File.join(dir, 'fr.xml') + Twine::Runner.run(%W(generate-string-file --format tizen test/fixtures/strings-1.txt #{output_path} --include-untranslated)) + assert_equal(ERB.new(File.read('test/fixtures/test-output-12.txt')).result, File.read(output_path)) + end + end + def test_consume_string_file_1 Dir.mktmpdir do |dir| output_path = File.join(dir, 'strings.txt') @@ -111,4 +119,20 @@ class TwineTest < Test::Unit::TestCase def test_generate_report_1 Twine::Runner.run(%w(generate-report test/fixtures/strings-1.txt)) end + + def test_json_line_breaks_consume + Dir.mktmpdir do |dir| + output_path = File.join(dir, 'strings.txt') + Twine::Runner.run(%W(consume-string-file test/fixtures/test-json-line-breaks/line-breaks.txt test/fixtures/test-json-line-breaks/line-breaks.json -l fr -o #{output_path})) + assert_equal(File.read('test/fixtures/test-json-line-breaks/consumed.txt'), File.read(output_path)) + end + end + + def test_json_line_breaks_generate + Dir.mktmpdir do |dir| + output_path = File.join(dir, 'en.json') + Twine::Runner.run(%W(generate-string-file test/fixtures/test-json-line-breaks/line-breaks.txt #{output_path})) + assert_equal(File.read('test/fixtures/test-json-line-breaks/generated.json'), File.read(output_path)) + end + end end diff --git a/tools/twine/twine.gemspec b/tools/twine/twine.gemspec index d26db2a47a..bd3ccc334b 100644 --- a/tools/twine/twine.gemspec +++ b/tools/twine/twine.gemspec @@ -19,6 +19,7 @@ Gem::Specification.new do |s| s.required_ruby_version = ">= 1.8.7" s.add_runtime_dependency('rubyzip', "~> 0.9.5") + s.add_runtime_dependency('safe_yaml', "~> 1.0.3") s.add_development_dependency('rake', "~> 0.9.2") s.executables = %w( twine )