diff --git a/generator/CMakeLists.txt b/generator/CMakeLists.txt index 3cfa2d1e0b..6975bfde92 100644 --- a/generator/CMakeLists.txt +++ b/generator/CMakeLists.txt @@ -47,8 +47,6 @@ set( collector_interface.hpp collector_tag.cpp collector_tag.hpp - data_version.cpp - data_version.hpp descriptions_section_builder.cpp descriptions_section_builder.hpp dumper.cpp @@ -91,8 +89,6 @@ set( intermediate_data.cpp intermediate_data.hpp intermediate_elements.hpp - key_value_storage.cpp - key_value_storage.hpp maxspeeds_builder.cpp maxspeeds_builder.hpp maxspeeds_collector.cpp @@ -184,8 +180,6 @@ set( traffic_generator.hpp transit_generator.cpp transit_generator.hpp - translation.cpp - translation.hpp translator.cpp translator.hpp translator_coastline.cpp diff --git a/generator/data_version.cpp b/generator/data_version.cpp deleted file mode 100644 index 747d92010d..0000000000 --- a/generator/data_version.cpp +++ /dev/null @@ -1,90 +0,0 @@ -#include "data_version.hpp" - -#include "base/file_name_utils.hpp" -#include "base/logging.hpp" - -#include "build_version.hpp" - -#include - -#include -#include -#include - -#include - -#include "platform/platform.hpp" - -namespace generator -{ -DataVersion::DataVersion(std::string const & planetFilePath) -{ - size_t planetTimestamp = 0; - - struct stat path_stat{}; - if (::stat(planetFilePath.c_str(), &path_stat) == 0) - planetTimestamp = base::TimeTToSecondsSinceEpoch(path_stat.st_mtime); - - m_json.reset(json_object()); - - ToJSONObject(*m_json, "time_generation_started", - base::TimeTToSecondsSinceEpoch( - std::chrono::system_clock::to_time_t(std::chrono::system_clock::now()))); - ToJSONObject(*m_json, "generator_build_time", omim::build_version::git::kTimestamp); - ToJSONObject(*m_json, "generator_git_hash", omim::build_version::git::kHash); - ToJSONObject(*m_json, "planet_md5", ReadWholeFile(planetFilePath + ".md5")); - ToJSONObject(*m_json, "planet_last_changeset_timestamp", ReadWholeFile(planetFilePath + ".timestamp")); - ToJSONObject(*m_json, "planet_file_timestamp", planetTimestamp); -} - -std::string DataVersion::GetVersionJson() const { return base::DumpToString(m_json); } - -void DataVersion::DumpToPath(std::string const & path) const -{ - std::string filePath = base::JoinPath(path, DataVersion::FileName()); - std::ofstream stream; - stream.exceptions(std::ios::failbit | std::ios::badbit); - stream.open(filePath); - stream << Key() << " " << GetVersionJson(); - - LOG(LINFO, ("Version of data has been written in", filePath)); -} - -// static -DataVersion DataVersion::LoadFromPath(std::string const & path) -{ - DataVersion result; - std::string filePath = base::JoinPath(path, DataVersion::FileName()); - std::string str = ReadWholeFile(filePath); - - CHECK_EQUAL(str.find(Key()), 0, ()); - - result.m_json = base::LoadFromString(str.substr(Key().length())); - LOG(LINFO, ("Version of data has been loaded from", filePath)); - - return result; -} -// static -std::string const & DataVersion::FileName() -{ - static std::string const fileName = "version.jsonl"; - return fileName; -} -// static -std::string const & DataVersion::Key() -{ - static std::string const key = "version"; - return key; -} -// static -std::string DataVersion::ReadWholeFile(std::string const & filePath) -{ - if (!GetPlatform().IsFileExistsByFullPath(filePath)) - return {}; - - std::ifstream stream; - stream.exceptions(std::fstream::failbit | std::fstream::badbit); - stream.open(filePath); - return std::string((std::istreambuf_iterator(stream)), std::istreambuf_iterator()); -} -} // namespace generator diff --git a/generator/data_version.hpp b/generator/data_version.hpp deleted file mode 100644 index 9413408be6..0000000000 --- a/generator/data_version.hpp +++ /dev/null @@ -1,29 +0,0 @@ -#pragma once - -#include "3party/jansson/myjansson.hpp" - -#include -#include - -#include "defines.hpp" - -namespace generator -{ -class DataVersion -{ -public: - static constexpr const char * kFileTag = INDEX_GENERATOR_DATA_VERSION_FILE_TAG; - - explicit DataVersion(std::string const & planetFilePath); - std::string GetVersionJson() const; - void DumpToPath(std::string const & path) const; - static DataVersion LoadFromPath(std::string const & path); - -private: - DataVersion() = default; - static std::string ReadWholeFile(std::string const & filePath); - static std::string const & FileName(); - static std::string const & Key(); - base::JSONPtr m_json; -}; -} // namespace generator diff --git a/generator/extract_addr/extract_addr.cpp b/generator/extract_addr/extract_addr.cpp index 80154e2e1a..a7c5cde696 100644 --- a/generator/extract_addr/extract_addr.cpp +++ b/generator/extract_addr/extract_addr.cpp @@ -1,5 +1,4 @@ #include "generator/feature_builder.hpp" -#include "generator/key_value_storage.hpp" #include "indexer/classificator.hpp" #include "indexer/classificator_loader.hpp" @@ -80,7 +79,7 @@ void PrintFeature(FeatureBuilder const & fb, uint64_t) ToJSONObject(*feature, "geometry", geometry); ToJSONObject(*feature, "properties", properties); - std::cout << generator::KeyValueStorage::Serialize(feature) << std::endl; + std::cout << base::DumpToString(feature, JSON_COMPACT | JSON_REAL_PRECISION(9 /* precision */)) << std::endl; } int main(int argc, char * argv[]) diff --git a/generator/generator_tests/CMakeLists.txt b/generator/generator_tests/CMakeLists.txt index 9f2f8a7884..6afc5da24a 100644 --- a/generator/generator_tests/CMakeLists.txt +++ b/generator/generator_tests/CMakeLists.txt @@ -39,7 +39,6 @@ set( srtm_parser_test.cpp tag_admixer_test.cpp tesselator_test.cpp - translation_test.cpp triangles_tree_coding_test.cpp types_helper.hpp ugc_test.cpp diff --git a/generator/generator_tests/feature_builder_test.cpp b/generator/generator_tests/feature_builder_test.cpp index 4705ecfd14..1f68c38141 100644 --- a/generator/generator_tests/feature_builder_test.cpp +++ b/generator/generator_tests/feature_builder_test.cpp @@ -10,7 +10,6 @@ #include "indexer/data_header.cpp" #include "indexer/classificator_loader.hpp" #include "indexer/feature_visibility.hpp" -#include "indexer/locality_object.hpp" #include "base/geo_object_id.hpp" @@ -261,15 +260,6 @@ UNIT_CLASS_TEST(TestWithClassificator, FeatureBuilder_SerializeLocalityObjectFor auto & buffer = holder.GetBuffer(); TEST(fb.PreSerializeAndRemoveUselessNamesForMwm(buffer), ()); fb.SerializeLocalityObject(serial::GeometryCodingParams(), buffer); - - using indexer::LocalityObject; - LocalityObject object; - object.Deserialize(buffer.m_buffer.data()); - - TEST_EQUAL(LocalityObject::FromStoredId(object.GetStoredId()), base::MakeOsmNode(1), ()); - object.ForEachPoint([] (auto && point) { - TEST(base::AlmostEqualAbs(point, m2::PointD(10.1, 15.8), 1e-7), ()); - }); } UNIT_TEST(FeatureBuilder_SerializeAccuratelyForIntermediate) diff --git a/generator/generator_tests/translation_test.cpp b/generator/generator_tests/translation_test.cpp deleted file mode 100644 index 2bb77e908e..0000000000 --- a/generator/generator_tests/translation_test.cpp +++ /dev/null @@ -1,57 +0,0 @@ -#include "testing/testing.hpp" - -#include "generator/translation.hpp" - -#include "coding/transliteration.hpp" - -#include "platform/platform.hpp" - -#include -#include -#include - -using namespace generator; - -// Transliteration tests --------------------------------------------------------------------------- -using Translations = std::vector>; -bool TestTransliteration(Translations const & translations, - std::string const & expectedTransliteration, std::string const & lang) -{ - StringUtf8Multilang name; - for (auto const & langAndTranslation : translations) - { - name.AddString(langAndTranslation.first, langAndTranslation.second); - } - return GetTranslatedOrTransliteratedName(name, StringUtf8Multilang::GetLangIndex(lang)) == - expectedTransliteration; -} - -UNIT_TEST(Transliteration) -{ - Transliteration & translit = Transliteration::Instance(); - translit.Init(GetPlatform().ResourcesDir()); - - Translations const scotlandTranslations = { - {"default", "Scotland"}, {"be", "Шатландыя"}, {"cs", "Skotsko"}, {"cy", "Yr Alban"}, - {"da", "Skotland"}, {"de", "Schottland"}, {"eo", "Skotlando"}, {"es", "Escocia"}, - {"eu", "Eskozia"}, {"fi", "Skotlanti"}, {"fr", "Écosse"}, {"ga", "Albain"}, - {"gd", "Alba"}, {"hr", "Škotska"}, {"ia", "Scotia"}, {"io", "Skotia"}, - {"ja", "スコットランド"}, {"ku", "Skotland"}, {"lfn", "Scotland"}, {"nl", "Schotland"}, - {"pl", "Szkocja"}, {"ru", "Шотландия"}, {"sco", "Scotland"}, {"sk", "Škótsko"}, - {"sr", "Шкотска"}, {"sv", "Skottland"}, {"tok", "Sukosi"}, {"tzl", "Escot"}, - {"uk", "Шотландія"}, {"vo", "Skotän"}, {"zh", "苏格兰"}}; - - Translations const michiganTranslations = { - {"default", "Michigan"}, {"ar", "ميشيغان"}, {"az", "Miçiqan"}, {"be", "Мічыган"}, - {"bg", "Мичиган"}, {"br", "Michigan"}, {"en", "Michigan"}, {"eo", "Miĉigano"}, - {"es", "Míchigan"}, {"fa", "میشیگان"}, {"haw", "Mikikana"}, {"he", "מישיגן"}, - {"hy", "Միչիգան"}, {"ja", "ミシガン州"}, {"ko", "미시간"}, {"lt", "Mičiganas"}, - {"lv", "Mičigana"}, {"nv", "Míshigin"}, {"pl", "Michigan"}, {"ru", "Мичиган"}, - {"sr", "Мичиген"}, {"ta", "மிச்சிகன்"}, {"th", "รัฐมิชิแกน"}, {"tl", "Misigan"}, - {"uk", "Мічиган"}, {"yi", "מישיגן"}, {"zh", "密歇根州"}}; - - TEST(TestTransliteration(scotlandTranslations, "Scotland", "en"), ()); - TEST(TestTransliteration(michiganTranslations, "Michigan", "en"), ()); - TEST(TestTransliteration(scotlandTranslations, "Шотландия", "ru"), ()); - TEST(TestTransliteration(michiganTranslations, "Мичиган", "ru"), ()); -} diff --git a/generator/generator_tool/generator_tool.cpp b/generator/generator_tool/generator_tool.cpp index da567a591e..d306d0677e 100644 --- a/generator/generator_tool/generator_tool.cpp +++ b/generator/generator_tool/generator_tool.cpp @@ -6,7 +6,6 @@ #include "generator/cities_boundaries_builder.hpp" #include "generator/cities_ids_builder.hpp" #include "generator/city_roads_generator.hpp" -#include "generator/data_version.hpp" #include "generator/descriptions_section_builder.hpp" #include "generator/dumper.hpp" #include "generator/feature_builder.hpp" @@ -45,7 +44,6 @@ #include "indexer/features_offsets_table.hpp" #include "indexer/features_vector.hpp" #include "indexer/index_builder.hpp" -#include "indexer/locality_index_builder.hpp" #include "indexer/map_style_reader.hpp" #include "indexer/rank_table.hpp" @@ -275,8 +273,6 @@ int GeneratorToolMain(int argc, char ** argv) return EXIT_FAILURE; } - DataVersion{FLAGS_osm_file_name}.DumpToPath(genInfo.m_intermediateDir); - // Generate .mwm.tmp files. if (FLAGS_generate_features || FLAGS_generate_world || diff --git a/generator/key_value_storage.cpp b/generator/key_value_storage.cpp deleted file mode 100644 index b886af8db1..0000000000 --- a/generator/key_value_storage.cpp +++ /dev/null @@ -1,133 +0,0 @@ -#include "generator/key_value_storage.hpp" - -#include "coding/reader.hpp" - -#include "base/exception.hpp" -#include "base/logging.hpp" - -#include -#include -#include - - -namespace generator -{ -KeyValueStorage::KeyValueStorage(std::string const & path, size_t cacheValuesCountLimit, - std::function const & pred) - : m_storage{path, std::ios_base::in | std::ios_base::out | std::ios_base::app} - , m_cacheValuesCountLimit{cacheValuesCountLimit} -{ - if (!m_storage) - MYTHROW(Reader::OpenException, ("Failed to open file", path)); - - std::string line; - std::streamoff lineNumber = 0; - while (std::getline(m_storage, line)) - { - ++lineNumber; - - uint64_t key; - auto value = std::string{}; - if (!ParseKeyValueLine(line, lineNumber, key, value)) - continue; - - std::shared_ptr json; - try - { - json = std::make_shared(base::LoadFromString(value)); - } - catch (base::Json::Exception const & e) - { - LOG(LWARNING, ("Cannot create base::Json in line", lineNumber, ":", e.Msg())); - continue; - } - - if (!pred({key, json})) - continue; - - if (m_cacheValuesCountLimit <= m_values.size()) - m_values.emplace(key, std::move(value)); - else - m_values.emplace(key, std::move(json)); - } - - m_storage.clear(); -} - -// static -bool KeyValueStorage::ParseKeyValueLine(std::string const & line, std::streamoff lineNumber, - uint64_t & key, std::string & value) -{ - auto const pos = line.find(" "); - if (pos == std::string::npos) - { - LOG(LWARNING, ("Cannot find separator in line", lineNumber)); - return false; - } - - std::string idStr = line.substr(0, pos); - - if (!strings::to_uint64(idStr, key, 16)) - { - LOG(LWARNING, ("Cannot parse id", line.substr(0, pos), "in line", lineNumber)); - return false; - } - - value = line.c_str() + pos + 1; - return true; -} - -// static -std::string KeyValueStorage::SerializeFullLine(uint64_t key, JsonValue && value) -{ - auto json = Serialize(value); - CHECK(!json.empty(), ()); - - std::stringstream result; - result << SerializeDref(key) << " " << json << "\n"; - return result.str(); -} - -void KeyValueStorage::Insert(uint64_t key, JsonValue && value) -{ - auto json = Serialize(value); - - CHECK(!json.empty(), ()); - - auto emplaceResult = m_values.emplace(key, std::move(json)); - - if (!emplaceResult.second) // it is ok for OSM relation with several outer borders - return; - - auto const & emplaceIterator = emplaceResult.first; - auto const & result = boost::get(emplaceIterator->second); - - m_storage << SerializeDref(key) << " " << result << "\n"; -} - -std::shared_ptr KeyValueStorage::Find(uint64_t key) const -{ - auto const it = m_values.find(key); - if (it == std::end(m_values)) - return {}; - - if (auto json = boost::get>(&it->second)) - return *json; - - auto const & jsonString = boost::get(it->second); - - auto json = std::make_shared(base::LoadFromString(jsonString)); - CHECK(json, ()); - return json; -} - -std::string KeyValueStorage::SerializeDref(uint64_t number) -{ - std::stringstream stream; - stream << std::setw(16) << std::setfill('0') << std::hex << std::uppercase << number; - - return stream.str(); -} - -size_t KeyValueStorage::Size() const { return m_values.size(); } -} // namespace generator diff --git a/generator/key_value_storage.hpp b/generator/key_value_storage.hpp deleted file mode 100644 index a489fd186a..0000000000 --- a/generator/key_value_storage.hpp +++ /dev/null @@ -1,80 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -#include "3party/jansson/myjansson.hpp" - -namespace generator -{ -class JsonValue -{ -public: - explicit JsonValue(json_t * value = nullptr) : m_handle{value} {} - explicit JsonValue(base::JSONPtr && value) : m_handle{std::move(value)} {} - - JsonValue(JsonValue const &) = delete; - JsonValue & operator=(JsonValue const &) = delete; - - operator json_t const *() const noexcept { return m_handle.get(); } - operator base::JSONPtr const &() noexcept { return m_handle; }; - base::JSONPtr MakeDeepCopyJson() const { return base::JSONPtr{json_deep_copy(m_handle.get())}; } - -private: - base::JSONPtr m_handle; -}; - -using KeyValue = std::pair>; - -class KeyValueStorage -{ -public: - // Longitude and latitude has maximum 3 digits before comma. So we have minimum 6 digits after - // comma. Nautical mile is good approximation for one angle minute, so we can rely, that final - // precision is 60 (minutes in degree) * 1852 (meters in one mile) / 1000000 = 0.111 = 111 - // millimeters. Also, if you are quizzed by nautical mile, just forget, precision was defined in - // https://jira.mail.ru/browse/MAPSB2B-41 - static uint32_t constexpr kDefaultPrecision = 9; - - explicit KeyValueStorage(std::string const & kvPath, size_t cacheValuesCountLimit, - std::function const & pred = DefaultPred); - - KeyValueStorage(KeyValueStorage &&) = default; - KeyValueStorage & operator=(KeyValueStorage &&) = default; - - KeyValueStorage(KeyValueStorage const &) = delete; - KeyValueStorage & operator=(KeyValueStorage const &) = delete; - - void Insert(uint64_t key, JsonValue && valueJson); - - static std::string SerializeFullLine(uint64_t key, JsonValue && valueJson); - - std::shared_ptr Find(uint64_t key) const; - size_t Size() const; - - static std::string Serialize(base::JSONPtr const & ptr) - { - return base::DumpToString(ptr, JSON_COMPACT | JSON_REAL_PRECISION(kDefaultPrecision)); - } - - static std::string SerializeDref(uint64_t number); - -private: - using Value = boost::variant, std::string>; - - static bool DefaultPred(KeyValue const &) { return true; } - static bool ParseKeyValueLine(std::string const & line, std::streamoff lineNumber, uint64_t & key, - std::string & value); - std::fstream m_storage; - std::unordered_map m_values; - size_t m_cacheValuesCountLimit; -}; -} // namespace generator diff --git a/generator/translation.cpp b/generator/translation.cpp deleted file mode 100644 index 772b9573ec..0000000000 --- a/generator/translation.cpp +++ /dev/null @@ -1,80 +0,0 @@ -#include "translation.hpp" - -#include "base/string_utils.hpp" - -#include "coding/transliteration.hpp" - -#include "platform/platform.hpp" - -#include - -namespace -{ -// Languages in order for better transliterations. This is kind -// of workaround before real made translations. -using Languages = std::vector; - -const std::unordered_map kPreferredLanguagesForTransliterate = { - {StringUtf8Multilang::GetLangIndex("ru"), {"ru", "uk", "be"}}, - {StringUtf8Multilang::GetLangIndex("en"), {"en", "da", "es", "fr"}}}; - -Languages kLocalelanguages = {"en", "ru"}; -} // namespace - -namespace -{ -struct TransliterationInitilizer -{ - TransliterationInitilizer() - { - Transliteration::Instance().Init(GetPlatform().ResourcesDir()); - } -}; -} // namespace -namespace generator -{ -std::string GetTranslatedOrTransliteratedName(StringUtf8Multilang const & name, - LanguageCode languageCode) -{ - static TransliterationInitilizer littleStaticHelperToPreventMisinitilizedTranslations; - - std::string s = GetName(name, languageCode); - if (!s.empty()) - return s; - - if (languageCode != StringUtf8Multilang::kEnglishCode) - return std::string(); - - s = GetName(name, StringUtf8Multilang::kInternationalCode); - if (!s.empty() && strings::IsASCIIString(s)) - return s; - - s = GetName(name, StringUtf8Multilang::kDefaultCode); - if (!s.empty() && strings::IsASCIIString(s)) - return s; - - auto const fn = [&s](int8_t code, std::string const & name) { - if (strings::IsASCIIString(name)) - { - s = name; - return base::ControlFlow::Break; - } - - if (code != StringUtf8Multilang::kDefaultCode && - Transliteration::Instance().Transliterate(name, code, s) && strings::IsASCIIString(s)) - { - return base::ControlFlow::Break; - } - - return base::ControlFlow::Continue; - }; - - if (kPreferredLanguagesForTransliterate.count(languageCode) && - !name.ForEachLanguage(kPreferredLanguagesForTransliterate.at(languageCode), fn)) - name.ForEach(fn); - - return std::string(); -} - -Languages const & Localizator::LocaleLanguages() const { return kLocalelanguages; } -} // namespace generator diff --git a/generator/translation.hpp b/generator/translation.hpp deleted file mode 100644 index c0ac396507..0000000000 --- a/generator/translation.hpp +++ /dev/null @@ -1,143 +0,0 @@ -#pragma once - -#include "coding/string_utf8_multilang.hpp" - -#include "3party/jansson/myjansson.hpp" - -#include -#include - -namespace generator -{ -using LanguageCode = int8_t; - -inline std::string GetName(StringUtf8Multilang const & name, LanguageCode lang) -{ - std::string s; - VERIFY(name.GetString(lang, s) != s.empty(), ()); - return s; -} - -/// This function will take the following steps: -/// 1. Return the |languageCode| name if it exists. -/// 1.1 Next steps only for english locale -/// 2. Try to get international name. -/// 3. Try to check if default name is ASCII and return it if succeeds. -/// 4. Return transliteration trying to use kPreferredLanguagesForTransliterate -/// first, then any, if it succeeds. -/// 5. Otherwise, return empty string. -std::string GetTranslatedOrTransliteratedName(StringUtf8Multilang const & name, - LanguageCode languageCode); - -class Localizator -{ -public: - class EasyObjectWithTranslation - { - public: - explicit EasyObjectWithTranslation(StringUtf8Multilang const & name) : m_name(name) {} - std::string GetTranslatedOrTransliteratedName(LanguageCode languageCode) const - { - return ::generator::GetTranslatedOrTransliteratedName(m_name, languageCode); - } - - std::string GetName(LanguageCode languageCode = StringUtf8Multilang::kDefaultCode) const - { - return ::generator::GetName(m_name, languageCode); - } - - private: - StringUtf8Multilang const m_name; - }; - - explicit Localizator(json_t & node) : m_node(GetOrCreateNode("locales", node)) {} - - template - void SetLocale(std::string const & label, Object const & objectWithName, - std::string const & level = std::string()) - { - RemoveLocale(DefaultLocaleName(), level, label); - auto const & name = objectWithName.GetName(); - if (!name.empty()) - AddLocale(DefaultLocaleName(), level, name, label); - - auto const & languages = LocaleLanguages(); - for (std::string const & language : languages) - { - RemoveLocale(language, level, label); - - std::string const & translation = objectWithName.GetTranslatedOrTransliteratedName( - StringUtf8Multilang::GetLangIndex(language)); - - if (translation.empty()) - continue; - - AddLocale(language, level, translation, label); - } - } - - template - void AddVerbose(Verboser && verboser, std::string const & level) - { - json_t & locale = GetOrCreateNode(DefaultLocaleName(), m_node); - json_t & node = GetOrCreateNode(level, locale); - verboser(node); - } - -private: - void AddLocale(std::string const & language, std::string const & level, std::string const & name, - std::string const & label) - { - json_t & locale = GetOrCreateNode(language, m_node); - - if (!level.empty()) - { - json_t & levelNode = GetOrCreateNode(level, locale); - ToJSONObject(levelNode, label, name); - } - else - { - ToJSONObject(locale, label, name); - } - } - - static std::string const & DefaultLocaleName() - { - static std::string const kDefaultLocaleName = "default"; - return kDefaultLocaleName; - } - - static json_t & GetOrCreateNode(std::string const & nodeName, json_t & root) - { - json_t * node = base::GetJSONOptionalField(&root, nodeName); - if (!node || base::JSONIsNull(node)) - { - node = json_object(); - ToJSONObject(root, nodeName, *node); - } - - return *node; - } - - void RemoveLocale(std::string const & language, std::string const & level, - std::string const & label) - { - json_t * node = base::GetJSONOptionalField(&m_node, language); - if (!node) - return; - - if (!level.empty()) - { - node = base::GetJSONOptionalField(node, level); - if (!node) - return; - } - - json_object_del(node, label.c_str()); - } - - std::vector const & LocaleLanguages() const; - - json_t & m_node; -}; -} // namespace generator