From cec8f91544707c624670b10eca6ecbc49d045dfc Mon Sep 17 00:00:00 2001 From: VladiMihaylenko Date: Tue, 19 Jun 2018 15:29:53 +0300 Subject: [PATCH] UGC migration. --- CMakeLists.txt | 1 + storage/pinger.cpp | 2 +- ugc/CMakeLists.txt | 2 + ugc/index_migration/utility.cpp | 87 +++++++++++ ugc/index_migration/utility.hpp | 17 +++ ugc/serdes.hpp | 41 +++++- ugc/serdes_json.hpp | 71 ++++++--- ugc/storage.cpp | 139 +++++++++++------- ugc/storage.hpp | 33 +---- ugc/types.hpp | 34 +++++ ugc/ugc_tests/CMakeLists.txt | 8 + .../migration/generate_migration_files.cpp | 88 +++++++++++ ugc/ugc_tests/storage_tests.cpp | 3 +- xcode/ugc/ugc.xcodeproj/project.pbxproj | 14 ++ 14 files changed, 434 insertions(+), 106 deletions(-) create mode 100644 ugc/index_migration/utility.cpp create mode 100644 ugc/index_migration/utility.hpp create mode 100644 ugc/ugc_tests/migration/generate_migration_files.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index b697ce263e..8a18201471 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -64,6 +64,7 @@ option(USE_TSAN "Enable Thread Sanitizer" OFF) option(PYBINDINGS "Create makefiles for building python bindings" OFF) option(SKIP_DESKTOP "Skip building of desktop application" OFF) option(BUILD_MAPSHOT "Build mapshot tool" OFF) +option(UGC_MIGRATION "Build migration file for ugc" OFF) option(USE_PCH "Use precompiled headers" OFF) if (CMAKE_CXX_COMPILER_ID MATCHES "Clang") diff --git a/storage/pinger.cpp b/storage/pinger.cpp index 4bd697624f..dae285c576 100644 --- a/storage/pinger.cpp +++ b/storage/pinger.cpp @@ -63,7 +63,7 @@ void Pinger::Ping(vector const & urls, Pinger::Pong const & pong) { base::WorkerThread t(size); for (size_t i = 0; i < size; ++i) - t.Push([ url = urls[i], &readyUrls, i ] { DoPing(url, i, readyUrls); }); + t.Push([url = urls[i], &readyUrls, i] { DoPing(url, i, readyUrls); }); t.Shutdown(base::WorkerThread::Exit::ExecPending); } diff --git a/ugc/CMakeLists.txt b/ugc/CMakeLists.txt index 27ac9e0c34..1e2093cdec 100644 --- a/ugc/CMakeLists.txt +++ b/ugc/CMakeLists.txt @@ -14,6 +14,8 @@ set( binary/serdes.hpp binary/ugc_holder.hpp binary/visitors.hpp + index_migration/utility.cpp + index_migration/utility.hpp loader.cpp loader.hpp serdes.hpp diff --git a/ugc/index_migration/utility.cpp b/ugc/index_migration/utility.cpp new file mode 100644 index 0000000000..9d48bbcc1f --- /dev/null +++ b/ugc/index_migration/utility.cpp @@ -0,0 +1,87 @@ +#include "ugc/index_migration/utility.hpp" +#include "ugc/serdes.hpp" + +#include "platform/platform.hpp" +#include "platform/settings.hpp" + +#include "coding/file_name_utils.hpp" +#include "coding/file_reader.hpp" + +#include +#include +#include +#include +#include +#include + +using namespace std; + +namespace +{ +string const kBinExt = ".bin"; + +using MigrationTable = unordered_map; +using MigrationTables = unordered_map; + +bool GetMigrationTable(int64_t tableVersion, MigrationTable & t) +{ + auto const fileName = to_string(tableVersion) + kBinExt; + try + { + auto reader = GetPlatform().GetReader(fileName); + NonOwningReaderSource source(*reader); + ugc::DeserializerV0 des(source); + des(t); + } + catch (RootException const & ex) + { + LOG(LWARNING, (ex.what())); + return false; + } + + if (t.empty()) + { + ASSERT(false, ()); + return false; + } + + return true; +} + +bool MigrateFromV0ToV1(ugc::UpdateIndexes & source) +{ + MigrationTables tables; + for (auto & index : source) + { + auto const version = index.m_dataVersion; + if (tables.find(version) == tables.end()) + { + MigrationTable t; + if (!GetMigrationTable(version, t)) + return false; + + tables.emplace(version, move(t)); + } + + auto & t = tables[version]; + index.m_type = t[index.m_type]; + index.m_matchingType = t[index.m_matchingType]; + index.m_synchronized = false; + index.m_version = ugc::IndexVersion::Latest; + } + + return true; +} +} // namespace + +namespace ugc +{ +namespace migration +{ +Result Migrate(UpdateIndexes & source) +{ + auto const result = MigrateFromV0ToV1(source); + return result ? Result::Success : Result::Failure; +} +} // namespace migration +} // namespace ugc diff --git a/ugc/index_migration/utility.hpp b/ugc/index_migration/utility.hpp new file mode 100644 index 0000000000..404ca834b0 --- /dev/null +++ b/ugc/index_migration/utility.hpp @@ -0,0 +1,17 @@ +#pragma once + +#include "ugc/types.hpp" + +namespace ugc +{ +namespace migration +{ +enum class Result +{ + Failure, + Success, +}; + +Result Migrate(UpdateIndexes & source); +} // namespace migration +} // namespace ugc diff --git a/ugc/serdes.hpp b/ugc/serdes.hpp index 191bc18a6d..2ae89cbbd1 100644 --- a/ugc/serdes.hpp +++ b/ugc/serdes.hpp @@ -11,6 +11,8 @@ #include #include +#include +#include #include namespace ugc @@ -82,13 +84,28 @@ public: } template - void operator()(vector const & vs, char const * /* name */ = nullptr) + void operator()(std::vector const & vs, char const * /* name */ = nullptr) { VisitVarUint(static_cast(vs.size())); for (auto const & v : vs) (*this)(v); } + template + void operator()(std::pair const & p, char const * /* name */ = nullptr) + { + (*this)(p.first); + (*this)(p.second); + } + + template + void operator()(std::unordered_map const & m, char const * /* name */ = nullptr) + { + VisitVarUint(static_cast(m.size())); + for (auto const & p : m) + (*this)(p); + } + template void operator()(R const & r, char const * /* name */ = nullptr) { @@ -170,7 +187,7 @@ public: } template - void operator()(vector & vs, char const * /* name */ = nullptr) + void operator()(std::vector & vs, char const * /* name */ = nullptr) { auto const size = DesVarUint(); vs.resize(size); @@ -178,6 +195,26 @@ public: (*this)(v); } + template + void operator()(std::pair & p, char const * /* name */ = nullptr) + { + (*this)(p.first); + (*this)(p.second); + } + + template + void operator()(std::unordered_map & m, char const * /* name */ = nullptr) + { + auto const size = DesVarUint(); + m.reserve(size); + for (int i = 0; i < size; ++i) + { + std::pair p; + (*this)(p); + m.emplace(p.first, p.second); + } + } + template void operator()(R & r, char const * /* name */ = nullptr) { diff --git a/ugc/serdes_json.hpp b/ugc/serdes_json.hpp index 306557a1c1..0a426dafa1 100644 --- a/ugc/serdes_json.hpp +++ b/ugc/serdes_json.hpp @@ -8,8 +8,18 @@ #include #include +#include #include +namespace +{ +template +using EnableIfEnum = std::enable_if_t::value>; + +template +using EnableIfNotEnum = std::enable_if_t::value>; +} // namespace + namespace ugc { template @@ -45,15 +55,6 @@ public: (*this)(ToDaysSinceEpoch(t), name); } - void operator()(Sentiment sentiment, char const * name = nullptr) - { - switch (sentiment) - { - case Sentiment::Negative: return (*this)(false, name); - case Sentiment::Positive: return (*this)(true, name); - } - } - template void operator()(vector const & vs, char const * name = nullptr) { @@ -63,12 +64,18 @@ public: }); } - template + template * = nullptr> void operator()(R const & r, char const * name = nullptr) { NewScopeWith(my::NewJSONObject(), name, [this, &r] { r.Visit(*this); }); } + template * = nullptr> + void operator()(T const & t, char const * name = nullptr) + { + (*this)(static_cast>(t), name); + } + void VisitRating(float const f, char const * name = nullptr) { CHECK_GREATER_OR_EQUAL(f, 0.0, ()); @@ -101,6 +108,12 @@ public: (*this)(pt.y, y); } + template + void operator()(Optional const & opt, Optional const &, char const * name = nullptr) + { + (*this)(opt, name); + } + private: template void NewScopeWith(my::JSONPtr json_object, char const * name, Fn && fn) @@ -128,9 +141,10 @@ class DeserializerJsonV0 public: DECLARE_EXCEPTION(Exception, RootException); - template ::value, Source>::type * = nullptr> + template + using EnableIfNotConvertibleToString = std::enable_if_t::value>; + + template * = nullptr> explicit DeserializerJsonV0(Source & source) { std::string src(source.Size(), '\0'); @@ -169,13 +183,6 @@ public: t = FromDaysSinceEpoch(d); } - void operator()(Sentiment & sentiment, char const * name = nullptr) - { - bool s = false; - FromJSONObject(m_json, name, s); - sentiment = s ? Sentiment::Positive : Sentiment::Negative; - } - template void operator()(vector & vs, char const * name = nullptr) { @@ -196,7 +203,7 @@ public: RestoreContext(context); } - template + template * = nullptr> void operator()(R & r, char const * name = nullptr) { json_t * context = SaveContext(name); @@ -204,6 +211,15 @@ public: RestoreContext(context); } + template * = nullptr> + void operator()(T & t, char const * name = nullptr) + { + using UndelyingType = std::underlying_type_t; + UndelyingType res; + FromJSONObject(m_json, name, res); + t = static_cast(res); + } + void VisitRating(float & f, char const * name = nullptr) { double d = 0.0; @@ -238,6 +254,19 @@ public: FromJSONObject(m_json, y, pt.y); } + template + void operator()(Optional & opt, Optional const & defaultValue, char const * name = nullptr) + { + auto json = my::GetJSONOptionalField(m_json, name); + if (!json) + { + opt = defaultValue; + return; + } + + (*this)(opt, name); + } + private: json_t * SaveContext(char const * name = nullptr) { diff --git a/ugc/storage.cpp b/ugc/storage.cpp index 0ceca67281..bb1bdba4c1 100644 --- a/ugc/storage.cpp +++ b/ugc/storage.cpp @@ -1,4 +1,5 @@ #include "ugc/storage.hpp" +#include "ugc/index_migration/utility.hpp" #include "ugc/serdes.hpp" #include "ugc/serdes_json.hpp" @@ -16,13 +17,13 @@ #include "coding/file_writer.hpp" #include "coding/internal/file_data.hpp" +#include "base/stl_helpers.hpp" + #include #include #include "3party/jansson/myjansson.hpp" -namespace ugc -{ using namespace std; namespace @@ -44,7 +45,7 @@ bool GetUGCFileSize(uint64_t & size) FileReader reader(GetUGCFilePath()); size = reader.Size(); } - catch (RootException const &) + catch (FileReader::Exception const &) { return false; } @@ -52,16 +53,16 @@ bool GetUGCFileSize(uint64_t & size) return true; } -void DeserializeUGCIndex(string const & jsonData, vector & res) +void DeserializeIndexes(string const & jsonData, ugc::UpdateIndexes & res) { if (jsonData.empty()) return; - DeserializerJsonV0 des(jsonData); + ugc::DeserializerJsonV0 des(jsonData); des(res); } -string SerializeUGCIndex(vector const & indexes) +string SerializeIndexes(ugc::UpdateIndexes const & indexes) { if (indexes.empty()) return string(); @@ -72,7 +73,7 @@ string SerializeUGCIndex(vector const & indexes) string data; { Sink sink(data); - SerializerJson ser(sink); + ugc::SerializerJson ser(sink); ser(index); } @@ -85,21 +86,23 @@ string SerializeUGCIndex(vector const & indexes) } template -Storage::SettingResult SetGenericUGCUpdate( - vector & indexes, size_t & numberOfDeleted, FeatureID const & id, - UGCUpdate const & ugc, - FeatureType const & featureType, - Version const version = Version::Latest) +ugc::Storage::SettingResult SetGenericUGCUpdate(UGCUpdate const & ugc, + FeatureType const & featureType, + FeatureID const & id, + ugc::UpdateIndexes & indexes, + size_t & numberOfDeleted, + ugc::Version const version = ugc::Version::Latest) { if (!ugc.IsValid()) - return Storage::SettingResult::InvalidUGC; + return ugc::Storage::SettingResult::InvalidUGC; auto const mercator = feature::GetCenter(featureType); feature::TypesHolder th(featureType); th.SortBySpec(); auto const optMatchingType = ftraits::UGC::GetType(th); CHECK(optMatchingType, ()); - auto const type = th.GetBestType(); + auto const & c = classif(); + auto const type = c.GetIndexForType(th.GetBestType()); for (auto & index : indexes) { if (type == index.m_type && mercator == index.m_mercator && !index.m_deleted) @@ -110,18 +113,19 @@ Storage::SettingResult SetGenericUGCUpdate( } } - Storage::UGCIndex index; + ugc::UpdateIndex index; uint64_t offset; if (!GetUGCFileSize(offset)) offset = 0; index.m_mercator = mercator; index.m_type = type; - index.m_matchingType = *optMatchingType; + index.m_matchingType = c.GetIndexForType(*optMatchingType); index.m_mwmName = id.GetMwmName(); index.m_dataVersion = id.GetMwmVersion(); index.m_featureId = id.m_index; index.m_offset = offset; + index.m_version = ugc::IndexVersion::Latest; auto const ugcFilePath = GetUGCFilePath(); try @@ -132,35 +136,38 @@ Storage::SettingResult SetGenericUGCUpdate( catch (FileWriter::Exception const & exception) { LOG(LERROR, ("Exception while writing file:", ugcFilePath, "reason:", exception.what())); - return Storage::SettingResult::WritingError; + return ugc::Storage::SettingResult::WritingError; } indexes.emplace_back(move(index)); - return Storage::SettingResult::Success; + return ugc::Storage::SettingResult::Success; } } // namespace +namespace ugc +{ UGCUpdate Storage::GetUGCUpdate(FeatureID const & id) const { - if (m_UGCIndexes.empty()) + if (m_indexes.empty()) return {}; auto const feature = GetFeature(id); auto const mercator = feature::GetCenter(*feature); feature::TypesHolder th(*feature); th.SortBySpec(); - auto const type = th.GetBestType(); + auto const & c = classif(); + auto const type = c.GetIndexForType(th.GetBestType()); auto const index = find_if( - m_UGCIndexes.begin(), m_UGCIndexes.end(), [type, &mercator](UGCIndex const & index) -> bool { + m_indexes.begin(), m_indexes.end(), [type, &mercator](UpdateIndex const & index) -> bool { return type == index.m_type && mercator == index.m_mercator && !index.m_deleted; }); - if (index == m_UGCIndexes.end()) + if (index == m_indexes.end()) return {}; auto const offset = index->m_offset; - auto const size = static_cast(UGCSizeAtIndex(distance(m_UGCIndexes.begin(), index))); + auto const size = static_cast(UGCSizeAtIndex(distance(m_indexes.begin(), index))); vector buf; buf.resize(size); auto const ugcFilePath = GetUGCFilePath(); @@ -185,8 +192,7 @@ UGCUpdate Storage::GetUGCUpdate(FeatureID const & id) const Storage::SettingResult Storage::SetUGCUpdate(FeatureID const & id, UGCUpdate const & ugc) { auto const feature = GetFeature(id); - return SetGenericUGCUpdate(m_UGCIndexes, m_numberOfDeleted, id, ugc, - *feature); + return SetGenericUGCUpdate(ugc, *feature, id, m_indexes, m_numberOfDeleted, Version::V1); } void Storage::Load() @@ -204,20 +210,45 @@ void Storage::Load() return; } - DeserializeUGCIndex(data, m_UGCIndexes); - for (auto const & i : m_UGCIndexes) + CHECK(!data.empty(), ()); + DeserializeIndexes(data, m_indexes); + if (m_indexes.empty()) + return; + + for (auto const & i : m_indexes) { if (i.m_deleted) ++m_numberOfDeleted; } -} -void Storage::SaveIndex() const -{ - if (m_UGCIndexes.empty()) + // We assume there is no situation when indexes from different version are stored in the vector + if (m_indexes.front().m_version == IndexVersion::Latest) return; - auto const jsonData = SerializeUGCIndex(m_UGCIndexes); + switch (migration::Migrate(m_indexes)) + { + case migration::Result::Failure: + LOG(LWARNING, ("Index migration failed")); + break; + case migration::Result::Success: + LOG(LINFO, ("Index migration successful")); + auto const newPath = indexFilePath + ".v0"; + my::RenameFileX(indexFilePath, newPath); + if (!SaveIndex()) + { + my::RenameFileX(newPath, indexFilePath); + LOG(LWARNING, ("Saving index file after indexes migration failed")); + } + break; + } +} + +bool Storage::SaveIndex() const +{ + if (m_indexes.empty()) + return false; + + auto const jsonData = SerializeIndexes(m_indexes); auto const indexFilePath = GetIndexFilePath(); try { @@ -227,12 +258,15 @@ void Storage::SaveIndex() const catch (FileWriter::Exception const & exception) { LOG(LERROR, ("Exception while writing file:", indexFilePath, "reason:", exception.what())); + return false; } + + return true; } void Storage::Defragmentation() { - auto const indexesSize = m_UGCIndexes.size(); + auto const indexesSize = m_indexes.size(); if (m_numberOfDeleted < indexesSize / 2) return; @@ -246,7 +280,7 @@ void Storage::Defragmentation() uint64_t actualOffset = 0; for (size_t i = 0; i < indexesSize; ++i) { - auto & index = m_UGCIndexes[i]; + auto & index = m_indexes[i]; if (index.m_deleted) continue; @@ -271,9 +305,7 @@ void Storage::Defragmentation() return; } - m_UGCIndexes.erase(remove_if(m_UGCIndexes.begin(), m_UGCIndexes.end(), - [](UGCIndex const & i) -> bool { return i.m_deleted; }), m_UGCIndexes.end()); - + my::EraseIf(m_indexes, [](UpdateIndex const & i) -> bool { return i.m_deleted; }); CHECK(my::DeleteFileX(ugcFilePath), ()); CHECK(my::RenameFileX(tmpUGCFilePath, ugcFilePath), ()); @@ -282,18 +314,18 @@ void Storage::Defragmentation() string Storage::GetUGCToSend() const { - if (m_UGCIndexes.empty()) + if (m_indexes.empty()) return string(); auto array = my::NewJSONArray(); - auto const indexesSize = m_UGCIndexes.size(); + auto const indexesSize = m_indexes.size(); auto const ugcFilePath = GetUGCFilePath(); FileReader r(ugcFilePath); vector buf; for (size_t i = 0; i < indexesSize; ++i) { buf.clear(); - auto const & index = m_UGCIndexes[i]; + auto const & index = m_indexes[i]; if (index.m_synchronized || index.m_deleted) continue; @@ -327,7 +359,9 @@ string Storage::GetUGCToSend() const ToJSONObject(*embeddedNode.get(), "data_version", index.m_dataVersion); ToJSONObject(*embeddedNode.get(), "mwm_name", index.m_mwmName); ToJSONObject(*embeddedNode.get(), "feature_id", index.m_featureId); - ToJSONObject(*embeddedNode.get(), "feature_type", classif().GetReadableObjectName(index.m_matchingType)); + auto const & c = classif(); + ToJSONObject(*embeddedNode.get(), "feature_type", + c.GetReadableObjectName(c.GetTypeForIndex(index.m_matchingType))); ToJSONObject(*serializedUgc.get(), "feature", *embeddedNode.release()); json_array_append_new(array.get(), serializedUgc.get_deep_copy()); } @@ -345,7 +379,7 @@ string Storage::GetUGCToSend() const size_t Storage::GetNumberOfUnsynchronized() const { size_t numberOfUnsynchronized = 0; - for (auto const & i : m_UGCIndexes) + for (auto const & i : m_indexes) { if (!i.m_deleted && !i.m_synchronized) ++numberOfUnsynchronized; @@ -355,11 +389,11 @@ size_t Storage::GetNumberOfUnsynchronized() const void Storage::MarkAllAsSynchronized() { - if (m_UGCIndexes.empty()) + if (m_indexes.empty()) return; size_t numberOfUnsynchronized = 0; - for (auto & index : m_UGCIndexes) + for (auto & index : m_indexes) { if (!index.m_synchronized) { @@ -378,15 +412,15 @@ void Storage::MarkAllAsSynchronized() uint64_t Storage::UGCSizeAtIndex(size_t const indexPosition) const { - CHECK(!m_UGCIndexes.empty(), ()); - auto const indexesSize = m_UGCIndexes.size(); + CHECK(!m_indexes.empty(), ()); + auto const indexesSize = m_indexes.size(); CHECK_LESS(indexPosition, indexesSize, ()); - auto const indexOffset = m_UGCIndexes[indexPosition].m_offset; + auto const indexOffset = m_indexes[indexPosition].m_offset; uint64_t nextOffset; if (indexPosition == indexesSize - 1) CHECK(GetUGCFileSize(nextOffset), ()); else - nextOffset = m_UGCIndexes[indexPosition + 1].m_offset; + nextOffset = m_indexes[indexPosition + 1].m_offset; CHECK_GREATER(nextOffset, indexOffset, ()); return nextOffset - indexOffset; @@ -409,8 +443,7 @@ Storage::SettingResult Storage::SetUGCUpdateForTesting(FeatureID const & id, v0::UGCUpdate const & ugc) { auto const feature = GetFeature(id); - return SetGenericUGCUpdate(m_UGCIndexes, m_numberOfDeleted, id, ugc, - *feature, Version::V0); + return SetGenericUGCUpdate(ugc, *feature, id, m_indexes, m_numberOfDeleted, Version::V0); } } // namespace ugc @@ -418,7 +451,7 @@ namespace lightweight { size_t GetNumberOfUnsentUGC() { - auto const indexFilePath = ugc::GetIndexFilePath(); + auto const indexFilePath = GetIndexFilePath(); if (!Platform::IsFileExistsByFullPath(indexFilePath)) return 0; @@ -434,8 +467,8 @@ size_t GetNumberOfUnsentUGC() return 0; } - vector index; - ugc::DeserializeUGCIndex(data, index); + ugc::UpdateIndexes index; + DeserializeIndexes(data, index); size_t number = 0; for (auto const & i : index) { diff --git a/ugc/storage.hpp b/ugc/storage.hpp index 3fc5a26be1..53d1908bc1 100644 --- a/ugc/storage.hpp +++ b/ugc/storage.hpp @@ -2,14 +2,10 @@ #include "ugc/types.hpp" -#include "geometry/point2d.hpp" - #include "base/thread_checker.hpp" -#include "base/visitor.hpp" #include #include -#include class DataSourceBase; class FeatureType; @@ -20,26 +16,7 @@ namespace ugc class Storage { public: - struct UGCIndex - { - DECLARE_VISITOR(visitor.VisitPoint(m_mercator, "x", "y"), visitor(m_type, "type"), - visitor(m_matchingType, "matching_type"), - visitor(m_offset, "offset"), visitor(m_deleted, "deleted"), - visitor(m_synchronized, "synchronized"), visitor(m_mwmName, "mwm_name"), - visitor(m_dataVersion, "data_version"), visitor(m_featureId, "feature_id")) - - m2::PointD m_mercator{}; - uint32_t m_type = 0; - uint32_t m_matchingType = 0; - uint64_t m_offset = 0; - bool m_deleted = false; - bool m_synchronized = false; - std::string m_mwmName; - int64_t m_dataVersion; - uint32_t m_featureId = 0; - }; - - explicit Storage(DataSourceBase const & index) : m_index(index) {} + explicit Storage(Index const & index) : m_index(index) {} UGCUpdate GetUGCUpdate(FeatureID const & id) const; @@ -51,7 +28,7 @@ public: }; SettingResult SetUGCUpdate(FeatureID const & id, UGCUpdate const & ugc); - void SaveIndex() const; + bool SaveIndex() const; std::string GetUGCToSend() const; void MarkAllAsSynchronized(); void Defragmentation(); @@ -59,7 +36,7 @@ public: size_t GetNumberOfUnsynchronized() const; /// Testing - std::vector const & GetIndexesForTesting() const { return m_UGCIndexes; } + UpdateIndexes const & GetIndexesForTesting() const { return m_indexes; } size_t GetNumberOfDeletedForTesting() const { return m_numberOfDeleted; } SettingResult SetUGCUpdateForTesting(FeatureID const & id, v0::UGCUpdate const & ugc); @@ -67,8 +44,8 @@ private: uint64_t UGCSizeAtIndex(size_t const indexPosition) const; std::unique_ptr GetFeature(FeatureID const & id) const; - DataSourceBase const & m_index; - std::vector m_UGCIndexes; + Index const & m_index; + UpdateIndexes m_indexes; size_t m_numberOfDeleted = 0; }; diff --git a/ugc/types.hpp b/ugc/types.hpp index 1c748100a5..f1948d5d61 100644 --- a/ugc/types.hpp +++ b/ugc/types.hpp @@ -1,5 +1,7 @@ #pragma once +#include "geometry/point2d.hpp" + #include "indexer/feature_decl.hpp" #include "coding/hex.hpp" @@ -416,5 +418,37 @@ inline std::string DebugPrint(UGCUpdate const & ugcUpdate) os << "days since epoch:" << ToDaysSinceEpoch(ugcUpdate.m_time) << " ]"; return os.str(); } + +enum class IndexVersion : uint8_t +{ + V0 = 0, + V1 = 1, + Latest = V1 +}; + +struct UpdateIndex +{ + DECLARE_VISITOR(visitor.VisitPoint(m_mercator, "x", "y"), visitor(m_type, "type"), + visitor(m_matchingType, "matching_type"), + visitor(m_offset, "offset"), visitor(m_deleted, "deleted"), + visitor(m_synchronized, "synchronized"), visitor(m_mwmName, "mwm_name"), + visitor(m_dataVersion, "data_version"), visitor(m_featureId, "feature_id"), + visitor(m_version, IndexVersion::V0, "version")) + + m2::PointD m_mercator = m2::PointD::Zero(); + // Index of the type from classificator.txt + uint32_t m_type = 0; + // Index of the type from ugc_types.txt + uint32_t m_matchingType = 0; + uint64_t m_offset = 0; + bool m_deleted = false; + bool m_synchronized = false; + std::string m_mwmName; + int64_t m_dataVersion = 0; + uint32_t m_featureId = 0; + IndexVersion m_version = IndexVersion::Latest; +}; + +using UpdateIndexes = std::vector; } // namespace ugc diff --git a/ugc/ugc_tests/CMakeLists.txt b/ugc/ugc_tests/CMakeLists.txt index c876856140..1918edcc63 100644 --- a/ugc/ugc_tests/CMakeLists.txt +++ b/ugc/ugc_tests/CMakeLists.txt @@ -11,6 +11,14 @@ set( utils.hpp ) +if (UGC_MIGRATION) + set( + SRC + ${SRC} + migration/generate_migration_files.cpp + ) +endif() + omim_add_test(${PROJECT_NAME} ${SRC}) omim_link_libraries( diff --git a/ugc/ugc_tests/migration/generate_migration_files.cpp b/ugc/ugc_tests/migration/generate_migration_files.cpp new file mode 100644 index 0000000000..9fae971393 --- /dev/null +++ b/ugc/ugc_tests/migration/generate_migration_files.cpp @@ -0,0 +1,88 @@ +#include "testing/testing.hpp" + +#include "coding/file_name_utils.hpp" +#include "coding/file_reader.hpp" +#include "coding/file_writer.hpp" + +#include "indexer/classificator.hpp" +#include "indexer/classificator_loader.hpp" + +#include "platform/platform.hpp" + +#include "ugc/serdes.hpp" + +#include +#include +#include + +using namespace std; + +namespace +{ +array const kVersions{{"171020", + "171117", + "171208", + "180110", + "180126", + "180209", + "180316", + "180417", + "180513", + "180527", + "180528"}}; + +string const kUGCMigrationDirName = "ugc_migration"; +string const kClassificatorFileName = "classificator.txt"; +string const kTypesFileName = "types.txt"; +string const kBinFileExtension = ".bin"; +} // namespace + +UNIT_TEST(UGC_GenerateMigrationFiles) +{ + auto & p = GetPlatform(); + auto const ugcDirPath = my::JoinPath(p.WritableDir(), kUGCMigrationDirName); + for (auto const & v : kVersions) + { + auto const folderPath = my::JoinPath(ugcDirPath, v); + string classificator; + { + auto const r = p.GetReader(my::JoinPath(folderPath, kClassificatorFileName)); + r->ReadAsString(classificator); + } + + string types; + { + auto const r = p.GetReader(my::JoinPath(folderPath, kTypesFileName)); + r->ReadAsString(types); + } + + classificator::LoadTypes(classificator, types); + Classificator const & c = classif(); + + unordered_map mapping; + auto const parse = [&c, &mapping](ClassifObject const * obj, uint32_t type) + { + if (c.IsTypeValid(type)) + mapping.emplace(type, c.GetIndexForType(type)); + }; + + c.ForEachTree(parse); + + auto const fileName = v + kBinFileExtension; + auto const filePath = my::JoinPath(ugcDirPath, fileName); + { + FileWriter sink(filePath, FileWriter::Op::OP_WRITE_TRUNCATE); + ugc::Serializer ser(sink); + ser(mapping); + } + + unordered_map res; + { + ReaderSource source(FileReader(filePath, true /* withExceptions */)); + ugc::DeserializerV0> des(source); + des(res); + } + + TEST_EQUAL(res, mapping, ()); + } +} diff --git a/ugc/ugc_tests/storage_tests.cpp b/ugc/ugc_tests/storage_tests.cpp index 2c20eb18ee..b6f0d4a6c7 100644 --- a/ugc/ugc_tests/storage_tests.cpp +++ b/ugc/ugc_tests/storage_tests.cpp @@ -314,7 +314,8 @@ UNIT_CLASS_TEST(StorageTest, ContentTest) ToJSONObject(*embeddedNode.get(), "data_version", lastIndex.m_dataVersion); ToJSONObject(*embeddedNode.get(), "mwm_name", lastIndex.m_mwmName); ToJSONObject(*embeddedNode.get(), "feature_id", lastIndex.m_featureId); - ToJSONObject(*embeddedNode.get(), "feature_type", classif().GetReadableObjectName(lastIndex.m_matchingType)); + auto const & c = classif(); + ToJSONObject(*embeddedNode.get(), "feature_type", c.GetReadableObjectName(c.GetTypeForIndex(lastIndex.m_matchingType))); ToJSONObject(*ugcNode.get(), "feature", *embeddedNode.release()); auto array = my::NewJSONArray(); diff --git a/xcode/ugc/ugc.xcodeproj/project.pbxproj b/xcode/ugc/ugc.xcodeproj/project.pbxproj index 2ca3ab876d..9ed1011ede 100644 --- a/xcode/ugc/ugc.xcodeproj/project.pbxproj +++ b/xcode/ugc/ugc.xcodeproj/project.pbxproj @@ -39,6 +39,7 @@ 671ED3B620D4081400D4317E /* libsuccinct.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 671ED3B520D4081400D4317E /* libsuccinct.a */; }; F6150E221EF90040000B955D /* api.cpp in Sources */ = {isa = PBXBuildFile; fileRef = F6150E1F1EF90040000B955D /* api.cpp */; }; F6150E5D1EFAAB45000B955D /* storage.cpp in Sources */ = {isa = PBXBuildFile; fileRef = F6150E5B1EFAAB45000B955D /* storage.cpp */; }; + F63EE67620D3F6B40025CC27 /* utility.cpp in Sources */ = {isa = PBXBuildFile; fileRef = F63EE67520D3F6B30025CC27 /* utility.cpp */; }; /* End PBXBuildFile section */ /* Begin PBXCopyFilesBuildPhase section */ @@ -98,6 +99,8 @@ F6150E211EF90040000B955D /* types.hpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; path = types.hpp; sourceTree = ""; }; F6150E5B1EFAAB45000B955D /* storage.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = storage.cpp; sourceTree = ""; }; F6150E5C1EFAAB45000B955D /* storage.hpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; path = storage.hpp; sourceTree = ""; }; + F63EE67420D3F6B30025CC27 /* utility.hpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; path = utility.hpp; sourceTree = ""; }; + F63EE67520D3F6B30025CC27 /* utility.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = utility.cpp; sourceTree = ""; }; F6F8E3A41EF83D7600F2DE8F /* libugc.a */ = {isa = PBXFileReference; explicitFileType = archive.ar; includeInIndex = 0; path = libugc.a; sourceTree = BUILT_PRODUCTS_DIR; }; F6F8E3B81EF83DF200F2DE8F /* common-debug.xcconfig */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.xcconfig; name = "common-debug.xcconfig"; path = "../common-debug.xcconfig"; sourceTree = ""; }; F6F8E3B91EF83DF200F2DE8F /* common-release.xcconfig */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.xcconfig; name = "common-release.xcconfig"; path = "../common-release.xcconfig"; sourceTree = ""; }; @@ -171,6 +174,15 @@ path = ../../ugc/ugc_tests; sourceTree = ""; }; + F63EE67320D3F6B30025CC27 /* index_migration */ = { + isa = PBXGroup; + children = ( + F63EE67420D3F6B30025CC27 /* utility.hpp */, + F63EE67520D3F6B30025CC27 /* utility.cpp */, + ); + path = index_migration; + sourceTree = ""; + }; F6F8E39B1EF83D7600F2DE8F = { isa = PBXGroup; children = ( @@ -195,6 +207,7 @@ F6F8E3A61EF83D7600F2DE8F /* ugc */ = { isa = PBXGroup; children = ( + F63EE67320D3F6B30025CC27 /* index_migration */, 3D74EF171F8B90B00081202C /* binary */, 3D74EF021F86840C0081202C /* loader.cpp */, 3D74EF031F86840C0081202C /* loader.hpp */, @@ -342,6 +355,7 @@ buildActionMask = 2147483647; files = ( F6150E221EF90040000B955D /* api.cpp in Sources */, + F63EE67620D3F6B40025CC27 /* utility.cpp in Sources */, 3D74EF1E1F8B94EA0081202C /* serdes.cpp in Sources */, 3D74EF041F86840C0081202C /* loader.cpp in Sources */, F6150E5D1EFAAB45000B955D /* storage.cpp in Sources */,