From 7b292e4e96f3c0be493a5ffdadec2ee5723a07d3 Mon Sep 17 00:00:00 2001 From: VladiMihaylenko Date: Thu, 5 Oct 2017 17:18:01 +0300 Subject: [PATCH] UGCUpdate storage --- 3party/jansson/myjansson.hpp | 2 +- map/framework.cpp | 2 +- search/editor_delegate.cpp | 1 - ugc/api.cpp | 8 +- ugc/api.hpp | 2 +- ugc/storage.cpp | 415 ++++++++++++++++++++++++++++++++++- ugc/storage.hpp | 38 +++- 7 files changed, 445 insertions(+), 23 deletions(-) diff --git a/3party/jansson/myjansson.hpp b/3party/jansson/myjansson.hpp index 989f17ec66..5ceb98fc1b 100644 --- a/3party/jansson/myjansson.hpp +++ b/3party/jansson/myjansson.hpp @@ -110,7 +110,7 @@ void ToJSONObject(json_t & root, std::string const & field, T const & value) json_object_set_new(&root, field.c_str(), ToJSON(value).release()); } -void ToJSONObject(json_t & root, std::string const & field, json_t & embedded) +inline void ToJSONObject(json_t & root, std::string const & field, json_t & embedded) { json_object_set_new(&root, field.c_str(), &embedded); } diff --git a/map/framework.cpp b/map/framework.cpp index db5ade8b7c..e04987f48f 100644 --- a/map/framework.cpp +++ b/map/framework.cpp @@ -1490,7 +1490,7 @@ void Framework::InitUGC() { ASSERT(!m_ugcApi.get(), ("InitUGC() must be called only once.")); - m_ugcApi = make_unique(m_model.GetIndex(), "FILENAME_PLACEHOLDER"); + m_ugcApi = make_unique(m_model.GetIndex()); } void Framework::InitSearchEngine() diff --git a/search/editor_delegate.cpp b/search/editor_delegate.cpp index aa982055df..b34cde6cde 100644 --- a/search/editor_delegate.cpp +++ b/search/editor_delegate.cpp @@ -4,7 +4,6 @@ #include "indexer/feature_decl.hpp" #include "indexer/index.hpp" -#include "indexer/index.hpp" #include "indexer/index_helpers.hpp" namespace search diff --git a/ugc/api.cpp b/ugc/api.cpp index 5336d73f59..f5bf0b3b72 100644 --- a/ugc/api.cpp +++ b/ugc/api.cpp @@ -1,7 +1,5 @@ #include "ugc/api.hpp" -#include "indexer/feature.hpp" - #include "platform/platform.hpp" #include @@ -11,11 +9,7 @@ using namespace ugc; namespace ugc { -Api::Api(Index const & index, std::string const & filename) - : m_storage(filename) - , m_loader(index) -{ -} +Api::Api(Index const & index) : m_storage(index) {} void Api::GetUGC(FeatureID const & id, UGCCallback callback) { diff --git a/ugc/api.hpp b/ugc/api.hpp index a9b50777b2..a6e3cc236b 100644 --- a/ugc/api.hpp +++ b/ugc/api.hpp @@ -18,7 +18,7 @@ class Api public: using UGCCallback = std::function; - explicit Api(Index const & index, std::string const & filename); + explicit Api(Index const & index); void GetUGC(FeatureID const & id, UGCCallback callback); diff --git a/ugc/storage.cpp b/ugc/storage.cpp index 53556d97ac..0f807594f2 100644 --- a/ugc/storage.cpp +++ b/ugc/storage.cpp @@ -1,31 +1,434 @@ #include "ugc/storage.hpp" +#include "ugc/serdes.hpp" +#include "ugc/serdes_json.hpp" + +#include "coding/file_reader.hpp" +#include "coding/file_writer.hpp" +#include "coding/internal/file_data.hpp" #include "indexer/feature_decl.hpp" +#include "indexer/index.hpp" + +#include "platform/platform.hpp" + +#include +#include + +#include "3party/jansson/myjansson.hpp" namespace ugc { -Storage::Storage(std::string const & filename) - : m_filename(filename) +using namespace std; +namespace +{ +string const kIndexFileName = "index.json"; +string const kUGCUpdateFileName = "ugc.update.bin"; +string const kTmpFileExtension = ".tmp"; + +char const kOffsetKey[] = "offset"; +char const kXKey[] = "x"; +char const kYKey[] = "y"; +char const kTypeKey[] = "type"; +char const kIsDeletedKey[] = "is_deleted"; +char const kIsSynchronizedKey[] = "is_synchronized"; +char const kMwmNameKey[] = "mwm_name"; +char const kDataVersionKey[] = "data_version"; +char const kFeatureIdKey[] = "feature_id"; +char const kFeatureKey[] = "feature"; + +string GetUGCFilePath() +{ + return GetPlatform().WritableDir() + kUGCUpdateFileName; +} + +string GetIndexFilePath() +{ + return GetPlatform().WritableDir() + kIndexFileName; +} + +bool GetUGCFileSize(uint64_t & size) +{ + return GetPlatform().GetFileSizeByName(kUGCUpdateFileName, size); +} + +void DeserializeUGCIndex(string const & jsonData, vector & res, size_t & numberOfDeleted) +{ + if (jsonData.empty()) + return; + + my::Json root(jsonData.c_str()); + if (!root.get() || !json_is_array(root.get())) + return; + + size_t const size = json_array_size(root.get()); + if (size == 0) + return; + + for (size_t i = 0; i < size; ++i) + { + auto node = json_array_get(root.get(), i); + if (!node) + return; + + Storage::UGCIndex index; + double x, y; + FromJSONObject(node, kXKey, x); + FromJSONObject(node, kYKey, y); + index.m_mercator = {x, y}; + + uint32_t type; + FromJSONObject(node, kTypeKey, type); + index.m_type = type; + + uint64_t offset; + FromJSONObject(node, kOffsetKey, offset); + index.m_offset = offset; + + bool isDeleted; + FromJSONObject(node, kIsDeletedKey, isDeleted); + index.m_isDeleted = isDeleted; + if (isDeleted) + numberOfDeleted++; + + bool isSynchronized; + FromJSONObject(node, kIsSynchronizedKey, isSynchronized); + index.m_isSynchronized = isDeleted; + + string mwmName; + FromJSONObject(node, kMwmNameKey, mwmName); + index.m_mwmName = mwmName; + + string mwmVersion; + FromJSONObject(node, kDataVersionKey, mwmVersion); + index.m_dataVersion = mwmName; + + uint32_t featureId; + FromJSONObject(node, kFeatureIdKey, featureId); + index.m_featureId = featureId; + + res.emplace_back(move(index)); + } +} + +string SerializeUGCIndex(vector const & indexes) +{ + if (indexes.empty()) + return string(); + + auto array = my::NewJSONArray(); + for (auto const & i : indexes) + { + auto node = my::NewJSONObject(); + auto const & mercator = i.m_mercator; + ToJSONObject(*node, kXKey, mercator.x); + ToJSONObject(*node, kYKey, mercator.y); + ToJSONObject(*node, kTypeKey, i.m_type); + ToJSONObject(*node, kOffsetKey, i.m_offset); + ToJSONObject(*node, kIsDeletedKey, i.m_isDeleted); + ToJSONObject(*node, kIsSynchronizedKey, i.m_isSynchronized); + ToJSONObject(*node, kMwmNameKey, i.m_mwmName); + ToJSONObject(*node, kDataVersionKey, i.m_dataVersion); + ToJSONObject(*node, kFeatureIdKey, i.m_featureId); + json_array_append_new(array.get(), node.release()); + } + + unique_ptr buffer(json_dumps(array.get(), JSON_COMPACT | JSON_ENSURE_ASCII)); + return string(buffer.get()); +} + +void SerializeUGCUpdate(json_t * node, UGCUpdate const & update, Storage::UGCIndex const & index) +{ + vector data; + using Sink = MemWriter>; + Sink sink(data); + SerializerJson ser(sink); + ser(update); + + my::Json serializedUgc(data.data()); + node = serializedUgc.get(); + auto embeddedNode = my::NewJSONObject(); + ToJSONObject(*embeddedNode, kDataVersionKey, index.m_dataVersion); + ToJSONObject(*embeddedNode, kMwmNameKey, index.m_mwmName); + ToJSONObject(*embeddedNode, kFeatureIdKey, index.m_featureId); + ToJSONObject(*node, kFeatureKey, *embeddedNode); + embeddedNode.release(); +} +} // namespace + +Storage::Storage(Index const & index) + : m_index(index) { Load(); } UGCUpdate Storage::GetUGCUpdate(FeatureID const & id) const { - // Dummy - return {}; + ASSERT_THREAD_CHECKER(m_threadChecker, ()); + if (m_UGCIndexes.empty()) + return {}; + + auto const feature = GetOriginalFeature(id); + CHECK(feature, ()); + auto const & mercator = feature->GetCenter(); + feature::TypesHolder th(*feature); + th.SortBySpec(); + auto const type = th.GetBestType(); + + auto const index = find_if(m_UGCIndexes.begin(), m_UGCIndexes.end(), [type, &mercator](UGCIndex const & index) -> bool + { + return type == index.m_type && mercator == index.m_mercator && !index.m_isDeleted; + }); + + if (index == m_UGCIndexes.end()) + return {}; + + auto const offset = index->m_offset; + auto const nextIndex = index + 1; + uint64_t nextOffset; + if (nextIndex == m_UGCIndexes.end()) + CHECK(GetPlatform().GetFileSizeByName(kUGCUpdateFileName, nextOffset), ()); + else + nextOffset = nextIndex->m_offset; + + auto const size = nextOffset - offset; + vector buf; + auto const ugcFilePath = GetUGCFilePath(); + try + { + FileReader r(ugcFilePath); + r.Read(offset, buf.data(), size); + } + catch (RootException const & exception) + { + LOG(LWARNING, ("Exception while reading file:", ugcFilePath)); + return {}; + } + + MemReader r(buf.data(), buf.size()); + NonOwningReaderSource source(r); + UGCUpdate update; + Deserialize(source, update); + return update; } void Storage::SetUGCUpdate(FeatureID const & id, UGCUpdate const & ugc) { - m_ugc[id] = ugc; + ASSERT_THREAD_CHECKER(m_threadChecker, ()); + auto const feature = GetOriginalFeature(id); + CHECK(feature, ()); + + auto const & mercator = feature->GetCenter(); + feature::TypesHolder th(*feature); + th.SortBySpec(); + auto const type = th.GetBestType(); + for (auto & index : m_UGCIndexes) + { + if (type == index.m_type && mercator == index.m_mercator && !index.m_isDeleted) + { + index.m_isDeleted = true; + m_numberOfDeleted++; + break; + } + } + // TODO: Call Defragmentation(). + + auto const ugcFilePath = GetUGCFilePath(); + try + { + FileWriter w(ugcFilePath); + Serialize(w, ugc); + } + catch (RootException const & exception) + { + LOG(LWARNING, ("Exception while writing file:", ugcFilePath)); + return; + } + + UGCIndex index; + index.m_mercator = mercator; + uint64_t offset; + if (GetUGCFileSize(offset)) + offset = 0; + + index.m_type = type; + index.m_mwmName = id.GetMwmName(); + index.m_dataVersion = id.GetMwmVersion(); + index.m_featureId = id.m_index; + index.m_offset = offset; + m_UGCIndexes.emplace_back(move(index)); } void Storage::Load() { + ASSERT_THREAD_CHECKER(m_threadChecker, ()); + string data; + auto const indexFilePath = GetIndexFilePath(); + try + { + FileReader r(indexFilePath); + r.ReadAsString(data); + } + catch (RootException const & exception) + { + LOG(LWARNING, ("Exception while reading file:", indexFilePath)); + return; + } + + DeserializeUGCIndex(data, m_UGCIndexes, m_numberOfDeleted); + sort(m_UGCIndexes.begin(), m_UGCIndexes.end(), [](UGCIndex const & lhs, UGCIndex const & rhs) -> bool { + return lhs.m_offset < rhs.m_offset; + }); } -void Storage::Save() +void Storage::SaveIndex() const { + ASSERT_THREAD_CHECKER(m_threadChecker, ()); + auto const jsonData = SerializeUGCIndex(m_UGCIndexes); + auto const indexFilePath = GetIndexFilePath(); + try + { + FileWriter w(indexFilePath); + w.Write(jsonData.c_str(), jsonData.length()); + } + catch (RootException const & exception) + { + LOG(LWARNING, ("Exception while writing file:", indexFilePath)); + } +} + +void Storage::Defragmentation() +{ + ASSERT_THREAD_CHECKER(m_threadChecker, ()); + auto const indexesSize = m_UGCIndexes.size(); + if (m_numberOfDeleted < indexesSize / 2) + return; + + auto const ugcFilePath = GetUGCFilePath(); + auto const tmpUGCFilePath = ugcFilePath + kTmpFileExtension; + FileReader r(ugcFilePath); + vector buf; + for (size_t i = 0; i < indexesSize; ++i) + { + auto const & index = m_UGCIndexes[i]; + if (index.m_isDeleted) + continue; + + auto const offset = index.m_offset; + uint64_t nextOffset; + if (i == indexesSize - 1) + GetUGCFileSize(nextOffset); + else + nextOffset = m_UGCIndexes[i + 1].m_offset; + + auto const bufSize = nextOffset - offset; + try + { + r.Read(offset, buf.data(), bufSize); + } + catch (RootException const & excpetion) + { + LOG(LWARNING, ("Exception while reading file:", ugcFilePath)); + return; + } + } + + try + { + FileWriter w(tmpUGCFilePath); + w.Write(buf.data(), buf.size()); + } + catch (RootException const & exception) + { + LOG(LWARNING, ("Exception while reading file:", tmpUGCFilePath)); + return; + } + + m_UGCIndexes.erase(remove_if(m_UGCIndexes.begin(), m_UGCIndexes.end(), [](UGCIndex const & i) -> bool { + return i.m_isDeleted; + })); + + CHECK(my::DeleteFileX(ugcFilePath), ()); + CHECK(my::RenameFileX(tmpUGCFilePath, ugcFilePath), ()); + + m_numberOfDeleted = 0; +} + +string Storage::GetUGCToSend() const +{ + ASSERT_THREAD_CHECKER(m_threadChecker, ()); + if (m_UGCIndexes.empty()) + return string(); + + auto array = my::NewJSONArray(); + auto const indexesSize = m_UGCIndexes.size(); + auto const ugcFilePath = GetUGCFilePath(); + FileReader r(ugcFilePath); + vector buf; + for (size_t i = 0; i < indexesSize; ++i) + { + buf.clear(); + auto const & index = m_UGCIndexes[i]; + if (index.m_isSynchronized) + continue; + + auto const offset = index.m_offset; + uint64_t nextOffset; + if (i == indexesSize - 1) + CHECK(GetUGCFileSize(nextOffset), ()); + else + nextOffset = m_UGCIndexes[i + 1].m_offset; + + auto const bufSize = nextOffset - offset; + try + { + r.Read(offset, buf.data(), bufSize); + } + catch (RootException const & excpetion) + { + LOG(LWARNING, ("Exception while reading file:", ugcFilePath)); + return string(); + } + + MemReader r(buf.data(), buf.size()); + NonOwningReaderSource source(r); + UGCUpdate update; + Deserialize(source, update); + + auto node = my::NewJSONObject(); + SerializeUGCUpdate(node.get(), update, index); + json_array_append_new(array.get(), node.release()); + } + + unique_ptr buffer(json_dumps(array.get(), JSON_COMPACT | JSON_ENSURE_ASCII)); + return string(buffer.get()); +} + +void Storage::MarkAllAsSynchronized() +{ + ASSERT_THREAD_CHECKER(m_threadChecker, ()); + if (m_UGCIndexes.empty()) + return; + + for (auto & index : m_UGCIndexes) + index.m_isSynchronized = true; + + auto const indexPath = GetIndexFilePath(); + my::DeleteFileX(indexPath); + SaveIndex(); +} + +unique_ptr Storage::GetOriginalFeature(FeatureID const & id) const +{ + ASSERT_THREAD_CHECKER(m_threadChecker, ()); + CHECK(id.IsValid(), ()); + + auto feature = make_unique(); + Index::FeaturesLoaderGuard const guard(m_index, id.m_mwmId); + if (!guard.GetOriginalFeatureByIndex(id.m_index, *feature)) + return unique_ptr(); + + feature->ParseEverything(); + return feature; } } // namespace ugc + diff --git a/ugc/storage.hpp b/ugc/storage.hpp index 0b97a0d616..45b6ba7b2d 100644 --- a/ugc/storage.hpp +++ b/ugc/storage.hpp @@ -2,9 +2,16 @@ #include "ugc/types.hpp" -#include -#include +#include "geometry/point2d.hpp" +#include "base/thread_checker.hpp" + +#include +#include +#include + +class Index; +class FeatureType; struct FeatureID; namespace ugc @@ -12,16 +19,35 @@ namespace ugc class Storage { public: - explicit Storage(std::string const & filename); + struct UGCIndex + { + m2::PointD m_mercator{}; + uint32_t m_type{}; + uint64_t m_offset{}; + bool m_isDeleted = false; + bool m_isSynchronized = false; + std::string m_mwmName; + std::string m_dataVersion; + uint32_t m_featureId{}; + }; + + explicit Storage(Index const & index); UGCUpdate GetUGCUpdate(FeatureID const & id) const; void SetUGCUpdate(FeatureID const & id, UGCUpdate const & ugc); - void Save(); + void SaveIndex() const; void Load(); + void Defragmentation(); + std::string GetUGCToSend() const; + void MarkAllAsSynchronized(); + + std::unique_ptr GetOriginalFeature(FeatureID const & id) const; private: - std::string m_filename; - std::map m_ugc; + Index const & m_index; + std::vector m_UGCIndexes; + size_t m_numberOfDeleted = 0; + ThreadChecker m_threadChecker; }; } // namespace ugc