kirylkaveryn/organicmaps
1
Fork 0
forked from organicmaps/organicmaps-tmp
Code Pull requests Activity

UGC migration.

Browse source
This commit is contained in:
VladiMihaylenko 2018-06-19 15:29:53 +03:00 • committed by mpimenov
parent 62e8e85c15
commit cec8f91544
14 changed files with 434 additions and 106 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
CMakeLists.txt
View file

@ -64,6 +64,7 @@ option(USE_TSAN "Enable Thread Sanitizer" OFF)
option(PYBINDINGS "Create makefiles for building python bindings" OFF)
option(SKIP_DESKTOP "Skip building of desktop application" OFF)
option(BUILD_MAPSHOT "Build mapshot tool" OFF)
option(UGC_MIGRATION "Build migration file for ugc" OFF)
option(USE_PCH "Use precompiled headers" OFF)
if (CMAKE_CXX_COMPILER_ID MATCHES "Clang")

2
storage/pinger.cpp
View file

@ -63,7 +63,7 @@ void Pinger::Ping(vector<string> const & urls, Pinger::Pong const & pong)
{
base::WorkerThread t(size);
for (size_t i = 0; i < size; ++i)
t.Push([ url = urls[i], &readyUrls, i ] { DoPing(url, i, readyUrls); });
t.Push([url = urls[i], &readyUrls, i] { DoPing(url, i, readyUrls); });
t.Shutdown(base::WorkerThread::Exit::ExecPending);
}

2
ugc/CMakeLists.txt
View file

@ -14,6 +14,8 @@ set(
binary/serdes.hpp
binary/ugc_holder.hpp
binary/visitors.hpp
index_migration/utility.cpp
index_migration/utility.hpp
loader.cpp
loader.hpp
serdes.hpp

87
ugc/index_migration/utility.cpp Normal file
View file

@ -0,0 +1,87 @@
#include "ugc/index_migration/utility.hpp"
#include "ugc/serdes.hpp"
#include "platform/platform.hpp"
#include "platform/settings.hpp"
#include "coding/file_name_utils.hpp"
#include "coding/file_reader.hpp"
#include <algorithm>
#include <cstdint>
#include <iterator>
#include <string>
#include <unordered_map>
#include <utility>
using namespace std;
namespace
{
string const kBinExt = ".bin";
using MigrationTable = unordered_map<uint32_t, uint32_t>;
using MigrationTables = unordered_map<int64_t, MigrationTable>;
bool GetMigrationTable(int64_t tableVersion, MigrationTable & t)
{
auto const fileName = to_string(tableVersion) + kBinExt;
try
{
auto reader = GetPlatform().GetReader(fileName);
NonOwningReaderSource source(*reader);
ugc::DeserializerV0<NonOwningReaderSource> des(source);
des(t);
}
catch (RootException const & ex)
{
LOG(LWARNING, (ex.what()));
return false;
}
if (t.empty())
{
ASSERT(false, ());
return false;
}
return true;
}
bool MigrateFromV0ToV1(ugc::UpdateIndexes & source)
{
MigrationTables tables;
for (auto & index : source)
{
auto const version = index.m_dataVersion;
if (tables.find(version) == tables.end())
{
MigrationTable t;
if (!GetMigrationTable(version, t))
return false;
tables.emplace(version, move(t));
}
auto & t = tables[version];
index.m_type = t[index.m_type];
index.m_matchingType = t[index.m_matchingType];
index.m_synchronized = false;
index.m_version = ugc::IndexVersion::Latest;
}
return true;
}
} // namespace
namespace ugc
{
namespace migration
{
Result Migrate(UpdateIndexes & source)
{
auto const result = MigrateFromV0ToV1(source);
return result ? Result::Success : Result::Failure;
}
} // namespace migration
} // namespace ugc

17
ugc/index_migration/utility.hpp Normal file
View file

@ -0,0 +1,17 @@
#pragma once
#include "ugc/types.hpp"
namespace ugc
{
namespace migration
{
enum class Result
{
Failure,
Success,
};
Result Migrate(UpdateIndexes & source);
} // namespace migration
} // namespace ugc

41
ugc/serdes.hpp
View file

@ -11,6 +11,8 @@
#include <cmath>
#include <cstdint>
#include <unordered_map>
#include <utility>
#include <vector>
namespace ugc
@ -82,13 +84,28 @@ public:
}
template <typename T>
void operator()(vector<T> const & vs, char const * /* name */ = nullptr)
void operator()(std::vector<T> const & vs, char const * /* name */ = nullptr)
{
VisitVarUint(static_cast<uint32_t>(vs.size()));
for (auto const & v : vs)
(*this)(v);
}
template <typename T, typename U>
void operator()(std::pair<T, U> const & p, char const * /* name */ = nullptr)
{
(*this)(p.first);
(*this)(p.second);
}
template <typename T, typename U>
void operator()(std::unordered_map<T, U> const & m, char const * /* name */ = nullptr)
{
VisitVarUint(static_cast<uint32_t>(m.size()));
for (auto const & p : m)
(*this)(p);
}
template <typename R>
void operator()(R const & r, char const * /* name */ = nullptr)
{
@ -170,7 +187,7 @@ public:
}
template <typename T>
void operator()(vector<T> & vs, char const * /* name */ = nullptr)
void operator()(std::vector<T> & vs, char const * /* name */ = nullptr)
{
auto const size = DesVarUint<uint32_t>();
vs.resize(size);
@ -178,6 +195,26 @@ public:
(*this)(v);
}
template <typename T, typename U>
void operator()(std::pair<T, U> & p, char const * /* name */ = nullptr)
{
(*this)(p.first);
(*this)(p.second);
}
template <typename T, typename U>
void operator()(std::unordered_map<T, U> & m, char const * /* name */ = nullptr)
{
auto const size = DesVarUint<uint32_t>();
m.reserve(size);
for (int i = 0; i < size; ++i)
{
std::pair<T, U> p;
(*this)(p);
m.emplace(p.first, p.second);
}
}
template <typename R>
void operator()(R & r, char const * /* name */ = nullptr)
{

71
ugc/serdes_json.hpp
View file

@ -8,8 +8,18 @@
#include <cstdint>
#include <cstdlib>
#include <type_traits>
#include <vector>
namespace
{
template <typename T>
using EnableIfEnum = std::enable_if_t<std::is_enum<T>::value>;
template <typename T>
using EnableIfNotEnum = std::enable_if_t<!std::is_enum<T>::value>;
} // namespace
namespace ugc
{
template <typename Sink>
@ -45,15 +55,6 @@ public:
(*this)(ToDaysSinceEpoch(t), name);
}
void operator()(Sentiment sentiment, char const * name = nullptr)
{
switch (sentiment)
{
case Sentiment::Negative: return (*this)(false, name);
case Sentiment::Positive: return (*this)(true, name);
}
}
template <typename T>
void operator()(vector<T> const & vs, char const * name = nullptr)
{
@ -63,12 +64,18 @@ public:
});
}
template <typename R>
template <typename R, EnableIfNotEnum<R> * = nullptr>
void operator()(R const & r, char const * name = nullptr)
{
NewScopeWith(my::NewJSONObject(), name, [this, &r] { r.Visit(*this); });
}
template <typename T, EnableIfEnum<T> * = nullptr>
void operator()(T const & t, char const * name = nullptr)
{
(*this)(static_cast<std::underlying_type_t<T>>(t), name);
}
void VisitRating(float const f, char const * name = nullptr)
{
CHECK_GREATER_OR_EQUAL(f, 0.0, ());
@ -101,6 +108,12 @@ public:
(*this)(pt.y, y);
}
template <typename Optional>
void operator()(Optional const & opt, Optional const &, char const * name = nullptr)
{
(*this)(opt, name);
}
private:
template <typename Fn>
void NewScopeWith(my::JSONPtr json_object, char const * name, Fn && fn)
@ -128,9 +141,10 @@ class DeserializerJsonV0
public:
DECLARE_EXCEPTION(Exception, RootException);
template <typename Source,
typename std::enable_if<
!std::is_convertible<Source, std::string>::value, Source>::type * = nullptr>
template <typename T>
using EnableIfNotConvertibleToString = std::enable_if_t<!std::is_convertible<T, std::string>::value>;
template <typename Source, EnableIfNotConvertibleToString<Source> * = nullptr>
explicit DeserializerJsonV0(Source & source)
{
std::string src(source.Size(), '\0');
@ -169,13 +183,6 @@ public:
t = FromDaysSinceEpoch(d);
}
void operator()(Sentiment & sentiment, char const * name = nullptr)
{
bool s = false;
FromJSONObject(m_json, name, s);
sentiment = s ? Sentiment::Positive : Sentiment::Negative;
}
template <typename T>
void operator()(vector<T> & vs, char const * name = nullptr)
{
@ -196,7 +203,7 @@ public:
RestoreContext(context);
}
template <typename R>
template <typename R, EnableIfNotEnum<R> * = nullptr>
void operator()(R & r, char const * name = nullptr)
{
json_t * context = SaveContext(name);
@ -204,6 +211,15 @@ public:
RestoreContext(context);
}
template <typename T, EnableIfEnum<T> * = nullptr>
void operator()(T & t, char const * name = nullptr)
{
using UndelyingType = std::underlying_type_t<T>;
UndelyingType res;
FromJSONObject(m_json, name, res);
t = static_cast<T>(res);
}
void VisitRating(float & f, char const * name = nullptr)
{
double d = 0.0;
@ -238,6 +254,19 @@ public:
FromJSONObject(m_json, y, pt.y);
}
template <typename Optional>
void operator()(Optional & opt, Optional const & defaultValue, char const * name = nullptr)
{
auto json = my::GetJSONOptionalField(m_json, name);
if (!json)
{
opt = defaultValue;
return;
}
(*this)(opt, name);
}
private:
json_t * SaveContext(char const * name = nullptr)
{

139
ugc/storage.cpp
View file

@ -1,4 +1,5 @@
#include "ugc/storage.hpp"
#include "ugc/index_migration/utility.hpp"
#include "ugc/serdes.hpp"
#include "ugc/serdes_json.hpp"
@ -16,13 +17,13 @@
#include "coding/file_writer.hpp"
#include "coding/internal/file_data.hpp"
#include "base/stl_helpers.hpp"
#include <algorithm>
#include <utility>
#include "3party/jansson/myjansson.hpp"
namespace ugc
{
using namespace std;
namespace
@ -44,7 +45,7 @@ bool GetUGCFileSize(uint64_t & size)
FileReader reader(GetUGCFilePath());
size = reader.Size();
}
catch (RootException const &)
catch (FileReader::Exception const &)
{
return false;
}
@ -52,16 +53,16 @@ bool GetUGCFileSize(uint64_t & size)
return true;
}
void DeserializeUGCIndex(string const & jsonData, vector<Storage::UGCIndex> & res)
void DeserializeIndexes(string const & jsonData, ugc::UpdateIndexes & res)
{
if (jsonData.empty())
return;
DeserializerJsonV0 des(jsonData);
ugc::DeserializerJsonV0 des(jsonData);
des(res);
}
string SerializeUGCIndex(vector<Storage::UGCIndex> const & indexes)
string SerializeIndexes(ugc::UpdateIndexes const & indexes)
{
if (indexes.empty())
return string();
@ -72,7 +73,7 @@ string SerializeUGCIndex(vector<Storage::UGCIndex> const & indexes)
string data;
{
Sink sink(data);
SerializerJson<Sink> ser(sink);
ugc::SerializerJson<Sink> ser(sink);
ser(index);
}
@ -85,21 +86,23 @@ string SerializeUGCIndex(vector<Storage::UGCIndex> const & indexes)
}
template <typename UGCUpdate>
Storage::SettingResult SetGenericUGCUpdate(
vector<Storage::UGCIndex> & indexes, size_t & numberOfDeleted, FeatureID const & id,
UGCUpdate const & ugc,
FeatureType const & featureType,
Version const version = Version::Latest)
ugc::Storage::SettingResult SetGenericUGCUpdate(UGCUpdate const & ugc,
FeatureType const & featureType,
FeatureID const & id,
ugc::UpdateIndexes & indexes,
size_t & numberOfDeleted,
ugc::Version const version = ugc::Version::Latest)
{
if (!ugc.IsValid())
return Storage::SettingResult::InvalidUGC;
return ugc::Storage::SettingResult::InvalidUGC;
auto const mercator = feature::GetCenter(featureType);
feature::TypesHolder th(featureType);
th.SortBySpec();
auto const optMatchingType = ftraits::UGC::GetType(th);
CHECK(optMatchingType, ());
auto const type = th.GetBestType();
auto const & c = classif();
auto const type = c.GetIndexForType(th.GetBestType());
for (auto & index : indexes)
{
if (type == index.m_type && mercator == index.m_mercator && !index.m_deleted)
@ -110,18 +113,19 @@ Storage::SettingResult SetGenericUGCUpdate(
}
}
Storage::UGCIndex index;
ugc::UpdateIndex index;
uint64_t offset;
if (!GetUGCFileSize(offset))
offset = 0;
index.m_mercator = mercator;
index.m_type = type;
index.m_matchingType = *optMatchingType;
index.m_matchingType = c.GetIndexForType(*optMatchingType);
index.m_mwmName = id.GetMwmName();
index.m_dataVersion = id.GetMwmVersion();
index.m_featureId = id.m_index;
index.m_offset = offset;
index.m_version = ugc::IndexVersion::Latest;
auto const ugcFilePath = GetUGCFilePath();
try
@ -132,35 +136,38 @@ Storage::SettingResult SetGenericUGCUpdate(
catch (FileWriter::Exception const & exception)
{
LOG(LERROR, ("Exception while writing file:", ugcFilePath, "reason:", exception.what()));
return Storage::SettingResult::WritingError;
return ugc::Storage::SettingResult::WritingError;
}
indexes.emplace_back(move(index));
return Storage::SettingResult::Success;
return ugc::Storage::SettingResult::Success;
}
} // namespace
namespace ugc
{
UGCUpdate Storage::GetUGCUpdate(FeatureID const & id) const
{
if (m_UGCIndexes.empty())
if (m_indexes.empty())
return {};
auto const feature = GetFeature(id);
auto const mercator = feature::GetCenter(*feature);
feature::TypesHolder th(*feature);
th.SortBySpec();
auto const type = th.GetBestType();
auto const & c = classif();
auto const type = c.GetIndexForType(th.GetBestType());
auto const index = find_if(
m_UGCIndexes.begin(), m_UGCIndexes.end(), [type, &mercator](UGCIndex const & index) -> bool {
m_indexes.begin(), m_indexes.end(), [type, &mercator](UpdateIndex const & index) -> bool {
return type == index.m_type && mercator == index.m_mercator && !index.m_deleted;
});
if (index == m_UGCIndexes.end())
if (index == m_indexes.end())
return {};
auto const offset = index->m_offset;
auto const size = static_cast<size_t>(UGCSizeAtIndex(distance(m_UGCIndexes.begin(), index)));
auto const size = static_cast<size_t>(UGCSizeAtIndex(distance(m_indexes.begin(), index)));
vector<uint8_t> buf;
buf.resize(size);
auto const ugcFilePath = GetUGCFilePath();
@ -185,8 +192,7 @@ UGCUpdate Storage::GetUGCUpdate(FeatureID const & id) const
Storage::SettingResult Storage::SetUGCUpdate(FeatureID const & id, UGCUpdate const & ugc)
{
auto const feature = GetFeature(id);
return SetGenericUGCUpdate(m_UGCIndexes, m_numberOfDeleted, id, ugc,
*feature);
return SetGenericUGCUpdate(ugc, *feature, id, m_indexes, m_numberOfDeleted, Version::V1);
}
void Storage::Load()
@ -204,20 +210,45 @@ void Storage::Load()
return;
}
DeserializeUGCIndex(data, m_UGCIndexes);
for (auto const & i : m_UGCIndexes)
CHECK(!data.empty(), ());
DeserializeIndexes(data, m_indexes);
if (m_indexes.empty())
return;
for (auto const & i : m_indexes)
{
if (i.m_deleted)
++m_numberOfDeleted;
}
}
void Storage::SaveIndex() const
{
if (m_UGCIndexes.empty())
// We assume there is no situation when indexes from different version are stored in the vector
if (m_indexes.front().m_version == IndexVersion::Latest)
return;
auto const jsonData = SerializeUGCIndex(m_UGCIndexes);
switch (migration::Migrate(m_indexes))
{
case migration::Result::Failure:
LOG(LWARNING, ("Index migration failed"));
break;
case migration::Result::Success:
LOG(LINFO, ("Index migration successful"));
auto const newPath = indexFilePath + ".v0";
my::RenameFileX(indexFilePath, newPath);
if (!SaveIndex())
{
my::RenameFileX(newPath, indexFilePath);
LOG(LWARNING, ("Saving index file after indexes migration failed"));
}
break;
}
}
bool Storage::SaveIndex() const
{
if (m_indexes.empty())
return false;
auto const jsonData = SerializeIndexes(m_indexes);
auto const indexFilePath = GetIndexFilePath();
try
{
@ -227,12 +258,15 @@ void Storage::SaveIndex() const
catch (FileWriter::Exception const & exception)
{
LOG(LERROR, ("Exception while writing file:", indexFilePath, "reason:", exception.what()));
return false;
}
return true;
}
void Storage::Defragmentation()
{
auto const indexesSize = m_UGCIndexes.size();
auto const indexesSize = m_indexes.size();
if (m_numberOfDeleted < indexesSize / 2)
return;
@ -246,7 +280,7 @@ void Storage::Defragmentation()
uint64_t actualOffset = 0;
for (size_t i = 0; i < indexesSize; ++i)
{
auto & index = m_UGCIndexes[i];
auto & index = m_indexes[i];
if (index.m_deleted)
continue;
@ -271,9 +305,7 @@ void Storage::Defragmentation()
return;
}
m_UGCIndexes.erase(remove_if(m_UGCIndexes.begin(), m_UGCIndexes.end(),
[](UGCIndex const & i) -> bool { return i.m_deleted; }), m_UGCIndexes.end());
my::EraseIf(m_indexes, [](UpdateIndex const & i) -> bool { return i.m_deleted; });
CHECK(my::DeleteFileX(ugcFilePath), ());
CHECK(my::RenameFileX(tmpUGCFilePath, ugcFilePath), ());
@ -282,18 +314,18 @@ void Storage::Defragmentation()
string Storage::GetUGCToSend() const
{
if (m_UGCIndexes.empty())
if (m_indexes.empty())
return string();
auto array = my::NewJSONArray();
auto const indexesSize = m_UGCIndexes.size();
auto const indexesSize = m_indexes.size();
auto const ugcFilePath = GetUGCFilePath();
FileReader r(ugcFilePath);
vector<uint8_t> buf;
for (size_t i = 0; i < indexesSize; ++i)
{
buf.clear();
auto const & index = m_UGCIndexes[i];
auto const & index = m_indexes[i];
if (index.m_synchronized || index.m_deleted)
continue;
@ -327,7 +359,9 @@ string Storage::GetUGCToSend() const
ToJSONObject(*embeddedNode.get(), "data_version", index.m_dataVersion);
ToJSONObject(*embeddedNode.get(), "mwm_name", index.m_mwmName);
ToJSONObject(*embeddedNode.get(), "feature_id", index.m_featureId);
ToJSONObject(*embeddedNode.get(), "feature_type", classif().GetReadableObjectName(index.m_matchingType));
auto const & c = classif();
ToJSONObject(*embeddedNode.get(), "feature_type",
c.GetReadableObjectName(c.GetTypeForIndex(index.m_matchingType)));
ToJSONObject(*serializedUgc.get(), "feature", *embeddedNode.release());
json_array_append_new(array.get(), serializedUgc.get_deep_copy());
}
@ -345,7 +379,7 @@ string Storage::GetUGCToSend() const
size_t Storage::GetNumberOfUnsynchronized() const
{
size_t numberOfUnsynchronized = 0;
for (auto const & i : m_UGCIndexes)
for (auto const & i : m_indexes)
{
if (!i.m_deleted && !i.m_synchronized)
++numberOfUnsynchronized;
@ -355,11 +389,11 @@ size_t Storage::GetNumberOfUnsynchronized() const
void Storage::MarkAllAsSynchronized()
{
if (m_UGCIndexes.empty())
if (m_indexes.empty())
return;
size_t numberOfUnsynchronized = 0;
for (auto & index : m_UGCIndexes)
for (auto & index : m_indexes)
{
if (!index.m_synchronized)
{
@ -378,15 +412,15 @@ void Storage::MarkAllAsSynchronized()
uint64_t Storage::UGCSizeAtIndex(size_t const indexPosition) const
{
CHECK(!m_UGCIndexes.empty(), ());
auto const indexesSize = m_UGCIndexes.size();
CHECK(!m_indexes.empty(), ());
auto const indexesSize = m_indexes.size();
CHECK_LESS(indexPosition, indexesSize, ());
auto const indexOffset = m_UGCIndexes[indexPosition].m_offset;
auto const indexOffset = m_indexes[indexPosition].m_offset;
uint64_t nextOffset;
if (indexPosition == indexesSize - 1)
CHECK(GetUGCFileSize(nextOffset), ());
else
nextOffset = m_UGCIndexes[indexPosition + 1].m_offset;
nextOffset = m_indexes[indexPosition + 1].m_offset;
CHECK_GREATER(nextOffset, indexOffset, ());
return nextOffset - indexOffset;
@ -409,8 +443,7 @@ Storage::SettingResult Storage::SetUGCUpdateForTesting(FeatureID const & id,
v0::UGCUpdate const & ugc)
{
auto const feature = GetFeature(id);
return SetGenericUGCUpdate(m_UGCIndexes, m_numberOfDeleted, id, ugc,
*feature, Version::V0);
return SetGenericUGCUpdate(ugc, *feature, id, m_indexes, m_numberOfDeleted, Version::V0);
}
} // namespace ugc
@ -418,7 +451,7 @@ namespace lightweight
{
size_t GetNumberOfUnsentUGC()
{
auto const indexFilePath = ugc::GetIndexFilePath();
auto const indexFilePath = GetIndexFilePath();
if (!Platform::IsFileExistsByFullPath(indexFilePath))
return 0;
@ -434,8 +467,8 @@ size_t GetNumberOfUnsentUGC()
return 0;
}
vector<ugc::Storage::UGCIndex> index;
ugc::DeserializeUGCIndex(data, index);
ugc::UpdateIndexes index;
DeserializeIndexes(data, index);
size_t number = 0;
for (auto const & i : index)
{

33
ugc/storage.hpp
View file

@ -2,14 +2,10 @@
#include "ugc/types.hpp"
#include "geometry/point2d.hpp"
#include "base/thread_checker.hpp"
#include "base/visitor.hpp"
#include <memory>
#include <string>
#include <vector>
class DataSourceBase;
class FeatureType;
@ -20,26 +16,7 @@ namespace ugc
class Storage
{
public:
struct UGCIndex
{
DECLARE_VISITOR(visitor.VisitPoint(m_mercator, "x", "y"), visitor(m_type, "type"),
visitor(m_matchingType, "matching_type"),
visitor(m_offset, "offset"), visitor(m_deleted, "deleted"),
visitor(m_synchronized, "synchronized"), visitor(m_mwmName, "mwm_name"),
visitor(m_dataVersion, "data_version"), visitor(m_featureId, "feature_id"))
m2::PointD m_mercator{};
uint32_t m_type = 0;
uint32_t m_matchingType = 0;
uint64_t m_offset = 0;
bool m_deleted = false;
bool m_synchronized = false;
std::string m_mwmName;
int64_t m_dataVersion;
uint32_t m_featureId = 0;
};
explicit Storage(DataSourceBase const & index) : m_index(index) {}
explicit Storage(Index const & index) : m_index(index) {}
UGCUpdate GetUGCUpdate(FeatureID const & id) const;
@ -51,7 +28,7 @@ public:
};
SettingResult SetUGCUpdate(FeatureID const & id, UGCUpdate const & ugc);
void SaveIndex() const;
bool SaveIndex() const;
std::string GetUGCToSend() const;
void MarkAllAsSynchronized();
void Defragmentation();
@ -59,7 +36,7 @@ public:
size_t GetNumberOfUnsynchronized() const;
/// Testing
std::vector<UGCIndex> const & GetIndexesForTesting() const { return m_UGCIndexes; }
UpdateIndexes const & GetIndexesForTesting() const { return m_indexes; }
size_t GetNumberOfDeletedForTesting() const { return m_numberOfDeleted; }
SettingResult SetUGCUpdateForTesting(FeatureID const & id, v0::UGCUpdate const & ugc);
@ -67,8 +44,8 @@ private:
uint64_t UGCSizeAtIndex(size_t const indexPosition) const;
std::unique_ptr<FeatureType> GetFeature(FeatureID const & id) const;
DataSourceBase const & m_index;
std::vector<UGCIndex> m_UGCIndexes;
Index const & m_index;
UpdateIndexes m_indexes;
size_t m_numberOfDeleted = 0;
};

34
ugc/types.hpp
View file

@ -1,5 +1,7 @@
#pragma once
#include "geometry/point2d.hpp"
#include "indexer/feature_decl.hpp"
#include "coding/hex.hpp"
@ -416,5 +418,37 @@ inline std::string DebugPrint(UGCUpdate const & ugcUpdate)
os << "days since epoch:" << ToDaysSinceEpoch(ugcUpdate.m_time) << " ]";
return os.str();
}
enum class IndexVersion : uint8_t
{
V0 = 0,
V1 = 1,
Latest = V1
};
struct UpdateIndex
{
DECLARE_VISITOR(visitor.VisitPoint(m_mercator, "x", "y"), visitor(m_type, "type"),
visitor(m_matchingType, "matching_type"),
visitor(m_offset, "offset"), visitor(m_deleted, "deleted"),
visitor(m_synchronized, "synchronized"), visitor(m_mwmName, "mwm_name"),
visitor(m_dataVersion, "data_version"), visitor(m_featureId, "feature_id"),
visitor(m_version, IndexVersion::V0, "version"))
m2::PointD m_mercator = m2::PointD::Zero();
// Index of the type from classificator.txt
uint32_t m_type = 0;
// Index of the type from ugc_types.txt
uint32_t m_matchingType = 0;
uint64_t m_offset = 0;
bool m_deleted = false;
bool m_synchronized = false;
std::string m_mwmName;
int64_t m_dataVersion = 0;
uint32_t m_featureId = 0;
IndexVersion m_version = IndexVersion::Latest;
};
using UpdateIndexes = std::vector<UpdateIndex>;
} // namespace ugc

8
ugc/ugc_tests/CMakeLists.txt
View file

@ -11,6 +11,14 @@ set(
utils.hpp
)
if (UGC_MIGRATION)
set(
SRC
${SRC}
migration/generate_migration_files.cpp
)
endif()
omim_add_test(${PROJECT_NAME} ${SRC})
omim_link_libraries(

88
ugc/ugc_tests/migration/generate_migration_files.cpp Normal file
View file

@ -0,0 +1,88 @@
#include "testing/testing.hpp"
#include "coding/file_name_utils.hpp"
#include "coding/file_reader.hpp"
#include "coding/file_writer.hpp"
#include "indexer/classificator.hpp"
#include "indexer/classificator_loader.hpp"
#include "platform/platform.hpp"
#include "ugc/serdes.hpp"
#include <array>
#include <string>
#include <unordered_map>
using namespace std;
namespace
{
array<string, 11> const kVersions{{"171020",
"171117",
"171208",
"180110",
"180126",
"180209",
"180316",
"180417",
"180513",
"180527",
"180528"}};
string const kUGCMigrationDirName = "ugc_migration";
string const kClassificatorFileName = "classificator.txt";
string const kTypesFileName = "types.txt";
string const kBinFileExtension = ".bin";
} // namespace
UNIT_TEST(UGC_GenerateMigrationFiles)
{
auto & p = GetPlatform();
auto const ugcDirPath = my::JoinPath(p.WritableDir(), kUGCMigrationDirName);
for (auto const & v : kVersions)
{
auto const folderPath = my::JoinPath(ugcDirPath, v);
string classificator;
{
auto const r = p.GetReader(my::JoinPath(folderPath, kClassificatorFileName));
r->ReadAsString(classificator);
}
string types;
{
auto const r = p.GetReader(my::JoinPath(folderPath, kTypesFileName));
r->ReadAsString(types);
}
classificator::LoadTypes(classificator, types);
Classificator const & c = classif();
unordered_map<uint32_t, uint32_t> mapping;
auto const parse = [&c, &mapping](ClassifObject const * obj, uint32_t type)
{
if (c.IsTypeValid(type))
mapping.emplace(type, c.GetIndexForType(type));
};
c.ForEachTree(parse);
auto const fileName = v + kBinFileExtension;
auto const filePath = my::JoinPath(ugcDirPath, fileName);
{
FileWriter sink(filePath, FileWriter::Op::OP_WRITE_TRUNCATE);
ugc::Serializer<FileWriter> ser(sink);
ser(mapping);
}
unordered_map<uint32_t, uint32_t> res;
{
ReaderSource<FileReader> source(FileReader(filePath, true /* withExceptions */));
ugc::DeserializerV0<ReaderSource<FileReader>> des(source);
des(res);
}
TEST_EQUAL(res, mapping, ());
}
}

3
ugc/ugc_tests/storage_tests.cpp
View file

@ -314,7 +314,8 @@ UNIT_CLASS_TEST(StorageTest, ContentTest)
ToJSONObject(*embeddedNode.get(), "data_version", lastIndex.m_dataVersion);
ToJSONObject(*embeddedNode.get(), "mwm_name", lastIndex.m_mwmName);
ToJSONObject(*embeddedNode.get(), "feature_id", lastIndex.m_featureId);
ToJSONObject(*embeddedNode.get(), "feature_type", classif().GetReadableObjectName(lastIndex.m_matchingType));
auto const & c = classif();
ToJSONObject(*embeddedNode.get(), "feature_type", c.GetReadableObjectName(c.GetTypeForIndex(lastIndex.m_matchingType)));
ToJSONObject(*ugcNode.get(), "feature", *embeddedNode.release());
auto array = my::NewJSONArray();

14
xcode/ugc/ugc.xcodeproj/project.pbxproj
View file

@ -39,6 +39,7 @@
671ED3B620D4081400D4317E /* libsuccinct.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 671ED3B520D4081400D4317E /* libsuccinct.a */; };
F6150E221EF90040000B955D /* api.cpp in Sources */ = {isa = PBXBuildFile; fileRef = F6150E1F1EF90040000B955D /* api.cpp */; };
F6150E5D1EFAAB45000B955D /* storage.cpp in Sources */ = {isa = PBXBuildFile; fileRef = F6150E5B1EFAAB45000B955D /* storage.cpp */; };
F63EE67620D3F6B40025CC27 /* utility.cpp in Sources */ = {isa = PBXBuildFile; fileRef = F63EE67520D3F6B30025CC27 /* utility.cpp */; };
/* End PBXBuildFile section */
/* Begin PBXCopyFilesBuildPhase section */
@ -98,6 +99,8 @@
F6150E211EF90040000B955D /* types.hpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; path = types.hpp; sourceTree = "<group>"; };
F6150E5B1EFAAB45000B955D /* storage.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = storage.cpp; sourceTree = "<group>"; };
F6150E5C1EFAAB45000B955D /* storage.hpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; path = storage.hpp; sourceTree = "<group>"; };
F63EE67420D3F6B30025CC27 /* utility.hpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; path = utility.hpp; sourceTree = "<group>"; };
F63EE67520D3F6B30025CC27 /* utility.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = utility.cpp; sourceTree = "<group>"; };
F6F8E3A41EF83D7600F2DE8F /* libugc.a */ = {isa = PBXFileReference; explicitFileType = archive.ar; includeInIndex = 0; path = libugc.a; sourceTree = BUILT_PRODUCTS_DIR; };
F6F8E3B81EF83DF200F2DE8F /* common-debug.xcconfig */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.xcconfig; name = "common-debug.xcconfig"; path = "../common-debug.xcconfig"; sourceTree = "<group>"; };
F6F8E3B91EF83DF200F2DE8F /* common-release.xcconfig */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.xcconfig; name = "common-release.xcconfig"; path = "../common-release.xcconfig"; sourceTree = "<group>"; };
@ -171,6 +174,15 @@
path = ../../ugc/ugc_tests;
sourceTree = "<group>";
};
F63EE67320D3F6B30025CC27 /* index_migration */ = {
isa = PBXGroup;
children = (
F63EE67420D3F6B30025CC27 /* utility.hpp */,
F63EE67520D3F6B30025CC27 /* utility.cpp */,
);
path = index_migration;
sourceTree = "<group>";
};
F6F8E39B1EF83D7600F2DE8F = {
isa = PBXGroup;
children = (
@ -195,6 +207,7 @@
F6F8E3A61EF83D7600F2DE8F /* ugc */ = {
isa = PBXGroup;
children = (
F63EE67320D3F6B30025CC27 /* index_migration */,
3D74EF171F8B90B00081202C /* binary */,
3D74EF021F86840C0081202C /* loader.cpp */,
3D74EF031F86840C0081202C /* loader.hpp */,
@ -342,6 +355,7 @@
buildActionMask = 2147483647;
files = (
F6150E221EF90040000B955D /* api.cpp in Sources */,
F63EE67620D3F6B40025CC27 /* utility.cpp in Sources */,
3D74EF1E1F8B94EA0081202C /* serdes.cpp in Sources */,
3D74EF041F86840C0081202C /* loader.cpp in Sources */,
F6150E5D1EFAAB45000B955D /* storage.cpp in Sources */,