AndrewShkrob/organicmaps
1
Fork 0
forked from organicmaps/organicmaps
Code Pull requests Activity

Index migration test

Browse source
This commit is contained in:
VladiMihaylenko 2018-06-21 16:21:44 +03:00 • committed by mpimenov
parent 673f9bdfdb
commit b8c9a8a020
6 changed files with 68 additions and 11 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
data/ugc_migration/test_index/v0/index.json Normal file
View file

File diff suppressed because one or more lines are too long

3
ugc/index_migration/utility.cpp
View file

@ -19,13 +19,14 @@ using namespace std;
namespace
{
string const kBinExt = ".bin";
string const kMigrationDirName = "ugc_migration";
using MigrationTable = unordered_map<uint32_t, uint32_t>;
using MigrationTables = unordered_map<int64_t, MigrationTable>;
bool GetMigrationTable(int64_t tableVersion, MigrationTable & t)
{
auto const fileName = to_string(tableVersion) + kBinExt;
auto const fileName = my::JoinPath(kMigrationDirName, to_string(tableVersion) + kBinExt);
try
{
auto reader = GetPlatform().GetReader(fileName);

40
ugc/storage.cpp
View file

@ -88,10 +88,9 @@ string SerializeIndexes(ugc::UpdateIndexes const & indexes)
template <typename UGCUpdate>
ugc::Storage::SettingResult SetGenericUGCUpdate(UGCUpdate const & ugc,
FeatureType const & featureType,
FeatureID const & id,
ugc::UpdateIndexes & indexes,
size_t & numberOfDeleted,
ugc::Version const version = ugc::Version::Latest)
ugc::Version const version)
{
if (!ugc.IsValid())
return ugc::Storage::SettingResult::InvalidUGC;
@ -118,6 +117,7 @@ ugc::Storage::SettingResult SetGenericUGCUpdate(UGCUpdate const & ugc,
if (!GetUGCFileSize(offset))
offset = 0;
auto const & id = featureType.GetID();
index.m_mercator = mercator;
index.m_type = type;
index.m_matchingType = c.GetIndexForType(*optMatchingType);
@ -192,7 +192,7 @@ UGCUpdate Storage::GetUGCUpdate(FeatureID const & id) const
Storage::SettingResult Storage::SetUGCUpdate(FeatureID const & id, UGCUpdate const & ugc)
{
auto const feature = GetFeature(id);
return SetGenericUGCUpdate(ugc, *feature, id, m_indexes, m_numberOfDeleted, Version::V1);
return SetGenericUGCUpdate(ugc, *feature, m_indexes, m_numberOfDeleted, Version::V1);
}
void Storage::Load()
@ -221,7 +221,12 @@ void Storage::Load()
++m_numberOfDeleted;
}
// We assume there is no situation when indexes from different version are stored in the vector
Migrate(indexFilePath);
}
void Storage::Migrate(string const & indexFilePath)
{
// We assume there is no situation when indexes from different versions are stored in the vector.
if (m_indexes.front().m_version == IndexVersion::Latest)
return;
@ -234,7 +239,7 @@ void Storage::Load()
LOG(LINFO, ("Index migration successful"));
auto const newPath = indexFilePath + ".v0";
my::RenameFileX(indexFilePath, newPath);
if (!SaveIndex())
if (!SaveIndex(indexFilePath))
{
my::RenameFileX(newPath, indexFilePath);
LOG(LWARNING, ("Saving index file after indexes migration failed"));
@ -243,13 +248,13 @@ void Storage::Load()
}
}
bool Storage::SaveIndex() const
bool Storage::SaveIndex(std::string const & pathToTargetFile /* = "" */) const
{
if (m_indexes.empty())
return false;
auto const indexFilePath = pathToTargetFile.empty() ? GetIndexFilePath() : pathToTargetFile;
auto const jsonData = SerializeIndexes(m_indexes);
auto const indexFilePath = GetIndexFilePath();
try
{
FileWriter w(indexFilePath);
@ -443,7 +448,26 @@ Storage::SettingResult Storage::SetUGCUpdateForTesting(FeatureID const & id,
v0::UGCUpdate const & ugc)
{
auto const feature = GetFeature(id);
return SetGenericUGCUpdate(ugc, *feature, id, m_indexes, m_numberOfDeleted, Version::V0);
return SetGenericUGCUpdate(ugc, *feature, m_indexes, m_numberOfDeleted, Version::V0);
}
void Storage::LoadForTesting(std::string const & testIndexFilePath)
{
string data;
try
{
FileReader r(testIndexFilePath);
r.ReadAsString(data);
}
catch (FileReader::Exception const & exception)
{
LOG(LWARNING, (exception.what()));
return;
}
CHECK(!data.empty(), ());
DeserializeIndexes(data, m_indexes);
Migrate(testIndexFilePath);
}
} // namespace ugc

4
ugc/storage.hpp
View file

@ -28,7 +28,7 @@ public:
};
SettingResult SetUGCUpdate(FeatureID const & id, UGCUpdate const & ugc);
bool SaveIndex() const;
bool SaveIndex(std::string const & pathToTargetFile = "") const;
std::string GetUGCToSend() const;
void MarkAllAsSynchronized();
void Defragmentation();
@ -39,10 +39,12 @@ public:
UpdateIndexes const & GetIndexesForTesting() const { return m_indexes; }
size_t GetNumberOfDeletedForTesting() const { return m_numberOfDeleted; }
SettingResult SetUGCUpdateForTesting(FeatureID const & id, v0::UGCUpdate const & ugc);
void LoadForTesting(std::string const & testIndexFilePath);
private:
uint64_t UGCSizeAtIndex(size_t const indexPosition) const;
std::unique_ptr<FeatureType> GetFeature(FeatureID const & id) const;
void Migrate(std::string const & indexFilePath);
Index const & m_index;
UpdateIndexes m_indexes;

2
ugc/ugc_tests/CMakeLists.txt
View file

@ -4,8 +4,8 @@ include_directories(${OMIM_ROOT}/3party/jansson/src)
set(
SRC
serdes_tests.cpp
serdes_binary_tests.cpp
serdes_tests.cpp
storage_tests.cpp
utils.cpp
utils.hpp

29
ugc/ugc_tests/storage_tests.cpp
View file

@ -438,3 +438,32 @@ UNIT_CLASS_TEST(StorageTest, GetNumberOfUnsentSeparately)
TEST_EQUAL(lightweight::GetNumberOfUnsentUGC(), 0, ());
TEST(DeleteIndexFile(), ());
}
UNIT_TEST(UGC_IndexMigrationFromV0ToV1Smoke)
{
auto & builder = MwmBuilder::Builder();
builder.Build({});
auto const versionString = "v0";
auto const indexFileName = "index.json";
auto const indexFilePath = my::JoinPath(GetPlatform().WritableDir(), "ugc_migration", "test_index", versionString,
indexFileName);
auto const v0IndexFilePath = indexFilePath + "." + versionString;
Storage s(builder.GetIndex());
s.LoadForTesting(indexFilePath);
uint64_t migratedIndexFileSize = 0;
uint64_t v0IndexFileSize = 0;
TEST(my::GetFileSize(indexFilePath, migratedIndexFileSize), ());
TEST(my::GetFileSize(v0IndexFilePath, v0IndexFileSize), ());
TEST_GREATER(migratedIndexFileSize, 0, ());
TEST_GREATER(v0IndexFileSize, 0, ());
auto const & indexes = s.GetIndexesForTesting();
for (auto const & i : indexes)
{
TEST_EQUAL(static_cast<uint8_t>(i.m_version), static_cast<uint8_t>(IndexVersion::Latest), ());
TEST(!i.m_synchronized, ());
}
my::DeleteFileX(indexFilePath);
my::RenameFileX(v0IndexFilePath, indexFilePath);
}