map-per/organicmaps
1
Fork 0
forked from organicmaps/organicmaps
Code Pull requests Activity

Added migration tables validation

Browse source
This commit is contained in:
VladiMihaylenko 2018-06-20 15:31:58 +03:00 • committed by mpimenov
parent cec8f91544
commit 673f9bdfdb
1 changed files with 122 additions and 28 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

150
ugc/ugc_tests/migration/generate_migration_files.cpp
View file

@ -3,6 +3,7 @@
#include "coding/file_name_utils.hpp"
#include "coding/file_reader.hpp"
#include "coding/file_writer.hpp"
#include "coding/zlib.hpp"
#include "indexer/classificator.hpp"
#include "indexer/classificator_loader.hpp"
@ -19,6 +20,9 @@ using namespace std;
namespace
{
using Source = ReaderSource<FileReader>;
using MigrationTable = unordered_map<uint32_t, uint32_t>;
array<string, 11> const kVersions{{"171020",
"171117",
"171208",
@ -32,35 +36,62 @@ array<string, 11> const kVersions{{"171020",
"180528"}};
string const kUGCMigrationDirName = "ugc_migration";
string const kClassificatorFileName = "classificator.txt";
string const kTypesFileName = "types.txt";
string const kClassificatorFileName = "classificator.gz";
string const kTypesFileName = "types.gz";
string const kBinFileExtension = ".bin";
string GetUGCDirPath()
{
return my::JoinPath(GetPlatform().WritableDir(), kUGCMigrationDirName);
}
void LoadClassificatorTypesForVersion(string const & version)
{
auto const ugcDirPath = GetUGCDirPath();
auto const folderPath = my::JoinPath(ugcDirPath, version);
auto const & p = GetPlatform();
using Inflate = coding::ZLib::Inflate;
string classificator;
{
auto const r = p.GetReader(my::JoinPath(folderPath, kClassificatorFileName));
string data;
r->ReadAsString(data);
Inflate inflate(Inflate::Format::GZip);
inflate(data.data(), data.size(), back_inserter(classificator));
}
string types;
{
auto const r = p.GetReader(my::JoinPath(folderPath, kTypesFileName));
string data;
r->ReadAsString(data);
Inflate inflate(Inflate::Format::GZip);
inflate(data.data(), data.size(), back_inserter(types));
}
classificator::LoadTypes(classificator, types);
}
void LoadTableForVersion(string const & version, MigrationTable & table)
{
Source source(FileReader(my::JoinPath(GetUGCDirPath(), version + kBinFileExtension)));
ugc::DeserializerV0<Source> des(source);
des(table);
}
} // namespace
UNIT_TEST(UGC_GenerateMigrationFiles)
{
auto & p = GetPlatform();
auto const ugcDirPath = my::JoinPath(p.WritableDir(), kUGCMigrationDirName);
auto const ugcDirPath = GetUGCDirPath();
for (auto const & v : kVersions)
{
auto const folderPath = my::JoinPath(ugcDirPath, v);
string classificator;
{
auto const r = p.GetReader(my::JoinPath(folderPath, kClassificatorFileName));
r->ReadAsString(classificator);
}
LoadClassificatorTypesForVersion(v);
auto const & c = classif();
string types;
{
auto const r = p.GetReader(my::JoinPath(folderPath, kTypesFileName));
r->ReadAsString(types);
}
classificator::LoadTypes(classificator, types);
Classificator const & c = classif();
unordered_map<uint32_t, uint32_t> mapping;
auto const parse = [&c, &mapping](ClassifObject const * obj, uint32_t type)
MigrationTable mapping;
auto const parse = [&c, &mapping](ClassifObject const *, uint32_t type)
{
if (c.IsTypeValid(type))
mapping.emplace(type, c.GetIndexForType(type));
@ -76,13 +107,76 @@ UNIT_TEST(UGC_GenerateMigrationFiles)
ser(mapping);
}
unordered_map<uint32_t, uint32_t> res;
{
ReaderSource<FileReader> source(FileReader(filePath, true /* withExceptions */));
ugc::DeserializerV0<ReaderSource<FileReader>> des(source);
des(res);
}
MigrationTable res;
LoadTableForVersion(v, res);
TEST_EQUAL(res, mapping, ());
}
}
UNIT_TEST(UGC_ValidateMigrationTables)
{
auto const ugcDirPath = GetUGCDirPath();
for (auto const & v : kVersions)
{
MigrationTable table;
LoadTableForVersion(v, table);
TEST(!table.empty(), ("Version", v));
LoadClassificatorTypesForVersion(v);
auto const & c = classif();
auto const parse = [&c, &table](ClassifObject const *, uint32_t type)
{
if (c.IsTypeValid(type))
TEST_EQUAL(table[type], c.GetIndexForType(type), ());
};
c.ForEachTree(parse);
}
}
UNIT_TEST(UGC_TypesFromDifferentVersions)
{
auto constexpr size = kVersions.size();
for (size_t i = 0; i < size; ++i)
{
auto const & v = kVersions[i];
MigrationTable table;
LoadTableForVersion(v, table);
LoadClassificatorTypesForVersion(v);
unordered_map<string, uint32_t> namesToTypes;
{
auto const & c = classif();
auto const parse = [&c, &namesToTypes](ClassifObject const *, uint32_t type)
{
if (c.IsTypeValid(type))
namesToTypes.emplace(c.GetReadableObjectName(type), type);
};
c.ForEachTree(parse);
}
for (size_t j = 0; j < size; ++j)
{
if (i == j)
continue;
auto const & otherV = kVersions[j];
MigrationTable otherTable;
LoadTableForVersion(otherV, otherTable);
LoadClassificatorTypesForVersion(otherV);
auto const & c = classif();
auto const parse = [&c, &namesToTypes, &otherTable, &table](ClassifObject const *, uint32_t otherType) {
if (c.IsTypeValid(otherType))
{
auto const otherReadableName = c.GetReadableObjectName(otherType);
if (namesToTypes.find(otherReadableName) != namesToTypes.end())
{
auto const type = namesToTypes[otherReadableName];
TEST_EQUAL(otherTable[otherType], table[type], ());
}
}
};
c.ForEachTree(parse);
}
}
}