diff --git a/ugc/ugc_tests/migration/generate_migration_files.cpp b/ugc/ugc_tests/migration/generate_migration_files.cpp index 9fae971393..3693b92aa8 100644 --- a/ugc/ugc_tests/migration/generate_migration_files.cpp +++ b/ugc/ugc_tests/migration/generate_migration_files.cpp @@ -3,6 +3,7 @@ #include "coding/file_name_utils.hpp" #include "coding/file_reader.hpp" #include "coding/file_writer.hpp" +#include "coding/zlib.hpp" #include "indexer/classificator.hpp" #include "indexer/classificator_loader.hpp" @@ -19,6 +20,9 @@ using namespace std; namespace { +using Source = ReaderSource; +using MigrationTable = unordered_map; + array const kVersions{{"171020", "171117", "171208", @@ -32,35 +36,62 @@ array const kVersions{{"171020", "180528"}}; string const kUGCMigrationDirName = "ugc_migration"; -string const kClassificatorFileName = "classificator.txt"; -string const kTypesFileName = "types.txt"; +string const kClassificatorFileName = "classificator.gz"; +string const kTypesFileName = "types.gz"; string const kBinFileExtension = ".bin"; + +string GetUGCDirPath() +{ + return my::JoinPath(GetPlatform().WritableDir(), kUGCMigrationDirName); +} + +void LoadClassificatorTypesForVersion(string const & version) +{ + auto const ugcDirPath = GetUGCDirPath(); + auto const folderPath = my::JoinPath(ugcDirPath, version); + auto const & p = GetPlatform(); + + using Inflate = coding::ZLib::Inflate; + + string classificator; + { + auto const r = p.GetReader(my::JoinPath(folderPath, kClassificatorFileName)); + string data; + r->ReadAsString(data); + Inflate inflate(Inflate::Format::GZip); + inflate(data.data(), data.size(), back_inserter(classificator)); + } + + string types; + { + auto const r = p.GetReader(my::JoinPath(folderPath, kTypesFileName)); + string data; + r->ReadAsString(data); + Inflate inflate(Inflate::Format::GZip); + inflate(data.data(), data.size(), back_inserter(types)); + } + + classificator::LoadTypes(classificator, types); +} + +void LoadTableForVersion(string const & version, MigrationTable & table) +{ + Source source(FileReader(my::JoinPath(GetUGCDirPath(), version + kBinFileExtension))); + ugc::DeserializerV0 des(source); + des(table); +} } // namespace UNIT_TEST(UGC_GenerateMigrationFiles) { - auto & p = GetPlatform(); - auto const ugcDirPath = my::JoinPath(p.WritableDir(), kUGCMigrationDirName); + auto const ugcDirPath = GetUGCDirPath(); for (auto const & v : kVersions) { - auto const folderPath = my::JoinPath(ugcDirPath, v); - string classificator; - { - auto const r = p.GetReader(my::JoinPath(folderPath, kClassificatorFileName)); - r->ReadAsString(classificator); - } + LoadClassificatorTypesForVersion(v); + auto const & c = classif(); - string types; - { - auto const r = p.GetReader(my::JoinPath(folderPath, kTypesFileName)); - r->ReadAsString(types); - } - - classificator::LoadTypes(classificator, types); - Classificator const & c = classif(); - - unordered_map mapping; - auto const parse = [&c, &mapping](ClassifObject const * obj, uint32_t type) + MigrationTable mapping; + auto const parse = [&c, &mapping](ClassifObject const *, uint32_t type) { if (c.IsTypeValid(type)) mapping.emplace(type, c.GetIndexForType(type)); @@ -76,13 +107,76 @@ UNIT_TEST(UGC_GenerateMigrationFiles) ser(mapping); } - unordered_map res; - { - ReaderSource source(FileReader(filePath, true /* withExceptions */)); - ugc::DeserializerV0> des(source); - des(res); - } - + MigrationTable res; + LoadTableForVersion(v, res); TEST_EQUAL(res, mapping, ()); } } + +UNIT_TEST(UGC_ValidateMigrationTables) +{ + auto const ugcDirPath = GetUGCDirPath(); + for (auto const & v : kVersions) + { + MigrationTable table; + LoadTableForVersion(v, table); + TEST(!table.empty(), ("Version", v)); + + LoadClassificatorTypesForVersion(v); + auto const & c = classif(); + auto const parse = [&c, &table](ClassifObject const *, uint32_t type) + { + if (c.IsTypeValid(type)) + TEST_EQUAL(table[type], c.GetIndexForType(type), ()); + }; + + c.ForEachTree(parse); + } +} + +UNIT_TEST(UGC_TypesFromDifferentVersions) +{ + auto constexpr size = kVersions.size(); + for (size_t i = 0; i < size; ++i) + { + auto const & v = kVersions[i]; + MigrationTable table; + LoadTableForVersion(v, table); + LoadClassificatorTypesForVersion(v); + unordered_map namesToTypes; + { + auto const & c = classif(); + auto const parse = [&c, &namesToTypes](ClassifObject const *, uint32_t type) + { + if (c.IsTypeValid(type)) + namesToTypes.emplace(c.GetReadableObjectName(type), type); + }; + + c.ForEachTree(parse); + } + + for (size_t j = 0; j < size; ++j) + { + if (i == j) + continue; + + auto const & otherV = kVersions[j]; + MigrationTable otherTable; + LoadTableForVersion(otherV, otherTable); + LoadClassificatorTypesForVersion(otherV); + auto const & c = classif(); + auto const parse = [&c, &namesToTypes, &otherTable, &table](ClassifObject const *, uint32_t otherType) { + if (c.IsTypeValid(otherType)) + { + auto const otherReadableName = c.GetReadableObjectName(otherType); + if (namesToTypes.find(otherReadableName) != namesToTypes.end()) + { + auto const type = namesToTypes[otherReadableName]; + TEST_EQUAL(otherTable[otherType], table[type], ()); + } + } + }; + c.ForEachTree(parse); + } + } +}