From 262b15bd42871775f49325e0f8959c49489ef62d Mon Sep 17 00:00:00 2001 From: VladiMihaylenko Date: Thu, 21 Jun 2018 19:43:24 +0300 Subject: [PATCH] Moved ugc migration files option to ugc_tests CMakeList. --- CMakeLists.txt | 1 - data/ugc_migration/test_index/v0/index.gz | Bin 0 -> 3232 bytes data/ugc_migration/test_index/v0/index.json | 1 - ugc/index_migration/utility.cpp | 4 +++ ugc/index_migration/utility.hpp | 3 ++- ugc/serdes_json.hpp | 4 +-- ugc/storage.cpp | 13 ++++++++-- ugc/storage.hpp | 4 +-- ugc/ugc_tests/CMakeLists.txt | 4 +++ ugc/ugc_tests/storage_tests.cpp | 26 +++++++++++++++----- 10 files changed, 45 insertions(+), 15 deletions(-) create mode 100644 data/ugc_migration/test_index/v0/index.gz delete mode 100644 data/ugc_migration/test_index/v0/index.json diff --git a/CMakeLists.txt b/CMakeLists.txt index 8a18201471..b697ce263e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -64,7 +64,6 @@ option(USE_TSAN "Enable Thread Sanitizer" OFF) option(PYBINDINGS "Create makefiles for building python bindings" OFF) option(SKIP_DESKTOP "Skip building of desktop application" OFF) option(BUILD_MAPSHOT "Build mapshot tool" OFF) -option(UGC_MIGRATION "Build migration file for ugc" OFF) option(USE_PCH "Use precompiled headers" OFF) if (CMAKE_CXX_COMPILER_ID MATCHES "Clang") diff --git a/data/ugc_migration/test_index/v0/index.gz b/data/ugc_migration/test_index/v0/index.gz new file mode 100644 index 0000000000000000000000000000000000000000..93f718d0be74d7c4536eab7b88790e903346e423 GIT binary patch literal 3232 zcmV;R3}5pfiwFox_aIvU18Ht#Wq2-Xb8l_{&0Ae>9M=u~D~6tp#pS&uFX^|U1%jXr z8ngxaP!z((UdKXhsgUG+75VQyoL%j3*7m3x$Fq&$Bo^)@bB15%ki7r+&&9j#{mVC3 z*WX{>|NQUW#S7|Q>+_4BFJ5ThuuCKBr5A1%Kc1XPY{SrUCzkQFN zz5DUq<@NR*{#O@&d4G3zwY~i2=I-Uqj~94+d%wN>VRw6Xb#onWLZy~ppML|2wm*}BgaIZ% ztddqMi2rUQ5G(6h#PCWZ5PZ|hh~1_-2L9p9DUhZkkmd%@>O@s4jX>aT>$w(LvIrxA znbpExh)35N&P;$P7f&%lZmj_e%-JPnFp~46Fd(sxOpKSJN;VrLc=wE?RXAjKF*nLc z){BzbswbOhN9t{~o}d-vV@+wYfP(y~FB?*)JL0`s`c zU?Om5I$+Wbq8?vEMGU#T<$P?*t@FqCw|Dv zGBt}cCwQEC!c#&7wbDV-lE;A`t&C@q5uiqp4I1>NI8?w~Mo?sdF@hv=2`jz_;;&^q zwwMk4CK&-r*-8tSgFbsCBRpsT;9*2gb;g(I<{6L5+7Wb$$0~SEX*f>vc)aLnAAAI_ z>eXQ3l(DPP%J!+?1IgbcL>lsefo~pp^x@1&{FsjRft!IN6y!N%?ra~owc`gwxTFdM zeF+1B>~tG@H2X8B89PG#As^s8;b9coUQ@;fE$!#TQxyuQH+FDR)fqex%tgn103raj z0JSp!N_eU)W<9?}{6Kj@wRL!X(1h9espUcK0FOrwfs{k}0n1K!m{!gQ_{_mBAMh9# zwP+}*j6IISnG=80I>ryOG{O#MqNu0|k1THrF2pm@3k7^7zC5^cbDP%-+GXr_HJL<%(LQ!6qz&VVMZoAO=Hw zCLtLH&}Zp53*w_GsBO@%`G_AR=W#LkMlGaP?|Cxw(cC53 zCdFJjTi{7-B}o5yybAmy=CRR`^j2?no(U#ufkv0BKxV6P^c2hiC#;7*VBw+M?7Y#jE@SJs7}s0(-8;nqO^6@ zY6n~tdFjJw=7@yWtMM^b(S)SnYm*Z=ed5CVGt>Dv@aD=GZAq|bMiNiGR%NHP8MRgjk&8ql~|M%Es`T1TJjvM-Z!cZZeGEnl&Se##7EZMM|^D? zE7nj=A=-g(;$6O=fA^>D{SQ~$MWOD|iidnPZxFr4b2KHC`eEn_2;D)nS}c$hEtEzQ z-u&sI{!|F07Dev>AzZ@P5b9pjg4O=abRkheuJ#akLj@SALjk~olCqWe1wDx6Rv_7c zAjUXRP|joOJ7Wd-)+4qth_7yzgNQBFShHZ~*-j%s z9Gdfc5K>S!i%L;S!QkGX9@dLMJSH^5nw>kg;R4tKFBe*mJF?xEIT?Rxs?;M7H0-p$ZH&(sj z-A=YATN%3zFc=}O5GLI6M1jh%nYdQ_TKh;vkZqcUCSr)yaV)<-b5g*2YF~4yk*8yn z&6$dTq41eDL3=DuLXVm}rU+3Z>QQ}g46wnoKmcIX$tGwkXKKZR=OTmlxG&MmtV{=# zo}g&NNJI9vN&W@1TyfGfVeLlI1gaJOVlua_7f`rmC~-F-gqZ1M)3~4MwgjMXq*S~Z ztIUn82Pi&5G0^>nN=i0qVaKk;nXVR~-hu6+!bJj@?Kyb(V2Tx!VETBpPxkX_ku!}F zCH5lV?*(S*bECvqo0hSq6>1m1JS0132Lq8@u^&H3*<(K*t^!GjYA{s4$(alV(p*Qa zE@qG~7TeP%dFP92qxFPgOr}SxpvD!vmmn3Vh1UIak80-H0}Euz^>eBUXY-RkkP)EnSZKSghcXC*oop4IvH=oJ(XA{Q|9Ql&eRf#2N-M zU>5|j&OoVgbue9F2~LK{*VZSuBB`l=fw2ELl+nSMHwbrVUepMMmaQ}X1@+#Yjva{7 zp%{n87Kvp4;<{iP;;Y-;?&a?C5BS^l{q6Q`9DDY=SMRQ_ukP^YdjNWN^ZMC0+n2jn z_>XL#ZLeQF``epa{P_vEk40#YTZEuD3AM=PW?cjDmVkGrpsY}woa<@aF#Ep;uA43$ zi5a|G<6_&~f&v`%!0yO421h5uU#Q3=x%jdyShM`*=K6kjz5D+5<_RHq%<%&ZjX7|Q z{qbQM|H&K=#vtVg167)1I}GHGC^I{hw5M~3MKDwk!ZctsVMLz9rgblx)SRAF-3AJ0 zqcviqvTIzdnd(v=`fe-0m^hX95<8I|9Wj~R4vTw0@&F@-ngSIED>};rhA-F+{X7^$ ziE2n|9GBUUp68d`SbSXGK~_g{Bp-fd4ip4&wGuVmO6WKUR>n|cG_S4UgX26*ive5Q znJt5i7@^ei1xi#cHq*KpI#A6RzhMu2+tgq_jgQI?_jiXz8mbw&Qi&yt>{-$l>>bYb zpPcb@@up}QZ zi+*yQ3hIG=b0P-?Pam_&gKNdT#JINVjn%*dEOk2C>78o$jF)n}mk0$^X}I@hB~t*g zy4WO--%I=yqA`Uv-oWP}Mt%6!#b?De>98jL)9_q$cRlv|ArGeU;{EAi|MlJNcK7yU S5Tx3#|NJjVy$a=ZL;wI~( * = nullptr> void operator()(T & t, char const * name = nullptr) { - using UndelyingType = std::underlying_type_t; - UndelyingType res; + using UnderlyingType = std::underlying_type_t; + UnderlyingType res; FromJSONObject(m_json, name, res); t = static_cast(res); } diff --git a/ugc/storage.cpp b/ugc/storage.cpp index f224bac351..927f34e85a 100644 --- a/ugc/storage.cpp +++ b/ugc/storage.cpp @@ -24,6 +24,8 @@ #include "3party/jansson/myjansson.hpp" +#include + using namespace std; namespace @@ -215,8 +217,14 @@ void Storage::Load() if (m_indexes.empty()) return; + boost::optional version; for (auto const & i : m_indexes) { + if (!version) + version = i.m_version; + else + CHECK_EQUAL(static_cast(*version), static_cast(i.m_version), ("Inconsistent index")); + if (i.m_deleted) ++m_numberOfDeleted; } @@ -226,12 +234,13 @@ void Storage::Load() void Storage::Migrate(string const & indexFilePath) { - // We assume there is no situation when indexes from different versions are stored in the vector. - if (m_indexes.front().m_version == IndexVersion::Latest) + if (m_indexes.empty()) return; switch (migration::Migrate(m_indexes)) { + case migration::Result::UpToDate: + break; case migration::Result::Failure: LOG(LWARNING, ("Index migration failed")); break; diff --git a/ugc/storage.hpp b/ugc/storage.hpp index 6010f20a55..9f1b2c337d 100644 --- a/ugc/storage.hpp +++ b/ugc/storage.hpp @@ -16,7 +16,7 @@ namespace ugc class Storage { public: - explicit Storage(Index const & index) : m_index(index) {} + explicit Storage(DataSourceBase const & index) : m_index(index) {} UGCUpdate GetUGCUpdate(FeatureID const & id) const; @@ -46,7 +46,7 @@ private: std::unique_ptr GetFeature(FeatureID const & id) const; void Migrate(std::string const & indexFilePath); - Index const & m_index; + DataSourceBase const & m_index; UpdateIndexes m_indexes; size_t m_numberOfDeleted = 0; }; diff --git a/ugc/ugc_tests/CMakeLists.txt b/ugc/ugc_tests/CMakeLists.txt index 1126623722..7b62970218 100644 --- a/ugc/ugc_tests/CMakeLists.txt +++ b/ugc/ugc_tests/CMakeLists.txt @@ -2,6 +2,8 @@ project(ugc_tests) include_directories(${OMIM_ROOT}/3party/jansson/src) +option(UGC_MIGRATION "Build migration file for ugc" OFF) + set( SRC serdes_binary_tests.cpp @@ -17,6 +19,8 @@ if (UGC_MIGRATION) ${SRC} migration/generate_migration_files.cpp ) + + message("Generating files for UGC index migration") endif() omim_add_test(${PROJECT_NAME} ${SRC}) diff --git a/ugc/ugc_tests/storage_tests.cpp b/ugc/ugc_tests/storage_tests.cpp index ce10551f3c..13ac9a384a 100644 --- a/ugc/ugc_tests/storage_tests.cpp +++ b/ugc/ugc_tests/storage_tests.cpp @@ -20,6 +20,7 @@ #include "coding/file_name_utils.hpp" #include "coding/internal/file_data.hpp" #include "coding/writer.hpp" +#include "coding/zlib.hpp" #include "platform/local_country_file_utils.hpp" #include "platform/platform.hpp" @@ -441,14 +442,27 @@ UNIT_CLASS_TEST(StorageTest, GetNumberOfUnsentSeparately) UNIT_TEST(UGC_IndexMigrationFromV0ToV1Smoke) { + auto & p = GetPlatform(); + auto const version = "v0"; + auto const indexFileName = "index.json"; + auto const folder = my::JoinPath(p.WritableDir(), "ugc_migration", "test_index", version); + auto const indexFilePath = my::JoinPath(folder, indexFileName); + { + using Inflate = coding::ZLib::Inflate; + auto const r = p.GetReader(my::JoinPath(folder, "index.gz")); + string data; + r->ReadAsString(data); + Inflate inflate(Inflate::Format::GZip); + string index; + inflate(data.data(), data.size(), back_inserter(index)); + FileWriter w(indexFilePath); + w.Write(index.data(), index.size()); + } + auto & builder = MwmBuilder::Builder(); builder.Build({}); - auto const versionString = "v0"; - auto const indexFileName = "index.json"; - auto const indexFilePath = my::JoinPath(GetPlatform().WritableDir(), "ugc_migration", "test_index", versionString, - indexFileName); - auto const v0IndexFilePath = indexFilePath + "." + versionString; + auto const v0IndexFilePath = indexFilePath + "." + version; Storage s(builder.GetIndex()); s.LoadForTesting(indexFilePath); uint64_t migratedIndexFileSize = 0; @@ -465,5 +479,5 @@ UNIT_TEST(UGC_IndexMigrationFromV0ToV1Smoke) } my::DeleteFileX(indexFilePath); - my::RenameFileX(v0IndexFilePath, indexFilePath); + my::DeleteFileX(v0IndexFilePath); }