From 0aa97bff1d9989fbd6d808307693059faec8bb73 Mon Sep 17 00:00:00 2001 From: Sergey Yershov Date: Tue, 2 Feb 2016 12:47:41 +0300 Subject: [PATCH] [new downloader] Add storage integration tests sources --- .../platform_tests_support.pro | 2 + .../write_dir_changer.cpp | 0 .../write_dir_changer.hpp | 0 .../migrate_tests.cpp | 114 +++++++ .../storage_group_download_tests.cpp | 292 ++++++++++++++++++ .../storage_http_tests.cpp | 104 +++++++ .../storage_integration_tests.pro | 32 ++ storage/storage_tests/storage_tests.pro | 2 - 8 files changed, 544 insertions(+), 2 deletions(-) rename {storage/storage_tests => platform/platform_tests_support}/write_dir_changer.cpp (100%) rename {storage/storage_tests => platform/platform_tests_support}/write_dir_changer.hpp (100%) create mode 100644 storage/storage_integration_tests/migrate_tests.cpp create mode 100644 storage/storage_integration_tests/storage_group_download_tests.cpp create mode 100644 storage/storage_integration_tests/storage_http_tests.cpp create mode 100644 storage/storage_integration_tests/storage_integration_tests.pro diff --git a/platform/platform_tests_support/platform_tests_support.pro b/platform/platform_tests_support/platform_tests_support.pro index 2f056ec392..1d7235af2c 100644 --- a/platform/platform_tests_support/platform_tests_support.pro +++ b/platform/platform_tests_support/platform_tests_support.pro @@ -10,8 +10,10 @@ SOURCES += \ scoped_dir.cpp \ scoped_file.cpp \ scoped_mwm.cpp \ + write_dir_changer.cpp \ HEADERS += \ scoped_dir.hpp \ scoped_file.hpp \ scoped_mwm.hpp \ + write_dir_changer.hpp \ diff --git a/storage/storage_tests/write_dir_changer.cpp b/platform/platform_tests_support/write_dir_changer.cpp similarity index 100% rename from storage/storage_tests/write_dir_changer.cpp rename to platform/platform_tests_support/write_dir_changer.cpp diff --git a/storage/storage_tests/write_dir_changer.hpp b/platform/platform_tests_support/write_dir_changer.hpp similarity index 100% rename from storage/storage_tests/write_dir_changer.hpp rename to platform/platform_tests_support/write_dir_changer.hpp diff --git a/storage/storage_integration_tests/migrate_tests.cpp b/storage/storage_integration_tests/migrate_tests.cpp new file mode 100644 index 0000000000..06bf23bd5f --- /dev/null +++ b/storage/storage_integration_tests/migrate_tests.cpp @@ -0,0 +1,114 @@ +#include "testing/testing.hpp" + +#include "map/framework.hpp" + +#include "platform/local_country_file_utils.hpp" +#include "platform/platform.hpp" +#include "platform/platform_tests_support/scoped_dir.hpp" + +#include "coding/file_name_utils.hpp" +#include "coding/internal/file_data.hpp" + +#include "base/scope_guard.hpp" +#include "base/string_utils.hpp" +#include "base/thread.hpp" + +#include "std/string.hpp" + +#include "write_dir_changer.hpp" + +using namespace platform; +using namespace storage; + +namespace +{ +static string const kMapTestDir = "map-tests"; +} + +UNIT_TEST(StorageFastMigrationTests) +{ + WritableDirChanger writableDirChanger(kMapTestDir); + + Framework f; + auto & s = f.Storage(); + + uint32_t version; + Settings::Get("LastMigration", version); + + TEST_GREATER_OR_EQUAL(s.GetCurrentDataVersion(), version, ()); + Settings::Clear(); +} + +UNIT_TEST(StorageMigrationTests) +{ + TCountriesVec const kOldCountries = {"Estonia"}; + TCountriesVec const kNewCountries = {"Estonia_East", "Estonia_West"}; + TCountriesVec const kPrefetchCountries = {"Russia_Moscow"}; + + WritableDirChanger writableDirChanger(kMapTestDir); + Settings::Clear(); + Settings::Set("DisableFastMigrate", true); + + Framework f; + auto & s = f.Storage(); + + auto statePrefetchChanged = [&](TCountryId const & id) + { + TStatus const nextStatus = f.Storage().m_prefetchStorage->CountryStatusEx(id); + LOG_SHORT(LINFO, (id, "status :", nextStatus)); + if (!f.Storage().m_prefetchStorage->IsDownloadInProgress()) + { + LOG_SHORT(LINFO, ("All prefetched. Ready to migrate.")); + testing::StopEventLoop(); + } + }; + + auto stateChanged = [&](TCountryId const & id) + { + if (!f.Storage().IsDownloadInProgress()) + { + LOG_SHORT(LINFO, ("All downloaded. Check consistency.")); + testing::StopEventLoop(); + } + }; + + auto progressChanged = [](TCountryId const & id, LocalAndRemoteSizeT const & sz) + { + LOG_SHORT(LINFO, (id, "downloading progress:", sz)); + }; + + // Somewhere in Moscow, Russia + ms::LatLon curPos(55.7, 37.7); + + s.SetDownloadingUrlsForTesting({"http://new-search.mapswithme.com/"}); + s.Subscribe(stateChanged, progressChanged); + for (auto const & countryId : kOldCountries) + s.DownloadNode(countryId); + + // Wait for downloading complete. + testing::EventLoop(); + + TEST_EQUAL(s.GetDownloadedFilesCount(), kOldCountries.size(), ()); + for (auto const & countryId : kOldCountries) + TEST(s.IsNodeDownloaded(countryId), (countryId)); + + f.PreMigrate(curPos, statePrefetchChanged, progressChanged); + // Wait for downloading complete. + testing::EventLoop(); + + TEST_EQUAL(s.GetDownloadedFilesCount(), kPrefetchCountries.size(), ()); + for (auto const & countryId : kPrefetchCountries) + TEST(s.m_prefetchStorage->IsNodeDownloaded(countryId), (countryId)); + + f.Migrate(); + // Wait for downloading complete. + testing::EventLoop(); + + TEST_EQUAL(s.GetDownloadedFilesCount(), kPrefetchCountries.size() + kNewCountries.size(), ()); + for (auto const & countryId : kNewCountries) + TEST(s.IsNodeDownloaded(countryId), (countryId)); + for (auto const & countryId : kPrefetchCountries) + TEST(s.IsNodeDownloaded(countryId), (countryId)); + + s.DeleteAllLocalMaps(); +} diff --git a/storage/storage_integration_tests/storage_group_download_tests.cpp b/storage/storage_integration_tests/storage_group_download_tests.cpp new file mode 100644 index 0000000000..534c0c657d --- /dev/null +++ b/storage/storage_integration_tests/storage_group_download_tests.cpp @@ -0,0 +1,292 @@ +#include "testing/testing.hpp" + +#include "map/framework.hpp" + +#include "platform/http_request.hpp" +#include "platform/local_country_file_utils.hpp" +#include "platform/platform.hpp" +#include "platform/platform_tests_support/scoped_dir.hpp" + +#include "coding/file_name_utils.hpp" + +#include "storage/storage.hpp" + +#include "storage/storage_integration_tests/write_dir_changer.hpp" + +#include "base/assert.hpp" + +#include "std/condition_variable.hpp" +#include "std/mutex.hpp" +#include "std/set.hpp" +#include "std/unique_ptr.hpp" + +#include + +using namespace platform; +using namespace storage; + +namespace +{ + +string const kTestWebServer = "http://new-search.mapswithme.com/"; + +string const kMapTestDir = "map-tests"; + +TCountryId const kGroupCountryId = "New Zealand"; +TCountriesSet const kLeafCountriesIds = {"Tokelau", + "New Zealand North_Auckland", + "New Zealand North_Wellington", + "New Zealand South_Canterbury", + "New Zealand South_Southland"}; + +string GetMwmFilePath(string const & version, TCountryId const & countryId) +{ + return my::JoinFoldersToPath({GetPlatform().WritableDir(), version}, + countryId + DATA_FILE_EXTENSION); +} + +string GetMwmDownloadingFilePath(string const & version, TCountryId const & countryId) +{ + return my::JoinFoldersToPath({GetPlatform().WritableDir(), version}, + countryId + DATA_FILE_EXTENSION READY_FILE_EXTENSION DOWNLOADING_FILE_EXTENSION); +} + +string GetMwmResumeFilePath(string const & version, TCountryId const & countryId) +{ + return my::JoinFoldersToPath({GetPlatform().WritableDir(), version}, + countryId + DATA_FILE_EXTENSION READY_FILE_EXTENSION RESUME_FILE_EXTENSION); +} + +void DownloadGroup(Storage & storage, bool oneByOne) +{ + Platform & platform = GetPlatform(); + + string const version = strings::to_string(storage.GetCurrentDataVersion()); + + // Get children nodes for the group node + TCountriesVec v; + storage.GetChildren(kGroupCountryId, v); + TCountriesSet const children(v.begin(), v.end()); + v.clear(); + + TCountriesSet changed; + auto onChangeCountryFn = [&](TCountryId const & countryId) + { + TEST(children.find(countryId) != children.end(), ()); + changed.insert(countryId); + if (!storage.IsDownloadInProgress()) + { + // end waiting when all chilren will be downloaded + QCoreApplication::exit(); + } + }; + + TCountriesSet downloaded; + auto onProgressFn = [&](TCountryId const & countryId, LocalAndRemoteSizeT const & mapSize) + { + TEST(children.find(countryId) != children.end(), ()); + if (mapSize.first == mapSize.second) + { + auto const res = downloaded.insert(countryId); + TEST_EQUAL(res.second, true, ()); // every child is downloaded only once + } + }; + + int const subsrcibtionId = storage.Subscribe(onChangeCountryFn, onProgressFn); + + // Check group node is not downloaded + storage.GetDownloadedChildren(storage.GetRootId(), v); + TEST(v.empty(), ()); + + // Check children nodes are not downloaded + storage.GetDownloadedChildren(kGroupCountryId, v); + TEST(v.empty(), ()); + + // Check there is no mwm or any other files for the children nodes + for (auto const & countryId : children) + { + string const mwmFullPath = GetMwmFilePath(version, countryId); + string const downloadingFullPath = GetMwmDownloadingFilePath(version, countryId); + string const resumeFullPath = GetMwmResumeFilePath(version, countryId); + TEST(!platform.IsFileExistsByFullPath(mwmFullPath), ()); + TEST(!platform.IsFileExistsByFullPath(downloadingFullPath), ()); + TEST(!platform.IsFileExistsByFullPath(resumeFullPath), ()); + } + + // Download the group + if (oneByOne) + { + for (auto const & countryId : children) + storage.DownloadNode(countryId); + } + else + { + storage.DownloadNode(kGroupCountryId); + } + // wait for downloading of all children + QCoreApplication::exec(); + + // Check all children nodes have been downloaded and changed. + TEST_EQUAL(changed, children, ()); + TEST_EQUAL(downloaded, children, ()); + + // Check state for the group node is set to UpToDate and NoError + NodeAttrs attrs; + storage.GetNodeAttrs(kGroupCountryId, attrs); + // TEST_EQUAL(NodeStatus::UpToDate, attrs.m_status, ()); // DOES NOT WORK + + // Check state for the all children nodes is set to UpToDate and NoError + for (auto const & countryId : children) + { + TEST_EQUAL(TStatus::EOnDisk, storage.CountryStatusEx(countryId), ()); + NodeAttrs attrs; + storage.GetNodeAttrs(countryId, attrs); + // TEST_EQUAL(NodeStatus::UpToDate, attrs.m_status, ()); // DOES NOT WORK + } + + // Check there is only mwm files are present and no any other for the children nodes + for (auto const & countryId : children) + { + string const mwmFullPath = GetMwmFilePath(version, countryId); + string const downloadingFullPath = GetMwmDownloadingFilePath(version, countryId); + string const resumeFullPath = GetMwmResumeFilePath(version, countryId); + TEST(platform.IsFileExistsByFullPath(mwmFullPath), ()); + TEST(!platform.IsFileExistsByFullPath(downloadingFullPath), ()); + TEST(!platform.IsFileExistsByFullPath(resumeFullPath), ()); + } + + // Check group is downloaded + storage.GetDownloadedChildren(storage.GetRootId(), v); + TEST_EQUAL(v, TCountriesVec({kGroupCountryId}), ()); + v.clear(); + + // Check all group children are downloaded + storage.GetDownloadedChildren(kGroupCountryId, v); + TEST_EQUAL(children, TCountriesSet(v.begin(), v.end()), ()); + v.clear(); + + storage.Unsubscribe(subsrcibtionId); +} + +void DeleteGroup(Storage & storage, bool oneByOne) +{ + Platform & platform = GetPlatform(); + + string const version = strings::to_string(storage.GetCurrentDataVersion()); + + // Get children nodes for the group node + TCountriesVec v; + storage.GetChildren(kGroupCountryId, v); + TCountriesSet const children(v.begin(), v.end()); + v.clear(); + + // Check group node is downloaded + storage.GetDownloadedChildren(storage.GetRootId(), v); + TEST_EQUAL(v, TCountriesVec({kGroupCountryId}), ()); + v.clear(); + + // Check children nodes are downloaded + storage.GetDownloadedChildren(kGroupCountryId, v); + TEST_EQUAL(children, TCountriesSet(v.begin(), v.end()), ()); + v.clear(); + + // Check there are mwm files for the children nodes + for (auto const & countryId : children) + { + string const mwmFullPath = GetMwmFilePath(version, countryId); + TEST(platform.IsFileExistsByFullPath(mwmFullPath), ()); + } + + // Delete the group + if (oneByOne) + { + for (auto const & countryId : children) + storage.DeleteNode(countryId); + } + else + { + storage.DeleteNode(kGroupCountryId); + } + + // Check state for the group node is set to UpToDate and NoError + NodeAttrs attrs; + storage.GetNodeAttrs(kGroupCountryId, attrs); + // TEST_EQUAL(NodeStatus::NotDownloaded, attrs.m_status, ()); // DOES NOT WORK + + // Check state for the all children nodes is set to UpToDate and NoError + for (auto const & countryId : children) + { + TEST_EQUAL(TStatus::ENotDownloaded, storage.CountryStatusEx(countryId), ()); + NodeAttrs attrs; + storage.GetNodeAttrs(countryId, attrs); + // TEST_EQUAL(NodeStatus::NotDownloaded, attrs.m_status, ()); // DOES NOT WORK + } + + // Check there are no mwm files for the children nodes + for (auto const & countryId : children) + { + string const mwmFullPath = GetMwmFilePath(version, countryId); + TEST(!platform.IsFileExistsByFullPath(mwmFullPath), ()); + } + + // Check group is not downloaded + storage.GetDownloadedChildren(storage.GetRootId(), v); + TEST(v.empty(), ()); + + // Check all children nodes are not downloaded + storage.GetDownloadedChildren(kGroupCountryId, v); + TEST(v.empty(), ()); +} + +void TestDownloadDelete(bool downloadOneByOne, bool deleteOneByOne) +{ + WritableDirChanger writableDirChanger(kMapTestDir); + + Storage storage(COUNTRIES_MIGRATE_FILE); + + TEST(version::IsSingleMwm(storage.GetCurrentDataVersion()), ()); + string const version = strings::to_string(storage.GetCurrentDataVersion()); + + auto onUpdatedFn = [&](LocalCountryFile const & localCountryFile) + { + TCountryId const countryId = localCountryFile.GetCountryName(); + TEST(kLeafCountriesIds.find(countryId) != kLeafCountriesIds.end(), ()); + }; + + storage.Init(onUpdatedFn); + storage.RegisterAllLocalMaps(); + storage.SetDownloadingUrlsForTesting({kTestWebServer}); + + tests_support::ScopedDir cleanupVersionDir(version); + + // Check children for the kGroupCountryId + TCountriesVec children; + storage.GetChildren(kGroupCountryId, children); + TEST_EQUAL(TCountriesSet(children.begin(), children.end()), kLeafCountriesIds, ()); + + DownloadGroup(storage, downloadOneByOne); + + DeleteGroup(storage, deleteOneByOne); +} + +} // namespace + +UNIT_TEST(SmallMwms_GroupDownloadDelete_Test1) +{ + TestDownloadDelete(false, false); +} + +UNIT_TEST(SmallMwms_GroupDownloadDelete_Test2) +{ + TestDownloadDelete(false, true); +} + +UNIT_TEST(SmallMwms_GroupDownloadDelete_Test3) +{ + TestDownloadDelete(true, false); +} + +UNIT_TEST(SmallMwms_GroupDownloadDelete_Test4) +{ + TestDownloadDelete(true, true); +} diff --git a/storage/storage_integration_tests/storage_http_tests.cpp b/storage/storage_integration_tests/storage_http_tests.cpp new file mode 100644 index 0000000000..a3e460d10c --- /dev/null +++ b/storage/storage_integration_tests/storage_http_tests.cpp @@ -0,0 +1,104 @@ +#include "testing/testing.hpp" + +#include "storage/storage.hpp" + +#include "platform/local_country_file_utils.hpp" +#include "platform/platform.hpp" +#include "platform/platform_tests_support/scoped_dir.hpp" + +#include "coding/file_name_utils.hpp" + +#include "base/scope_guard.hpp" +#include "base/string_utils.hpp" +#include "base/thread.hpp" + +#include "std/string.hpp" + +#include "write_dir_changer.hpp" + +using namespace platform; +using namespace storage; + +namespace +{ +string const kCountryId = "Angola"; +string const kMapTestDir = "map-tests"; + +void ChangeCountryFunction(TCountryId const & countryId) {} + +void ProgressFunction(TCountryId const & countryId, LocalAndRemoteSizeT const & mapSize) +{ + TEST_EQUAL(countryId, kCountryId, ()); + if (mapSize.first != mapSize.second) + return; + + testing::StopEventLoop(); +} + +void Update(LocalCountryFile const & localCountryFile) +{ + TEST_EQUAL(localCountryFile.GetCountryName(), kCountryId, ()); +} + +} // namespace + +UNIT_TEST(StorageDownloadNodeAndDeleteNodeTests) +{ + WritableDirChanger writableDirChanger(kMapTestDir); + + Storage storage(COUNTRIES_MIGRATE_FILE); + TEST(version::IsSingleMwm(storage.GetCurrentDataVersion()), ()); + + auto ChangeCountryFunction = [&](TCountryId const & countryId) + { + if (!storage.IsDownloadInProgress()) + { + // End wait for downloading complete. + testing::StopEventLoop(); + } + }; + + storage.Init(Update); + storage.RegisterAllLocalMaps(); + storage.Subscribe(ChangeCountryFunction, ProgressFunction); + storage.SetDownloadingUrlsForTesting({"http://new-search.mapswithme.com/"}); + string const version = strings::to_string(storage.GetCurrentDataVersion()); + tests_support::ScopedDir cleanupVersionDir(version); + MY_SCOPE_GUARD(cleanup, + bind(&Storage::DeleteNode, &storage, kCountryId)); + DeleteDownloaderFilesForCountry(storage.GetCurrentDataVersion(), + kMapTestDir, CountryFile(kCountryId)); + + string const mwmFullPath = my::JoinFoldersToPath({GetPlatform().WritableDir(), version}, + kCountryId + DATA_FILE_EXTENSION); + string const downloadingFullPath = my::JoinFoldersToPath( + {GetPlatform().WritableDir(), version}, + kCountryId + DATA_FILE_EXTENSION READY_FILE_EXTENSION DOWNLOADING_FILE_EXTENSION); + string const resumeFullPath = my::JoinFoldersToPath( + {GetPlatform().WritableDir(), version}, + kCountryId + DATA_FILE_EXTENSION READY_FILE_EXTENSION RESUME_FILE_EXTENSION); + Platform & platform = GetPlatform(); + + // Downloading to an empty directory. + storage.DownloadNode(kCountryId); + + // Wait for downloading complete. + testing::EventLoop(); + + TEST(platform.IsFileExistsByFullPath(mwmFullPath), ()); + TEST(!platform.IsFileExistsByFullPath(downloadingFullPath), ()); + TEST(!platform.IsFileExistsByFullPath(resumeFullPath), ()); + + // Downloading to directory with Angola.mwm. + storage.DownloadNode(kCountryId); + + // Wait for downloading complete. + testing::EventLoop(); + + TEST(platform.IsFileExistsByFullPath(mwmFullPath), ()); + TEST(!platform.IsFileExistsByFullPath(downloadingFullPath), ()); + TEST(!platform.IsFileExistsByFullPath(resumeFullPath), ()); + + storage.DeleteNode(kCountryId); + TEST(!platform.IsFileExistsByFullPath(mwmFullPath), ()); +} diff --git a/storage/storage_integration_tests/storage_integration_tests.pro b/storage/storage_integration_tests/storage_integration_tests.pro new file mode 100644 index 0000000000..a72f8ec610 --- /dev/null +++ b/storage/storage_integration_tests/storage_integration_tests.pro @@ -0,0 +1,32 @@ +# Storage library tests. + +TARGET = storage_integration_tests +CONFIG += console warn_on +CONFIG -= app_bundle +TEMPLATE = app + +ROOT_DIR = ../.. +DEPENDENCIES = map drape_frontend routing search storage indexer platform_tests_support drape platform geometry coding base \ + freetype expat fribidi tomcrypt jansson protobuf osrm stats_client minizip succinct + +include($$ROOT_DIR/common.pri) + +DEFINES *= OMIM_UNIT_TEST_WITH_QT_EVENT_LOOP + +QT *= core + +macx-* { + QT *= gui widgets network # needed for QApplication with event loop, to test async events (downloader, etc.) + LIBS *= "-framework IOKit" "-framework QuartzCore" "-framework SystemConfiguration" +} +win32*|linux* { + QT *= network +} + +HEADERS += \ + +SOURCES += \ + ../../testing/testingmain.cpp \ + migrate_tests.cpp \ + storage_group_download_tests.cpp \ + storage_http_tests.cpp \ diff --git a/storage/storage_tests/storage_tests.pro b/storage/storage_tests/storage_tests.pro index 3fb9cd1e24..c5576215ef 100644 --- a/storage/storage_tests/storage_tests.pro +++ b/storage/storage_tests/storage_tests.pro @@ -28,7 +28,6 @@ HEADERS += \ fake_map_files_downloader.hpp \ task_runner.hpp \ test_map_files_downloader.hpp \ - write_dir_changer.hpp \ SOURCES += \ ../../testing/testingmain.cpp \ @@ -40,4 +39,3 @@ SOURCES += \ storage_tests.cpp \ task_runner.cpp \ test_map_files_downloader.cpp \ - write_dir_changer.cpp \