[new downloader] Add storage integration tests sources

platform/platform_tests_support/platform_tests_support.pro

@@ -10,8 +10,10 @@ SOURCES += \
     scoped_dir.cpp \
     scoped_file.cpp \
     scoped_mwm.cpp \
+    write_dir_changer.cpp \
 
 HEADERS += \
     scoped_dir.hpp \
     scoped_file.hpp \
     scoped_mwm.hpp \
+    write_dir_changer.hpp \

storage/storage_integration_tests/migrate_tests.cpp

@@ -0,0 +1,114 @@
+#include "testing/testing.hpp"
+
+#include "map/framework.hpp"
+
+#include "platform/local_country_file_utils.hpp"
+#include "platform/platform.hpp"
+#include "platform/platform_tests_support/scoped_dir.hpp"
+
+#include "coding/file_name_utils.hpp"
+#include "coding/internal/file_data.hpp"
+
+#include "base/scope_guard.hpp"
+#include "base/string_utils.hpp"
+#include "base/thread.hpp"
+
+#include "std/string.hpp"
+
+#include "write_dir_changer.hpp"
+
+using namespace platform;
+using namespace storage;
+
+namespace
+{
+static string const kMapTestDir = "map-tests";
+}
+
+UNIT_TEST(StorageFastMigrationTests)
+{
+  WritableDirChanger writableDirChanger(kMapTestDir);
+
+  Framework f;
+  auto & s = f.Storage();
+
+  uint32_t version;
+  Settings::Get("LastMigration", version);
+
+  TEST_GREATER_OR_EQUAL(s.GetCurrentDataVersion(), version, ());
+  Settings::Clear();
+}
+
+UNIT_TEST(StorageMigrationTests)
+{
+  TCountriesVec const kOldCountries = {"Estonia"};
+  TCountriesVec const kNewCountries = {"Estonia_East", "Estonia_West"};
+  TCountriesVec const kPrefetchCountries = {"Russia_Moscow"};
+
+  WritableDirChanger writableDirChanger(kMapTestDir);
+  Settings::Clear();
+  Settings::Set("DisableFastMigrate", true);
+
+  Framework f;
+  auto & s = f.Storage();
+
+  auto statePrefetchChanged = [&](TCountryId const & id)
+  {
+    TStatus const nextStatus = f.Storage().m_prefetchStorage->CountryStatusEx(id);
+    LOG_SHORT(LINFO, (id, "status :", nextStatus));
+    if (!f.Storage().m_prefetchStorage->IsDownloadInProgress())
+    {
+      LOG_SHORT(LINFO, ("All prefetched. Ready to migrate."));
+      testing::StopEventLoop();
+    }
+  };
+
+  auto stateChanged = [&](TCountryId const & id)
+  {
+    if (!f.Storage().IsDownloadInProgress())
+    {
+      LOG_SHORT(LINFO, ("All downloaded. Check consistency."));
+      testing::StopEventLoop();
+    }
+  };
+
+  auto progressChanged = [](TCountryId const & id, LocalAndRemoteSizeT const & sz)
+  {
+    LOG_SHORT(LINFO, (id, "downloading progress:", sz));
+  };
+
+  // Somewhere in Moscow, Russia
+  ms::LatLon curPos(55.7, 37.7);
+
+  s.SetDownloadingUrlsForTesting({"http://new-search.mapswithme.com/"});
+  s.Subscribe(stateChanged, progressChanged);
+  for (auto const & countryId : kOldCountries)
+    s.DownloadNode(countryId);
+
+  // Wait for downloading complete.
+  testing::EventLoop();
+
+  TEST_EQUAL(s.GetDownloadedFilesCount(), kOldCountries.size(), ());
+  for (auto const & countryId : kOldCountries)
+    TEST(s.IsNodeDownloaded(countryId), (countryId));
+
+  f.PreMigrate(curPos, statePrefetchChanged, progressChanged);
+  // Wait for downloading complete.
+  testing::EventLoop();
+
+  TEST_EQUAL(s.GetDownloadedFilesCount(), kPrefetchCountries.size(), ());
+  for (auto const & countryId : kPrefetchCountries)
+    TEST(s.m_prefetchStorage->IsNodeDownloaded(countryId), (countryId));
+
+  f.Migrate();
+  // Wait for downloading complete.
+  testing::EventLoop();
+
+  TEST_EQUAL(s.GetDownloadedFilesCount(), kPrefetchCountries.size() + kNewCountries.size(), ());
+  for (auto const & countryId : kNewCountries)
+    TEST(s.IsNodeDownloaded(countryId), (countryId));
+  for (auto const & countryId : kPrefetchCountries)
+    TEST(s.IsNodeDownloaded(countryId), (countryId));
+
+  s.DeleteAllLocalMaps();
+}

storage/storage_integration_tests/storage_group_download_tests.cpp

@@ -0,0 +1,292 @@
+#include "testing/testing.hpp"
+
+#include "map/framework.hpp"
+
+#include "platform/http_request.hpp"
+#include "platform/local_country_file_utils.hpp"
+#include "platform/platform.hpp"
+#include "platform/platform_tests_support/scoped_dir.hpp"
+
+#include "coding/file_name_utils.hpp"
+
+#include "storage/storage.hpp"
+
+#include "storage/storage_integration_tests/write_dir_changer.hpp"
+
+#include "base/assert.hpp"
+
+#include "std/condition_variable.hpp"
+#include "std/mutex.hpp"
+#include "std/set.hpp"
+#include "std/unique_ptr.hpp"
+
+#include <QtCore/QCoreApplication>
+
+using namespace platform;
+using namespace storage;
+
+namespace
+{
+
+string const kTestWebServer = "http://new-search.mapswithme.com/";
+
+string const kMapTestDir = "map-tests";
+
+TCountryId const kGroupCountryId = "New Zealand";
+TCountriesSet const kLeafCountriesIds = {"Tokelau",
+                                         "New Zealand North_Auckland",
+                                         "New Zealand North_Wellington",
+                                         "New Zealand South_Canterbury",
+                                         "New Zealand South_Southland"};
+
+string GetMwmFilePath(string const & version, TCountryId const & countryId)
+{
+  return my::JoinFoldersToPath({GetPlatform().WritableDir(), version},
+                               countryId + DATA_FILE_EXTENSION);
+}
+
+string GetMwmDownloadingFilePath(string const & version, TCountryId const & countryId)
+{
+  return my::JoinFoldersToPath({GetPlatform().WritableDir(), version},
+                               countryId + DATA_FILE_EXTENSION READY_FILE_EXTENSION DOWNLOADING_FILE_EXTENSION);
+}
+
+string GetMwmResumeFilePath(string const & version, TCountryId const & countryId)
+{
+  return my::JoinFoldersToPath({GetPlatform().WritableDir(), version},
+                               countryId + DATA_FILE_EXTENSION READY_FILE_EXTENSION RESUME_FILE_EXTENSION);
+}
+
+void DownloadGroup(Storage & storage, bool oneByOne)
+{
+  Platform & platform = GetPlatform();
+
+  string const version = strings::to_string(storage.GetCurrentDataVersion());
+
+  //  Get children nodes for the group node
+  TCountriesVec v;
+  storage.GetChildren(kGroupCountryId, v);
+  TCountriesSet const children(v.begin(), v.end());
+  v.clear();
+
+  TCountriesSet changed;
+  auto onChangeCountryFn = [&](TCountryId const & countryId)
+  {
+    TEST(children.find(countryId) != children.end(), ());
+    changed.insert(countryId);
+    if (!storage.IsDownloadInProgress())
+    {
+      // end waiting when all chilren will be downloaded
+      QCoreApplication::exit();
+    }
+  };
+
+  TCountriesSet downloaded;
+  auto onProgressFn = [&](TCountryId const & countryId, LocalAndRemoteSizeT const & mapSize)
+  {
+    TEST(children.find(countryId) != children.end(), ());
+    if (mapSize.first == mapSize.second)
+    {
+      auto const res = downloaded.insert(countryId);
+      TEST_EQUAL(res.second, true, ()); // every child is downloaded only once
+    }
+  };
+
+  int const subsrcibtionId = storage.Subscribe(onChangeCountryFn, onProgressFn);
+
+  // Check group node is not downloaded
+  storage.GetDownloadedChildren(storage.GetRootId(), v);
+  TEST(v.empty(), ());
+
+  // Check children nodes are not downloaded
+  storage.GetDownloadedChildren(kGroupCountryId, v);
+  TEST(v.empty(), ());
+
+  // Check there is no mwm or any other files for the children nodes
+  for (auto const & countryId : children)
+  {
+    string const mwmFullPath = GetMwmFilePath(version, countryId);
+    string const downloadingFullPath = GetMwmDownloadingFilePath(version, countryId);
+    string const resumeFullPath = GetMwmResumeFilePath(version, countryId);
+    TEST(!platform.IsFileExistsByFullPath(mwmFullPath), ());
+    TEST(!platform.IsFileExistsByFullPath(downloadingFullPath), ());
+    TEST(!platform.IsFileExistsByFullPath(resumeFullPath), ());
+  }
+
+  // Download the group
+  if (oneByOne)
+  {
+    for (auto const & countryId : children)
+      storage.DownloadNode(countryId);
+  }
+  else
+  {
+    storage.DownloadNode(kGroupCountryId);
+  }
+  // wait for downloading of all children
+  QCoreApplication::exec();
+
+  // Check all children nodes have been downloaded and changed.
+  TEST_EQUAL(changed, children, ());
+  TEST_EQUAL(downloaded, children, ());
+
+  // Check state for the group node is set to UpToDate and NoError
+  NodeAttrs attrs;
+  storage.GetNodeAttrs(kGroupCountryId, attrs);
+  // TEST_EQUAL(NodeStatus::UpToDate, attrs.m_status, ()); // DOES NOT WORK
+
+  // Check state for the all children nodes is set to UpToDate and NoError
+  for (auto const & countryId : children)
+  {
+    TEST_EQUAL(TStatus::EOnDisk, storage.CountryStatusEx(countryId), ());
+    NodeAttrs attrs;
+    storage.GetNodeAttrs(countryId, attrs);
+    // TEST_EQUAL(NodeStatus::UpToDate, attrs.m_status, ()); // DOES NOT WORK
+  }
+
+  // Check there is only mwm files are present and no any other for the children nodes
+  for (auto const & countryId : children)
+  {
+    string const mwmFullPath = GetMwmFilePath(version, countryId);
+    string const downloadingFullPath = GetMwmDownloadingFilePath(version, countryId);
+    string const resumeFullPath = GetMwmResumeFilePath(version, countryId);
+    TEST(platform.IsFileExistsByFullPath(mwmFullPath), ());
+    TEST(!platform.IsFileExistsByFullPath(downloadingFullPath), ());
+    TEST(!platform.IsFileExistsByFullPath(resumeFullPath), ());
+  }
+
+  // Check group is downloaded
+  storage.GetDownloadedChildren(storage.GetRootId(), v);
+  TEST_EQUAL(v, TCountriesVec({kGroupCountryId}), ());
+  v.clear();
+
+  // Check all group children are downloaded
+  storage.GetDownloadedChildren(kGroupCountryId, v);
+  TEST_EQUAL(children, TCountriesSet(v.begin(), v.end()), ());
+  v.clear();
+
+  storage.Unsubscribe(subsrcibtionId);
+}
+
+void DeleteGroup(Storage & storage, bool oneByOne)
+{
+  Platform & platform = GetPlatform();
+
+  string const version = strings::to_string(storage.GetCurrentDataVersion());
+
+  //  Get children nodes for the group node
+  TCountriesVec v;
+  storage.GetChildren(kGroupCountryId, v);
+  TCountriesSet const children(v.begin(), v.end());
+  v.clear();
+
+  // Check group node is downloaded
+  storage.GetDownloadedChildren(storage.GetRootId(), v);
+  TEST_EQUAL(v, TCountriesVec({kGroupCountryId}), ());
+  v.clear();
+
+  // Check children nodes are downloaded
+  storage.GetDownloadedChildren(kGroupCountryId, v);
+  TEST_EQUAL(children, TCountriesSet(v.begin(), v.end()), ());
+  v.clear();
+
+  // Check there are mwm files for the children nodes
+  for (auto const & countryId : children)
+  {
+    string const mwmFullPath = GetMwmFilePath(version, countryId);
+    TEST(platform.IsFileExistsByFullPath(mwmFullPath), ());
+  }
+
+  // Delete the group
+  if (oneByOne)
+  {
+    for (auto const & countryId : children)
+      storage.DeleteNode(countryId);
+  }
+  else
+  {
+    storage.DeleteNode(kGroupCountryId);
+  }
+
+  // Check state for the group node is set to UpToDate and NoError
+  NodeAttrs attrs;
+  storage.GetNodeAttrs(kGroupCountryId, attrs);
+  // TEST_EQUAL(NodeStatus::NotDownloaded, attrs.m_status, ()); // DOES NOT WORK
+
+  // Check state for the all children nodes is set to UpToDate and NoError
+  for (auto const & countryId : children)
+  {
+    TEST_EQUAL(TStatus::ENotDownloaded, storage.CountryStatusEx(countryId), ());
+    NodeAttrs attrs;
+    storage.GetNodeAttrs(countryId, attrs);
+    // TEST_EQUAL(NodeStatus::NotDownloaded, attrs.m_status, ()); // DOES NOT WORK
+  }
+
+  // Check there are no mwm files for the children nodes
+  for (auto const & countryId : children)
+  {
+    string const mwmFullPath = GetMwmFilePath(version, countryId);
+    TEST(!platform.IsFileExistsByFullPath(mwmFullPath), ());
+  }
+
+  // Check group is not downloaded
+  storage.GetDownloadedChildren(storage.GetRootId(), v);
+  TEST(v.empty(), ());
+
+  // Check all children nodes are not downloaded
+  storage.GetDownloadedChildren(kGroupCountryId, v);
+  TEST(v.empty(), ());
+}
+
+void TestDownloadDelete(bool downloadOneByOne, bool deleteOneByOne)
+{
+  WritableDirChanger writableDirChanger(kMapTestDir);
+
+  Storage storage(COUNTRIES_MIGRATE_FILE);
+
+  TEST(version::IsSingleMwm(storage.GetCurrentDataVersion()), ());
+  string const version = strings::to_string(storage.GetCurrentDataVersion());
+
+  auto onUpdatedFn = [&](LocalCountryFile const & localCountryFile)
+  {
+    TCountryId const countryId = localCountryFile.GetCountryName();
+    TEST(kLeafCountriesIds.find(countryId) != kLeafCountriesIds.end(), ());
+  };
+
+  storage.Init(onUpdatedFn);
+  storage.RegisterAllLocalMaps();
+  storage.SetDownloadingUrlsForTesting({kTestWebServer});
+
+  tests_support::ScopedDir cleanupVersionDir(version);
+
+  // Check children for the kGroupCountryId
+  TCountriesVec children;
+  storage.GetChildren(kGroupCountryId, children);
+  TEST_EQUAL(TCountriesSet(children.begin(), children.end()), kLeafCountriesIds, ());
+
+  DownloadGroup(storage, downloadOneByOne);
+
+  DeleteGroup(storage, deleteOneByOne);
+}
+
+} // namespace
+
+UNIT_TEST(SmallMwms_GroupDownloadDelete_Test1)
+{
+  TestDownloadDelete(false, false);
+}
+
+UNIT_TEST(SmallMwms_GroupDownloadDelete_Test2)
+{
+  TestDownloadDelete(false, true);
+}
+
+UNIT_TEST(SmallMwms_GroupDownloadDelete_Test3)
+{
+  TestDownloadDelete(true, false);
+}
+
+UNIT_TEST(SmallMwms_GroupDownloadDelete_Test4)
+{
+  TestDownloadDelete(true, true);
+}

storage/storage_integration_tests/storage_http_tests.cpp

@@ -0,0 +1,104 @@
+#include "testing/testing.hpp"
+
+#include "storage/storage.hpp"
+
+#include "platform/local_country_file_utils.hpp"
+#include "platform/platform.hpp"
+#include "platform/platform_tests_support/scoped_dir.hpp"
+
+#include "coding/file_name_utils.hpp"
+
+#include "base/scope_guard.hpp"
+#include "base/string_utils.hpp"
+#include "base/thread.hpp"
+
+#include "std/string.hpp"
+
+#include "write_dir_changer.hpp"
+
+using namespace platform;
+using namespace storage;
+
+namespace
+{
+string const kCountryId = "Angola";
+string const kMapTestDir = "map-tests";
+
+void ChangeCountryFunction(TCountryId const & countryId) {}
+
+void ProgressFunction(TCountryId const & countryId, LocalAndRemoteSizeT const & mapSize)
+{
+  TEST_EQUAL(countryId, kCountryId, ());
+  if (mapSize.first != mapSize.second)
+    return;
+
+  testing::StopEventLoop();
+}
+
+void Update(LocalCountryFile const & localCountryFile)
+{
+  TEST_EQUAL(localCountryFile.GetCountryName(), kCountryId, ());
+}
+
+} // namespace
+
+UNIT_TEST(StorageDownloadNodeAndDeleteNodeTests)
+{
+  WritableDirChanger writableDirChanger(kMapTestDir);
+
+  Storage storage(COUNTRIES_MIGRATE_FILE);
+  TEST(version::IsSingleMwm(storage.GetCurrentDataVersion()), ());
+
+  auto ChangeCountryFunction = [&](TCountryId const & countryId)
+  {
+    if (!storage.IsDownloadInProgress())
+    {
+      // End wait for downloading complete.
+      testing::StopEventLoop();
+    }
+  };
+
+  storage.Init(Update);
+  storage.RegisterAllLocalMaps();
+  storage.Subscribe(ChangeCountryFunction, ProgressFunction);
+  storage.SetDownloadingUrlsForTesting({"http://new-search.mapswithme.com/"});
+  string const version = strings::to_string(storage.GetCurrentDataVersion());
+  tests_support::ScopedDir cleanupVersionDir(version);
+  MY_SCOPE_GUARD(cleanup,
+                 bind(&Storage::DeleteNode, &storage, kCountryId));
+  DeleteDownloaderFilesForCountry(storage.GetCurrentDataVersion(),
+                                  kMapTestDir, CountryFile(kCountryId));
+
+  string const mwmFullPath = my::JoinFoldersToPath({GetPlatform().WritableDir(), version},
+                                                   kCountryId + DATA_FILE_EXTENSION);
+  string const downloadingFullPath = my::JoinFoldersToPath(
+      {GetPlatform().WritableDir(), version},
+      kCountryId + DATA_FILE_EXTENSION READY_FILE_EXTENSION DOWNLOADING_FILE_EXTENSION);
+  string const resumeFullPath = my::JoinFoldersToPath(
+      {GetPlatform().WritableDir(), version},
+      kCountryId + DATA_FILE_EXTENSION READY_FILE_EXTENSION RESUME_FILE_EXTENSION);
+  Platform & platform = GetPlatform();
+
+  // Downloading to an empty directory.
+  storage.DownloadNode(kCountryId);
+
+  // Wait for downloading complete.
+  testing::EventLoop();
+
+  TEST(platform.IsFileExistsByFullPath(mwmFullPath), ());
+  TEST(!platform.IsFileExistsByFullPath(downloadingFullPath), ());
+  TEST(!platform.IsFileExistsByFullPath(resumeFullPath), ());
+
+  // Downloading to directory with Angola.mwm.
+  storage.DownloadNode(kCountryId);
+
+  // Wait for downloading complete.
+  testing::EventLoop();
+
+  TEST(platform.IsFileExistsByFullPath(mwmFullPath), ());
+  TEST(!platform.IsFileExistsByFullPath(downloadingFullPath), ());
+  TEST(!platform.IsFileExistsByFullPath(resumeFullPath), ());
+
+  storage.DeleteNode(kCountryId);
+  TEST(!platform.IsFileExistsByFullPath(mwmFullPath), ());
+}

storage/storage_integration_tests/storage_integration_tests.pro

@@ -0,0 +1,32 @@
+# Storage library tests.
+
+TARGET = storage_integration_tests
+CONFIG += console warn_on
+CONFIG -= app_bundle
+TEMPLATE = app
+
+ROOT_DIR = ../..
+DEPENDENCIES = map drape_frontend routing search storage indexer platform_tests_support drape platform geometry coding base \
+               freetype expat fribidi tomcrypt jansson protobuf osrm stats_client minizip succinct
+
+include($$ROOT_DIR/common.pri)
+
+DEFINES *= OMIM_UNIT_TEST_WITH_QT_EVENT_LOOP
+
+QT *= core
+
+macx-* {
+  QT *= gui widgets network # needed for QApplication with event loop, to test async events (downloader, etc.)
+  LIBS *= "-framework IOKit" "-framework QuartzCore" "-framework SystemConfiguration"
+}
+win32*|linux* {
+  QT *= network
+}
+
+HEADERS += \
+
+SOURCES += \
+  ../../testing/testingmain.cpp \
+  migrate_tests.cpp \
+  storage_group_download_tests.cpp \
+  storage_http_tests.cpp \

storage/storage_tests/storage_tests.pro

@@ -28,7 +28,6 @@ HEADERS += \
   fake_map_files_downloader.hpp \
   task_runner.hpp \
   test_map_files_downloader.hpp \
-  write_dir_changer.hpp \
 
 SOURCES += \
   ../../testing/testingmain.cpp \
@@ -40,4 +39,3 @@ SOURCES += \
   storage_tests.cpp \
   task_runner.cpp \
   test_map_files_downloader.cpp \
-  write_dir_changer.cpp \