From 849ff2df296d5f1ebcf68732f61ecce3b026dfcd Mon Sep 17 00:00:00 2001 From: Arsentiy Milchakov Date: Mon, 25 Nov 2019 15:23:34 +0300 Subject: [PATCH] [storage] downloading queue is moved from storage into downloader --- .../downloader_search_test.cpp | 18 +- storage/http_map_files_downloader.cpp | 121 ++++- storage/http_map_files_downloader.hpp | 17 +- storage/map_files_downloader.cpp | 61 ++- storage/map_files_downloader.hpp | 40 +- storage/map_files_downloader_with_ping.cpp | 11 +- storage/queued_country.cpp | 22 + storage/queued_country.hpp | 17 + storage/storage.cpp | 422 +++++++----------- storage/storage.hpp | 64 +-- .../storage_downloading_tests.cpp | 43 +- .../fake_map_files_downloader.cpp | 131 +++++- .../fake_map_files_downloader.hpp | 18 +- storage/storage_tests/storage_tests.cpp | 8 +- .../test_map_files_downloader.cpp | 4 +- .../test_map_files_downloader.hpp | 5 +- .../storage/storage.xcodeproj/project.pbxproj | 24 +- 17 files changed, 607 insertions(+), 419 deletions(-) diff --git a/search/search_integration_tests/downloader_search_test.cpp b/search/search_integration_tests/downloader_search_test.cpp index 70f35c9589..aa09e05908 100644 --- a/search/search_integration_tests/downloader_search_test.cpp +++ b/search/search_integration_tests/downloader_search_test.cpp @@ -13,6 +13,7 @@ #include "storage/map_files_downloader.hpp" #include "storage/queued_country.hpp" #include "storage/storage.hpp" +#include "storage/storage_defines.hpp" #include "platform/downloader_defines.hpp" @@ -89,17 +90,20 @@ public: downloader::Progress GetDownloadingProgress() override { return {}; } bool IsIdle() override { return false; } - - void Reset() override {} + + void Pause() override {} + void Resume() override {} + void Remove(storage::CountryId const & id) override {} + void Clear() override {} + + storage::Queue const & GetQueue() const override { return m_queue; } private: void GetServersList(ServersListCallback const & /* callback */) override {} - void Download(storage::QueuedCountry & queuedCountry, - FileDownloadedCallback const & /* onDownloaded */, - DownloadingProgressCallback const & /* onProgress */) override - { - } + void Download(storage::QueuedCountry & queuedCountry) override {} + + storage::Queue m_queue; }; class TestDelegate : public DownloaderSearchCallback::Delegate diff --git a/storage/http_map_files_downloader.cpp b/storage/http_map_files_downloader.cpp index e6693e3c6b..554aa4af2e 100644 --- a/storage/http_map_files_downloader.cpp +++ b/storage/http_map_files_downloader.cpp @@ -6,6 +6,7 @@ #include "base/assert.hpp" #include "base/string_utils.hpp" +#include #include using namespace std::placeholders; @@ -35,26 +36,46 @@ HttpMapFilesDownloader::~HttpMapFilesDownloader() CHECK_THREAD_CHECKER(m_checker, ()); } -void HttpMapFilesDownloader::Download(QueuedCountry & queuedCountry, - FileDownloadedCallback const & onDownloaded, - DownloadingProgressCallback const & onProgress) +void HttpMapFilesDownloader::Download(QueuedCountry & queuedCountry) { CHECK_THREAD_CHECKER(m_checker, ()); + m_queue.emplace_back(std::move(queuedCountry)); + + if (m_queue.size() != 1) + return; + + for (auto const subscriber : m_subscribers) + subscriber->OnStartDownloading(); + + Download(); +} + +void HttpMapFilesDownloader::Download() +{ + CHECK_THREAD_CHECKER(m_checker, ()); + + auto const & queuedCountry = m_queue.front(); + auto const urls = MakeUrlList(queuedCountry.GetRelativeUrl()); auto const path = queuedCountry.GetFileDownloadPath(); auto const size = queuedCountry.GetDownloadSize(); - m_request.reset(downloader::HttpRequest::GetFile( - urls, path, size, - std::bind(&HttpMapFilesDownloader::OnMapFileDownloaded, this, onDownloaded, _1), - std::bind(&HttpMapFilesDownloader::OnMapFileDownloadingProgress, this, onProgress, _1))); + m_request.reset(); + + if (IsDownloadingAllowed()) + { + m_request.reset(downloader::HttpRequest::GetFile( + urls, path, size, + std::bind(&HttpMapFilesDownloader::OnMapFileDownloaded, this, queuedCountry, _1), + std::bind(&HttpMapFilesDownloader::OnMapFileDownloadingProgress, this, queuedCountry, _1))); + } if (!m_request) { - // Mark the end of download with error. ErrorHttpRequest error(path); - OnMapFileDownloaded(onDownloaded, error); + auto const copy = queuedCountry; + OnMapFileDownloaded(copy, error); } } @@ -71,23 +92,93 @@ bool HttpMapFilesDownloader::IsIdle() return m_request.get() == nullptr; } -void HttpMapFilesDownloader::Reset() +void HttpMapFilesDownloader::Pause() { CHECK_THREAD_CHECKER(m_checker, ()); m_request.reset(); } -void HttpMapFilesDownloader::OnMapFileDownloaded(FileDownloadedCallback const & onDownloaded, +void HttpMapFilesDownloader::Resume() +{ + CHECK_THREAD_CHECKER(m_checker, ()); + + if (!m_request && !m_queue.empty()) + Download(); +} + +void HttpMapFilesDownloader::Remove(CountryId const & id) +{ + CHECK_THREAD_CHECKER(m_checker, ()); + + if (m_queue.empty()) + return; + + if (m_request && m_queue.front() == id) + { + m_request.reset(); + m_queue.pop_front(); + } + else + { + auto it = std::find(m_queue.begin(), m_queue.end(), id); + if (it != m_queue.end()) + m_queue.erase(it); + } + + if (m_queue.empty()) + { + m_request.reset(); + for (auto const subscriber : m_subscribers) + subscriber->OnFinishDownloading(); + } +} + +void HttpMapFilesDownloader::Clear() +{ + CHECK_THREAD_CHECKER(m_checker, ()); + + m_request.reset(); + m_queue.clear(); +} + +Queue const & HttpMapFilesDownloader::GetQueue() const +{ + CHECK_THREAD_CHECKER(m_checker, ()); + return m_queue; +} + +void HttpMapFilesDownloader::OnMapFileDownloaded(QueuedCountry const & queuedCountry, downloader::HttpRequest & request) { CHECK_THREAD_CHECKER(m_checker, ()); - onDownloaded(request.GetStatus(), request.GetProgress()); + // Because of this method calls deferred on original thread, + // it is possible the country is already removed from queue. + if (m_queue.empty() || m_queue.front().GetCountryId() != queuedCountry.GetCountryId()) + return; + + m_queue.pop_front(); + + queuedCountry.OnDownloadFinished(request.GetStatus()); + + if (!m_queue.empty()) + Download(); + else + { + m_request.reset(); + for (auto const subscriber : m_subscribers) + subscriber->OnFinishDownloading(); + } } -void HttpMapFilesDownloader::OnMapFileDownloadingProgress( - DownloadingProgressCallback const & onProgress, downloader::HttpRequest & request) +void HttpMapFilesDownloader::OnMapFileDownloadingProgress(QueuedCountry const & queuedCountry, + downloader::HttpRequest & request) { CHECK_THREAD_CHECKER(m_checker, ()); - onProgress(request.GetProgress()); + // Because of this method calls deferred on original thread, + // it is possible the country is already removed from queue. + if (m_queue.empty() || m_queue.front().GetCountryId() != queuedCountry.GetCountryId()) + return; + + queuedCountry.OnDownloadProgress(request.GetProgress()); } } // namespace storage diff --git a/storage/http_map_files_downloader.hpp b/storage/http_map_files_downloader.hpp index 05556e1dcd..5a1157c25c 100644 --- a/storage/http_map_files_downloader.hpp +++ b/storage/http_map_files_downloader.hpp @@ -25,19 +25,24 @@ public: // MapFilesDownloader overrides: downloader::Progress GetDownloadingProgress() override; bool IsIdle() override; - void Reset() override; + void Pause() override; + void Resume() override; + void Remove(CountryId const & id) override; + void Clear() override; + Queue const & GetQueue() const override; private: // MapFilesDownloaderWithServerList overrides: - void Download(QueuedCountry & queuedCountry, FileDownloadedCallback const & onDownloaded, - DownloadingProgressCallback const & onProgress) override; + void Download(QueuedCountry & queuedCountry) override; - void OnMapFileDownloaded(FileDownloadedCallback const & onDownloaded, - downloader::HttpRequest & request); - void OnMapFileDownloadingProgress(DownloadingProgressCallback const & onProgress, + void Download(); + + void OnMapFileDownloaded(QueuedCountry const & queuedCountry, downloader::HttpRequest & request); + void OnMapFileDownloadingProgress(QueuedCountry const & queuedCountry, downloader::HttpRequest & request); std::unique_ptr m_request; + Queue m_queue; DECLARE_THREAD_CHECKER(m_checker); }; diff --git a/storage/map_files_downloader.cpp b/storage/map_files_downloader.cpp index 4a3e6e1fcf..c14b9c82ff 100644 --- a/storage/map_files_downloader.cpp +++ b/storage/map_files_downloader.cpp @@ -12,27 +12,49 @@ #include "base/string_utils.hpp" #include "base/url_helpers.hpp" -#include - namespace storage { -void MapFilesDownloader::DownloadMapFile(QueuedCountry & queuedCountry, - FileDownloadedCallback const & onDownloaded, - DownloadingProgressCallback const & onProgress) +void MapFilesDownloader::DownloadMapFile(QueuedCountry & queuedCountry) { if (!m_serversList.empty()) { queuedCountry.ClarifyDownloadingType(); - Download(queuedCountry, onDownloaded, onProgress); + Download(queuedCountry); return; } - GetServersList([=](ServersList const & serversList) mutable - { - m_serversList = serversList; - queuedCountry.ClarifyDownloadingType(); - Download(queuedCountry, onDownloaded, onProgress); - }); + GetPlatform().RunTask(Platform::Thread::Network, [=]() + { + if (m_isServersListRequested) + { + GetPlatform().RunTask(Platform::Thread::Gui, [=]() mutable + { + queuedCountry.ClarifyDownloadingType(); + Download(queuedCountry); + }); + + return; + } + + m_isServersListRequested = true; + + GetServersList([=](ServersList const & serversList) mutable + { + m_serversList = serversList; + queuedCountry.ClarifyDownloadingType(); + Download(queuedCountry); + }); + }); +} + +void MapFilesDownloader::Subscribe(Subscriber * subscriber) +{ + m_subscribers.push_back(subscriber); +} + +void MapFilesDownloader::UnsubscribeAll() +{ + m_subscribers.clear(); } // static @@ -48,6 +70,16 @@ void MapFilesDownloader::SetServersList(ServersList const & serversList) m_serversList = serversList; } +void MapFilesDownloader::SetDownloadingPolicy(DownloadingPolicy * policy) +{ + m_downloadingPolicy = policy; +} + +bool MapFilesDownloader::IsDownloadingAllowed() const +{ + return m_downloadingPolicy == nullptr || m_downloadingPolicy->IsDownloadingAllowed(); +} + std::vector MapFilesDownloader::MakeUrlList(std::string const & relativeUrl) { std::vector urls; @@ -75,9 +107,6 @@ MapFilesDownloader::ServersList MapFilesDownloader::LoadServersList() void MapFilesDownloader::GetServersList(ServersListCallback const & callback) { - GetPlatform().RunTask(Platform::Thread::Network, [callback]() - { - callback(LoadServersList()); - }); + callback(LoadServersList()); } } // namespace storage diff --git a/storage/map_files_downloader.hpp b/storage/map_files_downloader.hpp index 653018fdf1..0b46f95e39 100644 --- a/storage/map_files_downloader.hpp +++ b/storage/map_files_downloader.hpp @@ -1,5 +1,6 @@ #pragma once +#include "storage/downloading_policy.hpp" #include "storage/queued_country.hpp" #include "platform/downloader_defines.hpp" @@ -15,6 +16,7 @@ namespace storage { +using Queue = std::list; // This interface encapsulates HTTP routines for receiving servers // URLs and downloading a single map file. class MapFilesDownloader @@ -23,18 +25,23 @@ public: // Denotes bytes downloaded and total number of bytes. using ServersList = std::vector; - using FileDownloadedCallback = - std::function; - using DownloadingProgressCallback = std::function; using ServersListCallback = platform::SafeCallback; + class Subscriber + { + public: + virtual ~Subscriber() = default; + + virtual void OnStartDownloading() = 0; + virtual void OnFinishDownloading() = 0; + }; + virtual ~MapFilesDownloader() = default; /// Asynchronously downloads a map file, periodically invokes /// onProgress callback and finally invokes onDownloaded /// callback. Both callbacks will be invoked on the main thread. - void DownloadMapFile(QueuedCountry & queuedCountry, FileDownloadedCallback const & onDownloaded, - DownloadingProgressCallback const & onProgress); + void DownloadMapFile(QueuedCountry & queuedCountry); /// Returns current downloading progress. virtual downloader::Progress GetDownloadingProgress() = 0; @@ -42,28 +49,41 @@ public: /// Returns true when downloader does not perform any job. virtual bool IsIdle() = 0; - /// Resets downloader to the idle state. - virtual void Reset() = 0; + virtual void Pause() = 0; + virtual void Resume() = 0; + virtual void Remove(CountryId const & id) = 0; + virtual void Clear() = 0; + + virtual Queue const & GetQueue() const = 0; + + void Subscribe(Subscriber * subscriber); + void UnsubscribeAll(); static std::string MakeFullUrlLegacy(std::string const & baseUrl, std::string const & fileName, int64_t dataVersion); void SetServersList(ServersList const & serversList); + void SetDownloadingPolicy(DownloadingPolicy * policy); protected: + bool IsDownloadingAllowed() const; std::vector MakeUrlList(std::string const & relativeUrl); // Synchronously loads list of servers by http client. static ServersList LoadServersList(); + std::vector m_subscribers; + private: - /// Default implementation asynchronously receives a list of all servers that can be asked + /// NOTE: this method will be called on network thread. + /// Default implementation receives a list of all servers that can be asked /// for a map file and invokes callback on the main thread. virtual void GetServersList(ServersListCallback const & callback); /// Asynchronously downloads the file from provided |urls| and saves result to |path|. - virtual void Download(QueuedCountry & queuedCountry, FileDownloadedCallback const & onDownloaded, - DownloadingProgressCallback const & onProgress) = 0; + virtual void Download(QueuedCountry & queuedCountry) = 0; ServersList m_serversList; + bool m_isServersListRequested = false; + DownloadingPolicy * m_downloadingPolicy = nullptr; }; } // namespace storage diff --git a/storage/map_files_downloader_with_ping.cpp b/storage/map_files_downloader_with_ping.cpp index a5da8f9ee5..612c20975f 100644 --- a/storage/map_files_downloader_with_ping.cpp +++ b/storage/map_files_downloader_with_ping.cpp @@ -12,14 +12,11 @@ void MapFilesDownloaderWithPing::GetServersList(ServersListCallback const & call { ASSERT(callback , ()); - GetPlatform().RunTask(Platform::Thread::Network, [callback]() - { - auto urls = LoadServersList(); + auto urls = LoadServersList(); - CHECK(!urls.empty(), ()); + CHECK(!urls.empty(), ()); - auto const availableEndpoints = Pinger::ExcludeUnavailableEndpoints(urls); - callback(availableEndpoints.empty() ? urls : availableEndpoints); - }); + auto const availableEndpoints = Pinger::ExcludeUnavailableEndpoints(urls); + callback(availableEndpoints.empty() ? urls : availableEndpoints); } } // namespace storage diff --git a/storage/queued_country.cpp b/storage/queued_country.cpp index 83c98f5227..fb4d6d3fa6 100644 --- a/storage/queued_country.cpp +++ b/storage/queued_country.cpp @@ -97,6 +97,28 @@ void QueuedCountry::ClarifyDownloadingType() } } +void QueuedCountry::SetOnFinishCallback(DownloadingFinishCallback const & onDownloaded) +{ + m_downloadingFinishCallback = onDownloaded; +} + +void QueuedCountry::SetOnProgressCallback(DownloadingProgressCallback const & onProgress) +{ + m_downloadingProgressCallback = onProgress; +} + +void QueuedCountry::OnDownloadFinished(downloader::DownloadStatus status) const +{ + if (m_downloadingFinishCallback) + m_downloadingFinishCallback(*this, status); +} + +void QueuedCountry::OnDownloadProgress(downloader::Progress const & progress) const +{ + if (m_downloadingProgressCallback) + m_downloadingProgressCallback(*this, progress); +} + bool QueuedCountry::operator==(CountryId const & countryId) const { return m_countryId == countryId; diff --git a/storage/queued_country.hpp b/storage/queued_country.hpp index 0b7e62ed79..7c28e5f770 100644 --- a/storage/queued_country.hpp +++ b/storage/queued_country.hpp @@ -5,11 +5,19 @@ #include "platform/country_defines.hpp" #include "platform/country_file.hpp" +#include "platform/downloader_defines.hpp" #include namespace storage { +class QueuedCountry; + +using DownloadingFinishCallback = + std::function; +using DownloadingProgressCallback = + std::function; + class QueuedCountry { public: @@ -28,6 +36,12 @@ public: void ClarifyDownloadingType(); + void SetOnFinishCallback(DownloadingFinishCallback const & onDownloaded); + void SetOnProgressCallback(DownloadingProgressCallback const & onProgress); + + void OnDownloadFinished(downloader::DownloadStatus status) const; + void OnDownloadProgress(downloader::Progress const & progress) const; + bool operator==(CountryId const & countryId) const; private: @@ -37,5 +51,8 @@ private: int64_t m_currentDataVersion; std::string m_dataDir; std::shared_ptr m_diffsDataSource; + + DownloadingFinishCallback m_downloadingFinishCallback; + DownloadingProgressCallback m_downloadingProgressCallback; }; } // namespace storage diff --git a/storage/storage.cpp b/storage/storage.cpp index 9e40082906..11a1381c51 100644 --- a/storage/storage.cpp +++ b/storage/storage.cpp @@ -82,9 +82,58 @@ bool ValidateIntegrity(LocalFilePtr mapLocalFile, string const & countryId, stri {"source", source}})); return false; } + +bool IsFileDownloaded(string const fileDownloadPath, MapFileType type) +{ + // Since a downloaded valid diff file may be either with .diff or .diff.ready extension, + // we have to check these both cases in order to find + // the diff file which is ready to apply. + // If there is a such file we have to cause the success download scenario. + string const readyFilePath = fileDownloadPath; + bool isDownloadedDiff = false; + if (type == MapFileType::Diff) + { + string filePath = readyFilePath; + base::GetNameWithoutExt(filePath); + isDownloadedDiff = GetPlatform().IsFileExistsByFullPath(filePath); + } + + // It may happen that the file already was downloaded, so there're + // no need to request servers list and download file. Let's + // switch to next file. + return isDownloadedDiff || GetPlatform().IsFileExistsByFullPath(readyFilePath); +} + +void SendStatisticsAfterDownloading(CountryId const & countryId, DownloadStatus status, + MapFileType type, int64_t dataVersion, + std::map> const & localFiles) +{ + // Send statistics to PushWoosh. We send these statistics only for the newly downloaded + // mwms, not for updated ones. + if (status == DownloadStatus::Completed && type != MapFileType::Diff) + { + auto const it = localFiles.find(countryId); + if (it == localFiles.end()) + { + auto & marketingService = GetPlatform().GetMarketingService(); + marketingService.SendPushWooshTag(marketing::kMapLastDownloaded, countryId); + auto const nowStr = marketingService.GetPushWooshTimestamp(); + marketingService.SendPushWooshTag(marketing::kMapLastDownloadedTimestamp, nowStr); + } + } + + alohalytics::LogEvent("$OnMapDownloadFinished", + alohalytics::TStringMap( + {{"name", countryId}, + {"status", status == DownloadStatus::Completed ? "ok" : "failed"}, + {"version", strings::to_string(dataVersion)}, + {"option", DebugPrint(type)}})); + GetPlatform().GetMarketingService().SendMarketingEvent(marketing::kDownloaderMapActionFinished, + {{"action", "download"}}); +} } // namespace -void GetQueuedCountries(Storage::Queue const & queue, CountriesSet & resultCountries) +void GetQueuedCountries(Queue const & queue, CountriesSet & resultCountries) { for (auto const & country : queue) resultCountries.insert(country.GetCountryId()); @@ -114,6 +163,9 @@ Storage::Storage(string const & pathToCountriesFile /* = COUNTRIES_FILE */, : m_downloader(make_unique()) , m_dataDir(dataDir) { + m_downloader->SetDownloadingPolicy(m_downloadingPolicy); + m_downloader->Subscribe(this); + SetLocale(languages::GetCurrentTwine()); LoadCountriesFile(pathToCountriesFile); CalcMaxMwmSizeBytes(); @@ -123,6 +175,8 @@ Storage::Storage(string const & referenceCountriesTxtJsonForTesting, unique_ptr mapDownloaderForTesting) : m_downloader(move(mapDownloaderForTesting)) { + m_downloader->Subscribe(this); + m_currentVersion = LoadCountriesFromBuffer(referenceCountriesTxtJsonForTesting, m_countries, m_affiliations, m_countryNameSynonyms, m_mwmTopCityGeoIds, m_mwmTopCountryGeoIds); @@ -138,6 +192,14 @@ void Storage::Init(UpdateCallback const & didDownload, DeleteCallback const & wi m_willDelete = willDelete; } +void Storage::SetDownloadingPolicy(DownloadingPolicy * policy) +{ + CHECK_THREAD_CHECKER(m_threadChecker, ()); + + m_downloadingPolicy = policy; + m_downloader->SetDownloadingPolicy(policy); +} + void Storage::DeleteAllLocalMaps(CountriesVec * existedCountries /* = nullptr */) { CHECK_THREAD_CHECKER(m_threadChecker, ()); @@ -167,8 +229,7 @@ void Storage::Clear() { CHECK_THREAD_CHECKER(m_threadChecker, ()); - m_downloader->Reset(); - m_queue.clear(); + m_downloader->Clear(); m_justDownloaded.clear(); m_failedCountries.clear(); m_localFiles.clear(); @@ -315,16 +376,13 @@ bool Storage::IsInnerNode(CountryId const & countryId) const LocalAndRemoteSize Storage::CountrySizeInBytes(CountryId const & countryId) const { - QueuedCountry const * queuedCountry = FindCountryInQueue(countryId); LocalFilePtr localFile = GetLatestLocalFile(countryId); CountryFile const & countryFile = GetCountryFile(countryId); - if (queuedCountry == nullptr) - { - return LocalAndRemoteSize(localFile ? localFile->GetSize(MapFileType::Map) : 0, - GetRemoteSize(countryFile, GetCurrentDataVersion())); - } - LocalAndRemoteSize sizes(0, GetRemoteSize(countryFile, GetCurrentDataVersion())); + + if (!IsCountryInQueue(countryId) && !IsDiffApplyingInProgressToCountry(countryId)) + sizes.first = localFile ? localFile->GetSize(MapFileType::Map) : 0; + if (!m_downloader->IsIdle() && IsCountryFirstInQueue(countryId)) { sizes.first = m_downloader->GetDownloadingProgress().first + @@ -387,15 +445,14 @@ Status Storage::CountryStatus(CountryId const & countryId) const if (IsCountryInQueue(countryId)) { if (IsCountryFirstInQueue(countryId)) - { - if (IsDiffApplyingInProgressToCountry(countryId)) - return Status::EApplying; return Status::EDownloading; - } return Status::EInQueue; } + if (IsDiffApplyingInProgressToCountry(countryId)) + return Status::EApplying; + return Status::EUnknown; } @@ -418,29 +475,13 @@ Status Storage::CountryStatusEx(CountryId const & countryId) const return Status::EOnDisk; } -void Storage::CountryStatusEx(CountryId const & countryId, Status & status, - MapFileType & type) const -{ - status = CountryStatusEx(countryId); - - if (status == Status::EOnDisk || status == Status::EOnDiskOutOfDate) - { - type = MapFileType::Map; - ASSERT(GetLatestLocalFile(countryId), - ("Invariant violation: local file out of sync with disk.")); - } -} - void Storage::SaveDownloadQueue() { CHECK_THREAD_CHECKER(m_threadChecker, ()); - if (!m_keepDownloadingQueue) - return; - ostringstream download; ostringstream update; - for (auto const & item : m_queue) + for (auto const & item : m_downloader->GetQueue()) { auto & ss = item.GetFileType() == MapFileType::Diff ? update : download; ss << (ss.str().empty() ? "" : ";") << item.GetCountryId(); @@ -452,9 +493,6 @@ void Storage::SaveDownloadQueue() void Storage::RestoreDownloadQueue() { - if (!m_keepDownloadingQueue) - return; - string download, update; if (!settings::Get(kDownloadQueueKey, download) && !settings::Get(kUpdateQueueKey, update)) return; @@ -483,23 +521,34 @@ void Storage::DownloadCountry(CountryId const & countryId, MapFileType type) { CHECK_THREAD_CHECKER(m_threadChecker, ()); - if (FindCountryInQueue(countryId) != nullptr) + if (IsCountryInQueue(countryId) || IsDiffApplyingInProgressToCountry(countryId)) return; m_failedCountries.erase(countryId); auto const countryFile = GetCountryFile(countryId); - m_queue.emplace_back(countryFile, countryId, type, GetCurrentDataVersion(), m_dataDir, - m_diffsDataSource); - if (m_queue.size() == 1) + // It's not even possible to prepare directory for files before + // downloading. Mark this country as failed and switch to next + // country. + if (!PreparePlaceForCountryFiles(GetCurrentDataVersion(), m_dataDir, countryFile)) { - if (m_startDownloadingCallback) - m_startDownloadingCallback(); - DownloadNextCountryFromQueue(); + OnMapDownloadFinished(countryId, DownloadStatus::Failed, type); + return; } - else + + if (IsFileDownloaded(GetFileDownloadPath(countryId, type), type)) { - NotifyStatusChangedForHierarchy(countryId); + OnMapDownloadFinished(countryId, DownloadStatus::Completed, type); + return; } + + QueuedCountry queuedCountry(countryFile, countryId, type, GetCurrentDataVersion(), m_dataDir, + m_diffsDataSource); + queuedCountry.SetOnFinishCallback(bind(&Storage::OnMapFileDownloadFinished, this, _1, _2)); + queuedCountry.SetOnProgressCallback(bind(&Storage::OnMapFileDownloadProgress, this, _1, _2)); + + m_downloader->DownloadMapFile(queuedCountry); + + NotifyStatusChangedForHierarchy(countryId); SaveDownloadQueue(); } @@ -515,9 +564,7 @@ void Storage::DeleteCountry(CountryId const & countryId, MapFileType type) NotifyStatusChangedForHierarchy(countryId); - // Kick possibly interrupted downloader. - if (!m_queue.empty() && m_downloader->IsIdle()) - DownloadNextCountryFromQueue(); + m_downloader->Resume(); } void Storage::DeleteCustomCountryVersion(LocalCountryFile const & localFile) @@ -565,106 +612,19 @@ void Storage::NotifyStatusChangedForHierarchy(CountryId const & countryId) }); } -void Storage::DownloadNextCountryFromQueue() -{ - CHECK_THREAD_CHECKER(m_threadChecker, ()); - - bool const stopDownload = !m_downloadingPolicy->IsDownloadingAllowed(); - - if (m_queue.empty()) - { - m_downloadingPolicy->ScheduleRetry(m_failedCountries, [this](CountriesSet const & needReload) { - for (auto const & country : needReload) - { - NodeStatuses status; - GetNodeStatuses(country, status); - if (status.m_error == NodeErrorCode::NoInetConnection) - RetryDownloadNode(country); - } - }); - CountriesVec localMaps; - GetLocalRealMaps(localMaps); - GetPlatform().GetMarketingService().SendPushWooshTag(marketing::kMapListing, localMaps); - if (!localMaps.empty()) - GetPlatform().GetMarketingService().SendPushWooshTag(marketing::kMapDownloadDiscovered); - return; - } - - QueuedCountry & queuedCountry = m_queue.front(); - CountryId const countryId = queuedCountry.GetCountryId(); - - // It's not even possible to prepare directory for files before - // downloading. Mark this country as failed and switch to next - // country. - if (stopDownload || - !PreparePlaceForCountryFiles(GetCurrentDataVersion(), m_dataDir, GetCountryFile(countryId))) - { - OnMapDownloadFinished(countryId, DownloadStatus::Failed, queuedCountry.GetFileType()); - return; - } - - DownloadNextFile(queuedCountry); - - // New status for the country, "Downloading" - NotifyStatusChangedForHierarchy(countryId); -} - -void Storage::DownloadNextFile(QueuedCountry const & country) -{ - CountryId const & countryId = country.GetCountryId(); - auto const opt = country.GetFileType(); - - string const readyFilePath = GetFileDownloadPath(countryId, opt); - uint64_t size; - auto & p = GetPlatform(); - - // Since a downloaded valid diff file may be either with .diff or .diff.ready extension, - // we have to check these both cases in order to find - // the diff file which is ready to apply. - // If there is a such file we have to cause the success download scenario. - bool isDownloadedDiff = false; - if (opt == MapFileType::Diff) - { - string filePath = readyFilePath; - base::GetNameWithoutExt(filePath); - isDownloadedDiff = p.GetFileSizeByFullPath(filePath, size); - } - - // It may happen that the file already was downloaded, so there're - // no need to request servers list and download file. Let's - // switch to next file. - if (isDownloadedDiff || p.GetFileSizeByFullPath(readyFilePath, size)) - { - if (m_latestDiffRequest) - { - // No need to kick the downloader: it will be kicked by - // the current diff application process when it is completed or cancelled. - return; - } - OnMapFileDownloadFinished(DownloadStatus::Completed, Progress(size, size)); - return; - } - - if (!m_queue.empty()) - { - m_downloader->DownloadMapFile(m_queue.front(), - bind(&Storage::OnMapFileDownloadFinished, this, _1, _2), - bind(&Storage::OnMapFileDownloadProgress, this, _1)); - } -} - bool Storage::IsDownloadInProgress() const { CHECK_THREAD_CHECKER(m_threadChecker, ()); - return !m_queue.empty(); + return !m_downloader->GetQueue().empty(); } CountryId Storage::GetCurrentDownloadingCountryId() const { CHECK_THREAD_CHECKER(m_threadChecker, ()); - return IsDownloadInProgress() ? m_queue.front().GetCountryId() : storage::CountryId(); + return IsDownloadInProgress() ? m_downloader->GetQueue().front().GetCountryId() + : storage::CountryId(); } void Storage::LoadCountriesFile(string const & pathToCountriesFile) @@ -709,33 +669,11 @@ void Storage::Unsubscribe(int slotId) } } -void Storage::OnMapFileDownloadFinished(DownloadStatus status, Progress const & progress) +void Storage::OnMapFileDownloadFinished(QueuedCountry const & queuedCountry, DownloadStatus status) { CHECK_THREAD_CHECKER(m_threadChecker, ()); - if (m_queue.empty()) - return; - - bool const success = status == DownloadStatus::Completed; - QueuedCountry & queuedCountry = m_queue.front(); - CountryId const countryId = queuedCountry.GetCountryId(); - - // Send statistics to PushWoosh. We send these statistics only for the newly downloaded - // mwms, not for updated ones. - if (success && queuedCountry.GetFileType() != MapFileType::Diff) - { - auto const it = m_localFiles.find(countryId); - if (it == m_localFiles.end()) - { - GetPlatform().GetMarketingService().SendPushWooshTag(marketing::kMapLastDownloaded, - countryId); - auto const nowStr = GetPlatform().GetMarketingService().GetPushWooshTimestamp(); - GetPlatform().GetMarketingService().SendPushWooshTag(marketing::kMapLastDownloadedTimestamp, - nowStr); - } - } - - OnMapDownloadFinished(countryId, status, queuedCountry.GetFileType()); + OnMapDownloadFinished(queuedCountry.GetCountryId(), status, queuedCountry.GetFileType()); } void Storage::ReportProgress(CountryId const & countryId, Progress const & p) @@ -752,7 +690,7 @@ void Storage::ReportProgressForHierarchy(CountryId const & countryId, Progress c // Reporting progress for the parents of the leaf with |countryId|. CountriesSet setQueue; - GetQueuedCountries(m_queue, setQueue); + GetQueuedCountries(m_downloader->GetQueue(), setQueue); auto calcProgress = [&](CountryId const & parentId, CountryTree::Node const & parentNode) { CountriesVec descendants; @@ -768,18 +706,15 @@ void Storage::ReportProgressForHierarchy(CountryId const & countryId, Progress c ForEachAncestorExceptForTheRoot(countryId, calcProgress); } -void Storage::OnMapFileDownloadProgress(Progress const & progress) +void Storage::OnMapFileDownloadProgress(QueuedCountry const & queuedCountry, + Progress const & progress) { CHECK_THREAD_CHECKER(m_threadChecker, ()); - // Queue can be empty because countries were deleted from queue. - if (m_queue.empty()) - return; - if (m_observers.empty()) return; - ReportProgressForHierarchy(m_queue.front().GetCountryId(), progress); + ReportProgressForHierarchy(queuedCountry.GetCountryId(), progress); } void Storage::RegisterDownloadedFiles(CountryId const & countryId, MapFileType type) @@ -793,6 +728,7 @@ void Storage::RegisterDownloadedFiles(CountryId const & countryId, MapFileType t if (!isSuccess) { + m_justDownloaded.erase(countryId); OnMapDownloadFailed(countryId); return; } @@ -802,14 +738,9 @@ void Storage::RegisterDownloadedFiles(CountryId const & countryId, MapFileType t DeleteCountryIndexes(*localFile); m_didDownload(countryId, localFile); - CHECK(!m_queue.empty(), ()); - PushToJustDownloaded(m_queue.begin()); - PopFromQueue(m_queue.begin()); SaveDownloadQueue(); - m_downloader->Reset(); NotifyStatusChangedForHierarchy(countryId); - DownloadNextCountryFromQueue(); }; if (type == MapFileType::Diff) @@ -856,14 +787,7 @@ void Storage::OnMapDownloadFinished(CountryId const & countryId, DownloadStatus CHECK_THREAD_CHECKER(m_threadChecker, ()); ASSERT(m_didDownload != nullptr, ("Storage::Init wasn't called")); - alohalytics::LogEvent("$OnMapDownloadFinished", - alohalytics::TStringMap( - {{"name", countryId}, - {"status", status == DownloadStatus::Completed ? "ok" : "failed"}, - {"version", strings::to_string(GetCurrentDataVersion())}, - {"option", DebugPrint(type)}})); - GetPlatform().GetMarketingService().SendMarketingEvent(marketing::kDownloaderMapActionFinished, - {{"action", "download"}}); + SendStatisticsAfterDownloading(countryId, status, type, GetCurrentDataVersion(), m_localFiles); if (status != DownloadStatus::Completed) { @@ -877,6 +801,7 @@ void Storage::OnMapDownloadFinished(CountryId const & countryId, DownloadStatus return; } + m_justDownloaded.insert(countryId); RegisterDownloadedFiles(countryId, type); } @@ -911,46 +836,39 @@ void Storage::GetOutdatedCountries(vector & countries) const } } -QueuedCountry * Storage::FindCountryInQueue(CountryId const & countryId) -{ - CHECK_THREAD_CHECKER(m_threadChecker, ()); - - auto it = find(m_queue.begin(), m_queue.end(), countryId); - return it == m_queue.end() ? nullptr : &*it; -} - -QueuedCountry const * Storage::FindCountryInQueue(CountryId const & countryId) const -{ - CHECK_THREAD_CHECKER(m_threadChecker, ()); - - auto it = find(m_queue.begin(), m_queue.end(), countryId); - return it == m_queue.end() ? nullptr : &*it; -} - bool Storage::IsCountryInQueue(CountryId const & countryId) const { - return FindCountryInQueue(countryId) != nullptr; + CHECK_THREAD_CHECKER(m_threadChecker, ()); + + auto const & queue = m_downloader->GetQueue(); + return find(queue.cbegin(), queue.cend(), countryId) != queue.cend(); } bool Storage::IsCountryFirstInQueue(CountryId const & countryId) const { CHECK_THREAD_CHECKER(m_threadChecker, ()); - return !m_queue.empty() && m_queue.front().GetCountryId() == countryId; + auto const & queue = m_downloader->GetQueue(); + return !queue.empty() && queue.front().GetCountryId() == countryId; } bool Storage::IsDiffApplyingInProgressToCountry(CountryId const & countryId) const { CHECK_THREAD_CHECKER(m_threadChecker, ()); - return countryId == m_latestDiffRequest; + return m_diffsBeingApplied.find(countryId) != m_diffsBeingApplied.cend(); } void Storage::SetLocale(string const & locale) { m_countryNameGetter.SetLocale(locale); } string Storage::GetLocale() const { return m_countryNameGetter.GetLocale(); } void Storage::SetDownloaderForTesting(unique_ptr downloader) { + if (m_downloader) + m_downloader->UnsubscribeAll(); + m_downloader = move(downloader); + m_downloader->Subscribe(this); + m_downloader->SetDownloadingPolicy(m_downloadingPolicy); } void Storage::SetEnabledIntegrityValidationForTesting(bool enabled) @@ -1051,27 +969,11 @@ bool Storage::DeleteCountryFilesFromDownloader(CountryId const & countryId) { CHECK_THREAD_CHECKER(m_threadChecker, ()); - QueuedCountry * queuedCountry = FindCountryInQueue(countryId); - if (!queuedCountry) - return false; - if (IsDiffApplyingInProgressToCountry(countryId)) - { - m_diffsCancellable.Cancel(); - m_diffsBeingApplied.erase(countryId); - } + m_diffsBeingApplied[countryId]->Cancel(); - if (IsCountryFirstInQueue(countryId)) - { - m_downloader->Reset(); - - // Remove all files the downloader has created for the country. - DeleteDownloaderFilesForCountry(GetCurrentDataVersion(), m_dataDir, GetCountryFile(countryId)); - } - - auto it = find(m_queue.begin(), m_queue.end(), countryId); - ASSERT(it != m_queue.end(), ()); - PopFromQueue(it); + m_downloader->Remove(countryId); + DeleteDownloaderFilesForCountry(GetCurrentDataVersion(), m_dataDir, GetCountryFile(countryId)); SaveDownloadQueue(); return true; @@ -1226,8 +1128,9 @@ void Storage::DownloadNode(CountryId const & countryId, bool isUpdate /* = false if (descendantNode.ChildrenCount() == 0 && GetNodeStatus(descendantNode).status != NodeStatus::OnDisk) { - DownloadCountry(descendantNode.Value().Name(), - isUpdate ? MapFileType::Diff : MapFileType::Map); + auto const countryId = descendantNode.Value().Name(); + + DownloadCountry(countryId, isUpdate ? MapFileType::Diff : MapFileType::Map); } }; @@ -1310,7 +1213,9 @@ void Storage::ApplyDiff(CountryId const & countryId, function()); + CHECK_EQUAL(emplaceResult.second, true, ()); + NotifyStatusChangedForHierarchy(countryId); diffs::ApplyDiffParams params; @@ -1330,12 +1237,10 @@ void Storage::ApplyDiff(CountryId const & countryId, functionsecond, [this, fn, countryId, diffFile](DiffApplicationResult result) { CHECK_THREAD_CHECKER(m_threadChecker, ()); - static string const kSourceKey = "diff"; if (result == DiffApplicationResult::Ok && m_integrityValidationEnabled && !ValidateIntegrity(diffFile, diffFile->GetCountryName(), kSourceKey)) @@ -1345,12 +1250,11 @@ void Storage::ApplyDiff(CountryId const & countryId, functionIsCancelled() && result == DiffApplicationResult::Ok) result = DiffApplicationResult::Cancelled; LOG(LINFO, ("Diff application result for", countryId, ":", result)); - m_latestDiffRequest = {}; m_diffsBeingApplied.erase(countryId); switch (result) { @@ -1364,8 +1268,6 @@ void Storage::ApplyDiff(CountryId const & countryId, functionIsIdle()) - DownloadNextCountryFromQueue(); break; } case DiffApplicationResult::Failed: @@ -1375,6 +1277,9 @@ void Storage::ApplyDiff(CountryId const & countryId, functionGetQueue().empty()) + OnFinishDownloading(); }); } @@ -1408,6 +1313,38 @@ void Storage::SetStartDownloadingCallback(StartDownloadingCallback const & cb) m_startDownloadingCallback = cb; } +void Storage::OnStartDownloading() +{ + if (m_startDownloadingCallback) + m_startDownloadingCallback(); +} + +void Storage::OnFinishDownloading() +{ + // When downloading is finished but diffs applying in progress + // it will be called from ApplyDiff callback. + if (!m_diffsBeingApplied.empty()) + return; + + m_justDownloaded.clear(); + + m_downloadingPolicy->ScheduleRetry(m_failedCountries, [this](CountriesSet const & needReload) { + for (auto const & country : needReload) + { + NodeStatuses status; + GetNodeStatuses(country, status); + if (status.m_error == NodeErrorCode::NoInetConnection) + RetryDownloadNode(country); + } + }); + CountriesVec localMaps; + GetLocalRealMaps(localMaps); + GetPlatform().GetMarketingService().SendPushWooshTag(marketing::kMapListing, localMaps); + if (!localMaps.empty()) + GetPlatform().GetMarketingService().SendPushWooshTag(marketing::kMapDownloadDiscovered); + return; +} + void Storage::OnDiffStatusReceived(diffs::NameDiffInfoMap && diffs) { m_diffsDataSource->SetDiffInfo(move(diffs)); @@ -1521,7 +1458,7 @@ void Storage::GetNodeAttrs(CountryId const & countryId, NodeAttrs & nodeAttrs) c } CountriesSet setQueue; - GetQueuedCountries(m_queue, setQueue); + GetQueuedCountries(m_downloader->GetQueue(), setQueue); nodeAttrs.m_downloadingProgress = CalculateProgress(downloadingMwm, subtree, downloadingMwmProgress, setQueue); } @@ -1654,7 +1591,7 @@ void Storage::CancelDownloadNode(CountryId const & countryId) LOG(LINFO, ("Cancelling the downloading of", countryId)); CountriesSet setQueue; - GetQueuedCountries(m_queue, setQueue); + GetQueuedCountries(m_downloader->GetQueue(), setQueue); ForEachInSubtree(countryId, [&](CountryId const & descendantId, bool /* groupNode */) { auto needNotify = false; @@ -1671,9 +1608,7 @@ void Storage::CancelDownloadNode(CountryId const & countryId) NotifyStatusChangedForHierarchy(countryId); }); - // Kick possibly interrupted downloader. - if (!m_queue.empty() && m_downloader->IsIdle()) - DownloadNextCountryFromQueue(); + m_downloader->Resume(); } void Storage::RetryDownloadNode(CountryId const & countryId) @@ -1736,19 +1671,6 @@ std::vector Storage::GetTopCountryGeoIds(CountryId const & co return result; } -void Storage::PushToJustDownloaded(Queue::iterator justDownloadedItem) -{ - m_justDownloaded.insert(justDownloadedItem->GetCountryId()); -} - -void Storage::PopFromQueue(Queue::iterator it) -{ - CHECK(!m_queue.empty(), ()); - m_queue.erase(it); - if (m_queue.empty()) - m_justDownloaded.clear(); -} - void Storage::GetQueuedChildren(CountryId const & parent, CountriesVec & queuedChildren) const { CountryTree::Node const * const node = m_countries.FindFirst(parent); @@ -1864,10 +1786,6 @@ MwmSize Storage::GetRemoteSize(CountryFile const & file, int64_t version) const void Storage::OnMapDownloadFailed(CountryId const & countryId) { m_failedCountries.insert(countryId); - auto it = find(m_queue.begin(), m_queue.end(), countryId); - if (it != m_queue.end()) - PopFromQueue(it); NotifyStatusChangedForHierarchy(countryId); - DownloadNextCountryFromQueue(); } } // namespace storage diff --git a/storage/storage.hpp b/storage/storage.hpp index 17c0d6d1a1..d00165d193 100644 --- a/storage/storage.hpp +++ b/storage/storage.hpp @@ -25,6 +25,7 @@ #include #include #include +#include #include #include @@ -146,16 +147,16 @@ struct NodeStatuses // Downloading of only one mwm at a time is supported, so while the // mwm at the top of the queue is being downloaded (or updated by // applying a diff file) all other mwms have to wait. -class Storage +class Storage : public MapFilesDownloader::Subscriber { public: struct StatusCallback; using StartDownloadingCallback = std::function; + using FinishDownloadingCallback = std::function; using UpdateCallback = std::function; using DeleteCallback = std::function; using ChangeCountryFunction = std::function; using ProgressFunction = std::function; - using Queue = std::list; private: /// We support only one simultaneous request at the moment @@ -166,20 +167,8 @@ private: CountryTree m_countries; - /// @todo. It appeared that our application uses m_queue from - /// different threads without any synchronization. To reproduce it - /// just download a map "from the map" on Android. (CountryStatus is - /// called from a different thread.) It's necessary to check if we - /// can call all the methods from a single thread using - /// RunOnUIThread. If not, at least use a syncronization object. - Queue m_queue; - - // Keep downloading queue between application restarts. - bool m_keepDownloadingQueue = true; - /// Set of mwm files which have been downloaded recently. - /// When a mwm file is downloaded it's moved from |m_queue| to |m_justDownloaded|. - /// When a new mwm file is added to |m_queue|, |m_justDownloaded| is cleared. + /// When a mwm file is downloaded it's added to |m_justDownloaded|. /// Note. This set is necessary for implementation of downloading progress of /// mwm group. CountriesSet m_justDownloaded; @@ -193,20 +182,12 @@ private: // folder. std::map m_localFilesForFakeCountries; - // Used to cancel an ongoing diff application. - // |m_diffsCancellable| is reset every time when a task to apply a diff is posted. - // We use the fact that at most one diff is being applied at a time and the - // calls to the diffs::ApplyDiff are coordinated from the storage thread. - base::Cancellable m_diffsCancellable; - - std::optional m_latestDiffRequest; - // Since the diffs applying runs on a different thread, the result // of diff application may return "Ok" when in fact the diff was - // cancelled. However, the storage thread knows for sure whether the - // latest request was to apply or to cancel the diff, and this knowledge + // cancelled. However, the storage thread knows for sure whether + // request was to apply or to cancel the diff, and this knowledge // is represented by |m_diffsBeingApplied|. - std::set m_diffsBeingApplied; + std::unordered_map> m_diffsBeingApplied; std::vector m_notAppliedDiffs; @@ -270,8 +251,6 @@ private: StartDownloadingCallback m_startDownloadingCallback; - void DownloadNextCountryFromQueue(); - void LoadCountriesFile(std::string const & pathToCountriesFile); void ReportProgress(CountryId const & countryId, downloader::Progress const & p); @@ -280,21 +259,19 @@ private: /// Called on the main thread by MapFilesDownloader when /// downloading of a map file succeeds/fails. - void OnMapFileDownloadFinished(downloader::DownloadStatus status, - downloader::Progress const & progress); + void OnMapFileDownloadFinished(QueuedCountry const & queuedCountry, + downloader::DownloadStatus status); /// Periodically called on the main thread by MapFilesDownloader /// during the downloading process. - void OnMapFileDownloadProgress(downloader::Progress const & progress); + void OnMapFileDownloadProgress(QueuedCountry const & queuedCountry, + downloader::Progress const & progress); void RegisterDownloadedFiles(CountryId const & countryId, MapFileType type); void OnMapDownloadFinished(CountryId const & countryId, downloader::DownloadStatus status, MapFileType type); - /// Initiates downloading of the next file from the queue. - void DownloadNextFile(QueuedCountry const & country); - public: ThreadChecker const & GetThreadChecker() const {return m_threadChecker;} @@ -317,7 +294,7 @@ public: void Init(UpdateCallback const & didDownload, DeleteCallback const & willDelete); - inline void SetDownloadingPolicy(DownloadingPolicy * policy) { m_downloadingPolicy = policy; } + void SetDownloadingPolicy(DownloadingPolicy * policy); /// @name Interface with clients (Android/iOS). /// \brief It represents the interface which can be used by clients (Android/iOS). @@ -556,7 +533,6 @@ public: bool IsDownloadInProgress() const; CountryId GetCurrentDownloadingCountryId() const; - void EnableKeepDownloadingQueue(bool enable) {m_keepDownloadingQueue = enable;} /// @param[out] res Populated with oudated countries. void GetOutdatedCountries(std::vector & countries) const; @@ -580,18 +556,15 @@ public: void SetStartDownloadingCallback(StartDownloadingCallback const & cb); + void OnStartDownloading() override; + void OnFinishDownloading() override; + private: friend struct UnitClass_StorageTest_DeleteCountry; void SaveDownloadQueue(); void RestoreDownloadQueue(); - // Returns a pointer to a country in the downloader's queue. - QueuedCountry * FindCountryInQueue(CountryId const & countryId); - - // Returns a pointer to a country in the downloader's queue. - QueuedCountry const * FindCountryInQueue(CountryId const & countryId) const; - // Returns true when country is in the downloader's queue. bool IsCountryInQueue(CountryId const & countryId) const; @@ -630,8 +603,6 @@ private: // downloaded files. std::string GetFileDownloadPath(CountryId const & countryId, MapFileType file) const; - void CountryStatusEx(CountryId const & countryId, Status & status, MapFileType & type) const; - /// Fast version, doesn't check if country is out of date Status CountryStatus(CountryId const & countryId) const; @@ -664,9 +635,6 @@ private: downloader::Progress const & downloadingMwmProgress, CountriesSet const & mwmsInQueue) const; - void PushToJustDownloaded(Queue::iterator justDownloadedItem); - void PopFromQueue(Queue::iterator it); - template void ForEachAncestorExceptForTheRoot(std::vector const & nodes, ToDo && toDo) const; @@ -686,7 +654,7 @@ private: void OnDiffStatusReceived(diffs::NameDiffInfoMap && diffs); }; -void GetQueuedCountries(Storage::Queue const & queue, CountriesSet & resultCountries); +void GetQueuedCountries(Queue const & queue, CountriesSet & resultCountries); template void Storage::ForEachInSubtree(CountryId const & root, ToDo && toDo) const diff --git a/storage/storage_integration_tests/storage_downloading_tests.cpp b/storage/storage_integration_tests/storage_downloading_tests.cpp index 0245200796..df639dadd0 100644 --- a/storage/storage_integration_tests/storage_downloading_tests.cpp +++ b/storage/storage_integration_tests/storage_downloading_tests.cpp @@ -90,7 +90,7 @@ UNIT_CLASS_TEST(Runner, SmallMwms_InterruptDownloadResumeDownload_Test) { Storage storage; - auto onProgressFn = [](CountryId const & countryId, LocalAndRemoteSize const & mapSize) { + auto const onProgressFn = [](CountryId const & countryId, LocalAndRemoteSize const & mapSize) { TEST_EQUAL(countryId, kCountryId, ()); // Interrupt download testing::StopEventLoop(); @@ -104,27 +104,52 @@ UNIT_CLASS_TEST(Runner, SmallMwms_InterruptDownloadResumeDownload_Test) testing::RunEventLoop(); TEST(storage.IsDownloadInProgress(), ()); + + NodeAttrs attrs; + storage.GetNodeAttrs(kCountryId, attrs); + TEST_EQUAL(NodeStatus::Downloading, attrs.m_status, ()); } // Continue download { Storage storage; - auto onProgressFn = [](CountryId const & countryId, LocalAndRemoteSize const & mapSize) { - TEST_EQUAL(countryId, kCountryId, ()); - }; + bool onProgressIsCalled = false; + NodeAttrs onProgressAttrs; + auto const onProgressFn = + [&](CountryId const & countryId, LocalAndRemoteSize const & mapSize) + { + TEST_EQUAL(countryId, kCountryId, ()); + + if (onProgressIsCalled) + return; + + onProgressIsCalled = true; + storage.GetNodeAttrs(kCountryId, onProgressAttrs); + testing::StopEventLoop(); + }; InitStorage(storage, onProgressFn); + storage.Init([](CountryId const &, storage::LocalFilePtr const localCountryFile) + { + TEST_EQUAL(localCountryFile->GetCountryName(), kCountryId, ()); + + testing::StopEventLoop(); + }, + [](CountryId const &, storage::LocalFilePtr const) + { + return false; + }); + + testing::RunEventLoop(); TEST(storage.IsDownloadInProgress(), ()); - NodeAttrs attrs; - storage.GetNodeAttrs(kCountryId, attrs); - TEST_EQUAL(NodeStatus::Downloading, attrs.m_status, ()); - - storage.DownloadNode(kCountryId); testing::RunEventLoop(); + TEST_EQUAL(NodeStatus::Downloading, onProgressAttrs.m_status, ()); + + NodeAttrs attrs; storage.GetNodeAttrs(kCountryId, attrs); TEST_EQUAL(NodeStatus::OnDisk, attrs.m_status, ()); } diff --git a/storage/storage_tests/fake_map_files_downloader.cpp b/storage/storage_tests/fake_map_files_downloader.cpp index d1379ddd76..e3175c40be 100644 --- a/storage/storage_tests/fake_map_files_downloader.cpp +++ b/storage/storage_tests/fake_map_files_downloader.cpp @@ -14,52 +14,117 @@ namespace storage int64_t const FakeMapFilesDownloader::kBlockSize; FakeMapFilesDownloader::FakeMapFilesDownloader(TaskRunner & taskRunner) - : m_progress(std::make_pair(0, 0)), m_idle(true), m_timestamp(0), m_taskRunner(taskRunner) + : m_progress(std::make_pair(0, 0)), m_timestamp(0), m_taskRunner(taskRunner) { SetServersList({"http://test-url/"}); } FakeMapFilesDownloader::~FakeMapFilesDownloader() { CHECK(m_checker.CalledOnOriginalThread(), ()); } -void FakeMapFilesDownloader::Download(QueuedCountry & queuedCountry, - FileDownloadedCallback const & onDownloaded, - DownloadingProgressCallback const & onProgress) +void FakeMapFilesDownloader::Download(QueuedCountry & queuedCountry) { -// Will be refactored in the followed commits. -// CHECK(m_checker.CalledOnOriginalThread(), ()); -// -// m_progress.first = 0; -// m_progress.second = size; -// m_idle = false; -// -// m_writer.reset(new FileWriter(path)); -// m_onDownloaded = onDownloaded; -// m_onProgress = onProgress; -// -// ++m_timestamp; -// m_taskRunner.PostTask(std::bind(&FakeMapFilesDownloader::DownloadNextChunk, this, m_timestamp)); + CHECK(m_checker.CalledOnOriginalThread(), ()); + + m_queue.push_back(queuedCountry); + + if (m_queue.size() != 1) + return; + + Download(); } downloader::Progress FakeMapFilesDownloader::GetDownloadingProgress() { CHECK(m_checker.CalledOnOriginalThread(), ()); + return m_progress; } bool FakeMapFilesDownloader::IsIdle() { CHECK(m_checker.CalledOnOriginalThread(), ()); - return m_idle; + + return m_writer.get() == nullptr; } -void FakeMapFilesDownloader::Reset() +void FakeMapFilesDownloader::Pause() { CHECK(m_checker.CalledOnOriginalThread(), ()); - m_idle = true; + m_writer.reset(); ++m_timestamp; } +void FakeMapFilesDownloader::Resume() +{ + CHECK(m_checker.CalledOnOriginalThread(), ()); + + if (m_queue.empty() || m_writer) + return; + + m_writer.reset(new FileWriter(m_queue.front().GetFileDownloadPath())); + ++m_timestamp; + m_taskRunner.PostTask(std::bind(&FakeMapFilesDownloader::DownloadNextChunk, this, m_timestamp)); +} + +void FakeMapFilesDownloader::Remove(CountryId const & id) +{ + CHECK(m_checker.CalledOnOriginalThread(), ()); + + if (m_queue.empty()) + return; + + if (m_writer && m_queue.front() == id) + { + m_writer.reset(); + m_queue.pop_front(); + } + else + { + auto it = std::find(m_queue.begin(), m_queue.end(), id); + if (it != m_queue.end()) + m_queue.erase(it); + } + + if (m_queue.empty()) + m_writer.reset(); + + ++m_timestamp; +} + +void FakeMapFilesDownloader::Clear() +{ + CHECK(m_checker.CalledOnOriginalThread(), ()); + + m_queue.clear(); + m_writer.reset(); + ++m_timestamp; +} + +Queue const & FakeMapFilesDownloader::GetQueue() const +{ + CHECK(m_checker.CalledOnOriginalThread(), ()); + + return m_queue; +} + +void FakeMapFilesDownloader::Download() +{ + auto const & queuedCountry = m_queue.front(); + if (!IsDownloadingAllowed()) + { + OnFileDownloaded(queuedCountry, downloader::DownloadStatus::Failed); + return; + } + + m_writer.reset(new FileWriter(queuedCountry.GetFileDownloadPath())); + + ++m_timestamp; + m_progress = {0, queuedCountry.GetDownloadSize()}; + m_writer.reset(new FileWriter(queuedCountry.GetFileDownloadPath())); + m_taskRunner.PostTask(std::bind(&FakeMapFilesDownloader::DownloadNextChunk, this, m_timestamp)); +} + void FakeMapFilesDownloader::DownloadNextChunk(uint64_t timestamp) { CHECK(m_checker.CalledOnOriginalThread(), ()); @@ -74,9 +139,7 @@ void FakeMapFilesDownloader::DownloadNextChunk(uint64_t timestamp) if (m_progress.first == m_progress.second) { - m_taskRunner.PostTask( - std::bind(m_onDownloaded, downloader::DownloadStatus::Completed, m_progress)); - Reset(); + OnFileDownloaded(m_queue.front(), downloader::DownloadStatus::Completed); return; } @@ -86,7 +149,27 @@ void FakeMapFilesDownloader::DownloadNextChunk(uint64_t timestamp) m_writer->Write(kZeroes.data(), bs); m_writer->Flush(); - m_taskRunner.PostTask(std::bind(m_onProgress, m_progress)); + m_taskRunner.PostTask([this, timestamp]() + { + CHECK(m_checker.CalledOnOriginalThread(), ()); + + if (timestamp != m_timestamp) + return; + + m_queue.front().OnDownloadProgress(m_progress); + }); m_taskRunner.PostTask(std::bind(&FakeMapFilesDownloader::DownloadNextChunk, this, timestamp)); } + +void FakeMapFilesDownloader::OnFileDownloaded(QueuedCountry const & queuedCountry, + downloader::DownloadStatus const & status) +{ + auto const country = queuedCountry; + m_queue.pop_front(); + + m_taskRunner.PostTask([country, status]() { country.OnDownloadFinished(status); }); + + if (!m_queue.empty()) + Download(); +} } // namespace storage diff --git a/storage/storage_tests/fake_map_files_downloader.hpp b/storage/storage_tests/fake_map_files_downloader.hpp index 07ee0f53a4..e99eac93c8 100644 --- a/storage/storage_tests/fake_map_files_downloader.hpp +++ b/storage/storage_tests/fake_map_files_downloader.hpp @@ -33,28 +33,32 @@ public: ~FakeMapFilesDownloader(); + // MapFilesDownloader overrides: downloader::Progress GetDownloadingProgress() override; bool IsIdle() override; - void Reset() override; + void Pause() override; + void Resume() override; + void Remove(CountryId const & id) override; + void Clear() override; + Queue const & GetQueue() const override; private: // MapFilesDownloader overrides: - void Download(QueuedCountry & queuedCountry, - FileDownloadedCallback const & onDownloaded, - DownloadingProgressCallback const & onProgress) override; + void Download(QueuedCountry & queuedCountry) override; + void Download(); void DownloadNextChunk(uint64_t requestId); + void OnFileDownloaded(QueuedCountry const & queuedCountry, + downloader::DownloadStatus const & status); downloader::Progress m_progress; - bool m_idle; std::unique_ptr m_writer; - FileDownloadedCallback m_onDownloaded; - DownloadingProgressCallback m_onProgress; uint64_t m_timestamp; TaskRunner & m_taskRunner; ThreadChecker m_checker; + Queue m_queue; }; } // namespace storage diff --git a/storage/storage_tests/storage_tests.cpp b/storage/storage_tests/storage_tests.cpp index df57e7244c..4d8929f7dc 100644 --- a/storage/storage_tests/storage_tests.cpp +++ b/storage/storage_tests/storage_tests.cpp @@ -29,8 +29,6 @@ #include "platform/platform_tests_support/scoped_file.hpp" #include "platform/platform_tests_support/writable_dir_changer.hpp" -#include "platform/platform_tests_support/scoped_dir.hpp" - #include "geometry/mercator.hpp" #include "coding/file_writer.hpp" @@ -272,7 +270,8 @@ protected: TEST_LESS(m_currStatus + 1, m_transitionList.size(), (m_countryFile)); TEST_EQUAL(nexStatus, m_transitionList[m_currStatus + 1], (m_countryFile)); ++m_currStatus; - if (m_transitionList[m_currStatus] == Status::EDownloading) + if (m_transitionList[m_currStatus] == Status::EDownloading || + m_transitionList[m_currStatus] == Status::EInQueue) { LocalAndRemoteSize localAndRemoteSize = m_storage.CountrySizeInBytes(m_countryId); m_totalBytesToDownload = localAndRemoteSize.second; @@ -775,8 +774,7 @@ UNIT_CLASS_TEST(StorageTest, DownloadedMap) auto algeriaCoastChecker = make_unique( storage, algeriaCoastCountryId, MapFileType::Map, - vector{Status::ENotDownloaded, Status::EInQueue, Status::EDownloading, - Status::EOnDisk}); + vector{Status::ENotDownloaded, Status::EInQueue, Status::EOnDisk}); algeriaCentralChecker->StartDownload(); algeriaCoastChecker->StartDownload(); diff --git a/storage/storage_tests/test_map_files_downloader.cpp b/storage/storage_tests/test_map_files_downloader.cpp index de537b62a4..1d5bdf2d4b 100644 --- a/storage/storage_tests/test_map_files_downloader.cpp +++ b/storage/storage_tests/test_map_files_downloader.cpp @@ -2,8 +2,8 @@ namespace storage { -void TestMapFilesDownloader::GetServersList(ServersListCallback const & callback) +TestMapFilesDownloader::TestMapFilesDownloader() : HttpMapFilesDownloader() { - callback({"http://localhost:34568/unit_tests/"}); + SetServersList({"http://localhost:34568/unit_tests/"}); } } // namespace storage diff --git a/storage/storage_tests/test_map_files_downloader.hpp b/storage/storage_tests/test_map_files_downloader.hpp index 1322e7987c..55ae068205 100644 --- a/storage/storage_tests/test_map_files_downloader.hpp +++ b/storage/storage_tests/test_map_files_downloader.hpp @@ -6,8 +6,7 @@ namespace storage { class TestMapFilesDownloader : public HttpMapFilesDownloader { -private: - // MapFilesDownloader overrides: - void GetServersList(ServersListCallback const & callback) override; +public: + TestMapFilesDownloader(); }; } // namespace storage diff --git a/xcode/storage/storage.xcodeproj/project.pbxproj b/xcode/storage/storage.xcodeproj/project.pbxproj index e1cdf6b083..9cc29c4096 100644 --- a/xcode/storage/storage.xcodeproj/project.pbxproj +++ b/xcode/storage/storage.xcodeproj/project.pbxproj @@ -30,13 +30,15 @@ 3DCD414920D80C1B00143533 /* lightweight_matching_tests.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 3DCD414820D80C1B00143533 /* lightweight_matching_tests.cpp */; }; 3DD84014233A38DE0008D0ED /* diff_scheme_loader.hpp in Headers */ = {isa = PBXBuildFile; fileRef = 3DD84012233A38DE0008D0ED /* diff_scheme_loader.hpp */; }; 3DD84015233A38DE0008D0ED /* diff_scheme_loader.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 3DD84013233A38DE0008D0ED /* diff_scheme_loader.cpp */; }; + 3DF528E12386B966000ED0D5 /* apply_diff.hpp in Headers */ = {isa = PBXBuildFile; fileRef = 3DF528DD2386B966000ED0D5 /* apply_diff.hpp */; }; + 3DF528E22386B966000ED0D5 /* diffs_data_source.hpp in Headers */ = {isa = PBXBuildFile; fileRef = 3DF528DE2386B966000ED0D5 /* diffs_data_source.hpp */; }; + 3DF528E32386B966000ED0D5 /* diffs_data_source.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 3DF528DF2386B966000ED0D5 /* diffs_data_source.cpp */; }; + 3DF528E42386B966000ED0D5 /* apply_diff.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 3DF528E02386B966000ED0D5 /* apply_diff.cpp */; }; 401ECED41F56C50900DFDF76 /* country_parent_getter.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 401ECED21F56C50900DFDF76 /* country_parent_getter.cpp */; }; 401ECED51F56C50900DFDF76 /* country_parent_getter.hpp in Headers */ = {isa = PBXBuildFile; fileRef = 401ECED31F56C50900DFDF76 /* country_parent_getter.hpp */; }; 402873422295A91F0036AA1C /* country_tree.hpp in Headers */ = {isa = PBXBuildFile; fileRef = 4028733F2295A91E0036AA1C /* country_tree.hpp */; }; 402873432295A91F0036AA1C /* country_tree.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 402873402295A91F0036AA1C /* country_tree.cpp */; }; 402873442295A91F0036AA1C /* downloader_search_params.hpp in Headers */ = {isa = PBXBuildFile; fileRef = 402873412295A91F0036AA1C /* downloader_search_params.hpp */; }; - 4586D0C71F48126A00DF9CE5 /* diff_manager.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 4586D0C51F48126A00DF9CE5 /* diff_manager.cpp */; }; - 4586D0C81F48126A00DF9CE5 /* diff_manager.hpp in Headers */ = {isa = PBXBuildFile; fileRef = 4586D0C61F48126A00DF9CE5 /* diff_manager.hpp */; }; 5670EAFA1FF1150C002495D8 /* libtransit.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 5670EAFB1FF1150C002495D8 /* libtransit.a */; }; 5670EAFC1FF11535002495D8 /* libmwm_diff.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 5670EAFD1FF11535002495D8 /* libmwm_diff.a */; }; 5670EAFE1FF11558002495D8 /* libbsdiff.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 5670EAFF1FF11558002495D8 /* libbsdiff.a */; }; @@ -216,14 +218,16 @@ 3DCD414820D80C1B00143533 /* lightweight_matching_tests.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = lightweight_matching_tests.cpp; sourceTree = ""; }; 3DD84012233A38DE0008D0ED /* diff_scheme_loader.hpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; name = diff_scheme_loader.hpp; path = diff_scheme/diff_scheme_loader.hpp; sourceTree = ""; }; 3DD84013233A38DE0008D0ED /* diff_scheme_loader.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = diff_scheme_loader.cpp; path = diff_scheme/diff_scheme_loader.cpp; sourceTree = ""; }; + 3DF528DD2386B966000ED0D5 /* apply_diff.hpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; name = apply_diff.hpp; path = diff_scheme/apply_diff.hpp; sourceTree = ""; }; + 3DF528DE2386B966000ED0D5 /* diffs_data_source.hpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; name = diffs_data_source.hpp; path = diff_scheme/diffs_data_source.hpp; sourceTree = ""; }; + 3DF528DF2386B966000ED0D5 /* diffs_data_source.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = diffs_data_source.cpp; path = diff_scheme/diffs_data_source.cpp; sourceTree = ""; }; + 3DF528E02386B966000ED0D5 /* apply_diff.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = apply_diff.cpp; path = diff_scheme/apply_diff.cpp; sourceTree = ""; }; 401ECED21F56C50900DFDF76 /* country_parent_getter.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = country_parent_getter.cpp; sourceTree = ""; }; 401ECED31F56C50900DFDF76 /* country_parent_getter.hpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; path = country_parent_getter.hpp; sourceTree = ""; }; 4028733F2295A91E0036AA1C /* country_tree.hpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; path = country_tree.hpp; sourceTree = ""; }; 402873402295A91F0036AA1C /* country_tree.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = country_tree.cpp; sourceTree = ""; }; 402873412295A91F0036AA1C /* downloader_search_params.hpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; path = downloader_search_params.hpp; sourceTree = ""; }; 4069AD9F21495A5A005EB75C /* categories_cuisines.txt */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text; path = categories_cuisines.txt; sourceTree = ""; }; - 4586D0C51F48126A00DF9CE5 /* diff_manager.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = diff_manager.cpp; path = diff_scheme/diff_manager.cpp; sourceTree = ""; }; - 4586D0C61F48126A00DF9CE5 /* diff_manager.hpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; name = diff_manager.hpp; path = diff_scheme/diff_manager.hpp; sourceTree = ""; }; 5670EAFB1FF1150C002495D8 /* libtransit.a */ = {isa = PBXFileReference; explicitFileType = archive.ar; path = libtransit.a; sourceTree = BUILT_PRODUCTS_DIR; }; 5670EAFD1FF11535002495D8 /* libmwm_diff.a */ = {isa = PBXFileReference; explicitFileType = archive.ar; path = libmwm_diff.a; sourceTree = BUILT_PRODUCTS_DIR; }; 5670EAFF1FF11558002495D8 /* libbsdiff.a */ = {isa = PBXFileReference; explicitFileType = archive.ar; path = libbsdiff.a; sourceTree = BUILT_PRODUCTS_DIR; }; @@ -583,6 +587,10 @@ children = ( 56DAC38323992819000BC50D /* queued_country.cpp */, 56DAC38423992819000BC50D /* queued_country.hpp */, + 3DF528E02386B966000ED0D5 /* apply_diff.cpp */, + 3DF528DD2386B966000ED0D5 /* apply_diff.hpp */, + 3DF528DF2386B966000ED0D5 /* diffs_data_source.cpp */, + 3DF528DE2386B966000ED0D5 /* diffs_data_source.hpp */, 3DD84013233A38DE0008D0ED /* diff_scheme_loader.cpp */, 3DD84012233A38DE0008D0ED /* diff_scheme_loader.hpp */, 3D497EA023292706000FB57D /* map_files_downloader_with_ping.cpp */, @@ -599,8 +607,6 @@ 56D0E47E1F8E40340084B18C /* routing_helpers.hpp */, 401ECED21F56C50900DFDF76 /* country_parent_getter.cpp */, 401ECED31F56C50900DFDF76 /* country_parent_getter.hpp */, - 4586D0C51F48126A00DF9CE5 /* diff_manager.cpp */, - 4586D0C61F48126A00DF9CE5 /* diff_manager.hpp */, F68CC5BD1F38B967007527C7 /* diff_types.hpp */, 67BE1DC31CD2180D00572709 /* downloading_policy.cpp */, 67BE1DC41CD2180D00572709 /* downloading_policy.hpp */, @@ -665,10 +671,12 @@ 56D8CB9A1CAC17A80003F420 /* test_defines.hpp in Headers */, F68CC5C01F38B967007527C7 /* diff_types.hpp in Headers */, 401ECED51F56C50900DFDF76 /* country_parent_getter.hpp in Headers */, + 3DF528E22386B966000ED0D5 /* diffs_data_source.hpp in Headers */, 34C9BCFD1C6CCF85000DC38D /* country_name_getter.hpp in Headers */, 6741251F1B4C05FA00A3E828 /* http_map_files_downloader.hpp in Headers */, 67BE1DC61CD2180D00572709 /* downloading_policy.hpp in Headers */, 675343191A3F5A2600A0A8C3 /* storage_defines.hpp in Headers */, + 3DF528E12386B966000ED0D5 /* apply_diff.hpp in Headers */, 67247FD41C60BA8A00EDE56A /* task_runner.hpp in Headers */, 56D0E4801F8E40340084B18C /* routing_helpers.hpp in Headers */, 56DAC38623992819000BC50D /* queued_country.hpp in Headers */, @@ -684,7 +692,6 @@ 402873442295A91F0036AA1C /* downloader_search_params.hpp in Headers */, 6753431B1A3F5A2600A0A8C3 /* storage.hpp in Headers */, 67247FD61C60BA8A00EDE56A /* test_map_files_downloader.hpp in Headers */, - 4586D0C81F48126A00DF9CE5 /* diff_manager.hpp in Headers */, 34B093231C61F9BA0066F4C3 /* storage_helpers.hpp in Headers */, 402873422295A91F0036AA1C /* country_tree.hpp in Headers */, ); @@ -847,12 +854,12 @@ 56D8CB991CAC17A80003F420 /* test_defines.cpp in Sources */, 3D497EA323292706000FB57D /* map_files_downloader_with_ping.cpp in Sources */, 3DD84015233A38DE0008D0ED /* diff_scheme_loader.cpp in Sources */, + 3DF528E42386B966000ED0D5 /* apply_diff.cpp in Sources */, 401ECED41F56C50900DFDF76 /* country_parent_getter.cpp in Sources */, 56D0E4811F8E40340084B18C /* routing_helpers.cpp in Sources */, 3D497EA123292706000FB57D /* map_files_downloader.cpp in Sources */, 67AF4A001BC579DD0048B1ED /* country_info_getter.cpp in Sources */, 34B093221C61F9BA0066F4C3 /* storage_helpers.cpp in Sources */, - 4586D0C71F48126A00DF9CE5 /* diff_manager.cpp in Sources */, 675343091A3F5A2600A0A8C3 /* country_decl.cpp in Sources */, 3DCD414720D80C0900143533 /* country_info_reader_light.cpp in Sources */, 56DAC38523992819000BC50D /* queued_country.cpp in Sources */, @@ -864,6 +871,7 @@ F6BC312C2034366100F677FE /* pinger.cpp in Sources */, 34C9BCFC1C6CCF85000DC38D /* country_name_getter.cpp in Sources */, 6741251E1B4C05FA00A3E828 /* http_map_files_downloader.cpp in Sources */, + 3DF528E32386B966000ED0D5 /* diffs_data_source.cpp in Sources */, ); runOnlyForDeploymentPostprocessing = 0; };