map-per/organicmaps
1
Fork 0
forked from organicmaps/organicmaps
Code Pull requests Activity

[storage] got rid of deferred downloading

Browse source
This commit is contained in:
Arsentiy Milchakov 2019-11-22 10:41:46 +03:00
parent 4478c24af3
commit 06249cca57
13 changed files with 73 additions and 111 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
search/search_integration_tests/downloader_search_test.cpp
View file

@ -11,6 +11,7 @@
#include "storage/downloader_search_params.hpp"
#include "storage/map_files_downloader.hpp"
#include "storage/queued_country.hpp"
#include "storage/storage.hpp"
#include "geometry/rect2d.hpp"
@ -92,7 +93,7 @@ public:
private:
void GetServersList(ServersListCallback const & /* callback */) override {}
void Download(vector<string> const & /* urls */, string const & /* path */, int64_t /* size */,
void Download(storage::QueuedCountry & queuedCountry,
FileDownloadedCallback const & /* onDownloaded */,
DownloadingProgressCallback const & /* onProgress */) override
{

1
storage/diff_scheme/diff_types.hpp
View file

@ -13,7 +13,6 @@ namespace diffs
// Status of the diffs data source as a whole.
enum class Status
{
Undefined,
NotAvailable,
Available
};

2
storage/diff_scheme/diffs_data_source.hpp
View file

@ -58,7 +58,7 @@ private:
ThreadChecker m_threadChecker;
Status m_status = Status::Undefined;
Status m_status = Status::NotAvailable;
NameDiffInfoMap m_diffs;
};
} // namespace diffs

8
storage/http_map_files_downloader.cpp
View file

@ -34,12 +34,16 @@ HttpMapFilesDownloader::~HttpMapFilesDownloader()
CHECK_THREAD_CHECKER(m_checker, ());
}
void HttpMapFilesDownloader::Download(std::vector<std::string> const & urls,
std::string const & path, int64_t size,
void HttpMapFilesDownloader::Download(QueuedCountry & queuedCountry,
FileDownloadedCallback const & onDownloaded,
DownloadingProgressCallback const & onProgress)
{
CHECK_THREAD_CHECKER(m_checker, ());
auto const urls = MakeUrlList(queuedCountry.GetRelativeUrl());
auto const path = queuedCountry.GetFileDownloadPath();
auto const size = queuedCountry.GetDownloadSize();
m_request.reset(downloader::HttpRequest::GetFile(
urls, path, size,
std::bind(&HttpMapFilesDownloader::OnMapFileDownloaded, this, onDownloaded, _1),

3
storage/http_map_files_downloader.hpp
View file

@ -29,8 +29,7 @@ public:
private:
// MapFilesDownloaderWithServerList overrides:
void Download(std::vector<std::string> const & urls, std::string const & path, int64_t size,
FileDownloadedCallback const & onDownloaded,
void Download(QueuedCountry & queuedCountry, FileDownloadedCallback const & onDownloaded,
DownloadingProgressCallback const & onProgress) override;
void OnMapFileDownloaded(FileDownloadedCallback const & onDownloaded,

51
storage/map_files_downloader.cpp
View file

@ -16,40 +16,23 @@
namespace storage
{
namespace
{
std::vector<std::string> MakeUrlList(MapFilesDownloader::ServersList const & servers,
std::string const & relativeUrl)
{
std::vector<std::string> urls;
urls.reserve(servers.size());
for (auto const & server : servers)
urls.emplace_back(base::url::Join(server, relativeUrl));
return urls;
}
} // namespace
void MapFilesDownloader::DownloadMapFile(QueuedCountry & country,
void MapFilesDownloader::DownloadMapFile(QueuedCountry & queuedCountry,
FileDownloadedCallback const & onDownloaded,
DownloadingProgressCallback const & onProgress)
{
if (m_serversList.empty())
if (!m_serversList.empty())
{
GetServersList([=](ServersList const & serversList)
{
m_serversList = serversList;
auto const urls = MakeUrlList(m_serversList, country.GetRelativeUrl());
Download(urls, country.GetFileDownloadPath(), country.GetDownloadSize(),
onDownloaded, onProgress);
});
}
else
{
auto const urls = MakeUrlList(m_serversList, country.GetRelativeUrl());
Download(urls, country.GetFileDownloadPath(), country.GetDownloadSize(), onDownloaded,
onProgress);
queuedCountry.ClarifyDownloadingType();
Download(queuedCountry, onDownloaded, onProgress);
return;
}
GetServersList([=](ServersList const & serversList) mutable
{
m_serversList = serversList;
queuedCountry.ClarifyDownloadingType();
Download(queuedCountry, onDownloaded, onProgress);
});
}
// static
@ -65,6 +48,16 @@ void MapFilesDownloader::SetServersList(ServersList const & serversList)
m_serversList = serversList;
}
std::vector<std::string> MapFilesDownloader::MakeUrlList(std::string const & relativeUrl)
{
std::vector<std::string> urls;
urls.reserve(m_serversList.size());
for (auto const & server : m_serversList)
urls.emplace_back(base::url::Join(server, relativeUrl));
return urls;
}
// static
MapFilesDownloader::ServersList MapFilesDownloader::LoadServersList()
{

5
storage/map_files_downloader.hpp
View file

@ -51,6 +51,8 @@ public:
void SetServersList(ServersList const & serversList);
protected:
std::vector<std::string> MakeUrlList(std::string const & relativeUrl);
// Synchronously loads list of servers by http client.
static ServersList LoadServersList();
@ -59,8 +61,7 @@ private:
/// for a map file and invokes callback on the main thread.
virtual void GetServersList(ServersListCallback const & callback);
/// Asynchronously downloads the file from provided |urls| and saves result to |path|.
virtual void Download(std::vector<std::string> const & urls, std::string const & path,
int64_t size, FileDownloadedCallback const & onDownloaded,
virtual void Download(QueuedCountry & queuedCountry, FileDownloadedCallback const & onDownloaded,
DownloadingProgressCallback const & onProgress) = 0;
ServersList m_serversList;

14
storage/queued_country.cpp
View file

@ -83,6 +83,20 @@ uint64_t QueuedCountry::GetDownloadSize() const
return GetRemoteSize(*m_diffsDataSource, m_countryFile, m_currentDataVersion);
}
void QueuedCountry::ClarifyDownloadingType()
{
if (m_fileType != MapFileType::Diff)
return;
using diffs::Status;
auto const status = m_diffsDataSource->GetStatus();
if (status == Status::NotAvailable ||
(status == Status::Available && !m_diffsDataSource->HasDiffFor(m_countryId)))
{
m_fileType = MapFileType::Map;
}
}
bool QueuedCountry::operator==(CountryId const & countryId) const
{
return m_countryId == countryId;

2
storage/queued_country.hpp
View file

@ -26,6 +26,8 @@ public:
std::string GetFileDownloadPath() const;
uint64_t GetDownloadSize() const;
void ClarifyDownloadingType();
bool operator==(CountryId const & countryId) const;
private:

60
storage/storage.cpp
View file

@ -646,7 +646,12 @@ void Storage::DownloadNextFile(QueuedCountry const & country)
return;
}
DoDownload();
if (!m_queue.empty())
{
m_downloader->DownloadMapFile(m_queue.front(),
bind(&Storage::OnMapFileDownloadFinished, this, _1, _2),
bind(&Storage::OnMapFileDownloadProgress, this, _1));
}
}
bool Storage::IsDownloadInProgress() const
@ -766,57 +771,6 @@ void Storage::ReportProgressForHierarchy(CountryId const & countryId,
ForEachAncestorExceptForTheRoot(countryId, calcProgress);
}
void Storage::DoDownload()
{
CHECK_THREAD_CHECKER(m_threadChecker, ());
// Queue can be empty because countries were deleted from queue.
if (m_queue.empty())
return;
QueuedCountry & queuedCountry = m_queue.front();
if (queuedCountry.GetFileType() == MapFileType::Diff)
{
using diffs::Status;
auto const status = m_diffsDataSource->GetStatus();
switch (status)
{
case Status::Undefined:
SetDeferDownloading();
return;
case Status::NotAvailable:
queuedCountry.SetFileType(MapFileType::Map);
break;
case Status::Available:
if (!m_diffsDataSource->HasDiffFor(queuedCountry.GetCountryId()))
queuedCountry.SetFileType(MapFileType::Map);
break;
}
}
m_downloader->DownloadMapFile(queuedCountry,
bind(&Storage::OnMapFileDownloadFinished, this, _1, _2),
bind(&Storage::OnMapFileDownloadProgress, this, _1));
}
void Storage::SetDeferDownloading()
{
CHECK_THREAD_CHECKER(m_threadChecker, ());
m_needToStartDeferredDownloading = true;
}
void Storage::DoDeferredDownloadIfNeeded()
{
CHECK_THREAD_CHECKER(m_threadChecker, ());
if (!m_needToStartDeferredDownloading)
return;
m_needToStartDeferredDownloading = false;
DoDownload();
}
void Storage::OnMapFileDownloadProgress(MapFilesDownloader::Progress const & progress)
{
CHECK_THREAD_CHECKER(m_threadChecker, ());
@ -1474,8 +1428,6 @@ void Storage::OnDiffStatusReceived(diffs::NameDiffInfoMap && diffs)
m_notAppliedDiffs.clear();
}
DoDeferredDownloadIfNeeded();
}
StatusAndError Storage::GetNodeStatusInfo(CountryTree::Node const & node,

4
storage/storage.hpp
View file

@ -278,10 +278,6 @@ private:
void ReportProgressForHierarchy(CountryId const & countryId,
MapFilesDownloader::Progress const & leafProgress);
void DoDownload();
void SetDeferDownloading();
void DoDeferredDownloadIfNeeded();
/// Called on the main thread by MapFilesDownloader when
/// downloading of a map file succeeds/fails.
void OnMapFileDownloadFinished(downloader::HttpRequest::Status status,

28
storage/storage_tests/fake_map_files_downloader.cpp
View file

@ -21,23 +21,23 @@ FakeMapFilesDownloader::FakeMapFilesDownloader(TaskRunner & taskRunner)
FakeMapFilesDownloader::~FakeMapFilesDownloader() { CHECK(m_checker.CalledOnOriginalThread(), ()); }
void FakeMapFilesDownloader::Download(std::vector<std::string> const & urls,
std::string const & path, int64_t size,
void FakeMapFilesDownloader::Download(QueuedCountry & queuedCountry,
FileDownloadedCallback const & onDownloaded,
DownloadingProgressCallback const & onProgress)
{
CHECK(m_checker.CalledOnOriginalThread(), ());
m_progress.first = 0;
m_progress.second = size;
m_idle = false;
m_writer.reset(new FileWriter(path));
m_onDownloaded = onDownloaded;
m_onProgress = onProgress;
++m_timestamp;
m_taskRunner.PostTask(std::bind(&FakeMapFilesDownloader::DownloadNextChunk, this, m_timestamp));
// Will be refactored in the followed commits.
// CHECK(m_checker.CalledOnOriginalThread(), ());
//
// m_progress.first = 0;
// m_progress.second = size;
// m_idle = false;
//
// m_writer.reset(new FileWriter(path));
// m_onDownloaded = onDownloaded;
// m_onProgress = onProgress;
//
// ++m_timestamp;
// m_taskRunner.PostTask(std::bind(&FakeMapFilesDownloader::DownloadNextChunk, this, m_timestamp));
}
MapFilesDownloader::Progress FakeMapFilesDownloader::GetDownloadingProgress()

3
storage/storage_tests/fake_map_files_downloader.hpp
View file

@ -1,6 +1,7 @@
#pragma once
#include "storage/map_files_downloader.hpp"
#include "storage/queued_country.hpp"
#include "coding/file_writer.hpp"
@ -36,7 +37,7 @@ public:
private:
// MapFilesDownloader overrides:
void Download(std::vector<std::string> const & urls, std::string const & path, int64_t size,
void Download(QueuedCountry & queuedCountry,
FileDownloadedCallback const & onDownloaded,
DownloadingProgressCallback const & onProgress) override;