pastk/organicmaps
1
Fork 0
forked from organicmaps/organicmaps
Code Pull requests Activity

Ping downloading urls

Browse source
This commit is contained in:
VladiMihaylenko 2018-02-15 13:12:06 +03:00 • committed by Vladimir Byko-Ianko
parent c9691d72ee
commit 0a5ee1e4f5
15 changed files with 192 additions and 40 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
search/search_integration_tests/downloader_search_test.cpp
View file

@ -79,10 +79,7 @@ class TestMapFilesDownloader : public storage::MapFilesDownloader
{
public:
// MapFilesDownloader overrides:
void GetServersList(int64_t const /* mapVersion */, string const & /* mapFileName */,
TServersListCallback const & /* callback */) override
{
}
void GetServersList(TServersListCallback const & /* callback */) override {}
void DownloadMapFile(vector<string> const & /* urls */, string const & /* path */,
int64_t /* size */, TFileDownloadedCallback const & /* onDownloaded */,

2
storage/CMakeLists.txt
View file

@ -32,6 +32,8 @@ set(
map_files_downloader.hpp
queued_country.cpp
queued_country.hpp
pinger.cpp
pinger.hpp
routing_helpers.cpp
routing_helpers.hpp
storage.cpp

7
storage/http_map_files_downloader.cpp
View file

@ -31,12 +31,11 @@ HttpMapFilesDownloader::~HttpMapFilesDownloader()
ASSERT_THREAD_CHECKER(m_checker, ());
}
void HttpMapFilesDownloader::GetServersList(int64_t const mapVersion, string const & mapFileName,
TServersListCallback const & callback)
void HttpMapFilesDownloader::GetServersList(TServersListCallback const & callback)
{
ASSERT_THREAD_CHECKER(m_checker, ());
m_request.reset(downloader::HttpRequest::PostJson(
GetPlatform().MetaServerUrl(), strings::to_string(mapVersion) + '/' + mapFileName,
m_request.reset(downloader::HttpRequest::Get(
GetPlatform().MetaServerUrl(),
bind(&HttpMapFilesDownloader::OnServersListDownloaded, this, callback, _1)));
}

3
storage/http_map_files_downloader.hpp
View file

@ -17,7 +17,8 @@ public:
virtual ~HttpMapFilesDownloader();
// MapFilesDownloader overrides:
void GetServersList(int64_t const mapVersion, string const & mapFileName, TServersListCallback const & callback) override;
void GetServersList(TServersListCallback const & callback) override;
void DownloadMapFile(vector<string> const & urls, string const & path, int64_t size,
TFileDownloadedCallback const & onDownloaded,
TDownloadingProgressCallback const & onProgress) override;

3
storage/map_files_downloader.hpp
View file

@ -24,8 +24,7 @@ public:
/// Asynchronously receives a list of all servers that can be asked
/// for a map file and invokes callback on the original thread.
virtual void GetServersList(int64_t const mapVersion, string const & mapFileName,
TServersListCallback const & callback) = 0;
virtual void GetServersList(TServersListCallback const & callback) = 0;
/// Asynchronously downloads a map file, periodically invokes
/// onProgress callback and finally invokes onDownloaded

75
storage/pinger.cpp Normal file
View file

@ -0,0 +1,75 @@
#include "storage/pinger.hpp"
#include "platform/http_client.hpp"
#include "platform/preferred_languages.hpp"
#include "base/assert.hpp"
#include "base/logging.hpp"
#include "base/stl_helpers.hpp"
#include "base/worker_thread.hpp"
#include "3party/Alohalytics/src/alohalytics.h"
#include <sstream>
#include <utility>
using namespace std;
namespace
{
auto constexpr kTimeoutInSeconds = 5.0;
void DoPing(string const & url, size_t index, vector<string> & readyUrls)
{
if (url.empty())
{
ASSERT(false, ("Metaserver returned an empty url."));
return;
}
platform::HttpClient request(url);
request.SetHttpMethod("HEAD");
request.SetTimeout(kTimeoutInSeconds);
if (request.RunHttpRequest() && !request.WasRedirected() && request.ErrorCode() == 200)
{
readyUrls[index] = url;
}
else
{
ostringstream ost;
ost << "Request to server " << url << " failed. Code = " << request.ErrorCode()
<< ", redirection = " << request.WasRedirected();
LOG(LINFO, (ost.str()));
}
}
void SendStatistics(size_t serversLeft)
{
alohalytics::Stats::Instance().LogEvent(
"Downloader_ServerList_check",
{{"lang", languages::GetCurrentNorm()}, {"servers", to_string(serversLeft)}});
}
} // namespace
namespace storage
{
// static
void Pinger::Ping(vector<string> const & urls, Pinger::Pong const & pong)
{
auto const size = urls.size();
CHECK_GREATER(size, 0, ());
vector<string> readyUrls(size);
{
base::WorkerThread t(size);
for (size_t i = 0; i < size; ++i)
t.Push([ url = urls[i], &readyUrls, i ] { DoPing(url, i, readyUrls); });
t.Shutdown(base::WorkerThread::Exit::ExecPending);
}
my::EraseIf(readyUrls, [](auto const & url) { return url.empty(); });
SendStatistics(readyUrls.size());
pong(move(readyUrls));
}
} // namespace storage

16
storage/pinger.hpp Normal file
View file

@ -0,0 +1,16 @@
#pragma once
#include "platform/safe_callback.hpp"
#include <string>
#include <vector>
namespace storage
{
class Pinger
{
public:
using Pong = platform::SafeCallback<void(std::vector<std::string> readyUrls)>;
static void Ping(std::vector<std::string> const & urls, Pong const & pong);
};
} // namespace storage

80
storage/storage.cpp
View file

@ -99,7 +99,7 @@ MapFilesDownloader::TProgress Storage::GetOverallProgress(TCountriesVec const &
Storage::Storage(string const & pathToCountriesFile /* = COUNTRIES_FILE */,
string const & dataDir /* = string() */)
: m_downloader(new HttpMapFilesDownloader())
: m_downloader(make_unique<HttpMapFilesDownloader>())
, m_currentSlotId(0)
, m_dataDir(dataDir)
, m_downloadMapOnTheMap(nullptr)
@ -108,6 +108,7 @@ Storage::Storage(string const & pathToCountriesFile /* = COUNTRIES_FILE */,
SetLocale(languages::GetCurrentTwine());
LoadCountriesFile(pathToCountriesFile, m_dataDir);
CalMaxMwmSizeBytes();
LoadServerListForSession();
}
Storage::Storage(string const & referenceCountriesTxtJsonForTesting,
@ -121,6 +122,7 @@ Storage::Storage(string const & referenceCountriesTxtJsonForTesting,
LoadCountriesFromBuffer(referenceCountriesTxtJsonForTesting, m_countries, m_affiliations);
CHECK_LESS_OR_EQUAL(0, m_currentVersion, ("Can't load test countries file"));
CalMaxMwmSizeBytes();
LoadServerListForTesting();
}
void Storage::Init(TUpdateCallback const & didDownload, TDeleteCallback const & willDelete)
@ -613,7 +615,6 @@ void Storage::DownloadNextCountryFromQueue()
void Storage::DownloadNextFile(QueuedCountry const & country)
{
TCountryId const & countryId = country.GetCountryId();
CountryFile const & countryFile = GetCountryFile(countryId);
auto const opt = country.GetCurrentFileOptions();
string const readyFilePath = GetFileDownloadPath(countryId, opt);
@ -641,9 +642,10 @@ void Storage::DownloadNextFile(QueuedCountry const & country)
return;
}
// send Country name for statistics
m_downloader->GetServersList(GetCurrentDataVersion(), countryFile.GetName(),
bind(&Storage::OnServerListDownloaded, this, _1));
if (m_sessionServerList)
DoDownload();
else
SetDeferDownloading();
}
void Storage::DeleteFromDownloader(TCountryId const & countryId)
@ -781,9 +783,10 @@ void Storage::ReportProgressForHierarchy(TCountryId const & countryId,
ForEachAncestorExceptForTheRoot(countryId, calcProgress);
}
void Storage::OnServerListDownloaded(vector<string> const & urls)
void Storage::DoDownload()
{
ASSERT_THREAD_CHECKER(m_threadChecker, ());
CHECK(m_sessionServerList, ());
// Queue can be empty because countries were deleted from queue.
if (m_queue.empty())
@ -796,9 +799,7 @@ void Storage::OnServerListDownloaded(vector<string> const & urls)
auto const status = m_diffManager.GetStatus();
switch (status)
{
case Status::Undefined:
m_deferredDownloads.push_back(urls);
return;
case Status::Undefined: SetDeferDownloading(); return;
case Status::NotAvailable:
queuedCountry.ResetToDefaultOptions();
break;
@ -810,7 +811,7 @@ void Storage::OnServerListDownloaded(vector<string> const & urls)
}
vector<string> const & downloadingUrls =
m_downloadingUrlsForTesting.empty() ? urls : m_downloadingUrlsForTesting;
m_downloadingUrlsForTesting.empty() ? *m_sessionServerList : m_downloadingUrlsForTesting;
vector<string> fileUrls;
fileUrls.reserve(downloadingUrls.size());
for (string const & url : downloadingUrls)
@ -823,6 +824,24 @@ void Storage::OnServerListDownloaded(vector<string> const & urls)
bind(&Storage::OnMapFileDownloadProgress, this, _1));
}
void Storage::SetDeferDownloading()
{
ASSERT_THREAD_CHECKER(m_threadChecker, ());
m_needToStartDeferredDownloading = true;
}
void Storage::DoDeferredDownloadIfNeeded()
{
ASSERT_THREAD_CHECKER(m_threadChecker, ());
if (!m_needToStartDeferredDownloading || !m_sessionServerList)
return;
m_needToStartDeferredDownloading = false;
DoDownload();
}
void Storage::OnMapFileDownloadProgress(MapFilesDownloader::TProgress const & progress)
{
ASSERT_THREAD_CHECKER(m_threadChecker, ());
@ -1083,6 +1102,7 @@ string Storage::GetLocale() const { return m_countryNameGetter.GetLocale(); }
void Storage::SetDownloaderForTesting(unique_ptr<MapFilesDownloader> && downloader)
{
m_downloader = move(downloader);
LoadServerListForTesting();
}
void Storage::SetCurrentDataVersionForTesting(int64_t currentVersion)
@ -1484,6 +1504,41 @@ void Storage::ApplyDiff(TCountryId const & countryId, function<void(bool isSucce
});
}
void Storage::LoadServerListForSession()
{
ASSERT_THREAD_CHECKER(m_threadChecker, ());
m_downloader->GetServersList([this](auto const & urls) { PingServerList(urls); });
}
void Storage::LoadServerListForTesting()
{
ASSERT_THREAD_CHECKER(m_threadChecker, ());
m_downloader->GetServersList([this](auto const & urls) { m_sessionServerList = urls; });
}
void Storage::PingServerList(vector<string> const & urls)
{
ASSERT_THREAD_CHECKER(m_threadChecker, ());
if (urls.empty())
return;
GetPlatform().RunTask(Platform::Thread::Network, [urls, this] {
Pinger::Ping(urls, [this, urls](auto readyUrls) {
ASSERT_THREAD_CHECKER(m_threadChecker, ());
if (readyUrls.empty())
m_sessionServerList = urls;
else
m_sessionServerList = move(readyUrls);
DoDeferredDownloadIfNeeded();
});
});
}
bool Storage::IsPossibleToAutoupdate() const
{
ASSERT_THREAD_CHECKER(m_threadChecker, ());
@ -1514,10 +1569,7 @@ void Storage::OnDiffStatusReceived(diffs::Status const status)
m_notAppliedDiffs.clear();
}
for (auto const & urls : m_deferredDownloads)
OnServerListDownloaded(urls);
m_deferredDownloads.clear();
DoDeferredDownloadIfNeeded();
}
StatusAndError Storage::GetNodeStatusInfo(

18
storage/storage.hpp
View file

@ -7,9 +7,9 @@
#include "storage/downloading_policy.hpp"
#include "storage/index.hpp"
#include "storage/map_files_downloader.hpp"
#include "storage/pinger.hpp"
#include "storage/queued_country.hpp"
#include "storage/storage_defines.hpp"
#include "storage/storage_defines.hpp"
#include "platform/local_country_file.hpp"
@ -17,7 +17,6 @@
#include "base/thread_checker.hpp"
#include "base/worker_thread.hpp"
#include <future>
#include "std/function.hpp"
#include "std/list.hpp"
#include "std/shared_ptr.hpp"
@ -26,6 +25,8 @@
#include "std/utility.hpp"
#include "std/vector.hpp"
#include <boost/optional.hpp>
namespace storage
{
struct CountryIdAndName
@ -250,7 +251,8 @@ private:
diffs::Manager m_diffManager;
vector<platform::LocalCountryFile> m_notAppliedDiffs;
vector<vector<string>> m_deferredDownloads;
bool m_needToStartDeferredDownloading = false;
boost::optional<vector<string>> m_sessionServerList;
StartDownloadingCallback m_startDownloadingCallback;
@ -263,9 +265,9 @@ private:
void ReportProgressForHierarchy(TCountryId const & countryId,
MapFilesDownloader::TProgress const & leafProgress);
/// Called on the main thread by MapFilesDownloader when list of
/// suitable servers is received.
void OnServerListDownloaded(vector<string> const & urls);
void DoDownload();
void SetDeferDownloading();
void DoDeferredDownloadIfNeeded();
/// Called on the main thread by MapFilesDownloader when
/// downloading of a map file succeeds/fails.
@ -673,6 +675,10 @@ private:
// Should be called once on startup, downloading process should be suspended until this method
// was not called. Do not call this method manually.
void OnDiffStatusReceived(diffs::Status const status) override;
void LoadServerListForSession();
void LoadServerListForTesting();
void PingServerList(vector<string> const & urls);
};
void GetQueuedCountries(Storage::TQueue const & queue, TCountriesSet & resultCountries);

5
storage/storage_tests/fake_map_files_downloader.cpp
View file

@ -20,13 +20,12 @@ FakeMapFilesDownloader::FakeMapFilesDownloader(TaskRunner & taskRunner)
FakeMapFilesDownloader::~FakeMapFilesDownloader() { CHECK(m_checker.CalledOnOriginalThread(), ()); }
void FakeMapFilesDownloader::GetServersList(int64_t const mapVersion, string const & mapFileName,
TServersListCallback const & callback)
void FakeMapFilesDownloader::GetServersList(TServersListCallback const & callback)
{
CHECK(m_checker.CalledOnOriginalThread(), ());
m_idle = false;
MY_SCOPE_GUARD(resetIdle, bind(&FakeMapFilesDownloader::Reset, this));
m_taskRunner.PostTask(bind(callback, m_servers));
callback(m_servers);
}
void FakeMapFilesDownloader::DownloadMapFile(vector<string> const & urls, string const & path,

3
storage/storage_tests/fake_map_files_downloader.hpp
View file

@ -26,7 +26,8 @@ public:
virtual ~FakeMapFilesDownloader();
// MapFilesDownloader overrides:
void GetServersList(int64_t const mapVersion, string const & mapFileName, TServersListCallback const & callback) override;
void GetServersList(TServersListCallback const & callback) override;
void DownloadMapFile(vector<string> const & urls, string const & path, int64_t size,
TFileDownloadedCallback const & onDownloaded,
TDownloadingProgressCallback const & onProgress) override;

1
storage/storage_tests/storage_tests.cpp
View file

@ -964,7 +964,6 @@ UNIT_CLASS_TEST(TwoComponentStorageTest, DeleteCountry)
map.Reset();
}
UNIT_TEST(StorageTest_FailedDownloading)
{
Storage storage;

3
storage/storage_tests/test_map_files_downloader.cpp
View file

@ -5,8 +5,7 @@
namespace storage
{
void TestMapFilesDownloader::GetServersList(int64_t const mapVersion, string const & mapFileName,
TServersListCallback const & callback)
void TestMapFilesDownloader::GetServersList(TServersListCallback const & callback)
{
vector<string> urls = {"http://localhost:34568/unit_tests/"};
callback(urls);

3
storage/storage_tests/test_map_files_downloader.hpp
View file

@ -8,7 +8,6 @@ class TestMapFilesDownloader : public HttpMapFilesDownloader
{
public:
// MapFilesDownloader overrides:
void GetServersList(int64_t const mapVersion, string const & mapFileName,
TServersListCallback const & callback) override;
void GetServersList(TServersListCallback const & callback) override;
};
} // namespace storage

8
xcode/storage/storage.xcodeproj/project.pbxproj
View file

@ -181,6 +181,8 @@
F68CC5BE1F38B967007527C7 /* diff_scheme_checker.cpp in Sources */ = {isa = PBXBuildFile; fileRef = F68CC5BB1F38B967007527C7 /* diff_scheme_checker.cpp */; };
F68CC5BF1F38B967007527C7 /* diff_scheme_checker.hpp in Headers */ = {isa = PBXBuildFile; fileRef = F68CC5BC1F38B967007527C7 /* diff_scheme_checker.hpp */; };
F68CC5C01F38B967007527C7 /* diff_types.hpp in Headers */ = {isa = PBXBuildFile; fileRef = F68CC5BD1F38B967007527C7 /* diff_types.hpp */; };
F6BC312B2034366100F677FE /* pinger.hpp in Headers */ = {isa = PBXBuildFile; fileRef = F6BC31292034366100F677FE /* pinger.hpp */; };
F6BC312C2034366100F677FE /* pinger.cpp in Sources */ = {isa = PBXBuildFile; fileRef = F6BC312A2034366100F677FE /* pinger.cpp */; };
/* End PBXBuildFile section */
/* Begin PBXFileReference section */
@ -315,6 +317,8 @@
F68CC5BB1F38B967007527C7 /* diff_scheme_checker.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = diff_scheme_checker.cpp; path = diff_scheme/diff_scheme_checker.cpp; sourceTree = "<group>"; };
F68CC5BC1F38B967007527C7 /* diff_scheme_checker.hpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; name = diff_scheme_checker.hpp; path = diff_scheme/diff_scheme_checker.hpp; sourceTree = "<group>"; };
F68CC5BD1F38B967007527C7 /* diff_types.hpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; name = diff_types.hpp; path = diff_scheme/diff_types.hpp; sourceTree = "<group>"; };
F6BC31292034366100F677FE /* pinger.hpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; path = pinger.hpp; sourceTree = "<group>"; };
F6BC312A2034366100F677FE /* pinger.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = pinger.cpp; sourceTree = "<group>"; };
/* End PBXFileReference section */
/* Begin PBXFrameworksBuildPhase section */
@ -542,6 +546,8 @@
675342E21A3F59EF00A0A8C3 /* storage */ = {
isa = PBXGroup;
children = (
F6BC312A2034366100F677FE /* pinger.cpp */,
F6BC31292034366100F677FE /* pinger.hpp */,
56D0E47F1F8E40340084B18C /* routing_helpers.cpp */,
56D0E47E1F8E40340084B18C /* routing_helpers.hpp */,
401ECED21F56C50900DFDF76 /* country_parent_getter.cpp */,
@ -633,6 +639,7 @@
56D0E4801F8E40340084B18C /* routing_helpers.hpp in Headers */,
6753430F1A3F5A2600A0A8C3 /* country.hpp in Headers */,
34B093261C61F9BA0066F4C3 /* app_store.hpp in Headers */,
F6BC312B2034366100F677FE /* pinger.hpp in Headers */,
6753430A1A3F5A2600A0A8C3 /* country_decl.hpp in Headers */,
678338D61C723B1A00FD6263 /* helpers.hpp in Headers */,
67247FCF1C60BA8A00EDE56A /* fake_map_files_downloader.hpp in Headers */,
@ -821,6 +828,7 @@
674125231B4C05FA00A3E828 /* storage_defines.cpp in Sources */,
67BE1DC51CD2180D00572709 /* downloading_policy.cpp in Sources */,
678338D41C723B1A00FD6263 /* country_name_getter_test.cpp in Sources */,
F6BC312C2034366100F677FE /* pinger.cpp in Sources */,
34C9BCFC1C6CCF85000DC38D /* country_name_getter.cpp in Sources */,
6741251E1B4C05FA00A3E828 /* http_map_files_downloader.cpp in Sources */,
);