gpesquero/organicmaps
1
Fork 0
forked from organicmaps/organicmaps
Code Pull requests Activity

[storage][diffs] use same diffs info into storage and queued counties for url logic incapsulation

Browse source
This commit is contained in:
Arsentiy Milchakov 2019-10-11 14:18:08 +03:00
parent dd93f2a493
commit 2e49266e04
16 changed files with 320 additions and 247 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
storage/CMakeLists.txt
View file

@ -17,6 +17,8 @@ set(
country_parent_getter.hpp
country_tree.cpp
country_tree.hpp
diff_scheme/apply_diff.cpp
diff_scheme/apply_diff.hpp
diff_scheme/diff_manager.cpp
diff_scheme/diff_manager.hpp
diff_scheme/diff_scheme_loader.cpp

85
storage/diff_scheme/apply_diff.cpp Normal file
View file

@ -0,0 +1,85 @@
#include "storage/diff_scheme/apply_diff.hpp"
#include "platform/platform.hpp"
#include "coding/internal/file_data.hpp"
#include "base/assert.hpp"
#include "base/cancellable.hpp"
#include "3party/Alohalytics/src/alohalytics.h"
namespace storage
{
namespace diffs
{
void ApplyDiff(ApplyDiffParams && p, base::Cancellable const & cancellable,
OnDiffApplicationFinished const & task)
{
using namespace generator::mwm_diff;
GetPlatform().RunTask(Platform::Thread::File, [p = std::move(p), &cancellable, task] {
CHECK(p.m_diffFile, ());
CHECK(p.m_oldMwmFile, ());
auto & diffReadyPath = p.m_diffReadyPath;
auto & diffFile = p.m_diffFile;
auto const diffPath = diffFile->GetPath(MapFileType::Diff);
auto result = DiffApplicationResult::Failed;
diffFile->SyncWithDisk();
auto const isOnDisk = diffFile->OnDisk(MapFileType::Diff);
auto const isFilePrepared = isOnDisk || base::RenameFileX(diffReadyPath, diffPath);
if (isFilePrepared)
{
// Sync with disk after renaming.
if (!isOnDisk)
diffFile->SyncWithDisk();
std::string const oldMwmPath = p.m_oldMwmFile->GetPath(MapFileType::Map);
std::string const newMwmPath = diffFile->GetPath(MapFileType::Map);
std::string const diffApplyingInProgressPath = newMwmPath + DIFF_APPLYING_FILE_EXTENSION;
result = generator::mwm_diff::ApplyDiff(oldMwmPath, diffApplyingInProgressPath, diffPath,
cancellable);
if (result == DiffApplicationResult::Ok &&
!base::RenameFileX(diffApplyingInProgressPath, newMwmPath))
{
result = DiffApplicationResult::Failed;
}
Platform::RemoveFileIfExists(diffApplyingInProgressPath);
if (result != DiffApplicationResult::Ok)
Platform::RemoveFileIfExists(newMwmPath);
}
switch (result)
{
case DiffApplicationResult::Ok:
diffFile->DeleteFromDisk(MapFileType::Diff);
break;
case DiffApplicationResult::Cancelled:
// The diff file will be deleted by storage.
// Another way would be to leave it on disk but all consequences
// of interacting with storage are much harder to be taken into account that way.
break;
case DiffApplicationResult::Failed:
diffFile->DeleteFromDisk(MapFileType::Diff);
alohalytics::Stats::Instance().LogEvent(
"Downloader_DiffScheme_error",
{{"type", "patching"},
{"error", isFilePrepared ? "Cannot apply diff" : "Cannot prepare file"}});
break;
}
GetPlatform().RunTask(Platform::Thread::Gui, [task, result]()
{
task(result);
});
});
}
} // namespace diffs
} // namespace storage

30
storage/diff_scheme/apply_diff.hpp Normal file
View file

@ -0,0 +1,30 @@
#pragma once
#include "generator/mwm_diff/diff.hpp"
#include "storage/storage_defines.hpp"
#include <functional>
namespace base
{
class Cancellable;
}
namespace storage
{
namespace diffs
{
struct ApplyDiffParams
{
std::string m_diffReadyPath;
LocalFilePtr m_diffFile;
LocalFilePtr m_oldMwmFile;
};
using OnDiffApplicationFinished = std::function<void(generator::mwm_diff::DiffApplicationResult)>;
void ApplyDiff(ApplyDiffParams && p, base::Cancellable const & cancellable,
OnDiffApplicationFinished const & task);
} // namespace diffs
} // namespace storage

111
storage/diff_scheme/diff_manager.cpp
View file

@ -1,12 +1,5 @@
#include "storage/diff_scheme/diff_manager.hpp"
#include "platform/platform.hpp"
#include "coding/internal/file_data.hpp"
#include "base/assert.hpp"
#include "base/cancellable.hpp"
#include <algorithm>
#include "3party/Alohalytics/src/alohalytics.h"
@ -24,8 +17,10 @@ namespace storage
{
namespace diffs
{
void Manager::Load(NameDiffInfoMap && info)
void DiffsDataSource::SetDiffInfo(NameDiffInfoMap && info)
{
CHECK_THREAD_CHECKER(m_threadChecker, ());
if (info.empty())
{
m_status = Status::NotAvailable;
@ -39,83 +34,17 @@ void Manager::Load(NameDiffInfoMap && info)
}
}
// static
void Manager::ApplyDiff(ApplyDiffParams && p, base::Cancellable const & cancellable,
Manager::OnDiffApplicationFinished const & task)
Status DiffsDataSource::GetStatus() const
{
using namespace generator::mwm_diff;
CHECK_THREAD_CHECKER(m_threadChecker, ());
GetPlatform().RunTask(Platform::Thread::File, [p = std::move(p), &cancellable, task] {
CHECK(p.m_diffFile, ());
CHECK(p.m_oldMwmFile, ());
auto & diffReadyPath = p.m_diffReadyPath;
auto & diffFile = p.m_diffFile;
auto const diffPath = diffFile->GetPath(MapFileType::Diff);
auto result = DiffApplicationResult::Failed;
diffFile->SyncWithDisk();
auto const isOnDisk = diffFile->OnDisk(MapFileType::Diff);
auto const isFilePrepared = isOnDisk || base::RenameFileX(diffReadyPath, diffPath);
if (isFilePrepared)
{
// Sync with disk after renaming.
if (!isOnDisk)
diffFile->SyncWithDisk();
std::string const oldMwmPath = p.m_oldMwmFile->GetPath(MapFileType::Map);
std::string const newMwmPath = diffFile->GetPath(MapFileType::Map);
std::string const diffApplyingInProgressPath = newMwmPath + DIFF_APPLYING_FILE_EXTENSION;
result = generator::mwm_diff::ApplyDiff(oldMwmPath, diffApplyingInProgressPath, diffPath,
cancellable);
if (result == DiffApplicationResult::Ok &&
!base::RenameFileX(diffApplyingInProgressPath, newMwmPath))
{
result = DiffApplicationResult::Failed;
}
Platform::RemoveFileIfExists(diffApplyingInProgressPath);
if (result != DiffApplicationResult::Ok)
Platform::RemoveFileIfExists(newMwmPath);
}
switch (result)
{
case DiffApplicationResult::Ok:
diffFile->DeleteFromDisk(MapFileType::Diff);
break;
case DiffApplicationResult::Cancelled:
// The diff file will be deleted by storage.
// Another way would be to leave it on disk but all consequences
// of interacting with storage are much harder to be taken into account that way.
break;
case DiffApplicationResult::Failed:
diffFile->DeleteFromDisk(MapFileType::Diff);
alohalytics::Stats::Instance().LogEvent(
"Downloader_DiffScheme_error",
{{"type", "patching"},
{"error", isFilePrepared ? "Cannot apply diff" : "Cannot prepare file"}});
break;
}
GetPlatform().RunTask(Platform::Thread::Gui, [task, result]()
{
task(result);
});
});
}
Status Manager::GetStatus() const
{
return m_status;
}
bool Manager::SizeFor(storage::CountryId const & countryId, uint64_t & size) const
bool DiffsDataSource::SizeFor(storage::CountryId const & countryId, uint64_t & size) const
{
CHECK_THREAD_CHECKER(m_threadChecker, ());
if (m_status != Status::Available)
return false;
@ -127,23 +56,31 @@ bool Manager::SizeFor(storage::CountryId const & countryId, uint64_t & size) con
return true;
}
bool Manager::SizeToDownloadFor(storage::CountryId const & countryId, uint64_t & size) const
bool DiffsDataSource::SizeToDownloadFor(storage::CountryId const & countryId, uint64_t & size) const
{
CHECK_THREAD_CHECKER(m_threadChecker, ());
return WithNotAppliedDiff(countryId, [&size](DiffInfo const & info) { size = info.m_size; });
}
bool Manager::VersionFor(storage::CountryId const & countryId, uint64_t & v) const
bool DiffsDataSource::VersionFor(storage::CountryId const & countryId, uint64_t & v) const
{
CHECK_THREAD_CHECKER(m_threadChecker, ());
return WithNotAppliedDiff(countryId, [&v](DiffInfo const & info) { v = info.m_version; });
}
bool Manager::HasDiffFor(storage::CountryId const & countryId) const
bool DiffsDataSource::HasDiffFor(storage::CountryId const & countryId) const
{
CHECK_THREAD_CHECKER(m_threadChecker, ());
return WithNotAppliedDiff(countryId, [](DiffInfo const &) {});
}
void Manager::MarkAsApplied(storage::CountryId const & countryId)
void DiffsDataSource::MarkAsApplied(storage::CountryId const & countryId)
{
CHECK_THREAD_CHECKER(m_threadChecker, ());
auto it = m_diffs.find(countryId);
if (it == m_diffs.end())
return;
@ -154,16 +91,20 @@ void Manager::MarkAsApplied(storage::CountryId const & countryId)
m_status = Status::NotAvailable;
}
void Manager::RemoveDiffForCountry(storage::CountryId const & countryId)
void DiffsDataSource::RemoveDiffForCountry(storage::CountryId const & countryId)
{
CHECK_THREAD_CHECKER(m_threadChecker, ());
m_diffs.erase(countryId);
if (m_diffs.empty() || !IsDiffsAvailable(m_diffs))
m_status = Status::NotAvailable;
}
void Manager::AbortDiffScheme()
void DiffsDataSource::AbortDiffScheme()
{
CHECK_THREAD_CHECKER(m_threadChecker, ());
m_status = Status::NotAvailable;
m_diffs.clear();
}

35
storage/diff_scheme/diff_manager.hpp
View file

@ -3,38 +3,21 @@
#include "storage/diff_scheme/diff_types.hpp"
#include "storage/storage_defines.hpp"
#include "generator/mwm_diff/diff.hpp"
#include "base/observer_list.hpp"
#include "base/thread_checker.hpp"
#include "base/thread_pool_delayed.hpp"
#include <functional>
#include <mutex>
#include <string>
#include <utility>
namespace base
{
class Cancellable;
}
#include <memory>
namespace storage
{
namespace diffs
{
class Manager final
class DiffsDataSource;
using DiffsSourcePtr = std::shared_ptr<diffs::DiffsDataSource>;
class DiffsDataSource final
{
public:
struct ApplyDiffParams
{
std::string m_diffReadyPath;
LocalFilePtr m_diffFile;
LocalFilePtr m_oldMwmFile;
};
using OnDiffApplicationFinished = std::function<void(generator::mwm_diff::DiffApplicationResult)>;
void SetDiffInfo(NameDiffInfoMap && info);
// If the diff is available, sets |size| to its size and returns true.
// Otherwise, returns false.
bool SizeFor(storage::CountryId const & countryId, uint64_t & size) const;
@ -58,10 +41,6 @@ public:
Status GetStatus() const;
void Load(NameDiffInfoMap && info);
static void ApplyDiff(ApplyDiffParams && p, base::Cancellable const & cancellable,
OnDiffApplicationFinished const & task);
private:
template <typename Fn>
bool WithNotAppliedDiff(storage::CountryId const & countryId, Fn && fn) const
@ -77,6 +56,8 @@ private:
return true;
}
ThreadChecker m_threadChecker;
Status m_status = Status::Undefined;
NameDiffInfoMap m_diffs;
};

20
storage/diff_scheme/diff_scheme_loader.cpp
View file

@ -53,34 +53,34 @@ NameDiffInfoMap DeserializeResponse(string const & response, LocalMapsInfo::Name
if (response.empty())
{
LOG(LERROR, ("Diff response shouldn't be empty."));
return NameDiffInfoMap{};
return {};
}
base::Json const json(response.c_str());
if (json.get() == nullptr)
return NameDiffInfoMap{};
return {};
auto const root = json_object_get(json.get(), kMwmsKey);
if (root == nullptr || !json_is_array(root))
return NameDiffInfoMap{};
return {};
auto const size = json_array_size(root);
if (size == 0 || size != nameVersionMap.size())
auto const count = json_array_size(root);
if (count == 0 || count != nameVersionMap.size())
{
LOG(LERROR, ("Diff list size in response must be equal to mwm list size in request."));
return NameDiffInfoMap{};
return {};
}
NameDiffInfoMap diffs;
for (size_t i = 0; i < size; ++i)
for (size_t i = 0; i < count; ++i)
{
auto const node = json_array_get(root, i);
if (!node)
{
LOG(LERROR, ("Incorrect server response."));
return NameDiffInfoMap{};
return {};
}
string name;
@ -94,7 +94,7 @@ NameDiffInfoMap DeserializeResponse(string const & response, LocalMapsInfo::Name
if (nameVersionMap.find(name) == nameVersionMap.end())
{
LOG(LERROR, ("Incorrect country name in response:", name));
return NameDiffInfoMap{};
return {};
}
DiffInfo info(size, nameVersionMap.at(name));
@ -107,7 +107,7 @@ NameDiffInfoMap DeserializeResponse(string const & response, LocalMapsInfo::Name
NameDiffInfoMap Load(LocalMapsInfo const & info)
{
if (info.m_localMaps.empty())
return NameDiffInfoMap();
return {};
platform::HttpClient request(DIFF_LIST_URL);
string const body = SerializeCheckerData(info);

8
storage/diff_scheme/diff_scheme_loader.hpp
View file

@ -8,6 +8,14 @@ namespace storage
{
namespace diffs
{
struct LocalMapsInfo final
{
using NameVersionMap = std::unordered_map<storage::CountryId, uint64_t>;
uint64_t m_currentDataVersion = 0;
NameVersionMap m_localMaps;
};
using DiffsReceivedCallback = std::function<void(diffs::NameDiffInfoMap && diffs)>;
class Loader final

10
storage/diff_scheme/diff_types.hpp
View file

@ -10,7 +10,7 @@ namespace storage
{
namespace diffs
{
// Status of the diff manager as a whole.
// Status of the diffs data source as a whole.
enum class Status
{
Undefined,
@ -28,13 +28,5 @@ struct DiffInfo final
};
using NameDiffInfoMap = std::unordered_map<storage::CountryId, DiffInfo>;
struct LocalMapsInfo final
{
using NameVersionMap = std::unordered_map<storage::CountryId, uint64_t>;
uint64_t m_currentDataVersion = 0;
NameVersionMap m_localMaps;
};
} // namespace diffs
} // namespace storage

31
storage/map_files_downloader.cpp
View file

@ -30,8 +30,7 @@ std::vector<std::string> MakeUrlList(MapFilesDownloader::ServersList const & ser
}
} // namespace
void MapFilesDownloader::DownloadMapFile(std::string const & relativeUrl,
std::string const & path, int64_t size,
void MapFilesDownloader::DownloadMapFile(QueuedCountry & country,
FileDownloadedCallback const & onDownloaded,
DownloadingProgressCallback const & onProgress)
{
@ -40,30 +39,19 @@ void MapFilesDownloader::DownloadMapFile(std::string const & relativeUrl,
GetServersList([=](ServersList const & serversList)
{
m_serversList = serversList;
auto const urls = MakeUrlList(m_serversList, relativeUrl);
Download(urls, path, size, onDownloaded, onProgress);
auto const urls = MakeUrlList(m_serversList, country.GetRelativeUrl());
Download(urls, country.GetFileDownloadPath(), country.GetDownloadSize(),
onDownloaded, onProgress);
});
}
else
{
auto const urls = MakeUrlList(m_serversList, relativeUrl);
Download(urls, path, size, onDownloaded, onProgress);
auto const urls = MakeUrlList(m_serversList, country.GetRelativeUrl());
Download(urls, country.GetFileDownloadPath(), country.GetDownloadSize(), onDownloaded,
onProgress);
}
}
// static
std::string MapFilesDownloader::MakeRelativeUrl(std::string const & fileName, int64_t dataVersion,
uint64_t diffVersion)
{
std::ostringstream url;
if (diffVersion != 0)
url << "diffs/" << dataVersion << "/" << diffVersion;
else
url << OMIM_OS_NAME "/" << dataVersion;
return base::url::Join(url.str(), UrlEncode(fileName));
}
// static
std::string MapFilesDownloader::MakeFullUrlLegacy(std::string const & baseUrl,
std::string const & fileName, int64_t dataVersion)
@ -77,11 +65,6 @@ void MapFilesDownloader::SetServersList(ServersList const & serversList)
m_serversList = serversList;
}
void MapFilesDownloader::SetDiffs(diffs::NameDiffInfoMap const & diffs)
{
m_diffs = diffs;
}
// static
MapFilesDownloader::ServersList MapFilesDownloader::LoadServersList()
{

9
storage/map_files_downloader.hpp
View file

@ -1,6 +1,6 @@
#pragma once
#include "storage/diff_scheme/diff_types.hpp"
#include "storage/queued_country.hpp"
#include "platform/http_request.hpp"
#include "platform/safe_callback.hpp"
@ -33,8 +33,7 @@ public:
/// Asynchronously downloads a map file, periodically invokes
/// onProgress callback and finally invokes onDownloaded
/// callback. Both callbacks will be invoked on the main thread.
void DownloadMapFile(std::string const & relativeUrl, std::string const & path, int64_t size,
FileDownloadedCallback const & onDownloaded,
void DownloadMapFile(QueuedCountry & queuedCountry, FileDownloadedCallback const & onDownloaded,
DownloadingProgressCallback const & onProgress);
/// Returns current downloading progress.
@ -46,13 +45,10 @@ public:
/// Resets downloader to the idle state.
virtual void Reset() = 0;
static std::string MakeRelativeUrl(std::string const & fileName, int64_t dataVersion,
uint64_t diffVersion);
static std::string MakeFullUrlLegacy(std::string const & baseUrl, std::string const & fileName,
int64_t dataVersion);
void SetServersList(ServersList const & serversList);
void SetDiffs(diffs::NameDiffInfoMap const & diffs);
protected:
// Synchronously loads list of servers by http client.
@ -68,6 +64,5 @@ private:
DownloadingProgressCallback const & onProgress) = 0;
ServersList m_serversList;
diffs::NameDiffInfoMap m_diffs;
};
} // namespace storage

62
storage/queued_country.cpp
View file

@ -1,13 +1,43 @@
#include "storage/queued_country.hpp"
#include "storage/storage_helpers.hpp"
#include "platform/local_country_file_utils.hpp"
#include "coding/url_encode.hpp"
#include "base/assert.hpp"
#include "base/url_helpers.hpp"
namespace
{
std::string MakeRelativeUrl(std::string const & fileName, int64_t dataVersion, uint64_t diffVersion)
{
std::ostringstream url;
if (diffVersion != 0)
url << "diffs/" << dataVersion << "/" << diffVersion;
else
url << OMIM_OS_NAME "/" << dataVersion;
return base::url::Join(url.str(), UrlEncode(fileName));
}
} // namespace
namespace storage
{
QueuedCountry::QueuedCountry(CountryId const & countryId, MapFileType type)
: m_countryId(countryId), m_fileType(type)
QueuedCountry::QueuedCountry(platform::CountryFile const & countryFile, CountryId const & countryId,
MapFileType type, int64_t currentDataVersion,
std::string const & dataDir,
diffs::DiffsSourcePtr const & diffs)
: m_countryFile(countryFile)
, m_countryId(countryId)
, m_fileType(type)
, m_currentDataVersion(currentDataVersion)
, m_dataDir(dataDir)
, m_diffsDataSource(diffs)
{
ASSERT(IsCountryIdValid(GetCountryId()), ("Only valid countries may be downloaded."));
ASSERT(m_diffsDataSource != nullptr, ());
}
void QueuedCountry::SetFileType(MapFileType type)
@ -25,6 +55,34 @@ CountryId const & QueuedCountry::GetCountryId() const
return m_countryId;
}
std::string QueuedCountry::GetRelativeUrl() const
{
auto const fileName = platform::GetFileName(m_countryFile.GetName(), m_fileType);
uint64_t diffVersion = 0;
if (m_fileType == MapFileType::Diff)
CHECK(m_diffsDataSource->VersionFor(m_countryId, diffVersion), ());
return MakeRelativeUrl(fileName, m_currentDataVersion, diffVersion);
}
std::string QueuedCountry::GetFileDownloadPath() const
{
return platform::GetFileDownloadPath(m_currentDataVersion, m_dataDir, m_countryFile, m_fileType);
}
uint64_t QueuedCountry::GetDownloadSize() const
{
uint64_t size;
if (m_fileType == MapFileType::Diff)
{
CHECK(m_diffsDataSource->SizeToDownloadFor(m_countryId, size), ());
return size;
}
return GetRemoteSize(*m_diffsDataSource, m_countryFile, m_currentDataVersion);
}
bool QueuedCountry::operator==(CountryId const & countryId) const
{
return m_countryId == countryId;

19
storage/queued_country.hpp
View file

@ -1,24 +1,39 @@
#pragma once
#include "storage/diff_scheme/diff_manager.hpp"
#include "storage/storage_defines.hpp"
#include "platform/country_defines.hpp"
#include "platform/country_file.hpp"
#include <string>
namespace storage
{
class QueuedCountry
{
public:
QueuedCountry(CountryId const & m_countryId, MapFileType type);
QueuedCountry(platform::CountryFile const & countryFile, CountryId const & m_countryId,
MapFileType type, int64_t currentDataVersion, std::string const & dataDir,
diffs::DiffsSourcePtr const & diffs);
void SetFileType(MapFileType type);
MapFileType GetFileType() const;
CountryId const & GetCountryId() const;
std::string GetRelativeUrl() const;
std::string GetFileDownloadPath() const;
uint64_t GetDownloadSize() const;
bool operator==(CountryId const & countryId) const;
private:
CountryId m_countryId;
platform::CountryFile const m_countryFile;
CountryId const m_countryId;
MapFileType m_fileType;
int64_t m_currentDataVersion;
std::string m_dataDir;
std::shared_ptr<diffs::DiffsDataSource> m_diffsDataSource;
};
} // namespace storage

106
storage/storage.cpp
View file

@ -1,7 +1,9 @@
#include "storage/storage.hpp"
#include "storage/diff_scheme/apply_diff.hpp"
#include "storage/diff_scheme/diff_scheme_loader.hpp"
#include "storage/http_map_files_downloader.hpp"
#include "storage/storage_helpers.hpp"
#include "defines.hpp"
@ -251,7 +253,7 @@ void Storage::RegisterAllLocalMaps(bool enableDiffs)
if (enableDiffs)
LoadDiffScheme();
// Note: call order is important, diffs loading must be called first.
// Since diffs downloading and servers list downloading
// Since diffs info downloading and servers list downloading
// are working on network thread, consecutive executing is guaranteed.
RestoreDownloadQueue();
}
@ -485,7 +487,9 @@ void Storage::DownloadCountry(CountryId const & countryId, MapFileType type)
return;
m_failedCountries.erase(countryId);
m_queue.push_back(QueuedCountry(countryId, type));
auto const countryFile = GetCountryFile(countryId);
m_queue.emplace_back(countryFile, countryId, type, GetCurrentDataVersion(), m_dataDir,
m_diffsDataSource);
if (m_queue.size() == 1)
{
if (m_startDownloadingCallback)
@ -507,9 +511,13 @@ void Storage::DeleteCountry(CountryId const & countryId, MapFileType type)
bool const deferredDelete = m_willDelete(countryId, localFile);
DeleteCountryFiles(countryId, type, deferredDelete);
DeleteCountryFilesFromDownloader(countryId);
m_diffManager.RemoveDiffForCountry(countryId);
m_diffsDataSource->RemoveDiffForCountry(countryId);
NotifyStatusChangedForHierarchy(countryId);
// Kick possibly interrupted downloader.
if (!m_queue.empty() && m_downloader->IsIdle())
DownloadNextCountryFromQueue();
}
void Storage::DeleteCustomCountryVersion(LocalCountryFile const & localFile)
@ -770,7 +778,7 @@ void Storage::DoDownload()
if (queuedCountry.GetFileType() == MapFileType::Diff)
{
using diffs::Status;
auto const status = m_diffManager.GetStatus();
auto const status = m_diffsDataSource->GetStatus();
switch (status)
{
case Status::Undefined:
@ -780,18 +788,13 @@ void Storage::DoDownload()
queuedCountry.SetFileType(MapFileType::Map);
break;
case Status::Available:
if (!m_diffManager.HasDiffFor(queuedCountry.GetCountryId()))
if (!m_diffsDataSource->HasDiffFor(queuedCountry.GetCountryId()))
queuedCountry.SetFileType(MapFileType::Map);
break;
}
}
auto const & id = queuedCountry.GetCountryId();
auto const options = queuedCountry.GetFileType();
auto const relativeUrl = GetDownloadRelativeUrl(id, options);
auto const filePath = GetFileDownloadPath(id, options);
m_downloader->DownloadMapFile(relativeUrl, filePath, GetDownloadSize(queuedCountry),
m_downloader->DownloadMapFile(queuedCountry,
bind(&Storage::OnMapFileDownloadFinished, this, _1, _2),
bind(&Storage::OnMapFileDownloadProgress, this, _1));
}
@ -915,7 +918,7 @@ void Storage::OnMapDownloadFinished(CountryId const & countryId, HttpRequest::St
{
if (status == HttpRequest::Status::FileNotFound && type == MapFileType::Diff)
{
m_diffManager.AbortDiffScheme();
m_diffsDataSource->AbortDiffScheme();
NotifyStatusChanged(GetRootId());
}
@ -926,18 +929,6 @@ void Storage::OnMapDownloadFinished(CountryId const & countryId, HttpRequest::St
RegisterDownloadedFiles(countryId, type);
}
string Storage::GetDownloadRelativeUrl(CountryId const & countryId, MapFileType type) const
{
auto const & countryFile = GetCountryFile(countryId);
auto const fileName = GetFileName(countryFile.GetName(), type);
uint64_t diffVersion = 0;
if (type == MapFileType::Diff)
CHECK(m_diffManager.VersionFor(countryId, diffVersion), ());
return MapFilesDownloader::MakeRelativeUrl(fileName, GetCurrentDataVersion(), diffVersion);
}
CountryId Storage::FindCountryIdByFile(string const & name) const
{
// @TODO(bykoianko) Probably it's worth to check here if name represent a node in the tree.
@ -1132,31 +1123,9 @@ bool Storage::DeleteCountryFilesFromDownloader(CountryId const & countryId)
PopFromQueue(it);
SaveDownloadQueue();
if (!m_queue.empty() && m_downloader->IsIdle())
{
// Kick possibly interrupted downloader.
if (IsCountryFirstInQueue(countryId))
DownloadNextFile(m_queue.front());
else
DownloadNextCountryFromQueue();
}
return true;
}
uint64_t Storage::GetDownloadSize(QueuedCountry const & queuedCountry) const
{
CountryId const & countryId = queuedCountry.GetCountryId();
uint64_t size;
if (queuedCountry.GetFileType() == MapFileType::Diff)
{
CHECK(m_diffManager.SizeToDownloadFor(countryId, size), ());
return size;
}
CountryFile const & file = GetCountryFile(countryId);
return GetRemoteSize(file, GetCurrentDataVersion());
}
string Storage::GetFileDownloadPath(CountryId const & countryId, MapFileType type) const
{
return platform::GetFileDownloadPath(GetCurrentDataVersion(), m_dataDir,
@ -1374,6 +1343,12 @@ void Storage::LoadDiffScheme()
localMapsInfo.m_localMaps.emplace(localFile->GetCountryName(), mapVersion);
}
if (localMapsInfo.m_localMaps.empty())
{
m_diffsDataSource->AbortDiffScheme();
return;
}
diffs::Loader::Load(move(localMapsInfo), [this](diffs::NameDiffInfoMap && diffs)
{
OnDiffStatusReceived(move(diffs));
@ -1388,7 +1363,7 @@ void Storage::ApplyDiff(CountryId const & countryId, function<void(bool isSucces
auto const diffLocalFile = PreparePlaceForCountryFiles(GetCurrentDataVersion(), m_dataDir,
GetCountryFile(countryId));
uint64_t version;
if (!diffLocalFile || !m_diffManager.VersionFor(countryId, version))
if (!diffLocalFile || !m_diffsDataSource->VersionFor(countryId, version))
{
fn(false);
return;
@ -1398,14 +1373,14 @@ void Storage::ApplyDiff(CountryId const & countryId, function<void(bool isSucces
m_diffsBeingApplied.insert(countryId);
NotifyStatusChangedForHierarchy(countryId);
diffs::Manager::ApplyDiffParams params;
diffs::ApplyDiffParams params;
params.m_diffFile = diffLocalFile;
params.m_diffReadyPath = GetFileDownloadPath(countryId, MapFileType::Diff);
params.m_oldMwmFile = GetLocalFile(countryId, version);
LocalFilePtr & diffFile = params.m_diffFile;
diffs::Manager::ApplyDiff(
diffs::ApplyDiff(
move(params), m_diffsCancellable,
[this, fn, countryId, diffFile](DiffApplicationResult result) {
CHECK_THREAD_CHECKER(m_threadChecker, ());
@ -1432,7 +1407,7 @@ void Storage::ApplyDiff(CountryId const & countryId, function<void(bool isSucces
{
RegisterCountryFiles(diffFile);
Platform::DisableBackupForFile(diffFile->GetPath(MapFileType::Map));
m_diffManager.MarkAsApplied(countryId);
m_diffsDataSource->MarkAsApplied(countryId);
fn(true);
break;
}
@ -1444,7 +1419,7 @@ void Storage::ApplyDiff(CountryId const & countryId, function<void(bool isSucces
}
case DiffApplicationResult::Failed:
{
m_diffManager.RemoveDiffForCountry(countryId);
m_diffsDataSource->RemoveDiffForCountry(countryId);
fn(false);
break;
}
@ -1455,7 +1430,7 @@ void Storage::ApplyDiff(CountryId const & countryId, function<void(bool isSucces
bool Storage::IsPossibleToAutoupdate() const
{
CHECK_THREAD_CHECKER(m_threadChecker, ());
if (m_diffManager.GetStatus() != diffs::Status::Available)
if (m_diffsDataSource->GetStatus() != diffs::Status::Available)
return false;
auto const currentVersion = GetCurrentDataVersion();
@ -1466,7 +1441,7 @@ bool Storage::IsPossibleToAutoupdate() const
auto const localFile = GetLatestLocalFile(countryId);
auto const mapVersion = localFile->GetVersion();
if (mapVersion != currentVersion && mapVersion > 0 &&
!m_diffManager.HasDiffFor(localFile->GetCountryName()))
!m_diffsDataSource->HasDiffFor(localFile->GetCountryName()))
{
return false;
}
@ -1484,15 +1459,14 @@ void Storage::SetStartDownloadingCallback(StartDownloadingCallback const & cb)
void Storage::OnDiffStatusReceived(diffs::NameDiffInfoMap && diffs)
{
m_downloader->SetDiffs(diffs);
m_diffManager.Load(move(diffs));
if (m_diffManager.GetStatus() != diffs::Status::NotAvailable)
m_diffsDataSource->SetDiffInfo(move(diffs));
if (m_diffsDataSource->GetStatus() == diffs::Status::Available)
{
for (auto const & localDiff : m_notAppliedDiffs)
{
auto const countryId = FindCountryIdByFile(localDiff.GetCountryName());
if (m_diffManager.HasDiffFor(countryId))
if (m_diffsDataSource->HasDiffFor(countryId))
UpdateNode(countryId);
else
localDiff.DeleteFromDisk(MapFileType::Diff);
@ -1748,6 +1722,10 @@ void Storage::CancelDownloadNode(CountryId const & countryId)
if (needNotify)
NotifyStatusChangedForHierarchy(countryId);
});
// Kick possibly interrupted downloader.
if (!m_queue.empty() && m_downloader->IsIdle())
DownloadNextCountryFromQueue();
}
void Storage::RetryDownloadNode(CountryId const & countryId)
@ -1755,7 +1733,7 @@ void Storage::RetryDownloadNode(CountryId const & countryId)
ForEachInSubtree(countryId, [this](CountryId const & descendantId, bool groupNode) {
if (!groupNode && m_failedCountries.count(descendantId) != 0)
{
bool const isUpdateRequest = m_diffManager.HasDiffFor(descendantId);
bool const isUpdateRequest = m_diffsDataSource->HasDiffFor(descendantId);
DownloadNode(descendantId, isUpdateRequest);
}
});
@ -1768,10 +1746,10 @@ bool Storage::GetUpdateInfo(CountryId const & countryId, UpdateInfo & updateInfo
return;
updateInfo.m_numberOfMwmFilesToUpdate += 1; // It's not a group mwm.
if (m_diffManager.HasDiffFor(node.Value().Name()))
if (m_diffsDataSource->HasDiffFor(node.Value().Name()))
{
uint64_t size;
m_diffManager.SizeToDownloadFor(node.Value().Name(), size);
m_diffsDataSource->SizeToDownloadFor(node.Value().Name(), size);
updateInfo.m_totalUpdateSizeInBytes += size;
}
else
@ -1931,12 +1909,8 @@ CountryId const Storage::GetTopmostParentFor(CountryId const & countryId) const
MwmSize Storage::GetRemoteSize(CountryFile const & file, int64_t version) const
{
uint64_t size;
if (m_diffManager.SizeFor(file.GetName(), size))
return size;
return file.GetRemoteSize();
return size;
ASSERT(m_diffsDataSource != nullptr, ());
return storage::GetRemoteSize(*m_diffsDataSource, file, version);
}
void Storage::OnMapDownloadFailed(CountryId const & countryId)

17
storage/storage.hpp
View file

@ -196,18 +196,22 @@ private:
// Used to cancel an ongoing diff application.
// |m_diffsCancellable| is reset every time when a task to apply a diff is posted.
// We use the fact that at most one diff is being applied at a time and the
// calls to the diff manager's ApplyDiff are coordinated from the storage thread.
// calls to the diffs::ApplyDiff are coordinated from the storage thread.
base::Cancellable m_diffsCancellable;
std::optional<CountryId> m_latestDiffRequest;
// Since the diff manager runs on a different thread, the result
// Since the diffs applying runs on a different thread, the result
// of diff application may return "Ok" when in fact the diff was
// cancelled. However, the storage thread knows for sure whether the
// latest request was to apply or to cancel the diff, and this knowledge
// is represented by |m_diffsBeingApplied|.
std::set<CountryId> m_diffsBeingApplied;
std::vector<platform::LocalCountryFile> m_notAppliedDiffs;
diffs::DiffsSourcePtr m_diffsDataSource = std::make_shared<diffs::DiffsDataSource>();
DownloadingPolicy m_defaultDownloadingPolicy;
DownloadingPolicy * m_downloadingPolicy = &m_defaultDownloadingPolicy;
@ -262,9 +266,6 @@ private:
ThreadChecker m_threadChecker;
diffs::Manager m_diffManager;
std::vector<platform::LocalCountryFile> m_notAppliedDiffs;
bool m_needToStartDeferredDownloading = false;
StartDownloadingCallback m_startDownloadingCallback;
@ -561,8 +562,6 @@ public:
CountryId GetCurrentDownloadingCountryId() const;
void EnableKeepDownloadingQueue(bool enable) {m_keepDownloadingQueue = enable;}
std::string GetDownloadRelativeUrl(CountryId const & countryId, MapFileType type) const;
/// @param[out] res Populated with oudated countries.
void GetOutdatedCountries(std::vector<Country const *> & countries) const;
@ -631,10 +630,6 @@ private:
// Removes country files from downloader.
bool DeleteCountryFilesFromDownloader(CountryId const & countryId);
// Returns download size of the currently downloading file for the
// queued country.
uint64_t GetDownloadSize(QueuedCountry const & queuedCountry) const;
// Returns a path to a place on disk downloader can use for
// downloaded files.
std::string GetFileDownloadPath(CountryId const & countryId, MapFileType file) const;

9
storage/storage_helpers.cpp
View file

@ -67,4 +67,13 @@ m2::RectD CalcLimitRect(CountryId const & countryId, Storage const & storage,
ASSERT(boundingBox.IsValid(), ());
return boundingBox;
}
MwmSize GetRemoteSize(diffs::DiffsDataSource const & diffsDataSource,
platform::CountryFile const & file, int64_t version)
{
uint64_t size;
if (diffsDataSource.SizeFor(file.GetName(), size))
return size;
return file.GetRemoteSize();
}
} // namespace storage

13
storage/storage_helpers.hpp
View file

@ -1,12 +1,14 @@
#pragma once
#include "storage/diff_scheme/diff_manager.hpp"
#include "storage/storage_defines.hpp"
#include "platform/country_defines.hpp"
#include "platform/country_file.hpp"
#include "geometry/point2d.hpp"
#include "geometry/rect2d.hpp"
#include "platform/country_defines.hpp"
#include "storage/storage_defines.hpp"
namespace storage
{
class CountryInfoGetter;
@ -29,4 +31,7 @@ bool IsEnoughSpaceForUpdate(CountryId const & countryId, Storage const & storage
/// \returns bounding box in mercator coordinates.
m2::RectD CalcLimitRect(CountryId const & countryId, Storage const & storage,
CountryInfoGetter const & countryInfoGetter);
MwmSize GetRemoteSize(diffs::DiffsDataSource const & diffsDataSource,
platform::CountryFile const & file, int64_t version);
} // namespace storage