kirylkaveryn/organicmaps
1
Fork 0
forked from organicmaps/organicmaps-tmp
Code Pull requests Activity

Apply diffs on startup

Browse source
This commit is contained in:
Arsentiy Milchakov 2017-08-22 14:57:15 +03:00 • committed by Vladimir Byko-Ianko
parent fd45889908
commit 1eb176f6fe
6 changed files with 206 additions and 116 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

43
platform/local_country_file_utils.cpp
View file

@ -138,6 +138,36 @@ inline string GetDataDirFullPath(string const & dataDir)
return dataDir.empty() ? platform.WritableDir()
: my::JoinFoldersToPath(platform.WritableDir(), dataDir);
}
void FindAllDiffsInDirectory(string const & dir, vector<LocalCountryFile> & diffs)
{
Platform & platform = GetPlatform();
Platform::TFilesWithType fwts;
platform.GetFilesByType(dir, Platform::FILE_TYPE_REGULAR, fwts);
for (auto const & fwt : fwts)
{
string name = fwt.first;
auto const isDiffReady =
strings::EndsWith(name, strings::to_string(DIFF_FILE_EXTENSION) + READY_FILE_EXTENSION);
auto const isDiff =
strings::EndsWith(name, DIFF_FILE_EXTENSION);
if (!isDiff && !isDiffReady)
continue;
my::GetNameWithoutExt(name);
if (isDiffReady)
my::GetNameWithoutExt(name);
LocalCountryFile localDiff(dir, CountryFile(name), 0 /* version */);
diffs.push_back(localDiff);
}
}
} // namespace
void DeleteDownloaderFilesForCountry(int64_t version, CountryFile const & countryFile)
@ -162,7 +192,6 @@ void FindAllLocalMapsInDirectoryAndCleanup(string const & directory, int64_t ver
int64_t latestVersion,
vector<LocalCountryFile> & localFiles)
{
vector<string> files;
Platform & platform = GetPlatform();
Platform::TFilesWithType fwts;
@ -224,6 +253,18 @@ void FindAllLocalMapsInDirectoryAndCleanup(string const & directory, int64_t ver
}
}
void FindAllDiffs(string const & dataDir, vector<LocalCountryFile> & diffs)
{
string const dir = GetDataDirFullPath(dataDir);
FindAllDiffsInDirectory(dir, diffs);
Platform::TFilesWithType fwts;
Platform::GetFilesByType(dir, Platform::FILE_TYPE_DIRECTORY, fwts);
for (auto const & fwt : fwts)
FindAllDiffsInDirectory(my::JoinFoldersToPath(dir, fwt.first /* subdir */), diffs);
}
void FindAllLocalMapsAndCleanup(int64_t latestVersion, vector<LocalCountryFile> & localFiles)
{
FindAllLocalMapsAndCleanup(latestVersion, string(), localFiles);

2
platform/local_country_file_utils.hpp
View file

@ -55,6 +55,8 @@ void FindAllLocalMapsAndCleanup(int64_t latestVersion, vector<LocalCountryFile>
void FindAllLocalMapsAndCleanup(int64_t latestVersion, string const & dataDir,
vector<LocalCountryFile> & localFiles);
void FindAllDiffs(string const & dataDir, vector<LocalCountryFile> & diffs);
// This method removes:
// * partially downloaded non-latest maps (with version less than |latestVersion|)
// * empty directories

30
storage/diff_scheme/diff_manager.cpp
View file

@ -39,9 +39,10 @@ void Manager::Load(LocalMapsInfo && info)
}
auto & observers = m_observers;
GetPlatform().RunOnGuiThread([observers]() mutable
auto status = m_status;
GetPlatform().RunOnGuiThread([observers, status]() mutable
{
observers.ForEach(&Observer::OnDiffStatusReceived);
observers.ForEach(&Observer::OnDiffStatusReceived, status);
});
});
}
@ -55,29 +56,30 @@ void Manager::ApplyDiff(ApplyDiffParams && p, std::function<void(bool const resu
auto & diffReadyPath = p.m_diffReadyPath;
auto & diffFile = p.m_diffFile;
auto const countryId = diffFile->GetCountryName();
auto const diffPath = diffFile->GetPath(MapOptions::Diff);
bool result = false;
if (!my::RenameFileX(diffReadyPath, diffFile->GetPath(MapOptions::Diff)))
{
diffFile->SyncWithDisk();
diffFile->DeleteFromDisk(MapOptions::Diff);
}
else
{
diffFile->SyncWithDisk();
diffFile->SyncWithDisk();
auto const isOnDisk = diffFile->OnDisk(MapOptions::Diff);
if (isOnDisk || my::RenameFileX(diffReadyPath, diffPath))
{
// Sync with disk after renaming.
if (!isOnDisk)
diffFile->SyncWithDisk();
string const oldMwmPath = p.m_oldMwmFile->GetPath(MapOptions::Map);
string const newMwmPath = diffFile->GetPath(MapOptions::Map);
string const diffPath = diffFile->GetPath(MapOptions::Diff);
result = generator::mwm_diff::ApplyDiff(oldMwmPath, newMwmPath, diffPath);
diffFile->DeleteFromDisk(MapOptions::Diff);
}
diffFile->DeleteFromDisk(MapOptions::Diff);
if (result)
{
std::lock_guard<std::mutex> lock(m_mutex);
m_diffs.erase(countryId);
m_diffs.erase(diffFile->GetCountryName());
if (m_diffs.empty())
m_status = Status::NotAvailable;
}

2
storage/diff_scheme/diff_manager.hpp
View file

@ -30,7 +30,7 @@ public:
public:
virtual ~Observer() = default;
virtual void OnDiffStatusReceived() = 0;
virtual void OnDiffStatusReceived(Status const status) = 0;
};
FileInfo const & InfoFor(storage::TCountryId const & countryId) const;

234
storage/storage.cpp
View file

@ -276,7 +276,8 @@ void Storage::RegisterAllLocalMaps(bool enableDiffs)
i = j;
}
FindAllDiffs(m_dataDir, m_notAppliedDiffs);
if (enableDiffs)
LoadDiffScheme();
RestoreDownloadQueue();
@ -465,7 +466,15 @@ void Storage::RestoreDownloadQueue()
return;
for (strings::SimpleTokenizer iter(token, ";"); iter; ++iter)
DownloadNode(*iter, isUpdate);
{
auto const diffIt = find_if(m_notAppliedDiffs.cbegin(), m_notAppliedDiffs.cend(),
[this, &iter](LocalCountryFile const & localDiff) {
return *iter == FindCountryIdByFile(localDiff.GetCountryName());
});
if (diffIt == m_notAppliedDiffs.end())
DownloadNode(*iter, isUpdate);
}
};
parse(download, false /* isUpdate */);
@ -809,97 +818,9 @@ void Storage::OnMapFileDownloadProgress(MapFilesDownloader::TProgress const & pr
ReportProgressForHierarchy(m_queue.front().GetCountryId(), progress);
}
void Storage::RegisterDownloadedFiles(TCountryId const & countryId, MapOptions files,
DownloadedFilesProcessingFn && fn)
void Storage::RegisterDownloadedFiles(TCountryId const & countryId, MapOptions options)
{
if (files == MapOptions::Diff)
{
diffs::Manager::ApplyDiffParams params;
params.m_diffFile =
PreparePlaceForCountryFiles(GetCurrentDataVersion(), m_dataDir, GetCountryFile(countryId));
params.m_diffReadyPath = GetFileDownloadPath(countryId, MapOptions::Diff);
params.m_oldMwmFile = GetLocalFile(countryId, m_diffManager.InfoFor(countryId).m_version);
TLocalFilePtr & diffFile = params.m_diffFile;
m_diffManager.ApplyDiff(move(params), [this, fn, diffFile] (bool const result)
{
GetPlatform().RunOnGuiThread([this, fn, diffFile, result]
{
if (result)
RegisterCountryFiles(diffFile);
fn(result);
});
});
return;
}
CountryFile const countryFile = GetCountryFile(countryId);
TLocalFilePtr localFile = GetLocalFile(countryId, GetCurrentDataVersion());
if (!localFile)
localFile = PreparePlaceForCountryFiles(GetCurrentDataVersion(), m_dataDir, countryFile);
if (!localFile)
{
LOG(LERROR, ("Local file data structure can't be prepared for downloaded file(", countryFile,
files, ")."));
fn(false /* isSuccess */);
return;
}
bool ok = true;
vector<MapOptions> mapOpt = {MapOptions::Map};
if (!version::IsSingleMwm(GetCurrentDataVersion()))
mapOpt.emplace_back(MapOptions::CarRouting);
for (MapOptions file : mapOpt)
{
if (!HasOptions(files, file))
continue;
string const path = GetFileDownloadPath(countryId, file);
if (!my::RenameFileX(path, localFile->GetPath(file)))
{
ok = false;
break;
}
}
if (!ok)
{
localFile->DeleteFromDisk(files);
fn(false);
return;
}
RegisterCountryFiles(localFile);
fn(true);
}
void Storage::OnMapDownloadFinished(TCountryId const & countryId, bool success, MapOptions files)
{
ASSERT_THREAD_CHECKER(m_threadChecker, ());
ASSERT(m_didDownload != nullptr, ("Storage::Init wasn't called"));
ASSERT_NOT_EQUAL(MapOptions::Nothing, files,
("This method should not be called for empty files set."));
alohalytics::LogEvent(
"$OnMapDownloadFinished",
alohalytics::TStringMap({{"name", countryId},
{"status", success ? "ok" : "failed"},
{"version", strings::to_string(GetCurrentDataVersion())},
{"option", DebugPrint(files)}}));
GetPlatform().GetMarketingService().SendMarketingEvent(marketing::kDownloaderMapActionFinished,
{{"action", "download"}});
if (!success)
{
m_failedCountries.insert(countryId);
NotifyStatusChangedForHierarchy(countryId);
return;
}
RegisterDownloadedFiles(countryId, files, [this, countryId](bool isSuccess)
auto const fn = [this, countryId](bool isSuccess)
{
ASSERT_THREAD_CHECKER(m_threadChecker, ());
if (!isSuccess)
@ -920,7 +841,78 @@ void Storage::OnMapDownloadFinished(TCountryId const & countryId, bool success,
m_downloader->Reset();
NotifyStatusChangedForHierarchy(countryId);
DownloadNextCountryFromQueue();
});
};
if (options == MapOptions::Diff)
{
ApplyDiff(countryId, fn);
return;
}
CountryFile const countryFile = GetCountryFile(countryId);
TLocalFilePtr localFile = GetLocalFile(countryId, GetCurrentDataVersion());
if (!localFile)
localFile = PreparePlaceForCountryFiles(GetCurrentDataVersion(), m_dataDir, countryFile);
if (!localFile)
{
LOG(LERROR, ("Local file data structure can't be prepared for downloaded file(", countryFile,
options, ")."));
fn(false /* isSuccess */);
return;
}
bool ok = true;
vector<MapOptions> mapOpt = {MapOptions::Map};
if (!version::IsSingleMwm(GetCurrentDataVersion()))
mapOpt.emplace_back(MapOptions::CarRouting);
for (MapOptions file : mapOpt)
{
if (!HasOptions(options, file))
continue;
string const path = GetFileDownloadPath(countryId, file);
if (!my::RenameFileX(path, localFile->GetPath(file)))
{
ok = false;
break;
}
}
if (!ok)
{
localFile->DeleteFromDisk(options);
fn(false);
return;
}
RegisterCountryFiles(localFile);
fn(true);
}
void Storage::OnMapDownloadFinished(TCountryId const & countryId, bool success, MapOptions options)
{
ASSERT_THREAD_CHECKER(m_threadChecker, ());
ASSERT(m_didDownload != nullptr, ("Storage::Init wasn't called"));
ASSERT_NOT_EQUAL(MapOptions::Nothing, options,
("This method should not be called for empty files set."));
alohalytics::LogEvent(
"$OnMapDownloadFinished",
alohalytics::TStringMap({{"name", countryId},
{"status", success ? "ok" : "failed"},
{"version", strings::to_string(GetCurrentDataVersion())},
{"option", DebugPrint(options)}}));
GetPlatform().GetMarketingService().SendMarketingEvent(marketing::kDownloaderMapActionFinished,
{{"action", "download"}});
if (!success)
{
m_failedCountries.insert(countryId);
NotifyStatusChangedForHierarchy(countryId);
return;
}
RegisterDownloadedFiles(countryId, options);
}
string Storage::GetFileDownloadUrl(string const & baseUrl,
@ -1419,14 +1411,68 @@ void Storage::LoadDiffScheme()
m_diffManager.Load(move(localMapsInfo));
}
void Storage::ApplyDiff(TCountryId const & countryId, function<void(bool isSuccess)> const & fn)
{
diffs::Manager::ApplyDiffParams params;
params.m_diffFile =
PreparePlaceForCountryFiles(GetCurrentDataVersion(), m_dataDir, GetCountryFile(countryId));
params.m_diffReadyPath = GetFileDownloadPath(countryId, MapOptions::Diff);
params.m_oldMwmFile = GetLocalFile(countryId, m_diffManager.InfoFor(countryId).m_version);
TLocalFilePtr & diffFile = params.m_diffFile;
m_diffManager.ApplyDiff(move(params), [this, fn, diffFile] (bool const result)
{
GetPlatform().RunOnGuiThread([this, fn, diffFile, result]
{
if (result)
RegisterCountryFiles(diffFile);
fn(result);
});
});
}
bool Storage::IsPossibleToAutoupdate() const
{
ASSERT_THREAD_CHECKER(m_threadChecker, ());
return m_diffManager.IsPossibleToAutoupdate();
}
void Storage::OnDiffStatusReceived()
void Storage::OnDiffStatusReceived(diffs::Status const status)
{
if (status != diffs::Status::NotAvailable)
{
for (auto const & localDiff : m_notAppliedDiffs)
{
auto const countryId = FindCountryIdByFile(localDiff.GetCountryName());
auto const registerDiffFn = [this, countryId](bool isSuccess)
{
ASSERT_THREAD_CHECKER(m_threadChecker, ());
if (!isSuccess)
{
m_failedCountries.insert(countryId);
NotifyStatusChangedForHierarchy(countryId);
return;
}
TLocalFilePtr localFile = GetLocalFile(countryId, GetCurrentDataVersion());
ASSERT(localFile, ());
DeleteCountryIndexes(*localFile);
m_didDownload(countryId, localFile);
NotifyStatusChangedForHierarchy(countryId);
};
if (m_diffManager.HasDiffFor(countryId))
ApplyDiff(countryId, registerDiffFn);
else
localDiff.DeleteFromDisk(MapOptions::Diff);
}
m_notAppliedDiffs.clear();
}
for (auto const & urls : m_deferredDownloads)
OnServerListDownloaded(urls);

11
storage/storage.hpp
View file

@ -247,6 +247,7 @@ private:
ThreadChecker m_threadChecker;
diffs::Manager m_diffManager;
vector<platform::LocalCountryFile> m_notAppliedDiffs;
vector<vector<string>> m_deferredDownloads;
@ -271,9 +272,7 @@ private:
/// during the downloading process.
void OnMapFileDownloadProgress(MapFilesDownloader::TProgress const & progress);
using DownloadedFilesProcessingFn = function<void(bool isSuccess)>;
void RegisterDownloadedFiles(TCountryId const & countryId, MapOptions files,
DownloadedFilesProcessingFn && fn);
void RegisterDownloadedFiles(TCountryId const & countryId, MapOptions files);
void OnMapDownloadFinished(TCountryId const & countryId, bool success, MapOptions files);
@ -508,8 +507,7 @@ public:
bool IsInnerNode(TCountryId const & countryId) const;
TLocalAndRemoteSize CountrySizeInBytes(TCountryId const & countryId, MapOptions opt) const;
TMwmSize GetRemoteSize(platform::CountryFile const & file, MapOptions opt,
int64_t version) const;
TMwmSize GetRemoteSize(platform::CountryFile const & file, MapOptions opt, int64_t version) const;
platform::CountryFile const & GetCountryFile(TCountryId const & countryId) const;
TLocalFilePtr GetLatestLocalFile(platform::CountryFile const & countryFile) const;
TLocalFilePtr GetLatestLocalFile(TCountryId const & countryId) const;
@ -660,8 +658,9 @@ private:
void CalMaxMwmSizeBytes();
void LoadDiffScheme();
void ApplyDiff(TCountryId const & countryId, function<void(bool isSuccess)> const & fn);
void OnDiffStatusReceived() override;
void OnDiffStatusReceived(diffs::Status const status) override;
};
void GetQueuedCountries(Storage::TQueue const & queue, TCountriesSet & resultCountries);