From 1eb176f6fefa4f0dfac0ce82865efafedb09655c Mon Sep 17 00:00:00 2001 From: Arsentiy Milchakov Date: Tue, 22 Aug 2017 14:57:15 +0300 Subject: [PATCH] Apply diffs on startup --- platform/local_country_file_utils.cpp | 43 ++++- platform/local_country_file_utils.hpp | 2 + storage/diff_scheme/diff_manager.cpp | 30 ++-- storage/diff_scheme/diff_manager.hpp | 2 +- storage/storage.cpp | 234 +++++++++++++++----------- storage/storage.hpp | 11 +- 6 files changed, 206 insertions(+), 116 deletions(-) diff --git a/platform/local_country_file_utils.cpp b/platform/local_country_file_utils.cpp index b25809a4a4..8593bdbb74 100644 --- a/platform/local_country_file_utils.cpp +++ b/platform/local_country_file_utils.cpp @@ -138,6 +138,36 @@ inline string GetDataDirFullPath(string const & dataDir) return dataDir.empty() ? platform.WritableDir() : my::JoinFoldersToPath(platform.WritableDir(), dataDir); } + +void FindAllDiffsInDirectory(string const & dir, vector & diffs) +{ + Platform & platform = GetPlatform(); + + Platform::TFilesWithType fwts; + platform.GetFilesByType(dir, Platform::FILE_TYPE_REGULAR, fwts); + + for (auto const & fwt : fwts) + { + string name = fwt.first; + + auto const isDiffReady = + strings::EndsWith(name, strings::to_string(DIFF_FILE_EXTENSION) + READY_FILE_EXTENSION); + auto const isDiff = + strings::EndsWith(name, DIFF_FILE_EXTENSION); + + if (!isDiff && !isDiffReady) + continue; + + my::GetNameWithoutExt(name); + + if (isDiffReady) + my::GetNameWithoutExt(name); + + LocalCountryFile localDiff(dir, CountryFile(name), 0 /* version */); + + diffs.push_back(localDiff); + } +} } // namespace void DeleteDownloaderFilesForCountry(int64_t version, CountryFile const & countryFile) @@ -162,7 +192,6 @@ void FindAllLocalMapsInDirectoryAndCleanup(string const & directory, int64_t ver int64_t latestVersion, vector & localFiles) { - vector files; Platform & platform = GetPlatform(); Platform::TFilesWithType fwts; @@ -224,6 +253,18 @@ void FindAllLocalMapsInDirectoryAndCleanup(string const & directory, int64_t ver } } +void FindAllDiffs(string const & dataDir, vector & diffs) +{ + string const dir = GetDataDirFullPath(dataDir); + FindAllDiffsInDirectory(dir, diffs); + + Platform::TFilesWithType fwts; + Platform::GetFilesByType(dir, Platform::FILE_TYPE_DIRECTORY, fwts); + + for (auto const & fwt : fwts) + FindAllDiffsInDirectory(my::JoinFoldersToPath(dir, fwt.first /* subdir */), diffs); +} + void FindAllLocalMapsAndCleanup(int64_t latestVersion, vector & localFiles) { FindAllLocalMapsAndCleanup(latestVersion, string(), localFiles); diff --git a/platform/local_country_file_utils.hpp b/platform/local_country_file_utils.hpp index 3d53918300..433dd643ac 100644 --- a/platform/local_country_file_utils.hpp +++ b/platform/local_country_file_utils.hpp @@ -55,6 +55,8 @@ void FindAllLocalMapsAndCleanup(int64_t latestVersion, vector void FindAllLocalMapsAndCleanup(int64_t latestVersion, string const & dataDir, vector & localFiles); +void FindAllDiffs(string const & dataDir, vector & diffs); + // This method removes: // * partially downloaded non-latest maps (with version less than |latestVersion|) // * empty directories diff --git a/storage/diff_scheme/diff_manager.cpp b/storage/diff_scheme/diff_manager.cpp index 7bc9347364..1fbd6bcf21 100644 --- a/storage/diff_scheme/diff_manager.cpp +++ b/storage/diff_scheme/diff_manager.cpp @@ -39,9 +39,10 @@ void Manager::Load(LocalMapsInfo && info) } auto & observers = m_observers; - GetPlatform().RunOnGuiThread([observers]() mutable + auto status = m_status; + GetPlatform().RunOnGuiThread([observers, status]() mutable { - observers.ForEach(&Observer::OnDiffStatusReceived); + observers.ForEach(&Observer::OnDiffStatusReceived, status); }); }); } @@ -55,29 +56,30 @@ void Manager::ApplyDiff(ApplyDiffParams && p, std::functionGetCountryName(); + auto const diffPath = diffFile->GetPath(MapOptions::Diff); bool result = false; - if (!my::RenameFileX(diffReadyPath, diffFile->GetPath(MapOptions::Diff))) - { - diffFile->SyncWithDisk(); - diffFile->DeleteFromDisk(MapOptions::Diff); - } - else - { - diffFile->SyncWithDisk(); + diffFile->SyncWithDisk(); + auto const isOnDisk = diffFile->OnDisk(MapOptions::Diff); + + if (isOnDisk || my::RenameFileX(diffReadyPath, diffPath)) + { + // Sync with disk after renaming. + if (!isOnDisk) + diffFile->SyncWithDisk(); + string const oldMwmPath = p.m_oldMwmFile->GetPath(MapOptions::Map); string const newMwmPath = diffFile->GetPath(MapOptions::Map); - string const diffPath = diffFile->GetPath(MapOptions::Diff); result = generator::mwm_diff::ApplyDiff(oldMwmPath, newMwmPath, diffPath); - diffFile->DeleteFromDisk(MapOptions::Diff); } + diffFile->DeleteFromDisk(MapOptions::Diff); + if (result) { std::lock_guard lock(m_mutex); - m_diffs.erase(countryId); + m_diffs.erase(diffFile->GetCountryName()); if (m_diffs.empty()) m_status = Status::NotAvailable; } diff --git a/storage/diff_scheme/diff_manager.hpp b/storage/diff_scheme/diff_manager.hpp index 46247f172a..0aa25f95ee 100644 --- a/storage/diff_scheme/diff_manager.hpp +++ b/storage/diff_scheme/diff_manager.hpp @@ -30,7 +30,7 @@ public: public: virtual ~Observer() = default; - virtual void OnDiffStatusReceived() = 0; + virtual void OnDiffStatusReceived(Status const status) = 0; }; FileInfo const & InfoFor(storage::TCountryId const & countryId) const; diff --git a/storage/storage.cpp b/storage/storage.cpp index 63ab7786e2..a7e8f28fa0 100644 --- a/storage/storage.cpp +++ b/storage/storage.cpp @@ -276,7 +276,8 @@ void Storage::RegisterAllLocalMaps(bool enableDiffs) i = j; } - + + FindAllDiffs(m_dataDir, m_notAppliedDiffs); if (enableDiffs) LoadDiffScheme(); RestoreDownloadQueue(); @@ -465,7 +466,15 @@ void Storage::RestoreDownloadQueue() return; for (strings::SimpleTokenizer iter(token, ";"); iter; ++iter) - DownloadNode(*iter, isUpdate); + { + auto const diffIt = find_if(m_notAppliedDiffs.cbegin(), m_notAppliedDiffs.cend(), + [this, &iter](LocalCountryFile const & localDiff) { + return *iter == FindCountryIdByFile(localDiff.GetCountryName()); + }); + + if (diffIt == m_notAppliedDiffs.end()) + DownloadNode(*iter, isUpdate); + } }; parse(download, false /* isUpdate */); @@ -809,97 +818,9 @@ void Storage::OnMapFileDownloadProgress(MapFilesDownloader::TProgress const & pr ReportProgressForHierarchy(m_queue.front().GetCountryId(), progress); } -void Storage::RegisterDownloadedFiles(TCountryId const & countryId, MapOptions files, - DownloadedFilesProcessingFn && fn) +void Storage::RegisterDownloadedFiles(TCountryId const & countryId, MapOptions options) { - if (files == MapOptions::Diff) - { - diffs::Manager::ApplyDiffParams params; - params.m_diffFile = - PreparePlaceForCountryFiles(GetCurrentDataVersion(), m_dataDir, GetCountryFile(countryId)); - params.m_diffReadyPath = GetFileDownloadPath(countryId, MapOptions::Diff); - params.m_oldMwmFile = GetLocalFile(countryId, m_diffManager.InfoFor(countryId).m_version); - - TLocalFilePtr & diffFile = params.m_diffFile; - - m_diffManager.ApplyDiff(move(params), [this, fn, diffFile] (bool const result) - { - GetPlatform().RunOnGuiThread([this, fn, diffFile, result] - { - if (result) - RegisterCountryFiles(diffFile); - - fn(result); - }); - }); - - return; - } - - CountryFile const countryFile = GetCountryFile(countryId); - TLocalFilePtr localFile = GetLocalFile(countryId, GetCurrentDataVersion()); - if (!localFile) - localFile = PreparePlaceForCountryFiles(GetCurrentDataVersion(), m_dataDir, countryFile); - if (!localFile) - { - LOG(LERROR, ("Local file data structure can't be prepared for downloaded file(", countryFile, - files, ").")); - fn(false /* isSuccess */); - return; - } - - bool ok = true; - vector mapOpt = {MapOptions::Map}; - if (!version::IsSingleMwm(GetCurrentDataVersion())) - mapOpt.emplace_back(MapOptions::CarRouting); - - for (MapOptions file : mapOpt) - { - if (!HasOptions(files, file)) - continue; - string const path = GetFileDownloadPath(countryId, file); - if (!my::RenameFileX(path, localFile->GetPath(file))) - { - ok = false; - break; - } - } - - if (!ok) - { - localFile->DeleteFromDisk(files); - fn(false); - return; - } - - RegisterCountryFiles(localFile); - fn(true); -} - -void Storage::OnMapDownloadFinished(TCountryId const & countryId, bool success, MapOptions files) -{ - ASSERT_THREAD_CHECKER(m_threadChecker, ()); - ASSERT(m_didDownload != nullptr, ("Storage::Init wasn't called")); - ASSERT_NOT_EQUAL(MapOptions::Nothing, files, - ("This method should not be called for empty files set.")); - - alohalytics::LogEvent( - "$OnMapDownloadFinished", - alohalytics::TStringMap({{"name", countryId}, - {"status", success ? "ok" : "failed"}, - {"version", strings::to_string(GetCurrentDataVersion())}, - {"option", DebugPrint(files)}})); - GetPlatform().GetMarketingService().SendMarketingEvent(marketing::kDownloaderMapActionFinished, - {{"action", "download"}}); - - if (!success) - { - m_failedCountries.insert(countryId); - NotifyStatusChangedForHierarchy(countryId); - return; - } - - RegisterDownloadedFiles(countryId, files, [this, countryId](bool isSuccess) + auto const fn = [this, countryId](bool isSuccess) { ASSERT_THREAD_CHECKER(m_threadChecker, ()); if (!isSuccess) @@ -920,7 +841,78 @@ void Storage::OnMapDownloadFinished(TCountryId const & countryId, bool success, m_downloader->Reset(); NotifyStatusChangedForHierarchy(countryId); DownloadNextCountryFromQueue(); - }); + }; + + if (options == MapOptions::Diff) + { + ApplyDiff(countryId, fn); + return; + } + + CountryFile const countryFile = GetCountryFile(countryId); + TLocalFilePtr localFile = GetLocalFile(countryId, GetCurrentDataVersion()); + if (!localFile) + localFile = PreparePlaceForCountryFiles(GetCurrentDataVersion(), m_dataDir, countryFile); + if (!localFile) + { + LOG(LERROR, ("Local file data structure can't be prepared for downloaded file(", countryFile, + options, ").")); + fn(false /* isSuccess */); + return; + } + + bool ok = true; + vector mapOpt = {MapOptions::Map}; + if (!version::IsSingleMwm(GetCurrentDataVersion())) + mapOpt.emplace_back(MapOptions::CarRouting); + + for (MapOptions file : mapOpt) + { + if (!HasOptions(options, file)) + continue; + string const path = GetFileDownloadPath(countryId, file); + if (!my::RenameFileX(path, localFile->GetPath(file))) + { + ok = false; + break; + } + } + + if (!ok) + { + localFile->DeleteFromDisk(options); + fn(false); + return; + } + + RegisterCountryFiles(localFile); + fn(true); +} + +void Storage::OnMapDownloadFinished(TCountryId const & countryId, bool success, MapOptions options) +{ + ASSERT_THREAD_CHECKER(m_threadChecker, ()); + ASSERT(m_didDownload != nullptr, ("Storage::Init wasn't called")); + ASSERT_NOT_EQUAL(MapOptions::Nothing, options, + ("This method should not be called for empty files set.")); + + alohalytics::LogEvent( + "$OnMapDownloadFinished", + alohalytics::TStringMap({{"name", countryId}, + {"status", success ? "ok" : "failed"}, + {"version", strings::to_string(GetCurrentDataVersion())}, + {"option", DebugPrint(options)}})); + GetPlatform().GetMarketingService().SendMarketingEvent(marketing::kDownloaderMapActionFinished, + {{"action", "download"}}); + + if (!success) + { + m_failedCountries.insert(countryId); + NotifyStatusChangedForHierarchy(countryId); + return; + } + + RegisterDownloadedFiles(countryId, options); } string Storage::GetFileDownloadUrl(string const & baseUrl, @@ -1419,14 +1411,68 @@ void Storage::LoadDiffScheme() m_diffManager.Load(move(localMapsInfo)); } +void Storage::ApplyDiff(TCountryId const & countryId, function const & fn) +{ + diffs::Manager::ApplyDiffParams params; + params.m_diffFile = + PreparePlaceForCountryFiles(GetCurrentDataVersion(), m_dataDir, GetCountryFile(countryId)); + params.m_diffReadyPath = GetFileDownloadPath(countryId, MapOptions::Diff); + params.m_oldMwmFile = GetLocalFile(countryId, m_diffManager.InfoFor(countryId).m_version); + + TLocalFilePtr & diffFile = params.m_diffFile; + + m_diffManager.ApplyDiff(move(params), [this, fn, diffFile] (bool const result) + { + GetPlatform().RunOnGuiThread([this, fn, diffFile, result] + { + if (result) + RegisterCountryFiles(diffFile); + + fn(result); + }); + }); +} + bool Storage::IsPossibleToAutoupdate() const { ASSERT_THREAD_CHECKER(m_threadChecker, ()); return m_diffManager.IsPossibleToAutoupdate(); } -void Storage::OnDiffStatusReceived() +void Storage::OnDiffStatusReceived(diffs::Status const status) { + if (status != diffs::Status::NotAvailable) + { + for (auto const & localDiff : m_notAppliedDiffs) + { + auto const countryId = FindCountryIdByFile(localDiff.GetCountryName()); + + auto const registerDiffFn = [this, countryId](bool isSuccess) + { + ASSERT_THREAD_CHECKER(m_threadChecker, ()); + if (!isSuccess) + { + m_failedCountries.insert(countryId); + NotifyStatusChangedForHierarchy(countryId); + return; + } + + TLocalFilePtr localFile = GetLocalFile(countryId, GetCurrentDataVersion()); + ASSERT(localFile, ()); + DeleteCountryIndexes(*localFile); + m_didDownload(countryId, localFile); + NotifyStatusChangedForHierarchy(countryId); + }; + + if (m_diffManager.HasDiffFor(countryId)) + ApplyDiff(countryId, registerDiffFn); + else + localDiff.DeleteFromDisk(MapOptions::Diff); + } + + m_notAppliedDiffs.clear(); + } + for (auto const & urls : m_deferredDownloads) OnServerListDownloaded(urls); diff --git a/storage/storage.hpp b/storage/storage.hpp index 251fcc0700..483860eec2 100644 --- a/storage/storage.hpp +++ b/storage/storage.hpp @@ -247,6 +247,7 @@ private: ThreadChecker m_threadChecker; diffs::Manager m_diffManager; + vector m_notAppliedDiffs; vector> m_deferredDownloads; @@ -271,9 +272,7 @@ private: /// during the downloading process. void OnMapFileDownloadProgress(MapFilesDownloader::TProgress const & progress); - using DownloadedFilesProcessingFn = function; - void RegisterDownloadedFiles(TCountryId const & countryId, MapOptions files, - DownloadedFilesProcessingFn && fn); + void RegisterDownloadedFiles(TCountryId const & countryId, MapOptions files); void OnMapDownloadFinished(TCountryId const & countryId, bool success, MapOptions files); @@ -508,8 +507,7 @@ public: bool IsInnerNode(TCountryId const & countryId) const; TLocalAndRemoteSize CountrySizeInBytes(TCountryId const & countryId, MapOptions opt) const; - TMwmSize GetRemoteSize(platform::CountryFile const & file, MapOptions opt, - int64_t version) const; + TMwmSize GetRemoteSize(platform::CountryFile const & file, MapOptions opt, int64_t version) const; platform::CountryFile const & GetCountryFile(TCountryId const & countryId) const; TLocalFilePtr GetLatestLocalFile(platform::CountryFile const & countryFile) const; TLocalFilePtr GetLatestLocalFile(TCountryId const & countryId) const; @@ -660,8 +658,9 @@ private: void CalMaxMwmSizeBytes(); void LoadDiffScheme(); + void ApplyDiff(TCountryId const & countryId, function const & fn); - void OnDiffStatusReceived() override; + void OnDiffStatusReceived(diffs::Status const status) override; }; void GetQueuedCountries(Storage::TQueue const & queue, TCountriesSet & resultCountries);