diff --git a/iphone/Maps/Classes/MapsAppDelegate.mm b/iphone/Maps/Classes/MapsAppDelegate.mm index f22a32cfad..54e28fe709 100644 --- a/iphone/Maps/Classes/MapsAppDelegate.mm +++ b/iphone/Maps/Classes/MapsAppDelegate.mm @@ -595,7 +595,9 @@ using namespace osm_auth_ios; --m_activeDownloadsCounter; if (m_activeDownloadsCounter <= 0) { - UIApplication.sharedApplication.networkActivityIndicatorVisible = NO; + dispatch_async(dispatch_get_main_queue(), ^{ + UIApplication.sharedApplication.networkActivityIndicatorVisible = NO; + }); m_activeDownloadsCounter = 0; if (UIApplication.sharedApplication.applicationState == UIApplicationStateBackground) { @@ -608,7 +610,9 @@ using namespace osm_auth_ios; - (void)enableDownloadIndicator { ++m_activeDownloadsCounter; - UIApplication.sharedApplication.networkActivityIndicatorVisible = YES; + dispatch_async(dispatch_get_main_queue(), ^{ + UIApplication.sharedApplication.networkActivityIndicatorVisible = YES; + }); } + (NSDictionary *)navigationBarTextAttributes diff --git a/map/bookmark_manager.cpp b/map/bookmark_manager.cpp index 25040cdb27..87fdbe4e3b 100644 --- a/map/bookmark_manager.cpp +++ b/map/bookmark_manager.cpp @@ -48,14 +48,14 @@ char const * KMZ_EXTENSION = ".kmz"; char const * kBookmarksExt = ".kmb"; // Returns extension with a dot in a lower case. -std::string const GetFileExt(std::string const & filePath) +std::string GetFileExt(std::string const & filePath) { std::string ext = my::GetFileExtension(filePath); std::transform(ext.begin(), ext.end(), ext.begin(), ::tolower); return ext; } -std::string const GetFileName(std::string const & filePath) +std::string GetFileName(std::string const & filePath) { std::string ret = filePath; my::GetNameFromFullPath(ret); @@ -142,6 +142,53 @@ BookmarkManager::SharingResult GetFileForSharing(df::MarkGroupID categoryId, std return BookmarkManager::SharingResult(categoryId, tmpFilePath); } + +bool ConvertBeforeUploading(std::string const & filePath, std::string const & convertedFilePath) +{ + //TODO: convert from kmb to kmz. + return CreateZipFromPathDeflatedAndDefaultCompression(filePath, convertedFilePath); +} + +bool ConvertAfterDownloading(std::string const & filePath, std::string const & convertedFilePath) +{ + ZipFileReader::FileListT files; + ZipFileReader::FilesList(filePath, files); + if (files.empty()) + return false; + + std::string const unarchievedPath = filePath + ".raw"; + MY_SCOPE_GUARD(fileGuard, bind(&FileWriter::DeleteFileX, unarchievedPath)); + ZipFileReader::UnzipFile(filePath, files.front().first, unarchievedPath); + if (!GetPlatform().IsFileExistsByFullPath(unarchievedPath)) + return false; + + kml::FileData kmlData; + try + { + kml::DeserializerKml des(kmlData); + FileReader reader(unarchievedPath); + des.Deserialize(reader); + } + catch (FileReader::Exception const & exc) + { + LOG(LWARNING, ("KML text deserialization failure: ", exc.what(), "file:", unarchievedPath)); + return false; + } + + try + { + kml::binary::SerializerKml ser(kmlData); + FileWriter writer(convertedFilePath); + ser.Serialize(writer); + } + catch (FileWriter::Exception const & exc) + { + LOG(LWARNING, ("KML binary serialization failure: ", exc.what(), "file:", convertedFilePath)); + return false; + } + + return true; +} } // namespace namespace migration @@ -349,13 +396,17 @@ void MigrateIfNeeded() } } // namespace migration +using namespace std::placeholders; + BookmarkManager::BookmarkManager(Callbacks && callbacks) : m_callbacks(std::move(callbacks)) , m_changesTracker(*this) , m_needTeardown(false) , m_nextGroupID(UserMark::BOOKMARK) , m_bookmarkCloud(Cloud::CloudParams("bmc.json", "bookmarks", "BookmarkCloudParam", - std::string(KMZ_EXTENSION))) + GetBookmarksDirectory(), std::string(kBookmarksExt), + std::bind(&ConvertBeforeUploading, _1, _2), + std::bind(&ConvertAfterDownloading, _1, _2))) { ASSERT(m_callbacks.m_getStringsBundle != nullptr, ()); m_userMarkLayers.reserve(UserMark::BOOKMARK); @@ -365,27 +416,12 @@ BookmarkManager::BookmarkManager(Callbacks && callbacks) m_selectionMark = CreateUserMark(m2::PointD{}); m_myPositionMark = CreateUserMark(m2::PointD{}); - m_bookmarkCloud.SetSynchronizationHandlers([]() - { - alohalytics::Stats::Instance().LogEvent("Bookmarks_sync_started"); - }, [](Cloud::SynchronizationResult result, std::string const & errorStr) - { - if (result == Cloud::SynchronizationResult::Success) - { - alohalytics::Stats::Instance().LogEvent("Bookmarks_sync_success"); - } - else - { - std::string const typeStr = (result == Cloud::SynchronizationResult::DiskError) ? "disk" : "network"; - alohalytics::TStringMap details {{"type", typeStr}, {"error", errorStr}}; - alohalytics::Stats::Instance().LogEvent("Bookmarks_sync_error", details); - } - }); -} - -BookmarkManager::~BookmarkManager() -{ - ClearCategories(); + using namespace std::placeholders; + m_bookmarkCloud.SetSynchronizationHandlers( + std::bind(&BookmarkManager::OnSynchronizationStarted, this, _1), + std::bind(&BookmarkManager::OnSynchronizationFinished, this, _1, _2, _3), + std::bind(&BookmarkManager::OnRestoreRequested, this, _1, _2), + std::bind(&BookmarkManager::OnRestoredFilesPrepared, this)); } BookmarkManager::EditSession BookmarkManager::GetEditSession() @@ -764,8 +800,16 @@ void BookmarkManager::LoadState() void BookmarkManager::ClearCategories() { ASSERT_THREAD_CHECKER(m_threadChecker, ()); + for (auto groupId : m_bmGroupsIdList) + { + ClearGroup(groupId); + m_changesTracker.OnDeleteGroup(groupId); + } + m_categories.clear(); m_bmGroupsIdList.clear(); + m_bookmarks.clear(); + m_tracks.clear(); } void BookmarkManager::LoadBookmarks() @@ -1637,6 +1681,74 @@ void BookmarkManager::ConvertAllKmlFiles(ConversionHandler && handler) const }); } +void BookmarkManager::SetCloudHandlers( + Cloud::SynchronizationStartedHandler && onSynchronizationStarted, + Cloud::SynchronizationFinishedHandler && onSynchronizationFinished, + Cloud::RestoreRequestedHandler && onRestoreRequested, + Cloud::RestoredFilesPreparedHandler && onRestoredFilesPrepared) +{ + m_onSynchronizationStarted = std::move(onSynchronizationStarted); + m_onSynchronizationFinished = std::move(onSynchronizationFinished); + m_onRestoreRequested = std::move(onRestoreRequested); + m_onRestoredFilesPrepared = std::move(onRestoredFilesPrepared); +} + +void BookmarkManager::OnSynchronizationStarted(Cloud::SynchronizationType type) +{ + GetPlatform().RunTask(Platform::Thread::Gui, [this, type]() + { + if (m_onSynchronizationStarted) + m_onSynchronizationStarted(type); + }); + + LOG(LINFO, ("Cloud Synchronization Started:", type)); +} + +void BookmarkManager::OnSynchronizationFinished(Cloud::SynchronizationType type, + Cloud::SynchronizationResult result, + std::string const & errorStr) +{ + GetPlatform().RunTask(Platform::Thread::Gui, [this, type, result, errorStr]() + { + if (m_onSynchronizationFinished) + m_onSynchronizationFinished(type, result, errorStr); + + if (type == Cloud::SynchronizationType::Restore && + result == Cloud::SynchronizationResult::Success) + { + // Reload bookmarks after restoring. + LoadBookmarks(); + } + }); + + LOG(LINFO, ("Cloud Synchronization Finished:", type, result, errorStr)); +} + +void BookmarkManager::OnRestoreRequested(Cloud::RestoringRequestResult result, + uint64_t backupTimestampInMs) +{ + GetPlatform().RunTask(Platform::Thread::Gui, [this, result, backupTimestampInMs]() + { + if (m_onRestoreRequested) + m_onRestoreRequested(result, backupTimestampInMs); + }); + + using namespace std::chrono; + LOG(LINFO, ("Cloud Restore Requested:", result, + time_point(milliseconds(backupTimestampInMs)))); +} + +void BookmarkManager::OnRestoredFilesPrepared() +{ + // This method is always called from UI-thread. + ClearCategories(); + + if (m_onRestoredFilesPrepared) + m_onRestoredFilesPrepared(); + + LOG(LINFO, ("Cloud Restore Files: Prepared")); +} + df::GroupIDSet BookmarkManager::MarksChangesTracker::GetAllGroupIds() const { auto const & groupIds = m_bmManager.GetBmGroupsIdList(); diff --git a/map/bookmark_manager.hpp b/map/bookmark_manager.hpp index e2dd565ce3..2bb00245bc 100644 --- a/map/bookmark_manager.hpp +++ b/map/bookmark_manager.hpp @@ -128,7 +128,6 @@ public: }; explicit BookmarkManager(Callbacks && callbacks); - ~BookmarkManager(); void SetDrapeEngine(ref_ptr engine); @@ -244,6 +243,12 @@ public: using ConversionHandler = platform::SafeCallback; void ConvertAllKmlFiles(ConversionHandler && handler) const; + // These handlers are always called from UI-thread. + void SetCloudHandlers(Cloud::SynchronizationStartedHandler && onSynchronizationStarted, + Cloud::SynchronizationFinishedHandler && onSynchronizationFinished, + Cloud::RestoreRequestedHandler && onRestoreRequested, + Cloud::RestoredFilesPreparedHandler && onRestoredFilesPrepared); + /// These functions are public for unit tests only. You shouldn't call them from client code. void SaveToKML(df::MarkGroupID groupId, std::ostream & s); void CreateCategories(KMLDataCollection && dataCollection, bool autoSave = true); @@ -397,6 +402,12 @@ private: void SaveToKML(BookmarkCategory * group, std::ostream & s); + void OnSynchronizationStarted(Cloud::SynchronizationType type); + void OnSynchronizationFinished(Cloud::SynchronizationType type, Cloud::SynchronizationResult result, + std::string const & errorStr); + void OnRestoreRequested(Cloud::RestoringRequestResult result, uint64_t backupTimestampInMs); + void OnRestoredFilesPrepared(); + ThreadChecker m_threadChecker; Callbacks m_callbacks; @@ -438,6 +449,10 @@ private: std::list m_bookmarkLoadingQueue; Cloud m_bookmarkCloud; + Cloud::SynchronizationStartedHandler m_onSynchronizationStarted; + Cloud::SynchronizationFinishedHandler m_onSynchronizationFinished; + Cloud::RestoreRequestedHandler m_onRestoreRequested; + Cloud::RestoredFilesPreparedHandler m_onRestoredFilesPrepared; DISALLOW_COPY_AND_MOVE(BookmarkManager); }; diff --git a/map/cloud.cpp b/map/cloud.cpp index 80ab3c1aec..a5936a6331 100644 --- a/map/cloud.cpp +++ b/map/cloud.cpp @@ -5,11 +5,11 @@ #include "coding/file_writer.hpp" #include "coding/internal/file_data.hpp" #include "coding/sha1.hpp" -#include "coding/zip_creator.hpp" #include "platform/network_policy.hpp" #include "platform/http_client.hpp" #include "platform/platform.hpp" +#include "platform/preferred_languages.hpp" #include "platform/settings.hpp" #include "platform/http_uploader.hpp" @@ -17,6 +17,8 @@ #include "base/logging.hpp" #include "base/stl_helpers.hpp" +#include "3party/Alohalytics/src/alohalytics.h" + #include #include #include @@ -37,18 +39,29 @@ uint32_t constexpr kRetryDegradationFactor = 2; uint64_t constexpr kMaxWwanUploadingSizeInBytes = 10 * 1024; // 10Kb uint64_t constexpr kMaxUploadingFileSizeInBytes = 100 * 1024 * 1024; // 100Mb +uint32_t constexpr kRequestTimeoutInSec = 5; + std::string const kServerUrl = CLOUD_URL; -std::string const kCloudServerVersion = "v1"; -std::string const kCloudServerCreateSnapshotMethod = "snapshot/create"; -std::string const kCloudServerUploadMethod = "file/upload/start"; -std::string const kCloudServerNotifyMethod = "file/upload/finish"; +std::string const kServerVersion = "v1"; +std::string const kServerCreateSnapshotMethod = "snapshot/create"; +std::string const kServerFinishSnapshotMethod = "snapshot/finish"; +std::string const kServerBestSnapshotMethod = "snapshot/best"; +std::string const kServerUploadMethod = "file/upload/start"; +std::string const kServerNotifyMethod = "file/upload/finish"; +std::string const kServerDownloadMethod = "file/download"; std::string const kApplicationJson = "application/json"; +std::string const kSnapshotFile = "snapshot.json"; std::string GetIndexFilePath(std::string const & indexName) { return my::JoinPath(GetPlatform().SettingsDir(), indexName); } +std::string GetRestoringFolder(std::string const & serverPathName) +{ + return my::JoinPath(GetPlatform().TmpDir(), serverPathName + "_restore"); +} + std::string BuildMethodUrl(std::string const & serverPathName, std::string const & methodName) { if (kServerUrl.empty()) @@ -56,7 +69,7 @@ std::string BuildMethodUrl(std::string const & serverPathName, std::string const std::ostringstream ss; ss << kServerUrl << "/" - << kCloudServerVersion << "/" + << kServerVersion << "/" << serverPathName << "/" << methodName << "/"; return ss.str(); @@ -115,31 +128,259 @@ bool CanUpload(uint64_t totalUploadingSize) totalUploadingSize <= kMaxWwanUploadingSizeInBytes; } } + +struct SnapshotCreateRequestData +{ + std::string m_deviceId; + std::string m_deviceName; + std::vector m_fileNames; + + explicit SnapshotCreateRequestData(std::vector const & files = {}) + : m_deviceId(GetPlatform().UniqueClientId()), m_deviceName(GetDeviceName()), m_fileNames(files) + {} + + DECLARE_VISITOR(visitor(m_deviceId, "device_id"), visitor(m_deviceName, "device_name"), + visitor(m_fileNames, "file_names")) +}; + +struct SnapshotRequestData +{ + std::string m_deviceId; + + SnapshotRequestData() : m_deviceId(GetPlatform().UniqueClientId()) {} + + DECLARE_VISITOR(visitor(m_deviceId, "device_id")) +}; + +struct UploadingRequestData +{ + std::string m_deviceId; + std::string m_fileName; + std::string m_locale; + + explicit UploadingRequestData(std::string const & filePath = {}) + : m_deviceId(GetPlatform().UniqueClientId()) + , m_fileName(ExtractFileNameWithoutExtension(filePath)) + , m_locale(languages::GetCurrentOrig()) + {} + + DECLARE_VISITOR(visitor(m_deviceId, "device_id"), visitor(m_fileName, "file_name"), + visitor(m_locale, "locale")) +}; + +struct NotifyRequestData +{ + std::string m_deviceId; + std::string m_fileName; + uint64_t m_fileSize = 0; + + NotifyRequestData(std::string const & filePath, uint64_t fileSize) + : m_deviceId(GetPlatform().UniqueClientId()) + , m_fileName(ExtractFileNameWithoutExtension(filePath)) + , m_fileSize(fileSize) + {} + + DECLARE_VISITOR(visitor(m_deviceId, "device_id"), visitor(m_fileName, "file_name"), + visitor(m_fileSize, "file_size")) +}; + +struct DownloadingRequestData +{ + std::string m_deviceId; + std::string m_fileName; + uint64_t m_datetime = 0; + + explicit DownloadingRequestData(std::string const & fileName, uint64_t datetime) + : m_deviceId(GetPlatform().UniqueClientId()), m_fileName(fileName), m_datetime(datetime) + {} + + DECLARE_VISITOR(visitor(m_deviceId, "device_id"), visitor(m_fileName, "file_name"), + visitor(m_datetime, "datetime")) +}; + +struct DownloadingResponseData +{ + std::string m_url; + std::string m_fallbackUrl; + + DECLARE_VISITOR(visitor(m_url, "url"), visitor(m_fallbackUrl, "fallback_url")) +}; + +struct DownloadingResult +{ + Cloud::RequestResult m_requestResult; + bool m_isMalformed = false; + DownloadingResponseData m_response; +}; + +using ResponseHandler = std::function; + +template +Cloud::RequestResult CloudRequestWithResult(std::string const & url, + std::string const & accessToken, + ResponseHandler const & responseHandler, + RequestDataArgs const & ... args) +{ + ASSERT(responseHandler != nullptr, ()); + + platform::HttpClient request(url); + request.SetTimeout(kRequestTimeoutInSec); + request.SetRawHeader("Accept", kApplicationJson); + request.SetRawHeader("Authorization", BuildAuthenticationToken(accessToken)); + request.SetBodyData(SerializeToJson(RequestDataType(args...)), kApplicationJson); + + if (request.RunHttpRequest() && !request.WasRedirected()) + { + int const resultCode = request.ErrorCode(); + if (resultCode == 403) + { + LOG(LWARNING, ("Access denied for", url)); + return {Cloud::RequestStatus::Forbidden, request.ServerResponse()}; + } + + return responseHandler(resultCode, request.ServerResponse()); + } + + return {Cloud::RequestStatus::NetworkError, request.ServerResponse()}; +} + +template +Cloud::RequestResult CloudRequest(std::string const & url, std::string const & accessToken, + RequestDataArgs const &... args) +{ + auto responseHandler = [](int code, std::string const & serverResponse) -> Cloud::RequestResult { + if (IsSuccessfulResultCode(code)) + return {Cloud::RequestStatus::Ok, {}}; + return {Cloud::RequestStatus::NetworkError, serverResponse}; + }; + + return CloudRequestWithResult(url, accessToken, responseHandler, args...); +} + +template +void ParseRequestJsonResult(std::string const & url, std::string const & serverResponse, + ResultType & result) +{ + try + { + DeserializeFromJson(serverResponse, result.m_response); + } + catch (my::Json::Exception const & exception) + { + LOG(LWARNING, ("Malformed server response", "url:", url, "response:", serverResponse)); + result.m_response = {}; + result.m_isMalformed = true; + } +} + +template +ResultType CloudRequestWithJsonResult(std::string const & url, std::string const & accessToken, + RequestDataArgs const &... args) +{ + ResultType result; + auto responseHandler = [&result, &url](int code, + std::string const & serverResponse) -> Cloud::RequestResult + { + if (IsSuccessfulResultCode(code)) + { + ParseRequestJsonResult(url, serverResponse, result); + return {Cloud::RequestStatus::Ok, {}}; + } + return {Cloud::RequestStatus::NetworkError, serverResponse}; + }; + result.m_requestResult = + CloudRequestWithResult(url, accessToken, responseHandler, args...); + return result; +} + +Cloud::SnapshotResponseData ReadSnapshotFile(std::string const & filename) +{ + if (!GetPlatform().IsFileExistsByFullPath(filename)) + return {}; + + std::string jsonStr; + try + { + FileReader r(filename); + r.ReadAsString(jsonStr); + } + catch (FileReader::Exception const & exception) + { + LOG(LWARNING, ("Exception while reading file:", filename, + "reason:", exception.what())); + return {}; + } + + if (jsonStr.empty()) + return {}; + + try + { + Cloud::SnapshotResponseData data; + DeserializeFromJson(jsonStr, data); + return data; + } + catch (my::Json::Exception const & exception) + { + LOG(LWARNING, ("Exception while parsing file:", filename, + "reason:", exception.what(), "json:", jsonStr)); + } + return {}; +} + +Cloud::RequestResult DownloadFile(std::string const & url, std::string const & fileName, + bool & successfullyWritten) +{ + successfullyWritten = true; + platform::HttpClient request(url); + request.SetTimeout(kRequestTimeoutInSec); + if (request.RunHttpRequest() && !request.WasRedirected()) + { + auto const & response = request.ServerResponse(); + int const resultCode = request.ErrorCode(); + if (resultCode == 403) + { + LOG(LWARNING, ("Access denied for", url)); + return {Cloud::RequestStatus::Forbidden, response}; + } + + if (IsSuccessfulResultCode(resultCode)) + { + try + { + FileWriter w(fileName); + w.Write(response.data(), response.length()); + } + catch (FileWriter::Exception const & exception) + { + LOG(LWARNING, ("Exception while writing file:", fileName, "reason:", exception.what())); + successfullyWritten = false; + } + return {Cloud::RequestStatus::Ok, {}}; + } + } + return {Cloud::RequestStatus::NetworkError, request.ServerResponse()}; +} + +bool CheckAndGetFileSize(std::string const & filePath, uint64_t & fileSize) +{ + if (!my::GetFileSize(filePath, fileSize)) + return false; + + // We do not work with files which size is more than kMaxUploadingFileSizeInBytes. + return fileSize <= kMaxUploadingFileSizeInBytes; +} } // namespace -Cloud::SnapshotRequestData::SnapshotRequestData(std::vector const & files) - : m_deviceId(GetPlatform().UniqueClientId()) - , m_deviceName(GetDeviceName()) - , m_fileNames(files) -{} - -Cloud::UploadingRequestData::UploadingRequestData(std::string const & filePath) - : m_deviceId(GetPlatform().UniqueClientId()) - , m_fileName(ExtractFileNameWithoutExtension(filePath)) -{} - -Cloud::NotifyRequestData::NotifyRequestData(std::string const & filePath, uint64_t fileSize) - : Cloud::UploadingRequestData(filePath) - , m_fileSize(fileSize) -{} - Cloud::Cloud(CloudParams && params) : m_params(std::move(params)) { ASSERT(!m_params.m_indexName.empty(), ()); ASSERT(!m_params.m_serverPathName.empty(), ()); ASSERT(!m_params.m_settingsParamName.empty(), ()); - ASSERT(!m_params.m_zipExtension.empty(), ()); + ASSERT(!m_params.m_restoredFileExtension.empty(), ()); + ASSERT(!m_params.m_restoringFolder.empty(), ()); int stateValue; if (!settings::Get(m_params.m_settingsParamName, stateValue)) @@ -165,11 +406,15 @@ void Cloud::SetInvalidTokenHandler(InvalidTokenHandler && onInvalidToken) } void Cloud::SetSynchronizationHandlers(SynchronizationStartedHandler && onSynchronizationStarted, - SynchronizationFinishedHandler && onSynchronizationFinished) + SynchronizationFinishedHandler && onSynchronizationFinished, + RestoreRequestedHandler && onRestoreRequested, + RestoredFilesPreparedHandler && onRestoredFilesPrepared) { std::lock_guard lock(m_mutex); m_onSynchronizationStarted = std::move(onSynchronizationStarted); m_onSynchronizationFinished = std::move(onSynchronizationFinished); + m_onRestoreRequested = std::move(onRestoreRequested); + m_onRestoredFilesPrepared = std::move(onRestoredFilesPrepared); } void Cloud::SetState(State state) @@ -259,7 +504,7 @@ bool Cloud::ReadIndex() catch (FileReader::Exception const & exception) { LOG(LWARNING, ("Exception while reading file:", indexFilePath, - "reason:", exception.what())); + "reason:", exception.what())); return false; } @@ -277,7 +522,8 @@ bool Cloud::ReadIndex() catch (my::Json::Exception const & exception) { LOG(LWARNING, ("Exception while parsing file:", indexFilePath, - "reason:", exception.what())); + "reason:", exception.what(), "json:", jsonStr)); + return false; } return true; } @@ -292,7 +538,7 @@ void Cloud::UpdateIndex(bool indexExists) if (!indexExists || h >= m_index.m_lastUpdateInHours + kUpdateTimeoutInHours) { for (auto const & path : m_files) - MarkModifiedImpl(path.second); + MarkModifiedImpl(path.second, true /* isOutdated */); // Erase disappeared files from index. my::EraseIf(m_index.m_entries, [this](EntryPtr const & entity) { @@ -317,6 +563,13 @@ uint64_t Cloud::CalculateUploadingSizeImpl() const return sz; } +bool Cloud::CanUploadImpl() const +{ + return m_state == State::Enabled && m_index.CanBeUploaded() && !m_accessToken.empty() && + !m_uploadingStarted && m_indexUpdated && m_restoringState == RestoringState::None && + CanUpload(CalculateUploadingSizeImpl()); +} + void Cloud::SortEntriesBeforeUploadingImpl() { std::sort(m_index.m_entries.begin(), m_index.m_entries.end(), @@ -326,30 +579,47 @@ void Cloud::SortEntriesBeforeUploadingImpl() }); } -void Cloud::MarkModifiedImpl(std::string const & filePath) +void Cloud::MarkModifiedImpl(std::string const & filePath, bool isOutdated) { uint64_t fileSize = 0; - if (!my::GetFileSize(filePath, fileSize)) - return; - - // We do not work with files which size is more than kMaxUploadingFileSizeInBytes. - if (fileSize > kMaxUploadingFileSizeInBytes) + if (!CheckAndGetFileSize(filePath, fileSize)) return; auto const fileName = ExtractFileNameWithoutExtension(filePath); auto entryPtr = GetEntryImpl(fileName); if (entryPtr) { - entryPtr->m_isOutdated = true; + entryPtr->m_isOutdated = isOutdated; entryPtr->m_sizeInBytes = fileSize; } else { m_index.m_entries.emplace_back( - std::make_shared(fileName, fileSize, true /* m_isOutdated */)); + std::make_shared(fileName, fileSize, isOutdated)); } } +void Cloud::UpdateIndexByRestoredFilesImpl(RestoredFilesCollection const & files, + uint64_t lastSyncTimestampInSec) +{ + m_index.m_isOutdated = false; + m_index.m_lastUpdateInHours = + static_cast(duration_cast(system_clock::now().time_since_epoch()).count()); + m_index.m_lastSyncTimestamp = lastSyncTimestampInSec; + m_index.m_entries.clear(); + for (auto const & f : files) + { + uint64_t fileSize = 0; + if (!CheckAndGetFileSize(f.m_filename, fileSize)) + continue; + + m_index.m_entries.emplace_back( + std::make_shared(ExtractFileNameWithoutExtension(f.m_filename), fileSize, + false /* isOutdated */, f.m_hash)); + } + SaveIndexImpl(); +} + Cloud::EntryPtr Cloud::GetEntryImpl(std::string const & fileName) const { auto it = std::find_if(m_index.m_entries.begin(), m_index.m_entries.end(), @@ -373,47 +643,51 @@ void Cloud::SaveIndexImpl() const } catch (FileWriter::Exception const & exception) { - LOG(LERROR, ("Exception while writing file:", indexFilePath, - "reason:", exception.what())); + LOG(LWARNING, ("Exception while writing file:", indexFilePath, + "reason:", exception.what())); } } void Cloud::ScheduleUploading() { std::vector snapshotFiles; + bool allUpdated = true; { std::lock_guard lock(m_mutex); - if (m_state != State::Enabled || !m_index.m_isOutdated || m_index.m_entries.empty() || - m_accessToken.empty() || m_uploadingStarted || !m_indexUpdated) - { - return; - } - - if (!CanUpload(CalculateUploadingSizeImpl())) + if (!CanUploadImpl()) return; SortEntriesBeforeUploadingImpl(); snapshotFiles.reserve(m_index.m_entries.size()); for (auto const & entry : m_index.m_entries) + { snapshotFiles.emplace_back(entry->m_name); + allUpdated = allUpdated && !entry->m_isOutdated; + } m_uploadingStarted = true; } - ThreadSafeCallback( - [this]() { return m_onSynchronizationStarted; }); + ThreadSafeCallback([this]() { return m_onSynchronizationStarted; }, + SynchronizationType::Backup); - // Create snapshot and begin uploading in case of success. - CreateSnapshotTask(kTaskTimeoutInSeconds, 0 /* attemptIndex */, - std::move(snapshotFiles), [this]() + if (allUpdated) { - auto entry = FindOutdatedEntry(); - if (entry != nullptr) - ScheduleUploadingTask(entry, kTaskTimeoutInSeconds, 0 /* attemptIndex */); - else - FinishUploading(SynchronizationResult::Success, {}); - }); + FinishSnapshotTask(kTaskTimeoutInSeconds, 0 /* attemptIndex */); + } + else + { + // Create snapshot and begin uploading in case of success. + CreateSnapshotTask(kTaskTimeoutInSeconds, 0 /* attemptIndex */, std::move(snapshotFiles), [this]() + { + auto entry = FindOutdatedEntry(); + if (entry != nullptr) + ScheduleUploadingTask(entry, kTaskTimeoutInSeconds, 0 /* attemptIndex */); + else + FinishUploading(SynchronizationResult::Success, {}); + }); + } } void Cloud::ScheduleUploadingTask(EntryPtr const & entry, uint32_t timeout, @@ -422,15 +696,19 @@ void Cloud::ScheduleUploadingTask(EntryPtr const & entry, uint32_t timeout, GetPlatform().RunDelayedTask(Platform::Thread::Network, seconds(timeout), [this, entry, timeout, attemptIndex]() { - #ifdef DEBUG + std::string entryName; + std::string entryHash; { std::lock_guard lock(m_mutex); + #ifdef DEBUG ASSERT(m_state == State::Enabled, ()); ASSERT(!m_accessToken.empty(), ()); ASSERT(m_uploadingStarted, ()); ASSERT(entry->m_isOutdated, ()); - } #endif + entryName = entry->m_name; + entryHash = entry->m_hash; + } if (kServerUrl.empty()) { @@ -439,7 +717,7 @@ void Cloud::ScheduleUploadingTask(EntryPtr const & entry, uint32_t timeout, } // Prepare file to uploading. - auto const uploadedName = PrepareFileToUploading(entry->m_name); + auto const uploadedName = PrepareFileToUploading(entryName); auto deleteAfterUploading = [uploadedName]() { if (!uploadedName.empty()) my::DeleteFileX(uploadedName); @@ -460,7 +738,7 @@ void Cloud::ScheduleUploadingTask(EntryPtr const & entry, uint32_t timeout, return; } - if (entry->m_hash != sha1) + if (entryHash != sha1) { uint64_t uploadedFileSize = 0; if (!my::GetFileSize(uploadedName, uploadedFileSize)) @@ -471,7 +749,12 @@ void Cloud::ScheduleUploadingTask(EntryPtr const & entry, uint32_t timeout, // Request uploading. auto const result = RequestUploading(uploadedName); - if (result.m_requestResult.m_status == RequestStatus::NetworkError) + if (result.m_isMalformed) + { + FinishUploading(SynchronizationResult::NetworkError, "Malformed uploading response"); + return; + } + else if (result.m_requestResult.m_status == RequestStatus::NetworkError) { // Retry uploading request up to kRetryMaxAttempts times. if (attemptIndex + 1 == kRetryMaxAttempts) @@ -503,7 +786,9 @@ void Cloud::ScheduleUploadingTask(EntryPtr const & entry, uint32_t timeout, auto const notificationResult = NotifyAboutUploading(uploadedName, uploadedFileSize); if (executeResult.m_status != RequestStatus::Ok) { - FinishUploading(SynchronizationResult::NetworkError, notificationResult.m_error); + FinishUploading(executeResult.m_status == RequestStatus::Forbidden ? + SynchronizationResult::AuthError : SynchronizationResult::NetworkError, + notificationResult.m_error); return; } } @@ -521,7 +806,7 @@ void Cloud::ScheduleUploadingTask(EntryPtr const & entry, uint32_t timeout, if (nextEntry != nullptr) ScheduleUploadingTask(nextEntry, kTaskTimeoutInSeconds, 0 /* attemptIndex */); else - FinishUploading(SynchronizationResult::Success, {}); + FinishSnapshotTask(kTaskTimeoutInSeconds, 0 /* attemptIndex */); }); } @@ -575,6 +860,53 @@ void Cloud::CreateSnapshotTask(uint32_t timeout, uint32_t attemptIndex, }); } +void Cloud::FinishSnapshotTask(uint32_t timeout, uint32_t attemptIndex) +{ + GetPlatform().RunDelayedTask(Platform::Thread::Network, seconds(timeout), + [this, timeout, attemptIndex]() + { + #ifdef DEBUG + { + std::lock_guard lock(m_mutex); + ASSERT(m_state == State::Enabled, ()); + ASSERT(!m_accessToken.empty(), ()); + ASSERT(m_uploadingStarted, ()); + } + #endif + + if (kServerUrl.empty()) + { + FinishUploading(SynchronizationResult::NetworkError, "Empty server url"); + return; + } + + auto const result = FinishSnapshot(); + if (result.m_status == RequestStatus::Ok) + { + FinishUploading(SynchronizationResult::Success, {}); + } + else if (result.m_status == RequestStatus::NetworkError) + { + // Retry request up to kRetryMaxAttempts times. + if (attemptIndex + 1 == kRetryMaxAttempts) + { + FinishUploading(SynchronizationResult::NetworkError, result.m_error); + return; + } + + auto const retryTimeout = attemptIndex == 0 ? kRetryTimeoutInSeconds + : timeout * kRetryDegradationFactor; + FinishSnapshotTask(retryTimeout, attemptIndex + 1); + return; + } + else if (result.m_status == RequestStatus::Forbidden) + { + FinishUploading(SynchronizationResult::AuthError, result.m_error); + return; + } + }); +} + std::string Cloud::PrepareFileToUploading(std::string const & fileName) { // 1. Get path to the original uploading file. @@ -585,9 +917,9 @@ std::string Cloud::PrepareFileToUploading(std::string const & fileName) if (it == m_files.end()) return {}; filePath = it->second; - if (!GetPlatform().IsFileExistsByFullPath(filePath)) - return {}; } + if (!GetPlatform().IsFileExistsByFullPath(filePath)) + return {}; // 2. Calculate SHA1 of the original uploading file. auto const originalSha1 = coding::SHA1::CalculateBase64(filePath); @@ -612,81 +944,65 @@ std::string Cloud::PrepareFileToUploading(std::string const & fileName) auto const outputPath = my::JoinFoldersToPath(GetPlatform().TmpDir(), name + ".uploaded"); - // 5. If the file is zipped return path to the temporary file. - auto ext = my::GetFileExtension(filePath); - strings::AsciiToLower(ext); - if (ext == m_params.m_zipExtension) + // 5. Convert temporary file and save to output path. + if (m_params.m_backupConverter != nullptr) + { + if (m_params.m_backupConverter(tmpPath, outputPath)) + return outputPath; + } + else { if (my::RenameFileX(tmpPath, outputPath)) - { - tmpFileGuard.release(); return outputPath; - } - return {}; } - - // 6. Zip file and return path. - if (CreateZipFromPathDeflatedAndDefaultCompression(tmpPath, outputPath)) - return outputPath; - return {}; } Cloud::RequestResult Cloud::CreateSnapshot(std::vector const & files) const { ASSERT(!files.empty(), ()); - auto const url = BuildMethodUrl(m_params.m_serverPathName, kCloudServerCreateSnapshotMethod); - platform::HttpClient request(url); - request.SetRawHeader("Accept", kApplicationJson); - request.SetRawHeader("Authorization", BuildAuthenticationToken(m_accessToken)); - request.SetBodyData(SerializeToJson(SnapshotRequestData(files)), kApplicationJson); + auto const url = BuildMethodUrl(m_params.m_serverPathName, kServerCreateSnapshotMethod); + return CloudRequest(url, GetAccessToken(), files); +} - if (request.RunHttpRequest() && !request.WasRedirected()) +Cloud::RequestResult Cloud::FinishSnapshot() const +{ + auto const url = BuildMethodUrl(m_params.m_serverPathName, kServerFinishSnapshotMethod); + return CloudRequest(url, GetAccessToken()); +} + +Cloud::SnapshotResult Cloud::GetBestSnapshot() const +{ + auto const url = BuildMethodUrl(m_params.m_serverPathName, kServerBestSnapshotMethod); + + SnapshotResult result; + auto responseHandler = [&result, &url]( + int code, std::string const & serverResponse) -> Cloud::RequestResult { - int const resultCode = request.ErrorCode(); - if (IsSuccessfulResultCode(resultCode)) - return {RequestStatus::Ok, {}}; - - if (resultCode == 403) + if (IsSuccessfulResultCode(code)) { - LOG(LWARNING, ("Access denied for", url)); - return {RequestStatus::Forbidden, request.ServerResponse()}; + ParseRequestJsonResult(url, serverResponse, result); + return {Cloud::RequestStatus::Ok, {}}; } - } - return {RequestStatus::NetworkError, request.ServerResponse()}; + // Server return 404 in case of snapshot absence. + if (code == 404) + return {Cloud::RequestStatus::Ok, {}}; + + return {Cloud::RequestStatus::NetworkError, serverResponse}; + }; + + result.m_requestResult = + CloudRequestWithResult(url, GetAccessToken(), responseHandler); + + return result; } Cloud::UploadingResult Cloud::RequestUploading(std::string const & filePath) const { - UploadingResult result; - - auto const url = BuildMethodUrl(m_params.m_serverPathName, kCloudServerUploadMethod); - platform::HttpClient request(url); - request.SetRawHeader("Accept", kApplicationJson); - request.SetRawHeader("Authorization", BuildAuthenticationToken(m_accessToken)); - request.SetBodyData(SerializeToJson(UploadingRequestData(filePath)), kApplicationJson); - - if (request.RunHttpRequest() && !request.WasRedirected()) - { - int const resultCode = request.ErrorCode(); - if (IsSuccessfulResultCode(resultCode)) - { - result.m_requestResult = {RequestStatus::Ok, {}}; - DeserializeFromJson(request.ServerResponse(), result.m_response); - return result; - } - - if (resultCode == 403) - { - LOG(LWARNING, ("Access denied for", url)); - result.m_requestResult = {RequestStatus::Forbidden, request.ServerResponse()}; - return result; - } - } - - result.m_requestResult = {RequestStatus::NetworkError, request.ServerResponse()}; - return result; + auto const url = BuildMethodUrl(m_params.m_serverPathName, kServerUploadMethod); + return CloudRequestWithJsonResult( + url, GetAccessToken(), filePath); } Cloud::RequestResult Cloud::ExecuteUploading(UploadingResponseData const & responseData, @@ -695,53 +1011,51 @@ Cloud::RequestResult Cloud::ExecuteUploading(UploadingResponseData const & respo ASSERT(!responseData.m_url.empty(), ()); ASSERT(!responseData.m_method.empty(), ()); - platform::HttpUploader request; - request.SetUrl(responseData.m_url); - request.SetMethod(responseData.m_method); - std::map params; - for (auto const & f : responseData.m_fields) + static std::string const kStatErrors[] = {"download_server", "fallback_server"}; + + std::vector urls; + urls.push_back(responseData.m_url); + if (!responseData.m_fallbackUrl.empty()) + urls.push_back(responseData.m_fallbackUrl); + + std::string errorStr; + int code; + for (size_t i = 0; i < urls.size(); ++i) { - ASSERT_EQUAL(f.size(), 2, ()); - params.insert(std::make_pair(f[0], f[1])); + platform::HttpUploader request; + request.SetUrl(urls[i]); + request.SetMethod(responseData.m_method); + for (auto const & f : responseData.m_fields) + { + ASSERT_EQUAL(f.size(), 2, ()); + request.SetParam(f[0], f[1]); + } + request.SetFilePath(filePath); + + auto const result = request.Upload(); + code = result.m_httpCode; + if (IsSuccessfulResultCode(code)) + return {RequestStatus::Ok, {}}; + + errorStr = strings::to_string(code) + " " + result.m_description; + if (code >= 500 && code < 600) + { + ASSERT_LESS_OR_EQUAL(i, ARRAY_SIZE(kStatErrors), ()); + alohalytics::TStringMap details{ + {"service", m_params.m_serverPathName}, {"type", kStatErrors[i]}, {"error", errorStr}}; + alohalytics::Stats::Instance().LogEvent("Cloud_Backup_error", details); + return {RequestStatus::NetworkError, errorStr}; + } } - request.SetParams(params); - request.SetFilePath(filePath); - auto const result = request.Upload(); - if (IsSuccessfulResultCode(result.m_httpCode)) - return {RequestStatus::Ok, {}}; - - auto const errorStr = strings::to_string(result.m_httpCode) + " " + result.m_description; - if (result.m_httpCode == 403) - return {RequestStatus::Forbidden, errorStr}; - - return {RequestStatus::NetworkError, errorStr}; + return {code == 403 ? RequestStatus::Forbidden : RequestStatus::NetworkError, errorStr}; } Cloud::RequestResult Cloud::NotifyAboutUploading(std::string const & filePath, uint64_t fileSize) const { - auto const url = BuildMethodUrl(m_params.m_serverPathName, kCloudServerNotifyMethod); - platform::HttpClient request(url); - request.SetRawHeader("Accept", kApplicationJson); - request.SetRawHeader("Authorization", BuildAuthenticationToken(m_accessToken)); - request.SetBodyData(SerializeToJson(NotifyRequestData(filePath, fileSize)), - kApplicationJson); - - if (request.RunHttpRequest() && !request.WasRedirected()) - { - int const resultCode = request.ErrorCode(); - if (IsSuccessfulResultCode(resultCode)) - return {RequestStatus::Ok, {}}; - - if (resultCode == 403) - { - LOG(LWARNING, ("Access denied for", url)); - return {RequestStatus::Forbidden, request.ServerResponse()}; - } - } - - return {RequestStatus::NetworkError, request.ServerResponse()}; + auto const url = BuildMethodUrl(m_params.m_serverPathName, kServerNotifyMethod); + return CloudRequest(url, GetAccessToken(), filePath, fileSize); } Cloud::EntryPtr Cloud::FindOutdatedEntry() const @@ -757,23 +1071,42 @@ Cloud::EntryPtr Cloud::FindOutdatedEntry() const void Cloud::FinishUploading(SynchronizationResult result, std::string const & errorStr) { - if (result == SynchronizationResult::AuthError) - ThreadSafeCallback([this]() { return m_onInvalidToken; }); - { std::lock_guard lock(m_mutex); - m_index.m_isOutdated = (result != SynchronizationResult::Success); - if (result == SynchronizationResult::Success) + if (!m_uploadingStarted) + return; + + if (result == SynchronizationResult::UserInterrupted) { - m_index.m_lastSyncTimestamp = static_cast( - duration_cast(system_clock::now().time_since_epoch()).count()); + // If the user interrupts uploading, we consider all files as up-to-dated. + // The files will be checked and uploaded (if necessary) next time. + m_index.m_isOutdated = false; + for (auto & entry : m_index.m_entries) + entry->m_isOutdated = false; + } + else + { + if (result == SynchronizationResult::Success) + { + m_index.m_isOutdated = false; + m_index.m_lastSyncTimestamp = static_cast( + duration_cast(system_clock::now().time_since_epoch()).count()); + } + else + { + m_index.m_isOutdated = true; + } } m_uploadingStarted = false; SaveIndexImpl(); } + if (result == SynchronizationResult::AuthError) + ThreadSafeCallback([this]() { return m_onInvalidToken; }); + ThreadSafeCallback( - [this]() { return m_onSynchronizationFinished; }, result, errorStr); + [this]() { return m_onSynchronizationFinished; }, SynchronizationType::Backup, result, + errorStr); } void Cloud::SetAccessToken(std::string const & token) @@ -781,3 +1114,454 @@ void Cloud::SetAccessToken(std::string const & token) std::lock_guard lock(m_mutex); m_accessToken = token; } + +std::string Cloud::GetAccessToken() const +{ + std::lock_guard lock(m_mutex); + return m_accessToken; +} + +void Cloud::RequestRestoring() +{ + auto const status = GetPlatform().ConnectionStatus(); + if (status == Platform::EConnectionType::CONNECTION_NONE) + return; + + { + std::lock_guard lock(m_mutex); + if (m_state != State::Enabled || !m_indexUpdated || m_accessToken.empty() || + m_restoringState != RestoringState::None) + { + return; + } + + m_restoringState = RestoringState::Requested; + } + + FinishUploading(SynchronizationResult::UserInterrupted, {}); + + ThreadSafeCallback( + [this]() { return m_onSynchronizationStarted; }, SynchronizationType::Restore); + + GetBestSnapshotTask(kTaskTimeoutInSeconds, 0 /* attemptIndex */); +} + +void Cloud::ApplyRestoring() +{ + auto const status = GetPlatform().ConnectionStatus(); + if (status == Platform::EConnectionType::CONNECTION_NONE) + return; + + { + std::lock_guard lock(m_mutex); + if (m_state != State::Enabled || m_accessToken.empty() || + m_restoringState != RestoringState::Requested || m_bestSnapshotData.m_deviceId.empty()) + { + return; + } + m_restoringState = RestoringState::Applying; + } + + GetPlatform().RunTask(Platform::Thread::File, [this]() + { + // Create folder if it does not exist. + auto const dirPath = GetRestoringFolder(m_params.m_serverPathName); + if (!GetPlatform().IsFileExistsByFullPath(dirPath) && !Platform::MkDirChecked(dirPath)) + { + FinishRestoring(SynchronizationResult::DiskError, "Unable create restoring directory"); + return; + } + + // Get list of files to download. + auto downloadingList = GetDownloadingList(dirPath); + if (downloadingList.empty()) + { + std::lock_guard lock(m_mutex); + if (m_restoringState != RestoringState::Applying) + return; + + // Try to complete restoring process. + CompleteRestoring(dirPath); + } + else + { + // Start downloading. + DownloadingTask(dirPath, false /* useFallbackUrl */, std::move(downloadingList)); + } + }); +} + +void Cloud::CancelRestoring() +{ + FinishRestoring(SynchronizationResult::UserInterrupted, {}); +} + +void Cloud::GetBestSnapshotTask(uint32_t timeout, uint32_t attemptIndex) +{ + GetPlatform().RunDelayedTask(Platform::Thread::Network, seconds(timeout), + [this, timeout, attemptIndex]() + { + #ifdef DEBUG + { + std::lock_guard lock(m_mutex); + ASSERT(m_state == State::Enabled, ()); + ASSERT(!m_accessToken.empty(), ()); + ASSERT_EQUAL(m_restoringState, RestoringState::Requested, ()); + } + #endif + + if (kServerUrl.empty()) + { + FinishRestoring(SynchronizationResult::NetworkError, "Empty server url"); + return; + } + + auto const result = GetBestSnapshot(); + if (result.m_isMalformed) + { + FinishRestoring(SynchronizationResult::NetworkError, "Malformed best snapshot response"); + } + else if (result.m_requestResult.m_status == RequestStatus::Ok) + { + ProcessSuccessfulSnapshot(result); + } + else if (result.m_requestResult.m_status == RequestStatus::NetworkError) + { + // Retry request up to kRetryMaxAttempts times. + if (attemptIndex + 1 == kRetryMaxAttempts) + { + FinishRestoring(SynchronizationResult::NetworkError, result.m_requestResult.m_error); + return; + } + + auto const retryTimeout = attemptIndex == 0 ? kRetryTimeoutInSeconds + : timeout * kRetryDegradationFactor; + GetBestSnapshotTask(retryTimeout, attemptIndex + 1); + } + else if (result.m_requestResult.m_status == RequestStatus::Forbidden) + { + FinishRestoring(SynchronizationResult::AuthError, result.m_requestResult.m_error); + } + }); +} + +void Cloud::ProcessSuccessfulSnapshot(SnapshotResult const & result) +{ + // Check if the backup is empty. + if (result.m_response.m_files.empty()) + { + ThreadSafeCallback([this]() { return m_onRestoreRequested; }, + RestoringRequestResult::NoBackup, + result.m_response.m_datetime); + FinishRestoring(SynchronizationResult::Success, {}); + return; + } + + // Check if there is enough space to download backup. + auto const totalSize = result.m_response.GetTotalSizeOfFiles(); + auto constexpr kSizeScalar = 10; + if (totalSize * kSizeScalar >= GetPlatform().GetWritableStorageSpace()) + { + ThreadSafeCallback([this]() { return m_onRestoreRequested; }, + RestoringRequestResult::NotEnoughDiskSpace, + result.m_response.m_datetime); + FinishRestoring(SynchronizationResult::DiskError, {}); + return; + } + + // Save snapshot data. + bool isInterrupted = false; + { + std::lock_guard lock(m_mutex); + m_bestSnapshotData = result.m_response; + isInterrupted = m_restoringState != RestoringState::Requested; + } + if (!isInterrupted) + { + ThreadSafeCallback([this]() { return m_onRestoreRequested; }, + RestoringRequestResult::BackupExists, + result.m_response.m_datetime); + } +} + +void Cloud::FinishRestoring(Cloud::SynchronizationResult result, std::string const & errorStr) +{ + { + std::lock_guard lock(m_mutex); + if (m_restoringState == RestoringState::None) + return; + + // We cannot interrupt restoring process on the finalizing step. + if (result == Cloud::SynchronizationResult::UserInterrupted && + m_restoringState == RestoringState::Finalizing) + { + return; + } + + m_bestSnapshotData = {}; + m_restoringState = RestoringState::None; + } + + if (result == SynchronizationResult::AuthError) + ThreadSafeCallback([this]() { return m_onInvalidToken; }); + + ThreadSafeCallback( + [this]() { return m_onSynchronizationFinished; }, SynchronizationType::Restore, result, + errorStr); +} + +std::list Cloud::GetDownloadingList(std::string const & restoringDirPath) +{ + auto const snapshotFile = my::JoinPath(restoringDirPath, kSnapshotFile); + auto const prevSnapshot = ReadSnapshotFile(snapshotFile); + + SnapshotResponseData currentSnapshot; + { + std::lock_guard lock(m_mutex); + currentSnapshot = m_bestSnapshotData; + } + + // Save to use in the next sessions. + try + { + auto jsonData = SerializeToJson(currentSnapshot); + FileWriter w(snapshotFile); + w.Write(jsonData.data(), jsonData.length()); + } + catch (FileWriter::Exception const & exception) + { + LOG(LWARNING, ("Exception while writing file:", snapshotFile, + "reason:", exception.what())); + FinishRestoring(SynchronizationResult::DiskError, "Could not save snapshot file"); + return {}; + } + + // If the snapshot from previous sessions is not valid, return files from new one. + if (currentSnapshot.m_deviceId != prevSnapshot.m_deviceId || + currentSnapshot.m_datetime != prevSnapshot.m_datetime || + currentSnapshot.m_files.size() != prevSnapshot.m_files.size()) + { + return std::list(currentSnapshot.m_files.begin(), + currentSnapshot.m_files.end()); + } + + // Check if some files were completely downloaded last time. + std::list result; + for (auto & f : currentSnapshot.m_files) + { + auto const restoringFile = my::JoinPath(restoringDirPath, f.m_fileName); + if (!GetPlatform().IsFileExistsByFullPath(restoringFile)) + { + result.push_back(std::move(f)); + continue; + } + + uint64_t fileSize = 0; + if (!my::GetFileSize(restoringFile, fileSize) || fileSize != f.m_fileSize) + result.push_back(std::move(f)); + } + + return result; +} + +void Cloud::DownloadingTask(std::string const & dirPath, bool useFallbackUrl, + std::list && files) +{ + GetPlatform().RunTask(Platform::Thread::Network, + [this, dirPath, useFallbackUrl, files = std::move(files)]() mutable + { + { + // Check if the process was interrupted. + std::lock_guard lock(m_mutex); + if (m_restoringState != RestoringState::Applying) + return; + } + + if (files.empty()) + { + CompleteRestoring(dirPath); + return; + } + + auto const f = files.front(); + files.erase(files.begin()); + auto const filePath = my::JoinPath(dirPath, f.m_fileName); + + auto const url = BuildMethodUrl(m_params.m_serverPathName, kServerDownloadMethod); + auto const result = CloudRequestWithJsonResult( + url, GetAccessToken(), f.m_fileName, f.m_datetime); + + if (result.m_isMalformed || result.m_response.m_url.empty()) + { + FinishRestoring(SynchronizationResult::NetworkError, "Malformed downloading file response"); + } + else if (result.m_requestResult.m_status == RequestStatus::Ok) + { + if (useFallbackUrl && result.m_response.m_fallbackUrl.empty()) + { + FinishRestoring(SynchronizationResult::NetworkError, "Fallback url is absent"); + return; + } + bool successfullyWritten = true; + auto const downloadResult = DownloadFile(useFallbackUrl ? result.m_response.m_fallbackUrl + : result.m_response.m_url, + filePath, successfullyWritten); + if (!successfullyWritten) + { + FinishRestoring(SynchronizationResult::DiskError, "Could not create downloaded file"); + } + else if (downloadResult.m_status == RequestStatus::Ok) + { + // Download next file. + DownloadingTask(dirPath, false /* useFallbackUrl */, std::move(files)); + } + else + { + alohalytics::TStringMap details{ + {"service", m_params.m_serverPathName}, + {"type", useFallbackUrl ? "fallback_server" : "download_server"}, + {"error", downloadResult.m_error}}; + alohalytics::Stats::Instance().LogEvent("Cloud_Restore_error", details); + + if (!useFallbackUrl) + { + // Retry to download by means of fallback url. + files.push_front(std::move(f)); + DownloadingTask(dirPath, true /* useFallbackUrl */, std::move(files)); + } + else + { + FinishRestoring(SynchronizationResult::NetworkError, "File downloader error"); + } + } + } + else if (result.m_requestResult.m_status == RequestStatus::NetworkError) + { + FinishRestoring(SynchronizationResult::NetworkError, result.m_requestResult.m_error); + } + else if (result.m_requestResult.m_status == RequestStatus::Forbidden) + { + FinishRestoring(SynchronizationResult::AuthError, result.m_requestResult.m_error); + } + }); +} + +void Cloud::CompleteRestoring(std::string const & dirPath) +{ + GetPlatform().RunTask(Platform::Thread::File, [this, dirPath]() + { + // Check files and convert them to expected format. + SnapshotResponseData currentSnapshot; + { + std::lock_guard lock(m_mutex); + currentSnapshot = m_bestSnapshotData; + } + + RestoredFilesCollection convertedFiles; + convertedFiles.reserve(currentSnapshot.m_files.size()); + for (auto & f : currentSnapshot.m_files) + { + auto const restoringFile = my::JoinPath(dirPath, f.m_fileName); + if (!GetPlatform().IsFileExistsByFullPath(restoringFile)) + { + FinishRestoring(SynchronizationResult::DiskError, "Restored file is absent"); + return; + } + + uint64_t fileSize = 0; + if (!my::GetFileSize(restoringFile, fileSize) || fileSize != f.m_fileSize) + { + std::string const str = "Restored file has incorrect size. Expected size = " + + strings::to_string(f.m_fileSize) + + ". Server size =" + strings::to_string(fileSize); + FinishRestoring(SynchronizationResult::DiskError, str); + return; + } + + auto const sha1 = coding::SHA1::CalculateBase64(restoringFile); + auto const fn = f.m_fileName + ".converted"; + auto const convertedFile = my::JoinPath(dirPath, fn); + if (m_params.m_restoreConverter != nullptr) + { + if (!m_params.m_restoreConverter(restoringFile, convertedFile)) + { + FinishRestoring(SynchronizationResult::DiskError, "Restored file conversion error"); + return; + } + } + else + { + if (!my::RenameFileX(restoringFile, convertedFile)) + { + FinishRestoring(SynchronizationResult::DiskError, "Restored file conversion error"); + return; + } + } + convertedFiles.emplace_back(fn, sha1); + } + + // Check if the process was interrupted and start finalizing. + { + std::lock_guard lock(m_mutex); + if (m_restoringState != RestoringState::Applying) + return; + + m_restoringState = RestoringState::Finalizing; + } + + GetPlatform().RunTask(Platform::Thread::Gui, + [this, dirPath, convertedFiles = std::move(convertedFiles)]() mutable + { + ThreadSafeCallback([this]() { return m_onRestoredFilesPrepared; }); + ApplyRestoredFiles(dirPath, std::move(convertedFiles)); + }); + }); +} + +void Cloud::ApplyRestoredFiles(std::string const & dirPath, RestoredFilesCollection && files) +{ + GetPlatform().RunTask(Platform::Thread::File, [this, dirPath, files = std::move(files)]() + { + // Delete all files in the destination folder. + if (GetPlatform().IsFileExistsByFullPath(m_params.m_restoringFolder) && + !GetPlatform().RmDirRecursively(m_params.m_restoringFolder)) + { + FinishRestoring(SynchronizationResult::DiskError, "Could not delete restoring folder"); + return; + } + + // Move files. + if (!GetPlatform().MkDirChecked(m_params.m_restoringFolder)) + { + FinishRestoring(SynchronizationResult::DiskError, "Could not create restoring folder"); + return; + } + + RestoredFilesCollection readyFiles; + readyFiles.reserve(files.size()); + for (auto const & f : files) + { + auto const restoredFile = my::JoinPath(dirPath, f.m_filename); + auto const finalFilename = + my::FilenameWithoutExt(f.m_filename) + m_params.m_restoredFileExtension; + auto const readyFile = my::JoinPath(m_params.m_restoringFolder, finalFilename); + if (!my::RenameFileX(restoredFile, readyFile)) + { + FinishRestoring(SynchronizationResult::DiskError, "Restored file moving error"); + return; + } + readyFiles.emplace_back(readyFile, f.m_hash); + } + + // Reset upload index to the restored state. + { + std::lock_guard lock(m_mutex); + auto const lastSyncTimestampInSec = m_bestSnapshotData.m_datetime / 1000; + UpdateIndexByRestoredFilesImpl(readyFiles, lastSyncTimestampInSec); + } + + // Delete temporary directory. + GetPlatform().RmDirRecursively(dirPath); + FinishRestoring(SynchronizationResult::Success, {}); + }); +} diff --git a/map/cloud.hpp b/map/cloud.hpp index 12b6fe78a6..762798c3ca 100644 --- a/map/cloud.hpp +++ b/map/cloud.hpp @@ -8,6 +8,7 @@ #include "base/visitor.hpp" #include +#include #include #include #include @@ -20,10 +21,13 @@ public: struct Entry { Entry() = default; + Entry(std::string const & name, uint64_t sizeInBytes, bool isOutdated) - : m_name(name) - , m_sizeInBytes(sizeInBytes) - , m_isOutdated(isOutdated) + : m_name(name), m_sizeInBytes(sizeInBytes), m_isOutdated(isOutdated) + {} + + Entry(std::string const & name, uint64_t sizeInBytes, bool isOutdated, std::string const & hash) + : m_name(name), m_sizeInBytes(sizeInBytes), m_hash(hash), m_isOutdated(isOutdated) {} bool operator==(Entry const & entry) const @@ -57,57 +61,59 @@ public: bool m_isOutdated = false; uint64_t m_lastSyncTimestamp = 0; // in seconds. + bool CanBeUploaded() const { return m_isOutdated && !m_entries.empty(); } + DECLARE_VISITOR_AND_DEBUG_PRINT(Index, visitor(m_entries, "entries"), visitor(m_lastUpdateInHours, "lastUpdateInHours"), visitor(m_isOutdated, "isOutdated"), visitor(m_lastSyncTimestamp, "lastSyncTimestamp")) }; - struct SnapshotRequestData - { - std::string m_deviceId; - std::string m_deviceName; - std::vector m_fileNames; - - explicit SnapshotRequestData(std::vector const & files = {}); - - DECLARE_VISITOR_AND_DEBUG_PRINT(SnapshotRequestData, visitor(m_deviceId, "device_id"), - visitor(m_deviceName, "device_name"), - visitor(m_fileNames, "file_names")) - }; - - struct UploadingRequestData - { - std::string m_deviceId; - std::string m_fileName; - - explicit UploadingRequestData(std::string const & filePath = {}); - - DECLARE_VISITOR_AND_DEBUG_PRINT(UploadingRequestData, visitor(m_deviceId, "device_id"), - visitor(m_fileName, "file_name")) - }; - struct UploadingResponseData { std::string m_url; + std::string m_fallbackUrl; std::vector> m_fields; std::string m_method; DECLARE_VISITOR_AND_DEBUG_PRINT(UploadingResponseData, visitor(m_url, "url"), + visitor(m_fallbackUrl, "fallback_url"), visitor(m_fields, "fields"), visitor(m_method, "method")) }; - struct NotifyRequestData : public UploadingRequestData + struct SnapshotFileData { + std::string m_fileName; uint64_t m_fileSize = 0; + uint64_t m_datetime = 0; - NotifyRequestData() = default; - NotifyRequestData(std::string const & filePath, uint64_t fileSize); - - DECLARE_VISITOR_AND_DEBUG_PRINT(NotifyRequestData, visitor(m_deviceId, "device_id"), + DECLARE_VISITOR_AND_DEBUG_PRINT(SnapshotFileData, visitor(m_fileName, "file_name"), - visitor(m_fileSize, "file_size")) + visitor(m_fileSize, "file_size"), + visitor(m_datetime, "datetime")) + }; + + struct SnapshotResponseData + { + std::string m_deviceId; + std::string m_deviceName; + uint64_t m_datetime = 0; + std::vector m_files; + + uint64_t GetTotalSizeOfFiles() const + { + uint64_t sz = 0; + for (auto const & f : m_files) + sz += f.m_fileSize; + return sz; + } + + DECLARE_VISITOR_AND_DEBUG_PRINT(SnapshotResponseData, + visitor(m_deviceId, "device_id"), + visitor(m_deviceName, "device_name"), + visitor(m_datetime, "datetime"), + visitor(m_files, "files")) }; enum class RequestStatus @@ -132,9 +138,17 @@ public: struct UploadingResult { RequestResult m_requestResult; + bool m_isMalformed = false; UploadingResponseData m_response; }; + struct SnapshotResult + { + RequestResult m_requestResult; + bool m_isMalformed = false; + SnapshotResponseData m_response; + }; + enum class State { // User never enabled or disabled synchronization via cloud. It is a default state. @@ -145,6 +159,12 @@ public: Enabled = 2 }; + enum class SynchronizationType + { + Backup = 0, + Restore = 1 + }; + enum class SynchronizationResult { // Synchronization was finished successfully. @@ -154,24 +174,48 @@ public: // Synchronization was interrupted by a network error. NetworkError = 2, // Synchronization was interrupted by a disk error. - DiskError = 3 + DiskError = 3, + // Synchronization was interrupted by the user. + UserInterrupted = 4 + }; + + enum class RestoringRequestResult + { + // There is a backup on the server. + BackupExists = 0, + // There is no backup on the server. + NoBackup = 1, + // Not enough space on the disk for the restoring. + NotEnoughDiskSpace = 2 }; using InvalidTokenHandler = std::function; - using SynchronizationStartedHandler = std::function; - using SynchronizationFinishedHandler = std::function; + using SynchronizationStartedHandler = std::function; + using SynchronizationFinishedHandler = std::function; + using RestoreRequestedHandler = std::function; + using RestoredFilesPreparedHandler = std::function; using SnapshotCompletionHandler = std::function; struct CloudParams { CloudParams() = default; CloudParams(std::string && indexName, std::string && serverPathName, - std::string && settingsParamName, std::string && zipExtension) + std::string && settingsParamName, std::string && restoringFolder, + std::string && restoredFileExtension, + FileConverter && backupConverter, + FileConverter && restoreConverter) : m_indexName(std::move(indexName)) , m_serverPathName(std::move(serverPathName)) , m_settingsParamName(std::move(settingsParamName)) - , m_zipExtension(std::move(zipExtension)) + , m_restoredFileExtension(std::move(restoredFileExtension)) + , m_restoringFolder(std::move(restoringFolder)) + , m_backupConverter(std::move(backupConverter)) + , m_restoreConverter(std::move(restoreConverter)) {} // Name of file in which cloud stores metadata. @@ -180,8 +224,14 @@ public: std::string m_serverPathName; // Name of parameter to store cloud's state in settings. std::string m_settingsParamName; - // Extension of zipped file. The first character must be '.' - std::string m_zipExtension; + // The extension of restored files. + std::string m_restoredFileExtension; + // The folder in which files will be restored. + std::string m_restoringFolder; + // This file converter is executed before uploading to the cloud. + FileConverter m_backupConverter; + // This file converter is executed after downloading from the cloud. + FileConverter m_restoreConverter; }; explicit Cloud(CloudParams && params); @@ -189,9 +239,12 @@ public: // Handler can be called from non-UI thread. void SetInvalidTokenHandler(InvalidTokenHandler && onInvalidToken); - // Handlers can be called from non-UI thread. + // Handlers can be called from non-UI thread except of ApplyRestoredFilesHandler. + // ApplyRestoredFilesHandler is always called from UI-thread. void SetSynchronizationHandlers(SynchronizationStartedHandler && onSynchronizationStarted, - SynchronizationFinishedHandler && onSynchronizationFinished); + SynchronizationFinishedHandler && onSynchronizationFinished, + RestoreRequestedHandler && onRestoreRequested, + RestoredFilesPreparedHandler && onRestoredFilesPrepared); void SetState(State state); State GetState() const; @@ -202,16 +255,35 @@ public: std::unique_ptr GetUserSubscriber(); + void RequestRestoring(); + void ApplyRestoring(); + void CancelRestoring(); + private: + struct RestoredFile + { + std::string m_filename; + std::string m_hash; + + RestoredFile() = default; + RestoredFile(std::string const & filename, std::string const & hash) + : m_filename(filename), m_hash(hash) + {} + }; + using RestoredFilesCollection = std::vector; + void LoadIndex(); bool ReadIndex(); void UpdateIndex(bool indexExists); void SaveIndexImpl() const; EntryPtr GetEntryImpl(std::string const & fileName) const; - void MarkModifiedImpl(std::string const & filePath); + void MarkModifiedImpl(std::string const & filePath, bool isOutdated); + void UpdateIndexByRestoredFilesImpl(RestoredFilesCollection const & files, + uint64_t lastSyncTimestampInSec); uint64_t CalculateUploadingSizeImpl() const; + bool CanUploadImpl() const; void SortEntriesBeforeUploadingImpl(); void ScheduleUploading(); void ScheduleUploadingTask(EntryPtr const & entry, uint32_t timeout, @@ -219,20 +291,34 @@ private: void CreateSnapshotTask(uint32_t timeout, uint32_t attemptIndex, std::vector && files, SnapshotCompletionHandler && handler); + void FinishSnapshotTask(uint32_t timeout, uint32_t attemptIndex); EntryPtr FindOutdatedEntry() const; void FinishUploading(SynchronizationResult result, std::string const & errorStr); void SetAccessToken(std::string const & token); + std::string GetAccessToken() const; // This function always returns path to a temporary file or the empty string // in case of a disk error. std::string PrepareFileToUploading(std::string const & fileName); RequestResult CreateSnapshot(std::vector const & files) const; + RequestResult FinishSnapshot() const; + SnapshotResult GetBestSnapshot() const; + void ProcessSuccessfulSnapshot(SnapshotResult const & result); UploadingResult RequestUploading(std::string const & filePath) const; RequestResult ExecuteUploading(UploadingResponseData const & responseData, std::string const & filePath); RequestResult NotifyAboutUploading(std::string const & filePath, uint64_t fileSize) const; + void GetBestSnapshotTask(uint32_t timeout, uint32_t attemptIndex); + void FinishRestoring(SynchronizationResult result, std::string const & errorStr); + std::list GetDownloadingList(std::string const & restoringDirPath); + void DownloadingTask(std::string const & dirPath, bool useFallbackUrl, + std::list && files); + void CompleteRestoring(std::string const & dirPath); + + void ApplyRestoredFiles(std::string const & dirPath, RestoredFilesCollection && files); + template void ThreadSafeCallback(HandlerGetterType && handlerGetter, HandlerArgs... handlerArgs) { @@ -249,11 +335,54 @@ private: InvalidTokenHandler m_onInvalidToken; SynchronizationStartedHandler m_onSynchronizationStarted; SynchronizationFinishedHandler m_onSynchronizationFinished; + RestoreRequestedHandler m_onRestoreRequested; + RestoredFilesPreparedHandler m_onRestoredFilesPrepared; State m_state; Index m_index; std::string m_accessToken; std::map m_files; bool m_uploadingStarted = false; + + enum RestoringState + { + None, + Requested, + Applying, + Finalizing + }; + RestoringState m_restoringState = RestoringState::None; bool m_indexUpdated = false; + SnapshotResponseData m_bestSnapshotData; mutable std::mutex m_mutex; }; + +inline std::string DebugPrint(Cloud::SynchronizationType type) +{ + switch (type) + { + case Cloud::SynchronizationType::Backup: return "Backup"; + case Cloud::SynchronizationType::Restore: return "Restore"; + } +} + +inline std::string DebugPrint(Cloud::SynchronizationResult result) +{ + switch (result) + { + case Cloud::SynchronizationResult::Success: return "Success"; + case Cloud::SynchronizationResult::AuthError: return "AuthError"; + case Cloud::SynchronizationResult::NetworkError: return "NetworkError"; + case Cloud::SynchronizationResult::DiskError: return "DiskError"; + case Cloud::SynchronizationResult::UserInterrupted: return "UserInterrupted"; + } +} + +inline std::string DebugPrint(Cloud::RestoringRequestResult result) +{ + switch (result) + { + case Cloud::RestoringRequestResult::BackupExists: return "BackupExists"; + case Cloud::RestoringRequestResult::NoBackup: return "NoBackup"; + case Cloud::RestoringRequestResult::NotEnoughDiskSpace: return "NotEnoughDiskSpace"; + } +} diff --git a/platform/http_uploader.hpp b/platform/http_uploader.hpp index be0df7d5ed..5f2e6dd67c 100644 --- a/platform/http_uploader.hpp +++ b/platform/http_uploader.hpp @@ -19,6 +19,7 @@ public: void SetMethod(std::string const & method) { m_method = method; } void SetUrl(std::string const & url) { m_url = url; } void SetParams(std::map const & params) { m_params = params; } + void SetParam(std::string const & key, std::string const & value) { m_params[key] = value; } void SetHeaders(std::map const & headers) { m_headers = headers; } void SetFileKey(std::string const & fileKey) { m_fileKey = fileKey; } void SetFilePath(std::string const & filePath) { m_filePath = filePath; }