From c9691d72eeb2514704ccf2d982892242c9f52d38 Mon Sep 17 00:00:00 2001 From: "r.kuznetsov" Date: Fri, 16 Feb 2018 15:23:46 +0300 Subject: [PATCH] Added SHA1 before uploading to the cloud --- map/bookmark_manager.cpp | 2 - map/cloud.cpp | 115 +++++++++++++++++++++++++-------------- map/cloud.hpp | 5 +- 3 files changed, 77 insertions(+), 45 deletions(-) diff --git a/map/bookmark_manager.cpp b/map/bookmark_manager.cpp index 6fed0508f6..d66e71ba06 100644 --- a/map/bookmark_manager.cpp +++ b/map/bookmark_manager.cpp @@ -1132,8 +1132,6 @@ bool BookmarkManager::SaveToKMLFile(df::MarkGroupID groupId) my::DeleteFileX(file); VERIFY(my::RenameFileX(fileTmp, file), (fileTmp, file)); - m_bookmarkCloud.MarkModified(file); - return true; } } diff --git a/map/cloud.cpp b/map/cloud.cpp index 8e45ccaba9..91ccc2585f 100644 --- a/map/cloud.cpp +++ b/map/cloud.cpp @@ -19,6 +19,9 @@ #include #include +#include "3party/liboauthcpp/src/SHA1.h" +#include "3party/liboauthcpp/src/base64.h" + #define STAGE_CLOUD_SERVER #include "private.h" @@ -68,6 +71,34 @@ std::string ExtractFileName(std::string const & filePath) my::GetNameFromFullPath(path); return path; } + +std::string CalculateSHA1(std::string const & filePath) +{ + uint32_t constexpr kFileBufferSize = 8192; + try + { + my::FileData file(filePath, my::FileData::OP_READ); + uint64_t const fileSize = file.Size(); + + CSHA1 sha1; + uint64_t currSize = 0; + unsigned char buffer[kFileBufferSize]; + while (currSize < fileSize) + { + auto const toRead = std::min(kFileBufferSize, static_cast(fileSize - currSize)); + file.Read(currSize, buffer, toRead); + sha1.Update(buffer, toRead); + currSize += toRead; + } + sha1.Final(); + return base64_encode(sha1.m_digest, ARRAY_SIZE(sha1.m_digest)); + } + catch (Reader::Exception const & ex) + { + LOG(LERROR, ("Error reading file:", filePath, ex.Msg())); + } + return {}; +} } // namespace Cloud::Cloud(CloudParams && params) @@ -154,16 +185,6 @@ void Cloud::Init(std::vector const & filePaths) GetPlatform().RunTask(Platform::Thread::File, [this]() { LoadIndex(); }); } -void Cloud::MarkModified(std::string const & filePath) -{ - std::lock_guard lock(m_mutex); - if (m_state != State::Enabled) - return; - - m_files[ExtractFileName(filePath)] = filePath; - MarkModifiedImpl(filePath, false /* checkSize */); -} - uint64_t Cloud::GetLastSynchronizationTimestampInMs() const { std::lock_guard lock(m_mutex); @@ -241,7 +262,7 @@ void Cloud::UpdateIndex(bool indexExists) if (!indexExists || h >= m_index.m_lastUpdateInHours + kUpdateTimeoutInHours) { for (auto const & path : m_files) - MarkModifiedImpl(path.second, indexExists /* checkSize */); + MarkModifiedImpl(path.second); m_index.m_lastUpdateInHours = h; m_index.m_isOutdated = true; @@ -272,7 +293,7 @@ void Cloud::SortEntriesBeforeUploadingImpl() }); } -void Cloud::MarkModifiedImpl(std::string const & filePath, bool checkSize) +void Cloud::MarkModifiedImpl(std::string const & filePath) { uint64_t fileSize = 0; if (!my::GetFileSize(filePath, fileSize)) @@ -286,7 +307,7 @@ void Cloud::MarkModifiedImpl(std::string const & filePath, bool checkSize) auto entryPtr = GetEntryImpl(fileName); if (entryPtr) { - entryPtr->m_isOutdated = checkSize ? (entryPtr->m_sizeInBytes != fileSize) : true; + entryPtr->m_isOutdated = true; entryPtr->m_sizeInBytes = fileSize; } else @@ -393,48 +414,60 @@ void Cloud::ScheduleUploadingTask(EntryPtr const & entry, uint32_t timeout, if (uploadedName.empty()) { - FinishUploading(SynchronizationResult::DiskError, {}); + FinishUploading(SynchronizationResult::DiskError, "File preparation error"); return; } - // Request uploading. - auto const result = RequestUploading(uploadingUrl, uploadedName); - if (result.m_requestResult.m_status == RequestStatus::NetworkError) + // Upload only if SHA1 is not equal to previous one. + auto const sha1 = CalculateSHA1(uploadedName); + if (sha1.empty()) { - // Retry uploading request up to kRetryMaxAttempts times. - if (attemptIndex + 1 == kRetryMaxAttempts) + FinishUploading(SynchronizationResult::DiskError, "SHA1 calculation error"); + return; + } + + if (entry->m_hash != sha1) + { + // Request uploading. + auto const result = RequestUploading(uploadingUrl, uploadedName); + if (result.m_requestResult.m_status == RequestStatus::NetworkError) { - FinishUploading(SynchronizationResult::NetworkError, result.m_requestResult.m_error); + // Retry uploading request up to kRetryMaxAttempts times. + if (attemptIndex + 1 == kRetryMaxAttempts) + { + FinishUploading(SynchronizationResult::NetworkError, result.m_requestResult.m_error); + return; + } + + auto const retryTimeout = attemptIndex == 0 ? kRetryTimeoutInSeconds + : timeout * kRetryDegradationFactor; + ScheduleUploadingTask(entry, retryTimeout, attemptIndex + 1); + return; + } + else if (result.m_requestResult.m_status == RequestStatus::Forbidden) + { + // Finish uploading and notify about invalid access token. + if (m_onInvalidToken != nullptr) + m_onInvalidToken(); + + FinishUploading(SynchronizationResult::AuthError, result.m_requestResult.m_error); return; } - auto const retryTimeout = attemptIndex == 0 ? kRetryTimeoutInSeconds - : timeout * kRetryDegradationFactor; - ScheduleUploadingTask(entry, retryTimeout, attemptIndex + 1); - return; - } - else if (result.m_requestResult.m_status == RequestStatus::Forbidden) - { - // Finish uploading and notify about invalid access token. - if (m_onInvalidToken != nullptr) - m_onInvalidToken(); - - FinishUploading(SynchronizationResult::AuthError, result.m_requestResult.m_error); - return; - } - - // Execute uploading. - auto const executeResult = ExecuteUploading(result.m_response, uploadedName); - if (executeResult.m_status != RequestStatus::Ok) - { - FinishUploading(SynchronizationResult::NetworkError, executeResult.m_error); - return; + // Execute uploading. + auto const executeResult = ExecuteUploading(result.m_response, uploadedName); + if (executeResult.m_status != RequestStatus::Ok) + { + FinishUploading(SynchronizationResult::NetworkError, executeResult.m_error); + return; + } } // Mark entry as not outdated. { std::lock_guard lock(m_mutex); entry->m_isOutdated = false; + entry->m_hash = sha1; SaveIndexImpl(); } diff --git a/map/cloud.hpp b/map/cloud.hpp index 66213d3e61..f0e1777bb8 100644 --- a/map/cloud.hpp +++ b/map/cloud.hpp @@ -39,10 +39,12 @@ public: std::string m_name; uint64_t m_sizeInBytes = 0; + std::string m_hash; bool m_isOutdated = false; DECLARE_VISITOR_AND_DEBUG_PRINT(Entry, visitor(m_name, "name"), visitor(m_sizeInBytes, "sizeInBytes"), + visitor(m_hash, "hash"), visitor(m_isOutdated, "isOutdated")) }; @@ -169,7 +171,6 @@ public: uint64_t GetLastSynchronizationTimestampInMs() const; void Init(std::vector const & filePaths); - void MarkModified(std::string const & filePath); std::unique_ptr GetUserSubscriber(); @@ -180,7 +181,7 @@ private: void SaveIndexImpl() const; EntryPtr GetEntryImpl(std::string const & fileName) const; - void MarkModifiedImpl(std::string const & filePath, bool checkSize); + void MarkModifiedImpl(std::string const & filePath); uint64_t CalculateUploadingSizeImpl() const; void SortEntriesBeforeUploadingImpl();