RedAuburn/organicmaps
1
Fork 0
forked from organicmaps/organicmaps
Code Pull requests Activity

Added cloud restoring

Browse source
This commit is contained in:
r.kuznetsov 2018-03-16 19:03:58 +03:00 • committed by Daria Volvenkova
parent 9a7f424bb1
commit f50347164c
6 changed files with 1282 additions and 237 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
iphone/Maps/Classes/MapsAppDelegate.mm
View file

@ -595,7 +595,9 @@ using namespace osm_auth_ios;
--m_activeDownloadsCounter;
if (m_activeDownloadsCounter <= 0)
{
UIApplication.sharedApplication.networkActivityIndicatorVisible = NO;
dispatch_async(dispatch_get_main_queue(), ^{
UIApplication.sharedApplication.networkActivityIndicatorVisible = NO;
});
m_activeDownloadsCounter = 0;
if (UIApplication.sharedApplication.applicationState == UIApplicationStateBackground)
{
@ -608,7 +610,9 @@ using namespace osm_auth_ios;
- (void)enableDownloadIndicator
{
++m_activeDownloadsCounter;
UIApplication.sharedApplication.networkActivityIndicatorVisible = YES;
dispatch_async(dispatch_get_main_queue(), ^{
UIApplication.sharedApplication.networkActivityIndicatorVisible = YES;
});
}
+ (NSDictionary *)navigationBarTextAttributes

160
map/bookmark_manager.cpp
View file

@ -48,14 +48,14 @@ char const * KMZ_EXTENSION = ".kmz";
char const * kBookmarksExt = ".kmb";
// Returns extension with a dot in a lower case.
std::string const GetFileExt(std::string const & filePath)
std::string GetFileExt(std::string const & filePath)
{
std::string ext = my::GetFileExtension(filePath);
std::transform(ext.begin(), ext.end(), ext.begin(), ::tolower);
return ext;
}
std::string const GetFileName(std::string const & filePath)
std::string GetFileName(std::string const & filePath)
{
std::string ret = filePath;
my::GetNameFromFullPath(ret);
@ -142,6 +142,53 @@ BookmarkManager::SharingResult GetFileForSharing(df::MarkGroupID categoryId, std
return BookmarkManager::SharingResult(categoryId, tmpFilePath);
}
bool ConvertBeforeUploading(std::string const & filePath, std::string const & convertedFilePath)
{
//TODO: convert from kmb to kmz.
return CreateZipFromPathDeflatedAndDefaultCompression(filePath, convertedFilePath);
}
bool ConvertAfterDownloading(std::string const & filePath, std::string const & convertedFilePath)
{
ZipFileReader::FileListT files;
ZipFileReader::FilesList(filePath, files);
if (files.empty())
return false;
std::string const unarchievedPath = filePath + ".raw";
MY_SCOPE_GUARD(fileGuard, bind(&FileWriter::DeleteFileX, unarchievedPath));
ZipFileReader::UnzipFile(filePath, files.front().first, unarchievedPath);
if (!GetPlatform().IsFileExistsByFullPath(unarchievedPath))
return false;
kml::FileData kmlData;
try
{
kml::DeserializerKml des(kmlData);
FileReader reader(unarchievedPath);
des.Deserialize(reader);
}
catch (FileReader::Exception const & exc)
{
LOG(LWARNING, ("KML text deserialization failure: ", exc.what(), "file:", unarchievedPath));
return false;
}
try
{
kml::binary::SerializerKml ser(kmlData);
FileWriter writer(convertedFilePath);
ser.Serialize(writer);
}
catch (FileWriter::Exception const & exc)
{
LOG(LWARNING, ("KML binary serialization failure: ", exc.what(), "file:", convertedFilePath));
return false;
}
return true;
}
} // namespace
namespace migration
@ -349,13 +396,17 @@ void MigrateIfNeeded()
}
} // namespace migration
using namespace std::placeholders;
BookmarkManager::BookmarkManager(Callbacks && callbacks)
: m_callbacks(std::move(callbacks))
, m_changesTracker(*this)
, m_needTeardown(false)
, m_nextGroupID(UserMark::BOOKMARK)
, m_bookmarkCloud(Cloud::CloudParams("bmc.json", "bookmarks", "BookmarkCloudParam",
std::string(KMZ_EXTENSION)))
GetBookmarksDirectory(), std::string(kBookmarksExt),
std::bind(&ConvertBeforeUploading, _1, _2),
std::bind(&ConvertAfterDownloading, _1, _2)))
{
ASSERT(m_callbacks.m_getStringsBundle != nullptr, ());
m_userMarkLayers.reserve(UserMark::BOOKMARK);
@ -365,27 +416,12 @@ BookmarkManager::BookmarkManager(Callbacks && callbacks)
m_selectionMark = CreateUserMark<StaticMarkPoint>(m2::PointD{});
m_myPositionMark = CreateUserMark<MyPositionMarkPoint>(m2::PointD{});
m_bookmarkCloud.SetSynchronizationHandlers([]()
{
alohalytics::Stats::Instance().LogEvent("Bookmarks_sync_started");
}, [](Cloud::SynchronizationResult result, std::string const & errorStr)
{
if (result == Cloud::SynchronizationResult::Success)
{
alohalytics::Stats::Instance().LogEvent("Bookmarks_sync_success");
}
else
{
std::string const typeStr = (result == Cloud::SynchronizationResult::DiskError) ? "disk" : "network";
alohalytics::TStringMap details {{"type", typeStr}, {"error", errorStr}};
alohalytics::Stats::Instance().LogEvent("Bookmarks_sync_error", details);
}
});
}
BookmarkManager::~BookmarkManager()
{
ClearCategories();
using namespace std::placeholders;
m_bookmarkCloud.SetSynchronizationHandlers(
std::bind(&BookmarkManager::OnSynchronizationStarted, this, _1),
std::bind(&BookmarkManager::OnSynchronizationFinished, this, _1, _2, _3),
std::bind(&BookmarkManager::OnRestoreRequested, this, _1, _2),
std::bind(&BookmarkManager::OnRestoredFilesPrepared, this));
}
BookmarkManager::EditSession BookmarkManager::GetEditSession()
@ -764,8 +800,16 @@ void BookmarkManager::LoadState()
void BookmarkManager::ClearCategories()
{
ASSERT_THREAD_CHECKER(m_threadChecker, ());
for (auto groupId : m_bmGroupsIdList)
{
ClearGroup(groupId);
m_changesTracker.OnDeleteGroup(groupId);
}
m_categories.clear();
m_bmGroupsIdList.clear();
m_bookmarks.clear();
m_tracks.clear();
}
void BookmarkManager::LoadBookmarks()
@ -1637,6 +1681,74 @@ void BookmarkManager::ConvertAllKmlFiles(ConversionHandler && handler) const
});
}
void BookmarkManager::SetCloudHandlers(
Cloud::SynchronizationStartedHandler && onSynchronizationStarted,
Cloud::SynchronizationFinishedHandler && onSynchronizationFinished,
Cloud::RestoreRequestedHandler && onRestoreRequested,
Cloud::RestoredFilesPreparedHandler && onRestoredFilesPrepared)
{
m_onSynchronizationStarted = std::move(onSynchronizationStarted);
m_onSynchronizationFinished = std::move(onSynchronizationFinished);
m_onRestoreRequested = std::move(onRestoreRequested);
m_onRestoredFilesPrepared = std::move(onRestoredFilesPrepared);
}
void BookmarkManager::OnSynchronizationStarted(Cloud::SynchronizationType type)
{
GetPlatform().RunTask(Platform::Thread::Gui, [this, type]()
{
if (m_onSynchronizationStarted)
m_onSynchronizationStarted(type);
});
LOG(LINFO, ("Cloud Synchronization Started:", type));
}
void BookmarkManager::OnSynchronizationFinished(Cloud::SynchronizationType type,
Cloud::SynchronizationResult result,
std::string const & errorStr)
{
GetPlatform().RunTask(Platform::Thread::Gui, [this, type, result, errorStr]()
{
if (m_onSynchronizationFinished)
m_onSynchronizationFinished(type, result, errorStr);
if (type == Cloud::SynchronizationType::Restore &&
result == Cloud::SynchronizationResult::Success)
{
// Reload bookmarks after restoring.
LoadBookmarks();
}
});
LOG(LINFO, ("Cloud Synchronization Finished:", type, result, errorStr));
}
void BookmarkManager::OnRestoreRequested(Cloud::RestoringRequestResult result,
uint64_t backupTimestampInMs)
{
GetPlatform().RunTask(Platform::Thread::Gui, [this, result, backupTimestampInMs]()
{
if (m_onRestoreRequested)
m_onRestoreRequested(result, backupTimestampInMs);
});
using namespace std::chrono;
LOG(LINFO, ("Cloud Restore Requested:", result,
time_point<system_clock>(milliseconds(backupTimestampInMs))));
}
void BookmarkManager::OnRestoredFilesPrepared()
{
// This method is always called from UI-thread.
ClearCategories();
if (m_onRestoredFilesPrepared)
m_onRestoredFilesPrepared();
LOG(LINFO, ("Cloud Restore Files: Prepared"));
}
df::GroupIDSet BookmarkManager::MarksChangesTracker::GetAllGroupIds() const
{
auto const & groupIds = m_bmManager.GetBmGroupsIdList();

17
map/bookmark_manager.hpp
View file

@ -128,7 +128,6 @@ public:
};
explicit BookmarkManager(Callbacks && callbacks);
~BookmarkManager();
void SetDrapeEngine(ref_ptr<df::DrapeEngine> engine);
@ -244,6 +243,12 @@ public:
using ConversionHandler = platform::SafeCallback<void(bool success)>;
void ConvertAllKmlFiles(ConversionHandler && handler) const;
// These handlers are always called from UI-thread.
void SetCloudHandlers(Cloud::SynchronizationStartedHandler && onSynchronizationStarted,
Cloud::SynchronizationFinishedHandler && onSynchronizationFinished,
Cloud::RestoreRequestedHandler && onRestoreRequested,
Cloud::RestoredFilesPreparedHandler && onRestoredFilesPrepared);
/// These functions are public for unit tests only. You shouldn't call them from client code.
void SaveToKML(df::MarkGroupID groupId, std::ostream & s);
void CreateCategories(KMLDataCollection && dataCollection, bool autoSave = true);
@ -397,6 +402,12 @@ private:
void SaveToKML(BookmarkCategory * group, std::ostream & s);
void OnSynchronizationStarted(Cloud::SynchronizationType type);
void OnSynchronizationFinished(Cloud::SynchronizationType type, Cloud::SynchronizationResult result,
std::string const & errorStr);
void OnRestoreRequested(Cloud::RestoringRequestResult result, uint64_t backupTimestampInMs);
void OnRestoredFilesPrepared();
ThreadChecker m_threadChecker;
Callbacks m_callbacks;
@ -438,6 +449,10 @@ private:
std::list<BookmarkLoaderInfo> m_bookmarkLoadingQueue;
Cloud m_bookmarkCloud;
Cloud::SynchronizationStartedHandler m_onSynchronizationStarted;
Cloud::SynchronizationFinishedHandler m_onSynchronizationFinished;
Cloud::RestoreRequestedHandler m_onRestoreRequested;
Cloud::RestoredFilesPreparedHandler m_onRestoredFilesPrepared;
DISALLOW_COPY_AND_MOVE(BookmarkManager);
};

1118
map/cloud.cpp
View file

File diff suppressed because it is too large Load diff

215
map/cloud.hpp
View file

@ -8,6 +8,7 @@
#include "base/visitor.hpp"
#include <functional>
#include <list>
#include <map>
#include <memory>
#include <mutex>
@ -20,10 +21,13 @@ public:
struct Entry
{
Entry() = default;
Entry(std::string const & name, uint64_t sizeInBytes, bool isOutdated)
: m_name(name)
, m_sizeInBytes(sizeInBytes)
, m_isOutdated(isOutdated)
: m_name(name), m_sizeInBytes(sizeInBytes), m_isOutdated(isOutdated)
{}
Entry(std::string const & name, uint64_t sizeInBytes, bool isOutdated, std::string const & hash)
: m_name(name), m_sizeInBytes(sizeInBytes), m_hash(hash), m_isOutdated(isOutdated)
{}
bool operator==(Entry const & entry) const
@ -57,57 +61,59 @@ public:
bool m_isOutdated = false;
uint64_t m_lastSyncTimestamp = 0; // in seconds.
bool CanBeUploaded() const { return m_isOutdated && !m_entries.empty(); }
DECLARE_VISITOR_AND_DEBUG_PRINT(Index, visitor(m_entries, "entries"),
visitor(m_lastUpdateInHours, "lastUpdateInHours"),
visitor(m_isOutdated, "isOutdated"),
visitor(m_lastSyncTimestamp, "lastSyncTimestamp"))
};
struct SnapshotRequestData
{
std::string m_deviceId;
std::string m_deviceName;
std::vector<std::string> m_fileNames;
explicit SnapshotRequestData(std::vector<std::string> const & files = {});
DECLARE_VISITOR_AND_DEBUG_PRINT(SnapshotRequestData, visitor(m_deviceId, "device_id"),
visitor(m_deviceName, "device_name"),
visitor(m_fileNames, "file_names"))
};
struct UploadingRequestData
{
std::string m_deviceId;
std::string m_fileName;
explicit UploadingRequestData(std::string const & filePath = {});
DECLARE_VISITOR_AND_DEBUG_PRINT(UploadingRequestData, visitor(m_deviceId, "device_id"),
visitor(m_fileName, "file_name"))
};
struct UploadingResponseData
{
std::string m_url;
std::string m_fallbackUrl;
std::vector<std::vector<std::string>> m_fields;
std::string m_method;
DECLARE_VISITOR_AND_DEBUG_PRINT(UploadingResponseData, visitor(m_url, "url"),
visitor(m_fallbackUrl, "fallback_url"),
visitor(m_fields, "fields"),
visitor(m_method, "method"))
};
struct NotifyRequestData : public UploadingRequestData
struct SnapshotFileData
{
std::string m_fileName;
uint64_t m_fileSize = 0;
uint64_t m_datetime = 0;
NotifyRequestData() = default;
NotifyRequestData(std::string const & filePath, uint64_t fileSize);
DECLARE_VISITOR_AND_DEBUG_PRINT(NotifyRequestData, visitor(m_deviceId, "device_id"),
DECLARE_VISITOR_AND_DEBUG_PRINT(SnapshotFileData,
visitor(m_fileName, "file_name"),
visitor(m_fileSize, "file_size"))
visitor(m_fileSize, "file_size"),
visitor(m_datetime, "datetime"))
};
struct SnapshotResponseData
{
std::string m_deviceId;
std::string m_deviceName;
uint64_t m_datetime = 0;
std::vector<SnapshotFileData> m_files;
uint64_t GetTotalSizeOfFiles() const
{
uint64_t sz = 0;
for (auto const & f : m_files)
sz += f.m_fileSize;
return sz;
}
DECLARE_VISITOR_AND_DEBUG_PRINT(SnapshotResponseData,
visitor(m_deviceId, "device_id"),
visitor(m_deviceName, "device_name"),
visitor(m_datetime, "datetime"),
visitor(m_files, "files"))
};
enum class RequestStatus
@ -132,9 +138,17 @@ public:
struct UploadingResult
{
RequestResult m_requestResult;
bool m_isMalformed = false;
UploadingResponseData m_response;
};
struct SnapshotResult
{
RequestResult m_requestResult;
bool m_isMalformed = false;
SnapshotResponseData m_response;
};
enum class State
{
// User never enabled or disabled synchronization via cloud. It is a default state.
@ -145,6 +159,12 @@ public:
Enabled = 2
};
enum class SynchronizationType
{
Backup = 0,
Restore = 1
};
enum class SynchronizationResult
{
// Synchronization was finished successfully.
@ -154,24 +174,48 @@ public:
// Synchronization was interrupted by a network error.
NetworkError = 2,
// Synchronization was interrupted by a disk error.
DiskError = 3
DiskError = 3,
// Synchronization was interrupted by the user.
UserInterrupted = 4
};
enum class RestoringRequestResult
{
// There is a backup on the server.
BackupExists = 0,
// There is no backup on the server.
NoBackup = 1,
// Not enough space on the disk for the restoring.
NotEnoughDiskSpace = 2
};
using InvalidTokenHandler = std::function<void()>;
using SynchronizationStartedHandler = std::function<void()>;
using SynchronizationFinishedHandler = std::function<void(SynchronizationResult,
using FileConverter = std::function<bool(std::string const & filePath,
std::string const & convertedFilePath)>;
using SynchronizationStartedHandler = std::function<void(SynchronizationType)>;
using SynchronizationFinishedHandler = std::function<void(SynchronizationType,
SynchronizationResult,
std::string const & error)>;
using RestoreRequestedHandler = std::function<void(RestoringRequestResult,
uint64_t backupTimestampInMs)>;
using RestoredFilesPreparedHandler = std::function<void()>;
using SnapshotCompletionHandler = std::function<void()>;
struct CloudParams
{
CloudParams() = default;
CloudParams(std::string && indexName, std::string && serverPathName,
std::string && settingsParamName, std::string && zipExtension)
std::string && settingsParamName, std::string && restoringFolder,
std::string && restoredFileExtension,
FileConverter && backupConverter,
FileConverter && restoreConverter)
: m_indexName(std::move(indexName))
, m_serverPathName(std::move(serverPathName))
, m_settingsParamName(std::move(settingsParamName))
, m_zipExtension(std::move(zipExtension))
, m_restoredFileExtension(std::move(restoredFileExtension))
, m_restoringFolder(std::move(restoringFolder))
, m_backupConverter(std::move(backupConverter))
, m_restoreConverter(std::move(restoreConverter))
{}
// Name of file in which cloud stores metadata.
@ -180,8 +224,14 @@ public:
std::string m_serverPathName;
// Name of parameter to store cloud's state in settings.
std::string m_settingsParamName;
// Extension of zipped file. The first character must be '.'
std::string m_zipExtension;
// The extension of restored files.
std::string m_restoredFileExtension;
// The folder in which files will be restored.
std::string m_restoringFolder;
// This file converter is executed before uploading to the cloud.
FileConverter m_backupConverter;
// This file converter is executed after downloading from the cloud.
FileConverter m_restoreConverter;
};
explicit Cloud(CloudParams && params);
@ -189,9 +239,12 @@ public:
// Handler can be called from non-UI thread.
void SetInvalidTokenHandler(InvalidTokenHandler && onInvalidToken);
// Handlers can be called from non-UI thread.
// Handlers can be called from non-UI thread except of ApplyRestoredFilesHandler.
// ApplyRestoredFilesHandler is always called from UI-thread.
void SetSynchronizationHandlers(SynchronizationStartedHandler && onSynchronizationStarted,
SynchronizationFinishedHandler && onSynchronizationFinished);
SynchronizationFinishedHandler && onSynchronizationFinished,
RestoreRequestedHandler && onRestoreRequested,
RestoredFilesPreparedHandler && onRestoredFilesPrepared);
void SetState(State state);
State GetState() const;
@ -202,16 +255,35 @@ public:
std::unique_ptr<User::Subscriber> GetUserSubscriber();
void RequestRestoring();
void ApplyRestoring();
void CancelRestoring();
private:
struct RestoredFile
{
std::string m_filename;
std::string m_hash;
RestoredFile() = default;
RestoredFile(std::string const & filename, std::string const & hash)
: m_filename(filename), m_hash(hash)
{}
};
using RestoredFilesCollection = std::vector<RestoredFile>;
void LoadIndex();
bool ReadIndex();
void UpdateIndex(bool indexExists);
void SaveIndexImpl() const;
EntryPtr GetEntryImpl(std::string const & fileName) const;
void MarkModifiedImpl(std::string const & filePath);
void MarkModifiedImpl(std::string const & filePath, bool isOutdated);
void UpdateIndexByRestoredFilesImpl(RestoredFilesCollection const & files,
uint64_t lastSyncTimestampInSec);
uint64_t CalculateUploadingSizeImpl() const;
bool CanUploadImpl() const;
void SortEntriesBeforeUploadingImpl();
void ScheduleUploading();
void ScheduleUploadingTask(EntryPtr const & entry, uint32_t timeout,
@ -219,20 +291,34 @@ private:
void CreateSnapshotTask(uint32_t timeout, uint32_t attemptIndex,
std::vector<std::string> && files,
SnapshotCompletionHandler && handler);
void FinishSnapshotTask(uint32_t timeout, uint32_t attemptIndex);
EntryPtr FindOutdatedEntry() const;
void FinishUploading(SynchronizationResult result, std::string const & errorStr);
void SetAccessToken(std::string const & token);
std::string GetAccessToken() const;
// This function always returns path to a temporary file or the empty string
// in case of a disk error.
std::string PrepareFileToUploading(std::string const & fileName);
RequestResult CreateSnapshot(std::vector<std::string> const & files) const;
RequestResult FinishSnapshot() const;
SnapshotResult GetBestSnapshot() const;
void ProcessSuccessfulSnapshot(SnapshotResult const & result);
UploadingResult RequestUploading(std::string const & filePath) const;
RequestResult ExecuteUploading(UploadingResponseData const & responseData,
std::string const & filePath);
RequestResult NotifyAboutUploading(std::string const & filePath, uint64_t fileSize) const;
void GetBestSnapshotTask(uint32_t timeout, uint32_t attemptIndex);
void FinishRestoring(SynchronizationResult result, std::string const & errorStr);
std::list<SnapshotFileData> GetDownloadingList(std::string const & restoringDirPath);
void DownloadingTask(std::string const & dirPath, bool useFallbackUrl,
std::list<SnapshotFileData> && files);
void CompleteRestoring(std::string const & dirPath);
void ApplyRestoredFiles(std::string const & dirPath, RestoredFilesCollection && files);
template <typename HandlerType, typename HandlerGetterType, typename... HandlerArgs>
void ThreadSafeCallback(HandlerGetterType && handlerGetter, HandlerArgs... handlerArgs)
{
@ -249,11 +335,54 @@ private:
InvalidTokenHandler m_onInvalidToken;
SynchronizationStartedHandler m_onSynchronizationStarted;
SynchronizationFinishedHandler m_onSynchronizationFinished;
RestoreRequestedHandler m_onRestoreRequested;
RestoredFilesPreparedHandler m_onRestoredFilesPrepared;
State m_state;
Index m_index;
std::string m_accessToken;
std::map<std::string, std::string> m_files;
bool m_uploadingStarted = false;
enum RestoringState
{
None,
Requested,
Applying,
Finalizing
};
RestoringState m_restoringState = RestoringState::None;
bool m_indexUpdated = false;
SnapshotResponseData m_bestSnapshotData;
mutable std::mutex m_mutex;
};
inline std::string DebugPrint(Cloud::SynchronizationType type)
{
switch (type)
{
case Cloud::SynchronizationType::Backup: return "Backup";
case Cloud::SynchronizationType::Restore: return "Restore";
}
}
inline std::string DebugPrint(Cloud::SynchronizationResult result)
{
switch (result)
{
case Cloud::SynchronizationResult::Success: return "Success";
case Cloud::SynchronizationResult::AuthError: return "AuthError";
case Cloud::SynchronizationResult::NetworkError: return "NetworkError";
case Cloud::SynchronizationResult::DiskError: return "DiskError";
case Cloud::SynchronizationResult::UserInterrupted: return "UserInterrupted";
}
}
inline std::string DebugPrint(Cloud::RestoringRequestResult result)
{
switch (result)
{
case Cloud::RestoringRequestResult::BackupExists: return "BackupExists";
case Cloud::RestoringRequestResult::NoBackup: return "NoBackup";
case Cloud::RestoringRequestResult::NotEnoughDiskSpace: return "NotEnoughDiskSpace";
}
}

1
platform/http_uploader.hpp
View file

@ -19,6 +19,7 @@ public:
void SetMethod(std::string const & method) { m_method = method; }
void SetUrl(std::string const & url) { m_url = url; }
void SetParams(std::map<std::string, std::string> const & params) { m_params = params; }
void SetParam(std::string const & key, std::string const & value) { m_params[key] = value; }
void SetHeaders(std::map<std::string, std::string> const & headers) { m_headers = headers; }
void SetFileKey(std::string const & fileKey) { m_fileKey = fileKey; }
void SetFilePath(std::string const & filePath) { m_filePath = filePath; }