pastk/organicmaps
1
Fork 0
forked from organicmaps/organicmaps
Code Pull requests Activity

[storage] Delete the downloaded diff files when cancelled; better logging.

Browse source
This commit is contained in:
Maxim Pimenov 2019-03-13 15:21:49 +03:00 • committed by mpimenov
parent e029c9f961
commit a4b6780b25
5 changed files with 91 additions and 72 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

62
platform/chunks_download_strategy.cpp
View file

@ -1,5 +1,7 @@
#include "platform/chunks_download_strategy.hpp"
#include "platform/platform.hpp"
#include "coding/file_writer.hpp"
#include "coding/file_reader.hpp"
#include "coding/varint.hpp"
@ -8,7 +10,6 @@
#include "std/algorithm.hpp"
namespace downloader
{
@ -80,7 +81,7 @@ void ChunksDownloadStrategy::SaveChunks(int64_t fileSize, string const & fName)
}
// Delete if no chunks or some error occured.
(void)FileWriter::DeleteFileX(fName);
UNUSED_VALUE(Platform::RemoveFileIfExists(fName));
}
int64_t ChunksDownloadStrategy::LoadOrInitChunks(string const & fName, int64_t fileSize,
@ -89,39 +90,42 @@ int64_t ChunksDownloadStrategy::LoadOrInitChunks(string const & fName, int64_t f
ASSERT ( fileSize > 0, () );
ASSERT ( chunkSize > 0, () );
try
if (Platform::IsFileExistsByFullPath(fName))
{
FileReader r(fName);
ReaderSource<FileReader> src(r);
int64_t const readSize = ReadVarInt<int64_t>(src);
if (readSize == fileSize)
try
{
// Load chunks.
uint64_t const size = src.Size();
int const stSize = sizeof(ChunkT);
size_t const count = size / stSize;
ASSERT_EQUAL(size, stSize * count, ());
FileReader r(fName);
ReaderSource<FileReader> src(r);
m_chunks.resize(count);
src.Read(&m_chunks[0], stSize * count);
// Reset status "downloading" to "free".
int64_t downloadedSize = 0;
for (size_t i = 0; i < count-1; ++i)
int64_t const readSize = ReadVarInt<int64_t>(src);
if (readSize == fileSize)
{
if (m_chunks[i].m_status != CHUNK_COMPLETE)
m_chunks[i].m_status = CHUNK_FREE;
else
downloadedSize += (m_chunks[i+1].m_pos - m_chunks[i].m_pos);
}
// Load chunks.
uint64_t const size = src.Size();
int const stSize = sizeof(ChunkT);
size_t const count = size / stSize;
ASSERT_EQUAL(size, stSize * count, ());
return downloadedSize;
m_chunks.resize(count);
src.Read(&m_chunks[0], stSize * count);
// Reset status "downloading" to "free".
int64_t downloadedSize = 0;
for (size_t i = 0; i < count - 1; ++i)
{
if (m_chunks[i].m_status != CHUNK_COMPLETE)
m_chunks[i].m_status = CHUNK_FREE;
else
downloadedSize += (m_chunks[i + 1].m_pos - m_chunks[i].m_pos);
}
return downloadedSize;
}
}
catch (FileReader::Exception const & e)
{
LOG(LDEBUG, (e.Msg()));
}
}
catch (FileReader::Exception const & e)
{
LOG(LDEBUG, (e.Msg()));
}
InitChunks(fileSize, chunkSize);

9
platform/http_request.cpp
View file

@ -269,11 +269,10 @@ class FileHttpRequest : public HttpRequest, public IHttpThreadCallback
// 3. Clean up resume file with chunks range on success
if (m_status == Status::Completed)
{
base::DeleteFileX(m_filePath + RESUME_FILE_EXTENSION);
Platform::RemoveFileIfExists(m_filePath + RESUME_FILE_EXTENSION);
// Rename finished file to it's original name.
if (Platform::IsFileExistsByFullPath(m_filePath))
base::DeleteFileX(m_filePath);
Platform::RemoveFileIfExists(m_filePath);
CHECK(base::RenameFileX(m_filePath + DOWNLOADING_FILE_EXTENSION, m_filePath),
(m_filePath, strerror(errno)));
@ -353,8 +352,8 @@ public:
if (m_doCleanProgressFiles)
{
base::DeleteFileX(m_filePath + DOWNLOADING_FILE_EXTENSION);
base::DeleteFileX(m_filePath + RESUME_FILE_EXTENSION);
Platform::RemoveFileIfExists(m_filePath + DOWNLOADING_FILE_EXTENSION);
Platform::RemoveFileIfExists(m_filePath + RESUME_FILE_EXTENSION);
}
}
}

76
platform/local_country_file_utils.cpp
View file

@ -10,8 +10,8 @@
#include "coding/reader.hpp"
#include "base/assert.hpp"
#include "base/string_utils.hpp"
#include "base/logging.hpp"
#include "base/string_utils.hpp"
#include "std/algorithm.hpp"
#include "std/cctype.hpp"
@ -24,7 +24,6 @@
namespace platform
{
namespace migrate
{
// Set of functions to support migration between different versions of MWM
@ -43,10 +42,7 @@ bool NeedMigrate()
return true;
}
void SetMigrationFlag()
{
settings::Set("LastMigration", kMinRequiredVersion);
}
void SetMigrationFlag() { settings::Set("LastMigration", kMinRequiredVersion); }
} // namespace migrate
namespace
@ -136,6 +132,16 @@ void FindAllDiffsInDirectory(string const & dir, vector<LocalCountryFile> & diff
diffs.push_back(localDiff);
}
}
string GetFilePath(int64_t version, string const & dataDir, CountryFile const & countryFile,
MapOptions options)
{
string const readyFile = GetFileName(countryFile.GetName(), options, version);
string const dir = GetDataDirFullPath(dataDir);
if (version == 0)
return base::JoinFoldersToPath(dir, readyFile);
return base::JoinFoldersToPath({dir, strings::to_string(version)}, readyFile);
}
} // namespace
void DeleteDownloaderFilesForCountry(int64_t version, CountryFile const & countryFile)
@ -146,13 +152,19 @@ void DeleteDownloaderFilesForCountry(int64_t version, CountryFile const & countr
void DeleteDownloaderFilesForCountry(int64_t version, string const & dataDir,
CountryFile const & countryFile)
{
for (MapOptions file : {MapOptions::Map, MapOptions::CarRouting, MapOptions::Diff})
for (MapOptions opt : {MapOptions::Map, MapOptions::CarRouting, MapOptions::Diff})
{
string const path = GetFileDownloadPath(version, dataDir, countryFile, file);
string const path = GetFileDownloadPath(version, dataDir, countryFile, opt);
ASSERT(strings::EndsWith(path, READY_FILE_EXTENSION), ());
base::DeleteFileX(path);
base::DeleteFileX(path + RESUME_FILE_EXTENSION);
base::DeleteFileX(path + DOWNLOADING_FILE_EXTENSION);
Platform::RemoveFileIfExists(path);
Platform::RemoveFileIfExists(path + RESUME_FILE_EXTENSION);
Platform::RemoveFileIfExists(path + DOWNLOADING_FILE_EXTENSION);
}
// Delete the diff that was downloaded but wasn't applied.
{
string const path = GetFilePath(version, dataDir, countryFile, MapOptions::Diff);
Platform::RemoveFileIfExists(path);
}
}
@ -262,8 +274,9 @@ void FindAllLocalMapsAndCleanup(int64_t latestVersion, string const & dataDir,
// World and WorldCoasts can be stored in app bundle or in resources
// directory, thus it's better to get them via Platform.
for (string const & file : { WORLD_FILE_NAME,
(migrate::NeedMigrate() ? WORLD_COASTS_OBSOLETE_FILE_NAME : WORLD_COASTS_FILE_NAME) })
for (string const & file :
{WORLD_FILE_NAME,
(migrate::NeedMigrate() ? WORLD_COASTS_OBSOLETE_FILE_NAME : WORLD_COASTS_FILE_NAME)})
{
auto i = localFiles.begin();
for (; i != localFiles.end(); ++i)
@ -279,8 +292,7 @@ void FindAllLocalMapsAndCleanup(int64_t latestVersion, string const & dataDir,
platform.GetReader(file + DATA_FILE_EXTENSION, GetSpecialFilesSearchScope()));
// Assume that empty path means the resource file.
LocalCountryFile worldFile{string(), CountryFile(file),
version::ReadVersionDate(reader)};
LocalCountryFile worldFile{string(), CountryFile(file), version::ReadVersionDate(reader)};
worldFile.m_files = MapOptions::Map;
if (i != localFiles.end())
{
@ -323,7 +335,8 @@ bool ParseVersion(string const & s, int64_t & version)
return true;
}
shared_ptr<LocalCountryFile> PreparePlaceForCountryFiles(int64_t version, CountryFile const & countryFile)
shared_ptr<LocalCountryFile> PreparePlaceForCountryFiles(int64_t version,
CountryFile const & countryFile)
{
return PreparePlaceForCountryFiles(version, string(), countryFile);
}
@ -345,17 +358,19 @@ string GetFileDownloadPath(int64_t version, CountryFile const & countryFile, Map
return GetFileDownloadPath(version, string(), countryFile, options);
}
string GetFileDownloadPath(int64_t version, string const & dataDir,
CountryFile const & countryFile, MapOptions options)
string GetFileDownloadPath(int64_t version, string const & dataDir, CountryFile const & countryFile,
MapOptions options)
{
string const readyFile = GetFileName(countryFile.GetName(), options, version) + READY_FILE_EXTENSION;
string const readyFile =
GetFileName(countryFile.GetName(), options, version) + READY_FILE_EXTENSION;
string const dir = GetDataDirFullPath(dataDir);
if (version == 0)
return base::JoinFoldersToPath(dir, readyFile);
return base::JoinFoldersToPath({dir, strings::to_string(version)}, readyFile);
}
unique_ptr<ModelReader> GetCountryReader(platform::LocalCountryFile const & file, MapOptions options)
unique_ptr<ModelReader> GetCountryReader(platform::LocalCountryFile const & file,
MapOptions options)
{
Platform & platform = GetPlatform();
// See LocalCountryFile comment for explanation.
@ -406,15 +421,9 @@ string CountryIndexes::GetPath(LocalCountryFile const & localFile, Index index)
char const * ext = nullptr;
switch (index)
{
case Index::Bits:
ext = kBitsExt;
break;
case Index::Nodes:
ext = kNodesExt;
break;
case Index::Offsets:
ext = kOffsetsExt;
break;
case Index::Bits: ext = kBitsExt; break;
case Index::Nodes: ext = kNodesExt; break;
case Index::Offsets: ext = kOffsetsExt; break;
}
return base::JoinFoldersToPath(IndexesDir(localFile), localFile.GetCountryName() + ext);
}
@ -460,12 +469,9 @@ string DebugPrint(CountryIndexes::Index index)
{
switch (index)
{
case CountryIndexes::Index::Bits:
return "Bits";
case CountryIndexes::Index::Nodes:
return "Nodes";
case CountryIndexes::Index::Offsets:
return "Offsets";
case CountryIndexes::Index::Bits: return "Bits";
case CountryIndexes::Index::Nodes: return "Nodes";
case CountryIndexes::Index::Offsets: return "Offsets";
}
UNREACHABLE();
}

9
storage/diff_scheme/diff_manager.cpp
View file

@ -94,7 +94,7 @@ void Manager::ApplyDiff(ApplyDiffParams && p, base::Cancellable const & cancella
base::DeleteFileX(diffApplyingInProgressPath);
if (result != DiffApplicationResult::Ok)
base::DeleteFileX(newMwmPath);
Platform::RemoveFileIfExists(newMwmPath);
}
switch (result)
@ -108,11 +108,14 @@ void Manager::ApplyDiff(ApplyDiffParams && p, base::Cancellable const & cancella
it->second.m_status = SingleDiffStatus::Applied;
break;
}
case DiffApplicationResult::Cancelled: break;
case DiffApplicationResult::Cancelled:
// The diff file will be deleted by storage.
// Another way would be to leave it on disk but all consequences
// of interacting with storage are much harder to be taken into account that way.
break;
case DiffApplicationResult::Failed:
{
diffFile->DeleteFromDisk(MapOptions::Diff);
alohalytics::Stats::Instance().LogEvent(
"Downloader_DiffScheme_error",
{{"type", "patching"},

7
storage/storage.cpp
View file

@ -167,6 +167,7 @@ void Storage::DeleteAllLocalMaps(CountriesVec * existedCountries /* = nullptr */
existedCountries->push_back(localFiles.first);
localFile->SyncWithDisk();
DeleteFromDiskWithIndexes(*localFile, MapOptions::MapWithCarRouting);
DeleteFromDiskWithIndexes(*localFile, MapOptions::Diff);
}
}
}
@ -287,6 +288,7 @@ void Storage::RegisterAllLocalMaps(bool enableDiffs)
LOG(LINFO, ("Removing obsolete", localFile));
localFile.SyncWithDisk();
DeleteFromDiskWithIndexes(localFile, MapOptions::MapWithCarRouting);
DeleteFromDiskWithIndexes(localFile, MapOptions::Diff);
++j;
}
@ -582,6 +584,7 @@ void Storage::DeleteCustomCountryVersion(LocalCountryFile const & localFile)
CountryFile const countryFile = localFile.GetCountryFile();
DeleteFromDiskWithIndexes(localFile, MapOptions::MapWithCarRouting);
DeleteFromDiskWithIndexes(localFile, MapOptions::Diff);
{
auto it = m_localFilesForFakeCountries.find(countryFile);
@ -1480,6 +1483,8 @@ void Storage::DownloadNode(CountryId const & countryId, bool isUpdate /* = false
{
CHECK_THREAD_CHECKER(m_threadChecker, ());
LOG(LINFO, ("Downloading", countryId));
CountryTreeNode const * const node = m_countries.FindFirst(countryId);
if (!node)
@ -1920,6 +1925,8 @@ void Storage::CancelDownloadNode(CountryId const & countryId)
{
CHECK_THREAD_CHECKER(m_threadChecker, ());
LOG(LINFO, ("Cancelling the downloading of", countryId));
CountriesSet setQueue;
GetQueuedCountries(m_queue, setQueue);