forked from organicmaps/organicmaps
Refactoring of downloader resume policy.
This commit is contained in:
vng
Alex Zolotarev
parent
3dfe083be6
commit
ffe2d85b22
5 changed files with 312 additions and 207 deletions
|
|
@ -1,92 +1,199 @@
|
|||
#include "chunks_download_strategy.hpp"
|
||||
|
||||
#include "../std/algorithm.hpp"
|
||||
#include "../defines.hpp"
|
||||
|
||||
#define INVALID_CHUNK -1
|
||||
#include "../coding/file_writer.hpp"
|
||||
#include "../coding/file_reader.hpp"
|
||||
|
||||
#include "../base/logging.hpp"
|
||||
|
||||
#include "../std/algorithm.hpp"
|
||||
|
||||
|
||||
namespace downloader
|
||||
{
|
||||
|
||||
ChunksDownloadStrategy::RangeT const ChunksDownloadStrategy::INVALID_RANGE = RangeT(INVALID_CHUNK, INVALID_CHUNK);
|
||||
|
||||
ChunksDownloadStrategy::ChunksDownloadStrategy(vector<string> const & urls, int64_t fileSize,
|
||||
int64_t chunkSize)
|
||||
: m_chunkSize(chunkSize)
|
||||
ChunksDownloadStrategy::ChunksDownloadStrategy(vector<string> const & urls)
|
||||
{
|
||||
// init servers list
|
||||
for (size_t i = 0; i < urls.size(); ++i)
|
||||
m_servers.push_back(make_pair(urls[i], INVALID_RANGE));
|
||||
|
||||
// init chunks which should be downloaded
|
||||
for (int64_t i = 0; i < fileSize; i += chunkSize)
|
||||
m_chunksToDownload.insert(RangeT(i, min(i + chunkSize - 1, fileSize - 1)));
|
||||
m_servers.push_back(ServerT(urls[i], SERVER_READY));
|
||||
}
|
||||
|
||||
void ChunksDownloadStrategy::SetChunksToDownload(RangesContainerT & chunks)
|
||||
pair<ChunksDownloadStrategy::ChunkT *, int>
|
||||
ChunksDownloadStrategy::GetChunk(RangeT const & range)
|
||||
{
|
||||
m_chunksToDownload.swap(chunks);
|
||||
}
|
||||
vector<ChunkT>::iterator i = lower_bound(m_chunks.begin(), m_chunks.end(), range.first, LessChunks());
|
||||
|
||||
void ChunksDownloadStrategy::ChunkFinished(bool successfully, RangeT const & range)
|
||||
{
|
||||
// find server which was downloading this chunk
|
||||
for (ServersT::iterator it = m_servers.begin(); it != m_servers.end(); ++it)
|
||||
if (i != m_chunks.end() && i->m_pos == range.first)
|
||||
{
|
||||
if (it->second == range)
|
||||
{
|
||||
if (successfully)
|
||||
it->second = INVALID_RANGE;
|
||||
else
|
||||
{
|
||||
// @TODO implement connection retry
|
||||
// remove failed server and mark chunk as not downloaded
|
||||
m_servers.erase(it);
|
||||
m_chunksToDownload.insert(range);
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
ChunksDownloadStrategy::ResultT ChunksDownloadStrategy::NextChunk(string & outUrl,
|
||||
RangeT & range)
|
||||
{
|
||||
if (m_servers.empty())
|
||||
return EDownloadFailed;
|
||||
|
||||
if (m_chunksToDownload.empty())
|
||||
{
|
||||
// no more chunks to download
|
||||
bool allChunksAreFinished = true;
|
||||
for (size_t i = 0; i < m_servers.size(); ++i)
|
||||
{
|
||||
if (m_servers[i].second != INVALID_RANGE)
|
||||
allChunksAreFinished = false;
|
||||
}
|
||||
if (allChunksAreFinished)
|
||||
return EDownloadSucceeded;
|
||||
else
|
||||
return ENoFreeServers;
|
||||
ASSERT_EQUAL ( (i+1)->m_pos, range.second + 1, () );
|
||||
return pair<ChunkT *, int>(&(*i), distance(m_chunks.begin(), i));
|
||||
}
|
||||
else
|
||||
{
|
||||
RangeT const nextChunk = *m_chunksToDownload.begin();
|
||||
for (size_t i = 0; i < m_servers.size(); ++i)
|
||||
{
|
||||
if (m_servers[i].second == INVALID_RANGE)
|
||||
{
|
||||
// found not used server
|
||||
m_servers[i].second = nextChunk;
|
||||
outUrl = m_servers[i].first;
|
||||
range = nextChunk;
|
||||
m_chunksToDownload.erase(m_chunksToDownload.begin());
|
||||
return ENextChunk;
|
||||
}
|
||||
}
|
||||
// if we're here, all servers are busy downloading
|
||||
return ENoFreeServers;
|
||||
LOG(LERROR, ("Downloader error. Invalid chunk range: ", range));
|
||||
return pair<ChunkT *, int>(0, -1);
|
||||
}
|
||||
}
|
||||
|
||||
void ChunksDownloadStrategy::InitChunks(int64_t fileSize, int64_t chunkSize, ChunkStatusT status)
|
||||
{
|
||||
// init chunks which should be downloaded
|
||||
m_chunks.reserve(fileSize / chunkSize + 2);
|
||||
for (int64_t i = 0; i < fileSize; i += chunkSize)
|
||||
m_chunks.push_back(ChunkT(i, status));
|
||||
m_chunks.push_back(ChunkT(fileSize, CHUNK_AUX));
|
||||
}
|
||||
|
||||
void ChunksDownloadStrategy::AddChunk(RangeT const & range, ChunkStatusT status)
|
||||
{
|
||||
ASSERT_LESS_OR_EQUAL ( range.first, range.second, () );
|
||||
if (m_chunks.empty())
|
||||
{
|
||||
ASSERT_EQUAL ( range.first, 0, () );
|
||||
m_chunks.push_back(ChunkT(range.first, status));
|
||||
}
|
||||
else
|
||||
{
|
||||
ASSERT_EQUAL ( m_chunks.back().m_pos, range.first, () );
|
||||
m_chunks.back().m_status = status;
|
||||
}
|
||||
|
||||
m_chunks.push_back(ChunkT(range.second + 1, CHUNK_AUX));
|
||||
}
|
||||
|
||||
void ChunksDownloadStrategy::SaveChunks(string const & fName)
|
||||
{
|
||||
if (!m_chunks.empty())
|
||||
{
|
||||
try
|
||||
{
|
||||
FileWriter w(fName + RESUME_FILE_EXTENSION);
|
||||
w.Write(&m_chunks[0], sizeof(ChunkT) * m_chunks.size());
|
||||
return;
|
||||
}
|
||||
catch(FileWriter::Exception const & e)
|
||||
{
|
||||
LOG(LERROR, (e.Msg()));
|
||||
}
|
||||
}
|
||||
|
||||
// Delete if no chunks or some error occured.
|
||||
(void)FileWriter::DeleteFileX(fName + RESUME_FILE_EXTENSION);
|
||||
}
|
||||
|
||||
int64_t ChunksDownloadStrategy::LoadOrInitChunks( string const & fName,
|
||||
int64_t fileSize, int64_t chunkSize)
|
||||
{
|
||||
ASSERT ( fileSize > 0, () );
|
||||
ASSERT ( chunkSize > 0, () );
|
||||
|
||||
try
|
||||
{
|
||||
FileReader r(fName + RESUME_FILE_EXTENSION);
|
||||
|
||||
// Load chunks.
|
||||
size_t const count = r.Size() / sizeof(ChunkT);
|
||||
m_chunks.resize(count);
|
||||
r.Read(0, &m_chunks[0], sizeof(ChunkT) * count);
|
||||
|
||||
// Reset status "downloading" to "free".
|
||||
int64_t downloadedSize = 0;
|
||||
for (size_t i = 0; i < count-1; ++i)
|
||||
{
|
||||
if (m_chunks[i].m_status != CHUNK_COMPLETE)
|
||||
m_chunks[i].m_status = CHUNK_FREE;
|
||||
else
|
||||
downloadedSize += m_chunks[i+1].m_pos - m_chunks[i].m_pos;
|
||||
}
|
||||
|
||||
return downloadedSize;
|
||||
}
|
||||
catch(FileReader::Exception const & e)
|
||||
{
|
||||
// Usually - file not exists.
|
||||
LOG(LDEBUG, (e.Msg()));
|
||||
}
|
||||
|
||||
InitChunks(fileSize, chunkSize);
|
||||
return 0;
|
||||
}
|
||||
|
||||
void ChunksDownloadStrategy::ChunkFinished(bool success, RangeT const & range)
|
||||
{
|
||||
pair<ChunkT *, int> res = GetChunk(range);
|
||||
|
||||
// find server which was downloading this chunk
|
||||
if (res.first)
|
||||
{
|
||||
for (size_t s = 0; s < m_servers.size(); ++s)
|
||||
{
|
||||
if (m_servers[s].m_chunkIndex == res.second)
|
||||
{
|
||||
if (success)
|
||||
{
|
||||
// mark server as free and chunk as ready
|
||||
m_servers[s].m_chunkIndex = SERVER_READY;
|
||||
res.first->m_status = CHUNK_COMPLETE;
|
||||
}
|
||||
else
|
||||
{
|
||||
// remove failed server and mark chunk as free
|
||||
m_servers.erase(m_servers.begin() + s);
|
||||
res.first->m_status = CHUNK_FREE;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
ChunksDownloadStrategy::ResultT
|
||||
ChunksDownloadStrategy::NextChunk(string & outUrl, RangeT & range)
|
||||
{
|
||||
// If no servers at all.
|
||||
if (m_servers.empty())
|
||||
return EDownloadFailed;
|
||||
|
||||
// Find first free server.
|
||||
ServerT * server = 0;
|
||||
for (size_t i = 0; i < m_servers.size(); ++i)
|
||||
{
|
||||
if (m_servers[i].m_chunkIndex == SERVER_READY)
|
||||
{
|
||||
server = &m_servers[i];
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (server == 0)
|
||||
return ENoFreeServers;
|
||||
|
||||
bool allChunksDownloaded = true;
|
||||
|
||||
// Find first free chunk.
|
||||
for (size_t i = 0; i < m_chunks.size()-1; ++i)
|
||||
{
|
||||
switch (m_chunks[i].m_status)
|
||||
{
|
||||
case CHUNK_FREE:
|
||||
server->m_chunkIndex = i;
|
||||
outUrl = server->m_url;
|
||||
|
||||
range.first = m_chunks[i].m_pos;
|
||||
range.second = m_chunks[i+1].m_pos - 1;
|
||||
|
||||
m_chunks[i].m_status = CHUNK_DOWNLOADING;
|
||||
return ENextChunk;
|
||||
|
||||
case CHUNK_DOWNLOADING:
|
||||
allChunksDownloaded = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return (allChunksDownloaded ? EDownloadSucceeded : ENoFreeServers);
|
||||
}
|
||||
|
||||
} // namespace downloader
|
||||
|
|
|
|||
|
|
@ -14,24 +14,55 @@ namespace downloader
|
|||
class ChunksDownloadStrategy
|
||||
{
|
||||
public:
|
||||
typedef pair<int64_t, int64_t> RangeT;
|
||||
typedef set<RangeT> RangesContainerT;
|
||||
enum ChunkStatusT { CHUNK_FREE = 0, CHUNK_DOWNLOADING = 1, CHUNK_COMPLETE = 2, CHUNK_AUX = -1 };
|
||||
|
||||
private:
|
||||
int64_t m_chunkSize;
|
||||
static RangeT const INVALID_RANGE;
|
||||
/// <server url, currently downloading range or INVALID_RANGE if url is not used>
|
||||
typedef vector<pair<string, RangeT> > ServersT;
|
||||
ServersT m_servers;
|
||||
RangesContainerT m_chunksToDownload;
|
||||
struct ChunkT
|
||||
{
|
||||
/// position of chunk in file
|
||||
int64_t m_pos;
|
||||
/// @see ChunkStatusT
|
||||
int8_t m_status;
|
||||
|
||||
ChunkT() : m_pos(-1), m_status(-1) {}
|
||||
ChunkT(int64_t pos, int8_t st) : m_pos(pos), m_status(st) {}
|
||||
};
|
||||
|
||||
vector<ChunkT> m_chunks;
|
||||
|
||||
static const int SERVER_READY = -1;
|
||||
struct ServerT
|
||||
{
|
||||
string m_url;
|
||||
int m_chunkIndex;
|
||||
|
||||
ServerT(string const & url, int ind) : m_url(url), m_chunkIndex(ind) {}
|
||||
};
|
||||
|
||||
vector<ServerT> m_servers;
|
||||
|
||||
struct LessChunks
|
||||
{
|
||||
bool operator() (ChunkT const & r1, ChunkT const & r2) const { return r1.m_pos < r2.m_pos; }
|
||||
bool operator() (ChunkT const & r1, int64_t const & r2) const { return r1.m_pos < r2; }
|
||||
bool operator() (int64_t const & r1, ChunkT const & r2) const { return r1 < r2.m_pos; }
|
||||
};
|
||||
|
||||
typedef pair<int64_t, int64_t> RangeT;
|
||||
pair<ChunkT *, int> GetChunk(RangeT const & range);
|
||||
|
||||
public:
|
||||
/// @param[in] chunksToDownload used for resume
|
||||
ChunksDownloadStrategy(vector<string> const & urls, int64_t fileSize, int64_t chunkSize);
|
||||
ChunksDownloadStrategy(vector<string> const & urls);
|
||||
|
||||
void InitChunks(int64_t fileSize, int64_t chunkSize, ChunkStatusT status = CHUNK_FREE);
|
||||
void AddChunk(RangeT const & range, ChunkStatusT status);
|
||||
|
||||
void SaveChunks(string const & fName);
|
||||
/// @return Already downloaded size.
|
||||
int64_t LoadOrInitChunks(string const & fName, int64_t fileSize, int64_t chunkSize);
|
||||
|
||||
void ChunkFinished(bool success, RangeT const & range);
|
||||
|
||||
int64_t ChunkSize() const { return m_chunkSize; }
|
||||
/// Should be called when each chunk is completed
|
||||
void ChunkFinished(bool successfully, RangeT const & range);
|
||||
enum ResultT
|
||||
{
|
||||
ENextChunk,
|
||||
|
|
@ -41,9 +72,6 @@ public:
|
|||
};
|
||||
/// Should be called until returns ENextChunk
|
||||
ResultT NextChunk(string & outUrl, RangeT & range);
|
||||
/// Used for resume support - external code knows when it's necessary
|
||||
void SetChunksToDownload(RangesContainerT & chunks);
|
||||
RangesContainerT const & ChunksLeft() const { return m_chunksToDownload; }
|
||||
};
|
||||
|
||||
} // namespace downloader
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@
|
|||
#include "../defines.hpp"
|
||||
|
||||
#ifdef DEBUG
|
||||
#include "../base/thread.hpp"
|
||||
#include "../base/thread.hpp"
|
||||
#endif
|
||||
|
||||
#include "../base/std_serialization.hpp"
|
||||
|
|
@ -13,25 +13,31 @@
|
|||
|
||||
#include "../coding/file_writer_stream.hpp"
|
||||
#include "../coding/file_reader_stream.hpp"
|
||||
#include "../coding/internal/file_data.hpp"
|
||||
|
||||
#include "../std/scoped_ptr.hpp"
|
||||
|
||||
#include "../3party/jansson/myjansson.hpp"
|
||||
|
||||
|
||||
#ifdef OMIM_OS_IPHONE
|
||||
|
||||
#include <sys/xattr.h>
|
||||
#endif
|
||||
|
||||
void DisableiCloudBackupForFile(string const & filePath)
|
||||
void DisableBackupForFile(string const & filePath)
|
||||
{
|
||||
#ifdef OMIM_OS_IPHONE
|
||||
// We need to disable iCloud backup for downloaded files.
|
||||
// This is the reason for rejecting from the AppStore
|
||||
|
||||
static char const * attrName = "com.apple.MobileBackup";
|
||||
u_int8_t attrValue = 1;
|
||||
const int result = setxattr(filePath.c_str(), attrName, &attrValue, sizeof(attrValue), 0, 0);
|
||||
if (result != 0)
|
||||
LOG(LWARNING, ("Error while disabling iCloud backup for file", filePath));
|
||||
#endif
|
||||
}
|
||||
|
||||
#endif // OMIM_OS_IPHONE
|
||||
|
||||
class HttpThread;
|
||||
|
||||
|
|
@ -71,7 +77,7 @@ class MemoryHttpRequest : public HttpRequest, public IHttpThreadCallback
|
|||
}
|
||||
|
||||
public:
|
||||
MemoryHttpRequest(string const & url, CallbackT onFinish, CallbackT onProgress)
|
||||
MemoryHttpRequest(string const & url, CallbackT const & onFinish, CallbackT const & onProgress)
|
||||
: HttpRequest(onFinish, onProgress), m_writer(m_downloadedData)
|
||||
{
|
||||
m_thread = CreateNativeHttpThread(url, *this);
|
||||
|
|
@ -111,7 +117,7 @@ class FileHttpRequest : public HttpRequest, public IHttpThreadCallback
|
|||
ChunksDownloadStrategy::ResultT StartThreads()
|
||||
{
|
||||
string url;
|
||||
ChunksDownloadStrategy::RangeT range;
|
||||
pair<int64_t, int64_t> range;
|
||||
ChunksDownloadStrategy::ResultT result;
|
||||
while ((result = m_strategy.NextChunk(url, range)) == ChunksDownloadStrategy::ENextChunk)
|
||||
m_threads.push_back(make_pair(CreateNativeHttpThread(url, *this, range.first, range.second,
|
||||
|
|
@ -133,23 +139,25 @@ class FileHttpRequest : public HttpRequest, public IHttpThreadCallback
|
|||
|
||||
virtual void OnWrite(int64_t offset, void const * buffer, size_t size)
|
||||
{
|
||||
#ifdef DEBUG
|
||||
#ifdef DEBUG
|
||||
static threads::ThreadID const id = threads::GetCurrentThreadID();
|
||||
ASSERT_EQUAL(id, threads::GetCurrentThreadID(), ("OnWrite called from different threads"));
|
||||
#endif
|
||||
#endif
|
||||
|
||||
m_writer->Seek(offset);
|
||||
m_writer->Write(buffer, size);
|
||||
}
|
||||
|
||||
/// Called by each http thread
|
||||
/// Called for each chunk by one main (GUI) thread.
|
||||
virtual void OnFinish(long httpCode, int64_t begRange, int64_t endRange)
|
||||
{
|
||||
#ifdef DEBUG
|
||||
static threads::ThreadID const id = threads::GetCurrentThreadID();
|
||||
ASSERT_EQUAL(id, threads::GetCurrentThreadID(), ("OnFinish called from different threads"));
|
||||
#endif
|
||||
|
||||
bool const isChunkOk = (httpCode == 200);
|
||||
m_strategy.ChunkFinished(isChunkOk, ChunksDownloadStrategy::RangeT(begRange, endRange));
|
||||
m_strategy.ChunkFinished(isChunkOk, make_pair(begRange, endRange));
|
||||
|
||||
// remove completed chunk from the list, beg is the key
|
||||
RemoveHttpThreadByKey(begRange);
|
||||
|
|
@ -170,102 +178,52 @@ class FileHttpRequest : public HttpRequest, public IHttpThreadCallback
|
|||
m_status = ECompleted;
|
||||
|
||||
if (isChunkOk)
|
||||
{ // save information for download resume
|
||||
{
|
||||
// save information for download resume
|
||||
++m_goodChunksCount;
|
||||
if (m_status != ECompleted && m_goodChunksCount % 10 == 0)
|
||||
SaveRanges(m_filePath + RESUME_FILE_EXTENSION, m_strategy.ChunksLeft());
|
||||
m_strategy.SaveChunks(m_filePath);
|
||||
}
|
||||
|
||||
if (m_status != EInProgress)
|
||||
{
|
||||
m_writer.reset();
|
||||
|
||||
// clean up resume file with chunks range on success
|
||||
if (m_strategy.ChunksLeft().empty())
|
||||
if (m_status == ECompleted)
|
||||
{
|
||||
FileWriter::DeleteFileX(m_filePath + RESUME_FILE_EXTENSION);
|
||||
// rename finished file to it's original name
|
||||
rename((m_filePath + DOWNLOADING_FILE_EXTENSION).c_str(), m_filePath.c_str());
|
||||
#ifdef OMIM_OS_IPHONE
|
||||
// We need to disable iCloud backup for downloaded files.
|
||||
// This is the reason for rejecting from the AppStore
|
||||
DisableiCloudBackupForFile(m_filePath.c_str());
|
||||
#endif
|
||||
my::DeleteFileX(m_filePath + RESUME_FILE_EXTENSION);
|
||||
|
||||
// Rename finished file to it's original name.
|
||||
CHECK(my::RenameFileX((m_filePath + DOWNLOADING_FILE_EXTENSION).c_str(), m_filePath.c_str()), ());
|
||||
|
||||
DisableBackupForFile(m_filePath.c_str());
|
||||
}
|
||||
else // or save "chunks left" otherwise
|
||||
SaveRanges(m_filePath + RESUME_FILE_EXTENSION, m_strategy.ChunksLeft());
|
||||
m_strategy.SaveChunks(m_filePath);
|
||||
|
||||
m_onFinish(*this);
|
||||
}
|
||||
}
|
||||
|
||||
/// @return true if ranges are present and loaded
|
||||
static bool LoadRanges(string const & file, ChunksDownloadStrategy::RangesContainerT & ranges)
|
||||
{
|
||||
ranges.clear();
|
||||
try
|
||||
{
|
||||
FileReaderStream frs(file);
|
||||
frs >> ranges;
|
||||
}
|
||||
catch (std::exception const &)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
return !ranges.empty();
|
||||
}
|
||||
|
||||
static void SaveRanges(string const & file, ChunksDownloadStrategy::RangesContainerT const & ranges)
|
||||
{
|
||||
// Delete resume file if ranges are empty
|
||||
if (ranges.empty())
|
||||
FileWriter::DeleteFileX(file);
|
||||
else
|
||||
{
|
||||
FileWriterStream fws(file);
|
||||
fws << ranges;
|
||||
}
|
||||
#ifdef OMIM_OS_IPHONE
|
||||
DisableiCloudBackupForFile(file);
|
||||
#endif
|
||||
}
|
||||
|
||||
struct CalcRanges
|
||||
{
|
||||
int64_t & m_summ;
|
||||
CalcRanges(int64_t & summ) : m_summ(summ) {}
|
||||
void operator()(ChunksDownloadStrategy::RangeT const & range)
|
||||
{
|
||||
m_summ += (range.second - range.first) + 1;
|
||||
}
|
||||
};
|
||||
|
||||
public:
|
||||
FileHttpRequest(vector<string> const & urls, string const & filePath, int64_t fileSize,
|
||||
CallbackT onFinish, CallbackT onProgress, int64_t chunkSize, bool doCleanProgressFiles)
|
||||
: HttpRequest(onFinish, onProgress), m_strategy(urls, fileSize, chunkSize),
|
||||
m_filePath(filePath),
|
||||
CallbackT const & onFinish, CallbackT const & onProgress,
|
||||
int64_t chunkSize, bool doCleanProgressFiles)
|
||||
: HttpRequest(onFinish, onProgress), m_strategy(urls), m_filePath(filePath),
|
||||
m_writer(new FileWriter(filePath + DOWNLOADING_FILE_EXTENSION, FileWriter::OP_WRITE_EXISTING)),
|
||||
m_goodChunksCount(0),
|
||||
m_doCleanProgressFiles(doCleanProgressFiles)
|
||||
m_goodChunksCount(0), m_doCleanProgressFiles(doCleanProgressFiles)
|
||||
{
|
||||
ASSERT_GREATER(fileSize, 0, ("At the moment only known file sizes are supported"));
|
||||
ASSERT(!urls.empty(), ("Urls list shouldn't be empty"));
|
||||
m_progress.second = fileSize;
|
||||
ASSERT ( !urls.empty(), () );
|
||||
|
||||
// Resume support - load chunks which should be downloaded (if they're present)
|
||||
ChunksDownloadStrategy::RangesContainerT ranges;
|
||||
if (LoadRanges(filePath + RESUME_FILE_EXTENSION, ranges))
|
||||
{
|
||||
// fix progress
|
||||
int64_t sizeLeft = 0;
|
||||
for_each(ranges.begin(), ranges.end(), CalcRanges(sizeLeft));
|
||||
m_progress.first = fileSize - sizeLeft;
|
||||
m_strategy.SetChunksToDownload(ranges);
|
||||
}
|
||||
m_progress.first = m_strategy.LoadOrInitChunks(m_filePath, fileSize, chunkSize);
|
||||
m_progress.second = fileSize;
|
||||
|
||||
#ifdef OMIM_OS_IPHONE
|
||||
m_writer->Flush();
|
||||
DisableiCloudBackupForFile(filePath + DOWNLOADING_FILE_EXTENSION);
|
||||
DisableBackupForFile(filePath + DOWNLOADING_FILE_EXTENSION);
|
||||
#endif
|
||||
|
||||
StartThreads();
|
||||
}
|
||||
|
||||
|
|
@ -275,13 +233,14 @@ public:
|
|||
DeleteNativeHttpThread(it->first);
|
||||
|
||||
if (m_status == EInProgress)
|
||||
{ // means that client canceled download process
|
||||
// so delete all temporary files
|
||||
{
|
||||
// means that client canceled download process, so delete all temporary files
|
||||
m_writer.reset();
|
||||
|
||||
if (m_doCleanProgressFiles)
|
||||
{
|
||||
FileWriter::DeleteFileX(m_filePath + DOWNLOADING_FILE_EXTENSION);
|
||||
FileWriter::DeleteFileX(m_filePath + RESUME_FILE_EXTENSION);
|
||||
CHECK(my::DeleteFileX(m_filePath + DOWNLOADING_FILE_EXTENSION), ());
|
||||
my::DeleteFileX(m_filePath + RESUME_FILE_EXTENSION);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -293,7 +252,7 @@ public:
|
|||
};
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
HttpRequest::HttpRequest(CallbackT onFinish, CallbackT onProgress)
|
||||
HttpRequest::HttpRequest(CallbackT const & onFinish, CallbackT const & onProgress)
|
||||
: m_status(EInProgress), m_progress(make_pair(0, -1)),
|
||||
m_onFinish(onFinish), m_onProgress(onProgress)
|
||||
{
|
||||
|
|
@ -303,19 +262,20 @@ HttpRequest::~HttpRequest()
|
|||
{
|
||||
}
|
||||
|
||||
HttpRequest * HttpRequest::Get(string const & url, CallbackT onFinish, CallbackT onProgress)
|
||||
HttpRequest * HttpRequest::Get(string const & url, CallbackT const & onFinish, CallbackT const & onProgress)
|
||||
{
|
||||
return new MemoryHttpRequest(url, onFinish, onProgress);
|
||||
}
|
||||
|
||||
HttpRequest * HttpRequest::PostJson(string const & url, string const & postData,
|
||||
CallbackT onFinish, CallbackT onProgress)
|
||||
CallbackT const & onFinish, CallbackT const & onProgress)
|
||||
{
|
||||
return new MemoryHttpRequest(url, postData, onFinish, onProgress);
|
||||
}
|
||||
|
||||
HttpRequest * HttpRequest::GetFile(vector<string> const & urls, string const & filePath, int64_t fileSize,
|
||||
CallbackT onFinish, CallbackT onProgress, int64_t chunkSize, bool doCleanProgressFiles)
|
||||
CallbackT const & onFinish, CallbackT const & onProgress,
|
||||
int64_t chunkSize, bool doCleanProgressFiles)
|
||||
{
|
||||
return new FileHttpRequest(urls, filePath, fileSize, onFinish, onProgress, chunkSize, doCleanProgressFiles);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -30,7 +30,7 @@ protected:
|
|||
CallbackT m_onFinish;
|
||||
CallbackT m_onProgress;
|
||||
|
||||
explicit HttpRequest(CallbackT onFinish, CallbackT onProgress);
|
||||
explicit HttpRequest(CallbackT const & onFinish, CallbackT const & onProgress);
|
||||
|
||||
public:
|
||||
virtual ~HttpRequest() = 0;
|
||||
|
|
@ -41,14 +41,17 @@ public:
|
|||
virtual string const & Data() const = 0;
|
||||
|
||||
/// Response saved to memory buffer and retrieved with Data()
|
||||
static HttpRequest * Get(string const & url, CallbackT onFinish,
|
||||
CallbackT onProgress = CallbackT());
|
||||
static HttpRequest * Get(string const & url,
|
||||
CallbackT const & onFinish,
|
||||
CallbackT const & onProgress = CallbackT());
|
||||
/// Content-type for request is always "application/json"
|
||||
static HttpRequest * PostJson(string const & url, string const & postData,
|
||||
CallbackT onFinish, CallbackT onProgress = CallbackT());
|
||||
CallbackT const & onFinish,
|
||||
CallbackT const & onProgress = CallbackT());
|
||||
static HttpRequest * GetFile(vector<string> const & urls, string const & filePath,
|
||||
int64_t projectedFileSize,
|
||||
CallbackT onFinish, CallbackT onProgress = CallbackT(),
|
||||
CallbackT const & onFinish,
|
||||
CallbackT const & onProgress = CallbackT(),
|
||||
int64_t chunkSize = 512 * 1024,
|
||||
bool doCleanProgressFiles = true);
|
||||
};
|
||||
|
|
|
|||
|
|
@ -210,29 +210,34 @@ UNIT_TEST(ChunksDownloadStrategy)
|
|||
string const S1 = "UrlOfServer1";
|
||||
string const S2 = "UrlOfServer2";
|
||||
string const S3 = "UrlOfServer3";
|
||||
ChunksDownloadStrategy::RangeT const R1(0, 249), R2(250, 499), R3(500, 749), R4(750, 800);
|
||||
|
||||
typedef pair<int64_t, int64_t> RangeT;
|
||||
RangeT const R1(0, 249), R2(250, 499), R3(500, 749), R4(750, 800);
|
||||
|
||||
vector<string> servers;
|
||||
servers.push_back(S1);
|
||||
servers.push_back(S2);
|
||||
servers.push_back(S3);
|
||||
|
||||
int64_t const FILE_SIZE = 800;
|
||||
int64_t const CHUNK_SIZE = 250;
|
||||
ChunksDownloadStrategy strategy(servers, FILE_SIZE, CHUNK_SIZE);
|
||||
ChunksDownloadStrategy strategy(servers);
|
||||
strategy.InitChunks(FILE_SIZE, CHUNK_SIZE);
|
||||
|
||||
string s1;
|
||||
ChunksDownloadStrategy::RangeT r1;
|
||||
RangeT r1;
|
||||
TEST_EQUAL(strategy.NextChunk(s1, r1), ChunksDownloadStrategy::ENextChunk, ());
|
||||
|
||||
string s2;
|
||||
ChunksDownloadStrategy::RangeT r2;
|
||||
RangeT r2;
|
||||
TEST_EQUAL(strategy.NextChunk(s2, r2), ChunksDownloadStrategy::ENextChunk, ());
|
||||
|
||||
string s3;
|
||||
ChunksDownloadStrategy::RangeT r3;
|
||||
RangeT r3;
|
||||
TEST_EQUAL(strategy.NextChunk(s3, r3), ChunksDownloadStrategy::ENextChunk, ());
|
||||
|
||||
string sEmpty;
|
||||
ChunksDownloadStrategy::RangeT rEmpty;
|
||||
RangeT rEmpty;
|
||||
TEST_EQUAL(strategy.NextChunk(sEmpty, rEmpty), ChunksDownloadStrategy::ENoFreeServers, ());
|
||||
TEST_EQUAL(strategy.NextChunk(sEmpty, rEmpty), ChunksDownloadStrategy::ENoFreeServers, ());
|
||||
|
||||
|
|
@ -246,7 +251,7 @@ UNIT_TEST(ChunksDownloadStrategy)
|
|||
strategy.ChunkFinished(true, r1);
|
||||
|
||||
string s4;
|
||||
ChunksDownloadStrategy::RangeT r4;
|
||||
RangeT r4;
|
||||
TEST_EQUAL(strategy.NextChunk(s4, r4), ChunksDownloadStrategy::ENextChunk, ());
|
||||
TEST_EQUAL(s4, s1, ());
|
||||
TEST(r4 != r1 && r4 != r2 && r4 != r3, (r4));
|
||||
|
|
@ -261,7 +266,7 @@ UNIT_TEST(ChunksDownloadStrategy)
|
|||
strategy.ChunkFinished(true, r4);
|
||||
|
||||
string s5;
|
||||
ChunksDownloadStrategy::RangeT r5;
|
||||
RangeT r5;
|
||||
TEST_EQUAL(strategy.NextChunk(s5, r5), ChunksDownloadStrategy::ENextChunk, ());
|
||||
TEST_EQUAL(s5, s4, (s5, s4));
|
||||
TEST_EQUAL(r5, r2, ());
|
||||
|
|
@ -285,19 +290,25 @@ UNIT_TEST(ChunksDownloadStrategyFAIL)
|
|||
{
|
||||
string const S1 = "UrlOfServer1";
|
||||
string const S2 = "UrlOfServer2";
|
||||
ChunksDownloadStrategy::RangeT const R1(0, 249), R2(250, 499), R3(500, 749), R4(750, 800);
|
||||
|
||||
typedef pair<int64_t, int64_t> RangeT;
|
||||
RangeT const R1(0, 249), R2(250, 499), R3(500, 749), R4(750, 800);
|
||||
|
||||
vector<string> servers;
|
||||
servers.push_back(S1);
|
||||
servers.push_back(S2);
|
||||
|
||||
int64_t const FILE_SIZE = 800;
|
||||
int64_t const CHUNK_SIZE = 250;
|
||||
ChunksDownloadStrategy strategy(servers, FILE_SIZE, CHUNK_SIZE);
|
||||
ChunksDownloadStrategy strategy(servers);
|
||||
strategy.InitChunks(FILE_SIZE, CHUNK_SIZE);
|
||||
|
||||
string s1;
|
||||
ChunksDownloadStrategy::RangeT r1;
|
||||
RangeT r1;
|
||||
TEST_EQUAL(strategy.NextChunk(s1, r1), ChunksDownloadStrategy::ENextChunk, ());
|
||||
|
||||
string s2;
|
||||
ChunksDownloadStrategy::RangeT r2;
|
||||
RangeT r2;
|
||||
TEST_EQUAL(strategy.NextChunk(s2, r2), ChunksDownloadStrategy::ENextChunk, ());
|
||||
TEST_EQUAL(strategy.NextChunk(s2, r2), ChunksDownloadStrategy::ENoFreeServers, ());
|
||||
|
||||
|
|
@ -507,11 +518,13 @@ UNIT_TEST(DownloadResumeChunks)
|
|||
TEST_EQUAL(sha2::digest256(s), SHA256, ());
|
||||
uint64_t size;
|
||||
TEST(!Platform::GetFileSizeByFullPath(RESUME_FILENAME, size), ("No resume file on success"));
|
||||
// to substitute temporary not fully downloaded file
|
||||
my::RenameFileX(FILENAME, DOWNLOADING_FILENAME);
|
||||
}
|
||||
|
||||
// 2nd step - mark some file blocks as not downloaded
|
||||
{
|
||||
// to substitute temporary not fully downloaded file
|
||||
my::RenameFileX(FILENAME, DOWNLOADING_FILENAME);
|
||||
|
||||
FileWriter f(DOWNLOADING_FILENAME, FileWriter::OP_WRITE_EXISTING);
|
||||
f.Seek(beg1);
|
||||
char b1[end1 - beg1 + 1];
|
||||
|
|
@ -523,21 +536,15 @@ UNIT_TEST(DownloadResumeChunks)
|
|||
for (size_t i = 0; i < ARRAY_SIZE(b2); ++i) b2[i] = 0;
|
||||
f.Write(b2, ARRAY_SIZE(b2));
|
||||
|
||||
vector<pair<int64_t, int64_t> > originalRanges;
|
||||
originalRanges.push_back(make_pair(beg1, end1));
|
||||
originalRanges.push_back(make_pair(beg2, end2));
|
||||
{
|
||||
FileWriterStream fws(RESUME_FILENAME);
|
||||
fws << originalRanges;
|
||||
}
|
||||
// check if ranges are stored correctly
|
||||
vector<pair<int64_t, int64_t> > loadedRanges;
|
||||
{
|
||||
FileReaderStream frs(RESUME_FILENAME);
|
||||
frs >> loadedRanges;
|
||||
}
|
||||
TEST_EQUAL(originalRanges, loadedRanges, ());
|
||||
ChunksDownloadStrategy strategy((vector<string>()));
|
||||
strategy.AddChunk(make_pair(int64_t(0), beg1-1), ChunksDownloadStrategy::CHUNK_COMPLETE);
|
||||
strategy.AddChunk(make_pair(beg1, end1), ChunksDownloadStrategy::CHUNK_FREE);
|
||||
strategy.AddChunk(make_pair(end1+1, beg2-1), ChunksDownloadStrategy::CHUNK_COMPLETE);
|
||||
strategy.AddChunk(make_pair(beg2, end2), ChunksDownloadStrategy::CHUNK_FREE);
|
||||
|
||||
strategy.SaveChunks(FILENAME);
|
||||
}
|
||||
|
||||
// 3rd step - check that resume works
|
||||
{
|
||||
ResumeChecker checker;
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue