pastk/organicmaps
1
Fork 0
forked from organicmaps/organicmaps
Code Pull requests Activity

[mac][downloader] Finished basic chunks support

Browse source
This commit is contained in:
Alex Zolotarev 2011-11-07 23:03:58 +03:00 • committed by Alex Zolotarev
parent 4ca29d9593
commit cf4d77fa0b
9 changed files with 247 additions and 117 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
platform/chunks_download_strategy.hpp
View file

@ -22,6 +22,8 @@ class ChunksDownloadStrategy
public:
ChunksDownloadStrategy(vector<string> const & urls, int64_t fileSize, int64_t chunkSize = 512 * 1024);
int64_t ChunkSize() const { return m_chunkSize; }
/// Should be called when each chunk is completed
void ChunkFinished(bool successfully, int64_t begRange, int64_t endRange);
enum ResultT
{
@ -30,6 +32,7 @@ public:
EDownloadFailed,
EDownloadSucceeded
};
/// Should be called until returns ENextChunk
ResultT NextChunk(string & outUrl, int64_t & begRange, int64_t & endRange);
};

150
platform/http_request.cpp
View file

@ -1,23 +1,26 @@
#include "http_request.hpp"
#include "chunks_download_strategy.hpp"
#include "http_thread_callback.hpp"
#ifdef DEBUG
#include "../base/thread.hpp"
#include "../base/logging.hpp"
#endif
#include "../coding/writer.hpp"
class HttpThread;
namespace downloader
{
/// @return 0 if creation failed
HttpRequestImpl * CreateNativeHttpRequest(string const & url,
IHttpRequestImplCallback & callback,
HttpThread * CreateNativeHttpThread(string const & url,
IHttpThreadCallback & callback,
int64_t begRange = 0,
int64_t endRange = -1,
int64_t expectedSize = -1,
string const & postBody = string());
void DeleteNativeHttpRequest(HttpRequestImpl * request);
void DeleteNativeHttpThread(HttpThread * request);
//////////////////////////////////////////////////////////////////////////////////////////
HttpRequest::HttpRequest(Writer & writer, CallbackT onFinish, CallbackT onProgress)
@ -28,55 +31,64 @@ HttpRequest::HttpRequest(Writer & writer, CallbackT onFinish, CallbackT onProgre
HttpRequest::~HttpRequest()
{
for (ThreadsContainerT::iterator it = m_threads.begin(); it != m_threads.end(); ++it)
DeleteNativeHttpRequest(*it);
}
void HttpRequest::OnSizeKnown(int64_t projectedSize)
//////////////////////////////////////////////////////////////////////////////////////////////////////////
class SimpleHttpRequest : public HttpRequest, public IHttpThreadCallback
{
LOG(LDEBUG, ("Projected size", projectedSize));
}
HttpThread * m_thread;
void HttpRequest::OnWrite(int64_t offset, void const * buffer, size_t size)
{
#ifdef DEBUG
static threads::ThreadID id = threads::GetCurrentThreadID();
ASSERT_EQUAL(id, threads::GetCurrentThreadID(), ("OnWrite called from different threads"));
#endif
m_writer.Seek(offset);
m_writer.Write(buffer, size);
m_progress.first += size;
if (m_onProgress)
m_onProgress(*this);
}
virtual void OnWrite(int64_t, void const * buffer, size_t size)
{
m_writer.Write(buffer, size);
m_progress.first += size;
if (m_onProgress)
m_onProgress(*this);
}
void HttpRequest::OnFinish(long httpCode, int64_t begRange, int64_t endRange)
{
m_status = (httpCode == 200) ? ECompleted : EFailed;
ASSERT(m_onFinish, ());
m_onFinish(*this);
}
virtual void OnFinish(long httpCode, int64_t begRange, int64_t endRange)
{
m_status = (httpCode == 200) ? ECompleted : EFailed;
m_onFinish(*this);
}
public:
SimpleHttpRequest(string const & url, Writer & writer, CallbackT onFinish, CallbackT onProgress)
: HttpRequest(writer, onFinish, onProgress)
{
m_thread = CreateNativeHttpThread(url, *this);
}
SimpleHttpRequest(string const & url, Writer & writer, string const & postData,
CallbackT onFinish, CallbackT onProgress)
: HttpRequest(writer, onFinish, onProgress)
{
m_thread = CreateNativeHttpThread(url, *this, 0, -1, -1, postData);
}
virtual ~SimpleHttpRequest()
{
DeleteNativeHttpThread(m_thread);
}
};
HttpRequest * HttpRequest::Get(string const & url, Writer & writer, CallbackT onFinish, CallbackT onProgress)
{
HttpRequest * self = new HttpRequest(writer, onFinish, onProgress);
self->m_threads.push_back(CreateNativeHttpRequest(url, *self));
return self;
return new SimpleHttpRequest(url, writer, onFinish, onProgress);
}
HttpRequest * HttpRequest::Post(string const & url, Writer & writer, string const & postData,
CallbackT onFinish, CallbackT onProgress)
{
HttpRequest * self = new HttpRequest(writer, onFinish, onProgress);
self->m_threads.push_back(CreateNativeHttpRequest(url, *self, 0, -1, postData));
return self;
return new SimpleHttpRequest(url, writer, postData, onFinish, onProgress);
}
//////////////////////////////////////////////////////////////////////////////////////////////////////////
class ChunksHttpRequest : public HttpRequest
////////////////////////////////////////////////////////////////////////////////////////////////
class ChunksHttpRequest : public HttpRequest, public IHttpThreadCallback
{
ChunksDownloadStrategy m_strategy;
typedef list<pair<HttpThread *, int64_t> > ThreadsContainerT;
ThreadsContainerT m_threads;
ChunksDownloadStrategy::ResultT StartThreads()
{
@ -84,27 +96,77 @@ class ChunksHttpRequest : public HttpRequest
int64_t beg, end;
ChunksDownloadStrategy::ResultT result;
while ((result = m_strategy.NextChunk(url, beg, end)) == ChunksDownloadStrategy::ENextChunk)
m_threads.push_back(CreateNativeHttpRequest(url, *this, beg, end));
m_threads.push_back(make_pair(CreateNativeHttpThread(url, *this, beg, end, m_progress.second), beg));
return result;
}
void RemoveHttpThreadByKey(int64_t begRange)
{
for (ThreadsContainerT::iterator it = m_threads.begin(); it != m_threads.end(); ++it)
if (it->second == begRange)
{
DeleteNativeHttpThread(it->first);
m_threads.erase(it);
return;
}
ASSERT(false, ("Tried to remove invalid thread?"));
}
virtual void OnWrite(int64_t offset, void const * buffer, size_t size)
{
#ifdef DEBUG
static threads::ThreadID const id = threads::GetCurrentThreadID();
ASSERT_EQUAL(id, threads::GetCurrentThreadID(), ("OnWrite called from different threads"));
#endif
m_writer.Seek(offset);
m_writer.Write(buffer, size);
}
virtual void OnFinish(long httpCode, int64_t begRange, int64_t endRange)
{
#ifdef DEBUG
static threads::ThreadID const id = threads::GetCurrentThreadID();
ASSERT_EQUAL(id, threads::GetCurrentThreadID(), ("OnFinish called from different threads"));
#endif
m_strategy.ChunkFinished(httpCode == 200, begRange, endRange);
// remove completed chunk from the list, beg is the key
RemoveHttpThreadByKey(begRange);
ChunksDownloadStrategy::ResultT const result = StartThreads();
// report progress
if (result == ChunksDownloadStrategy::EDownloadSucceeded
|| result == ChunksDownloadStrategy::ENoFreeServers)
{
m_progress.first += m_strategy.ChunkSize();
if (m_onProgress)
m_onProgress(*this);
}
if (result == ChunksDownloadStrategy::EDownloadFailed)
m_status = EFailed;
else if (result == ChunksDownloadStrategy::EDownloadSucceeded)
m_status = ECompleted;
if (m_status != EInProgress)
m_onFinish(*this);
}
public:
ChunksHttpRequest(vector<string> const & urls, Writer & writer, int64_t fileSize,
CallbackT onFinish, CallbackT onProgress, int64_t chunkSize)
: HttpRequest(writer, onFinish, onProgress), m_strategy(urls, fileSize, chunkSize)
{
ASSERT(!urls.empty(), ("Urls list shouldn't be empty"));
// store expected file size for future checks
m_progress.second = fileSize;
StartThreads();
}
protected:
virtual void OnFinish(long httpCode, int64_t begRange, int64_t endRange)
virtual ~ChunksHttpRequest()
{
m_strategy.ChunkFinished(httpCode == 200, begRange, endRange);
ChunksDownloadStrategy::ResultT const result = StartThreads();
if (result != ChunksDownloadStrategy::ENoFreeServers)
HttpRequest::OnFinish(result == ChunksDownloadStrategy::EDownloadSucceeded ? 200 : -2,
0, -1);
for (ThreadsContainerT::iterator it = m_threads.begin(); it != m_threads.end(); ++it)
DeleteNativeHttpThread(it->first);
}
};

24
platform/http_request.hpp
View file

@ -2,20 +2,16 @@
#include "../std/function.hpp"
#include "../std/string.hpp"
#include "../std/list.hpp"
#include "../std/vector.hpp"
#include "../std/utility.hpp"
#include "http_request_impl_callback.hpp"
class Writer;
class HttpRequestImpl;
namespace downloader
{
/// Request will be canceled on delete
class HttpRequest : public IHttpRequestImplCallback
/// Request in progress will be canceled on delete
class HttpRequest
{
public:
enum StatusT
@ -27,31 +23,19 @@ public:
/// <current, total>, total can be -1 if size is unknown
typedef pair<int64_t, int64_t> ProgressT;
typedef function<void(HttpRequest &)> CallbackT;
private:
protected:
StatusT m_status;
ProgressT m_progress;
Writer & m_writer;
CallbackT m_onFinish;
CallbackT m_onProgress;
protected:
typedef list<HttpRequestImpl *> ThreadsContainerT;
ThreadsContainerT m_threads;
explicit HttpRequest(Writer & writer, CallbackT onFinish, CallbackT onProgress);
/// @name Callbacks for internal native downloading threads
//@{
virtual void OnSizeKnown(int64_t projectedSize);
virtual void OnWrite(int64_t offset, void const * buffer, size_t size);
virtual void OnFinish(long httpCode, int64_t begRange, int64_t endRange);
//@}
public:
virtual ~HttpRequest();
virtual ~HttpRequest() = 0;
StatusT Status() const { return m_status; }
ProgressT Progress() const { return m_progress; }

22
platform/http_request_impl_apple.h
View file

@ -1,22 +0,0 @@
#pragma once
#import <Foundation/Foundation.h>
#include "../std/string.hpp"
namespace downloader { class IHttpRequestImplCallback; }
@interface HttpRequestImpl : NSObject
{
downloader::IHttpRequestImplCallback * m_callback;
NSURLConnection * m_connection;
int64_t m_begRange, m_endRange;
int64_t m_downloadedBytes;
}
- (id) initWith:(string const &)url callback:(downloader::IHttpRequestImplCallback &)cb
begRange:(int64_t)beg endRange:(int64_t)end contentType:(string const &)ct
postBody:(string const &)pb;
- (void) cancel;
@end

22
platform/http_thread_apple.h Normal file
View file

@ -0,0 +1,22 @@
#pragma once
#import <Foundation/Foundation.h>
#include "../std/string.hpp"
namespace downloader { class IHttpThreadCallback; }
@interface HttpThread : NSObject
{
downloader::IHttpThreadCallback * m_callback;
NSURLConnection * m_connection;
int64_t m_begRange, m_endRange;
int64_t m_downloadedBytes;
int64_t m_expectedSize;
}
- (id) initWith:(string const &)url callback:(downloader::IHttpThreadCallback &)cb begRange:(int64_t)beg
endRange:(int64_t)end expectedSize:(int64_t)size postBody:(string const &)pb;
- (void) cancel;
@end

64
platform/http_request_impl_apple.mm → platform/http_thread_apple.mm
View file

@ -1,13 +1,13 @@
#import "http_request_impl_apple.h"
#import "http_thread_apple.h"
#include "http_request_impl_callback.hpp"
#include "http_thread_callback.hpp"
#include "platform.hpp"
#include "../base/logging.hpp"
#define TIMEOUT_IN_SECONDS 15.0
@implementation HttpRequestImpl
@implementation HttpThread
- (void) dealloc
{
@ -22,8 +22,8 @@
[m_connection cancel];
}
- (id) initWith:(string const &)url callback:(downloader::IHttpRequestImplCallback &)cb
begRange:(int64_t)beg endRange:(int64_t)end postBody:(string const &)pb
- (id) initWith:(string const &)url callback:(downloader::IHttpThreadCallback &)cb begRange:(int64_t)beg
endRange:(int64_t)end expectedSize:(int64_t)size postBody:(string const &)pb
{
self = [super init];
@ -31,12 +31,14 @@
m_begRange = beg;
m_endRange = end;
m_downloadedBytes = 0;
m_expectedSize = size;
NSMutableURLRequest * request = [NSMutableURLRequest requestWithURL:
[NSURL URLWithString:[NSString stringWithUTF8String:url.c_str()]]
cachePolicy:NSURLRequestReloadIgnoringLocalCacheData timeoutInterval:TIMEOUT_IN_SECONDS];
if (beg > 0)
// use Range header only if we don't download whole file from start
if (!(beg == 0 && end < 0))
{
NSString * val;
if (end > 0)
@ -82,6 +84,19 @@
return self;
}
/// @return -1 if can't decode
+ (int64_t) getContentRange:(NSDictionary *)httpHeader
{
NSString * cr = [httpHeader valueForKey:@"Content-Range"];
if (cr)
{
NSArray * arr = [cr componentsSeparatedByString:@"/"];
if (arr && [arr count])
return [(NSString *)[arr objectAtIndex:[arr count] - 1] longLongValue];
}
return -1;
}
- (void) connection: (NSURLConnection *)connection didReceiveResponse: (NSURLResponse *)response
{
// This method is called when the server has determined that it
@ -92,11 +107,6 @@
{
NSInteger const statusCode = [(NSHTTPURLResponse *)response statusCode];
LOG(LDEBUG, ("Got response with status code", statusCode));
#ifdef DEBUG
// NSDictionary * fields = [(NSHTTPURLResponse *)response allHeaderFields];
// for (id key in fields)
// NSLog(@"%@: %@", key, [fields objectForKey:key]);
#endif
if (statusCode < 200 || statusCode > 299)
{
LOG(LWARNING, ("Received HTTP error, canceling download", statusCode));
@ -104,8 +114,23 @@
m_callback->OnFinish(statusCode, m_begRange, m_endRange);
return;
}
else if (m_expectedSize > 0)
{
// get full file expected size from Content-Range header
int64_t sizeOnServer = [HttpThread getContentRange:[(NSHTTPURLResponse *)response allHeaderFields]];
// if it's absent, use Content-Length instead
if (sizeOnServer < 0)
sizeOnServer = [response expectedContentLength];
if (sizeOnServer > 0 & m_expectedSize != sizeOnServer)
{
m_callback->OnSizeKnown([response expectedContentLength]);
LOG(LWARNING, ("Canceling download - server replied with invalid size",
sizeOnServer, "!=", m_expectedSize));
[m_connection cancel];
m_callback->OnFinish(-2, m_begRange, m_endRange);
return;
}
}
}
else
{ // in theory, we should never be here
@ -138,16 +163,17 @@
///////////////////////////////////////////////////////////////////////////////////////
namespace downloader
{
HttpRequestImpl * CreateNativeHttpRequest(string const & url,
downloader::IHttpRequestImplCallback & cb,
int64_t beg,
int64_t end,
string const & pb)
HttpThread * CreateNativeHttpThread(string const & url,
downloader::IHttpThreadCallback & cb,
int64_t beg,
int64_t end,
int64_t size,
string const & pb)
{
return [[HttpRequestImpl alloc] initWith:url callback:cb begRange:beg endRange:end postBody:pb];
return [[HttpThread alloc] initWith:url callback:cb begRange:beg endRange:end expectedSize:size postBody:pb];
}
void DeleteNativeHttpRequest(HttpRequestImpl * request)
void DeleteNativeHttpThread(HttpThread * request)
{
[request cancel];
[request release];

4
platform/http_request_impl_callback.hpp → platform/http_thread_callback.hpp
View file

@ -3,11 +3,9 @@
namespace downloader
{
class IHttpRequestImplCallback
class IHttpThreadCallback
{
public:
/// Called before OnWrite, projectedSize can be -1 if server doesn't support it
virtual void OnSizeKnown(int64_t projectedSize) = 0;
virtual void OnWrite(int64_t offset, void const * buffer, size_t size) = 0;
virtual void OnFinish(long httpCode, int64_t begRange, int64_t endRange) = 0;
};

11
platform/platform.pro
View file

@ -40,10 +40,10 @@ include($$ROOT_DIR/common.pri)
macx*|iphone* {
HEADERS += apple_download.h \
http_request_impl_apple.h
http_thread_apple.h
OBJECTIVE_SOURCES += apple_download.mm \
apple_download_manager.mm \
http_request_impl_apple.mm
http_thread_apple.mm
}
win32*|linux* {
@ -67,8 +67,8 @@ HEADERS += \
languages.hpp \
url_generator.hpp \
http_request.hpp \
http_request_impl_callback.hpp \
chunks_download_strategy.hpp
http_thread_callback.hpp \
chunks_download_strategy.hpp \
SOURCES += \
preferred_languages.cpp \
@ -77,5 +77,4 @@ SOURCES += \
languages.cpp \
url_generator.cpp \
http_request.cpp \
chunks_download_strategy.cpp
chunks_download_strategy.cpp \

64
platform/platform_tests/downloader_test.cpp
View file

@ -291,15 +291,35 @@ UNIT_TEST(DownloadChunks)
vector<string> urls;
urls.push_back(TEST_URL1);
urls.push_back(TEST_URL1);
int64_t FILESIZE = 5;
{ // should use only one thread
scoped_ptr<HttpRequest> request(HttpRequest::GetChunks(urls, writer, 5, onFinish, onProgress));
scoped_ptr<HttpRequest> request(HttpRequest::GetChunks(urls, writer, FILESIZE, onFinish, onProgress));
// wait until download is finished
QCoreApplication::exec();
observer.TestOk();
TEST_EQUAL(buffer.size(), FILESIZE, ());
TEST_EQUAL(buffer, "Test1", (buffer));
}
observer.Reset();
writer.Seek(0);
buffer.clear();
urls.clear();
urls.push_back(TEST_URL_BIG_FILE);
urls.push_back(TEST_URL_BIG_FILE);
urls.push_back(TEST_URL_BIG_FILE);
FILESIZE = 5;
{ // 3 threads - fail, because of invalid size
scoped_ptr<HttpRequest> request(HttpRequest::GetChunks(urls, writer, FILESIZE, onFinish, onProgress, 2048));
// wait until download is finished
QCoreApplication::exec();
observer.TestFailed();
TEST_EQUAL(buffer.size(), 0, ());
}
string const SHA256 = "EE6AE6A2A3619B2F4A397326BEC32583DE2196D9D575D66786CB3B6F9D04A633";
observer.Reset();
@ -310,12 +330,50 @@ UNIT_TEST(DownloadChunks)
urls.push_back(TEST_URL_BIG_FILE);
urls.push_back(TEST_URL_BIG_FILE);
urls.push_back(TEST_URL_BIG_FILE);
FILESIZE = 47684;
{ // 3 threads
scoped_ptr<HttpRequest> request(HttpRequest::GetChunks(urls, writer, 5, onFinish, onProgress, 2048));
{ // 3 threads - succeeded
scoped_ptr<HttpRequest> request(HttpRequest::GetChunks(urls, writer, FILESIZE, onFinish, onProgress, 2048));
// wait until download is finished
QCoreApplication::exec();
observer.TestOk();
TEST_EQUAL(buffer.size(), FILESIZE, ());
TEST_EQUAL(sha2::digest256(buffer), SHA256, (buffer));
}
observer.Reset();
writer.Seek(0);
buffer.clear();
urls.clear();
urls.push_back(TEST_URL_BIG_FILE);
urls.push_back(TEST_URL1);
urls.push_back(TEST_URL_404);
FILESIZE = 47684;
{ // 3 threads with only one valid url - succeeded
scoped_ptr<HttpRequest> request(HttpRequest::GetChunks(urls, writer, FILESIZE, onFinish, onProgress, 2048));
// wait until download is finished
QCoreApplication::exec();
observer.TestOk();
TEST_EQUAL(buffer.size(), FILESIZE, ());
TEST_EQUAL(sha2::digest256(buffer), SHA256, (buffer));
}
observer.Reset();
writer.Seek(0);
buffer.clear();
urls.clear();
urls.push_back(TEST_URL_BIG_FILE);
urls.push_back(TEST_URL_BIG_FILE);
FILESIZE = 12345;
{ // 2 threads and all points to file with invalid size - fail
scoped_ptr<HttpRequest> request(HttpRequest::GetChunks(urls, writer, FILESIZE, onFinish, onProgress, 2048));
// wait until download is finished
QCoreApplication::exec();
observer.TestFailed();
TEST_EQUAL(buffer.size(), 0, ());
}
}