diff --git a/platform/chunks_download_strategy.hpp b/platform/chunks_download_strategy.hpp index 1ca82f43a5..dad166ed5a 100644 --- a/platform/chunks_download_strategy.hpp +++ b/platform/chunks_download_strategy.hpp @@ -22,6 +22,8 @@ class ChunksDownloadStrategy public: ChunksDownloadStrategy(vector const & urls, int64_t fileSize, int64_t chunkSize = 512 * 1024); + int64_t ChunkSize() const { return m_chunkSize; } + /// Should be called when each chunk is completed void ChunkFinished(bool successfully, int64_t begRange, int64_t endRange); enum ResultT { @@ -30,6 +32,7 @@ public: EDownloadFailed, EDownloadSucceeded }; + /// Should be called until returns ENextChunk ResultT NextChunk(string & outUrl, int64_t & begRange, int64_t & endRange); }; diff --git a/platform/http_request.cpp b/platform/http_request.cpp index 50c3cc32d7..ce2a9895e9 100644 --- a/platform/http_request.cpp +++ b/platform/http_request.cpp @@ -1,23 +1,26 @@ #include "http_request.hpp" #include "chunks_download_strategy.hpp" +#include "http_thread_callback.hpp" #ifdef DEBUG #include "../base/thread.hpp" - #include "../base/logging.hpp" #endif #include "../coding/writer.hpp" +class HttpThread; + namespace downloader { /// @return 0 if creation failed -HttpRequestImpl * CreateNativeHttpRequest(string const & url, - IHttpRequestImplCallback & callback, +HttpThread * CreateNativeHttpThread(string const & url, + IHttpThreadCallback & callback, int64_t begRange = 0, int64_t endRange = -1, + int64_t expectedSize = -1, string const & postBody = string()); -void DeleteNativeHttpRequest(HttpRequestImpl * request); +void DeleteNativeHttpThread(HttpThread * request); ////////////////////////////////////////////////////////////////////////////////////////// HttpRequest::HttpRequest(Writer & writer, CallbackT onFinish, CallbackT onProgress) @@ -28,55 +31,64 @@ HttpRequest::HttpRequest(Writer & writer, CallbackT onFinish, CallbackT onProgre HttpRequest::~HttpRequest() { - for (ThreadsContainerT::iterator it = m_threads.begin(); it != m_threads.end(); ++it) - DeleteNativeHttpRequest(*it); } - -void HttpRequest::OnSizeKnown(int64_t projectedSize) +////////////////////////////////////////////////////////////////////////////////////////////////////////// +class SimpleHttpRequest : public HttpRequest, public IHttpThreadCallback { - LOG(LDEBUG, ("Projected size", projectedSize)); -} + HttpThread * m_thread; -void HttpRequest::OnWrite(int64_t offset, void const * buffer, size_t size) -{ -#ifdef DEBUG - static threads::ThreadID id = threads::GetCurrentThreadID(); - ASSERT_EQUAL(id, threads::GetCurrentThreadID(), ("OnWrite called from different threads")); -#endif - m_writer.Seek(offset); - m_writer.Write(buffer, size); - m_progress.first += size; - if (m_onProgress) - m_onProgress(*this); -} + virtual void OnWrite(int64_t, void const * buffer, size_t size) + { + m_writer.Write(buffer, size); + m_progress.first += size; + if (m_onProgress) + m_onProgress(*this); + } -void HttpRequest::OnFinish(long httpCode, int64_t begRange, int64_t endRange) -{ - m_status = (httpCode == 200) ? ECompleted : EFailed; - ASSERT(m_onFinish, ()); - m_onFinish(*this); -} + virtual void OnFinish(long httpCode, int64_t begRange, int64_t endRange) + { + m_status = (httpCode == 200) ? ECompleted : EFailed; + m_onFinish(*this); + } + +public: + SimpleHttpRequest(string const & url, Writer & writer, CallbackT onFinish, CallbackT onProgress) + : HttpRequest(writer, onFinish, onProgress) + { + m_thread = CreateNativeHttpThread(url, *this); + } + + SimpleHttpRequest(string const & url, Writer & writer, string const & postData, + CallbackT onFinish, CallbackT onProgress) + : HttpRequest(writer, onFinish, onProgress) + { + m_thread = CreateNativeHttpThread(url, *this, 0, -1, -1, postData); + } + + virtual ~SimpleHttpRequest() + { + DeleteNativeHttpThread(m_thread); + } +}; HttpRequest * HttpRequest::Get(string const & url, Writer & writer, CallbackT onFinish, CallbackT onProgress) { - HttpRequest * self = new HttpRequest(writer, onFinish, onProgress); - self->m_threads.push_back(CreateNativeHttpRequest(url, *self)); - return self; + return new SimpleHttpRequest(url, writer, onFinish, onProgress); } HttpRequest * HttpRequest::Post(string const & url, Writer & writer, string const & postData, CallbackT onFinish, CallbackT onProgress) { - HttpRequest * self = new HttpRequest(writer, onFinish, onProgress); - self->m_threads.push_back(CreateNativeHttpRequest(url, *self, 0, -1, postData)); - return self; + return new SimpleHttpRequest(url, writer, postData, onFinish, onProgress); } -////////////////////////////////////////////////////////////////////////////////////////////////////////// -class ChunksHttpRequest : public HttpRequest +//////////////////////////////////////////////////////////////////////////////////////////////// +class ChunksHttpRequest : public HttpRequest, public IHttpThreadCallback { ChunksDownloadStrategy m_strategy; + typedef list > ThreadsContainerT; + ThreadsContainerT m_threads; ChunksDownloadStrategy::ResultT StartThreads() { @@ -84,27 +96,77 @@ class ChunksHttpRequest : public HttpRequest int64_t beg, end; ChunksDownloadStrategy::ResultT result; while ((result = m_strategy.NextChunk(url, beg, end)) == ChunksDownloadStrategy::ENextChunk) - m_threads.push_back(CreateNativeHttpRequest(url, *this, beg, end)); + m_threads.push_back(make_pair(CreateNativeHttpThread(url, *this, beg, end, m_progress.second), beg)); return result; } + void RemoveHttpThreadByKey(int64_t begRange) + { + for (ThreadsContainerT::iterator it = m_threads.begin(); it != m_threads.end(); ++it) + if (it->second == begRange) + { + DeleteNativeHttpThread(it->first); + m_threads.erase(it); + return; + } + ASSERT(false, ("Tried to remove invalid thread?")); + } + + virtual void OnWrite(int64_t offset, void const * buffer, size_t size) + { + #ifdef DEBUG + static threads::ThreadID const id = threads::GetCurrentThreadID(); + ASSERT_EQUAL(id, threads::GetCurrentThreadID(), ("OnWrite called from different threads")); + #endif + m_writer.Seek(offset); + m_writer.Write(buffer, size); + } + + virtual void OnFinish(long httpCode, int64_t begRange, int64_t endRange) + { +#ifdef DEBUG + static threads::ThreadID const id = threads::GetCurrentThreadID(); + ASSERT_EQUAL(id, threads::GetCurrentThreadID(), ("OnFinish called from different threads")); +#endif + m_strategy.ChunkFinished(httpCode == 200, begRange, endRange); + + // remove completed chunk from the list, beg is the key + RemoveHttpThreadByKey(begRange); + + ChunksDownloadStrategy::ResultT const result = StartThreads(); + // report progress + if (result == ChunksDownloadStrategy::EDownloadSucceeded + || result == ChunksDownloadStrategy::ENoFreeServers) + { + m_progress.first += m_strategy.ChunkSize(); + if (m_onProgress) + m_onProgress(*this); + } + + if (result == ChunksDownloadStrategy::EDownloadFailed) + m_status = EFailed; + else if (result == ChunksDownloadStrategy::EDownloadSucceeded) + m_status = ECompleted; + + if (m_status != EInProgress) + m_onFinish(*this); + } + public: ChunksHttpRequest(vector const & urls, Writer & writer, int64_t fileSize, CallbackT onFinish, CallbackT onProgress, int64_t chunkSize) : HttpRequest(writer, onFinish, onProgress), m_strategy(urls, fileSize, chunkSize) { ASSERT(!urls.empty(), ("Urls list shouldn't be empty")); + // store expected file size for future checks + m_progress.second = fileSize; StartThreads(); } -protected: - virtual void OnFinish(long httpCode, int64_t begRange, int64_t endRange) + virtual ~ChunksHttpRequest() { - m_strategy.ChunkFinished(httpCode == 200, begRange, endRange); - ChunksDownloadStrategy::ResultT const result = StartThreads(); - if (result != ChunksDownloadStrategy::ENoFreeServers) - HttpRequest::OnFinish(result == ChunksDownloadStrategy::EDownloadSucceeded ? 200 : -2, - 0, -1); + for (ThreadsContainerT::iterator it = m_threads.begin(); it != m_threads.end(); ++it) + DeleteNativeHttpThread(it->first); } }; diff --git a/platform/http_request.hpp b/platform/http_request.hpp index 85980e4718..7e07554ddb 100644 --- a/platform/http_request.hpp +++ b/platform/http_request.hpp @@ -2,20 +2,16 @@ #include "../std/function.hpp" #include "../std/string.hpp" -#include "../std/list.hpp" #include "../std/vector.hpp" #include "../std/utility.hpp" -#include "http_request_impl_callback.hpp" - class Writer; -class HttpRequestImpl; namespace downloader { -/// Request will be canceled on delete -class HttpRequest : public IHttpRequestImplCallback +/// Request in progress will be canceled on delete +class HttpRequest { public: enum StatusT @@ -27,31 +23,19 @@ public: /// , total can be -1 if size is unknown typedef pair ProgressT; - typedef function CallbackT; -private: +protected: StatusT m_status; ProgressT m_progress; Writer & m_writer; CallbackT m_onFinish; CallbackT m_onProgress; -protected: - typedef list ThreadsContainerT; - ThreadsContainerT m_threads; - explicit HttpRequest(Writer & writer, CallbackT onFinish, CallbackT onProgress); - /// @name Callbacks for internal native downloading threads - //@{ - virtual void OnSizeKnown(int64_t projectedSize); - virtual void OnWrite(int64_t offset, void const * buffer, size_t size); - virtual void OnFinish(long httpCode, int64_t begRange, int64_t endRange); - //@} - public: - virtual ~HttpRequest(); + virtual ~HttpRequest() = 0; StatusT Status() const { return m_status; } ProgressT Progress() const { return m_progress; } diff --git a/platform/http_request_impl_apple.h b/platform/http_request_impl_apple.h deleted file mode 100644 index 3424f5ffc5..0000000000 --- a/platform/http_request_impl_apple.h +++ /dev/null @@ -1,22 +0,0 @@ -#pragma once - -#import - -#include "../std/string.hpp" - -namespace downloader { class IHttpRequestImplCallback; } - -@interface HttpRequestImpl : NSObject -{ - downloader::IHttpRequestImplCallback * m_callback; - NSURLConnection * m_connection; - int64_t m_begRange, m_endRange; - int64_t m_downloadedBytes; -} - -- (id) initWith:(string const &)url callback:(downloader::IHttpRequestImplCallback &)cb - begRange:(int64_t)beg endRange:(int64_t)end contentType:(string const &)ct - postBody:(string const &)pb; - -- (void) cancel; -@end diff --git a/platform/http_thread_apple.h b/platform/http_thread_apple.h new file mode 100644 index 0000000000..50e2e7a391 --- /dev/null +++ b/platform/http_thread_apple.h @@ -0,0 +1,22 @@ +#pragma once + +#import + +#include "../std/string.hpp" + +namespace downloader { class IHttpThreadCallback; } + +@interface HttpThread : NSObject +{ + downloader::IHttpThreadCallback * m_callback; + NSURLConnection * m_connection; + int64_t m_begRange, m_endRange; + int64_t m_downloadedBytes; + int64_t m_expectedSize; +} + +- (id) initWith:(string const &)url callback:(downloader::IHttpThreadCallback &)cb begRange:(int64_t)beg + endRange:(int64_t)end expectedSize:(int64_t)size postBody:(string const &)pb; + +- (void) cancel; +@end diff --git a/platform/http_request_impl_apple.mm b/platform/http_thread_apple.mm similarity index 65% rename from platform/http_request_impl_apple.mm rename to platform/http_thread_apple.mm index 69cc03e17b..fabe9d655f 100644 --- a/platform/http_request_impl_apple.mm +++ b/platform/http_thread_apple.mm @@ -1,13 +1,13 @@ -#import "http_request_impl_apple.h" +#import "http_thread_apple.h" -#include "http_request_impl_callback.hpp" +#include "http_thread_callback.hpp" #include "platform.hpp" #include "../base/logging.hpp" #define TIMEOUT_IN_SECONDS 15.0 -@implementation HttpRequestImpl +@implementation HttpThread - (void) dealloc { @@ -22,8 +22,8 @@ [m_connection cancel]; } -- (id) initWith:(string const &)url callback:(downloader::IHttpRequestImplCallback &)cb - begRange:(int64_t)beg endRange:(int64_t)end postBody:(string const &)pb +- (id) initWith:(string const &)url callback:(downloader::IHttpThreadCallback &)cb begRange:(int64_t)beg + endRange:(int64_t)end expectedSize:(int64_t)size postBody:(string const &)pb { self = [super init]; @@ -31,12 +31,14 @@ m_begRange = beg; m_endRange = end; m_downloadedBytes = 0; + m_expectedSize = size; NSMutableURLRequest * request = [NSMutableURLRequest requestWithURL: [NSURL URLWithString:[NSString stringWithUTF8String:url.c_str()]] cachePolicy:NSURLRequestReloadIgnoringLocalCacheData timeoutInterval:TIMEOUT_IN_SECONDS]; - if (beg > 0) + // use Range header only if we don't download whole file from start + if (!(beg == 0 && end < 0)) { NSString * val; if (end > 0) @@ -82,6 +84,19 @@ return self; } +/// @return -1 if can't decode ++ (int64_t) getContentRange:(NSDictionary *)httpHeader +{ + NSString * cr = [httpHeader valueForKey:@"Content-Range"]; + if (cr) + { + NSArray * arr = [cr componentsSeparatedByString:@"/"]; + if (arr && [arr count]) + return [(NSString *)[arr objectAtIndex:[arr count] - 1] longLongValue]; + } + return -1; +} + - (void) connection: (NSURLConnection *)connection didReceiveResponse: (NSURLResponse *)response { // This method is called when the server has determined that it @@ -92,11 +107,6 @@ { NSInteger const statusCode = [(NSHTTPURLResponse *)response statusCode]; LOG(LDEBUG, ("Got response with status code", statusCode)); -#ifdef DEBUG -// NSDictionary * fields = [(NSHTTPURLResponse *)response allHeaderFields]; -// for (id key in fields) -// NSLog(@"%@: %@", key, [fields objectForKey:key]); -#endif if (statusCode < 200 || statusCode > 299) { LOG(LWARNING, ("Received HTTP error, canceling download", statusCode)); @@ -104,8 +114,23 @@ m_callback->OnFinish(statusCode, m_begRange, m_endRange); return; } + else if (m_expectedSize > 0) + { + // get full file expected size from Content-Range header + int64_t sizeOnServer = [HttpThread getContentRange:[(NSHTTPURLResponse *)response allHeaderFields]]; + // if it's absent, use Content-Length instead + if (sizeOnServer < 0) + sizeOnServer = [response expectedContentLength]; + if (sizeOnServer > 0 & m_expectedSize != sizeOnServer) + { - m_callback->OnSizeKnown([response expectedContentLength]); + LOG(LWARNING, ("Canceling download - server replied with invalid size", + sizeOnServer, "!=", m_expectedSize)); + [m_connection cancel]; + m_callback->OnFinish(-2, m_begRange, m_endRange); + return; + } + } } else { // in theory, we should never be here @@ -138,16 +163,17 @@ /////////////////////////////////////////////////////////////////////////////////////// namespace downloader { -HttpRequestImpl * CreateNativeHttpRequest(string const & url, - downloader::IHttpRequestImplCallback & cb, - int64_t beg, - int64_t end, - string const & pb) +HttpThread * CreateNativeHttpThread(string const & url, + downloader::IHttpThreadCallback & cb, + int64_t beg, + int64_t end, + int64_t size, + string const & pb) { - return [[HttpRequestImpl alloc] initWith:url callback:cb begRange:beg endRange:end postBody:pb]; + return [[HttpThread alloc] initWith:url callback:cb begRange:beg endRange:end expectedSize:size postBody:pb]; } -void DeleteNativeHttpRequest(HttpRequestImpl * request) +void DeleteNativeHttpThread(HttpThread * request) { [request cancel]; [request release]; diff --git a/platform/http_request_impl_callback.hpp b/platform/http_thread_callback.hpp similarity index 58% rename from platform/http_request_impl_callback.hpp rename to platform/http_thread_callback.hpp index 397ac032b5..544a2edaad 100644 --- a/platform/http_request_impl_callback.hpp +++ b/platform/http_thread_callback.hpp @@ -3,11 +3,9 @@ namespace downloader { -class IHttpRequestImplCallback +class IHttpThreadCallback { public: - /// Called before OnWrite, projectedSize can be -1 if server doesn't support it - virtual void OnSizeKnown(int64_t projectedSize) = 0; virtual void OnWrite(int64_t offset, void const * buffer, size_t size) = 0; virtual void OnFinish(long httpCode, int64_t begRange, int64_t endRange) = 0; }; diff --git a/platform/platform.pro b/platform/platform.pro index 83f0d5c99f..4e3c995ab4 100644 --- a/platform/platform.pro +++ b/platform/platform.pro @@ -40,10 +40,10 @@ include($$ROOT_DIR/common.pri) macx*|iphone* { HEADERS += apple_download.h \ - http_request_impl_apple.h + http_thread_apple.h OBJECTIVE_SOURCES += apple_download.mm \ apple_download_manager.mm \ - http_request_impl_apple.mm + http_thread_apple.mm } win32*|linux* { @@ -67,8 +67,8 @@ HEADERS += \ languages.hpp \ url_generator.hpp \ http_request.hpp \ - http_request_impl_callback.hpp \ - chunks_download_strategy.hpp + http_thread_callback.hpp \ + chunks_download_strategy.hpp \ SOURCES += \ preferred_languages.cpp \ @@ -77,5 +77,4 @@ SOURCES += \ languages.cpp \ url_generator.cpp \ http_request.cpp \ - chunks_download_strategy.cpp - + chunks_download_strategy.cpp \ diff --git a/platform/platform_tests/downloader_test.cpp b/platform/platform_tests/downloader_test.cpp index f3be092826..b3b0ebea67 100644 --- a/platform/platform_tests/downloader_test.cpp +++ b/platform/platform_tests/downloader_test.cpp @@ -291,15 +291,35 @@ UNIT_TEST(DownloadChunks) vector urls; urls.push_back(TEST_URL1); urls.push_back(TEST_URL1); + int64_t FILESIZE = 5; { // should use only one thread - scoped_ptr request(HttpRequest::GetChunks(urls, writer, 5, onFinish, onProgress)); + scoped_ptr request(HttpRequest::GetChunks(urls, writer, FILESIZE, onFinish, onProgress)); // wait until download is finished QCoreApplication::exec(); observer.TestOk(); + TEST_EQUAL(buffer.size(), FILESIZE, ()); TEST_EQUAL(buffer, "Test1", (buffer)); } + observer.Reset(); + writer.Seek(0); + buffer.clear(); + + urls.clear(); + urls.push_back(TEST_URL_BIG_FILE); + urls.push_back(TEST_URL_BIG_FILE); + urls.push_back(TEST_URL_BIG_FILE); + FILESIZE = 5; + + { // 3 threads - fail, because of invalid size + scoped_ptr request(HttpRequest::GetChunks(urls, writer, FILESIZE, onFinish, onProgress, 2048)); + // wait until download is finished + QCoreApplication::exec(); + observer.TestFailed(); + TEST_EQUAL(buffer.size(), 0, ()); + } + string const SHA256 = "EE6AE6A2A3619B2F4A397326BEC32583DE2196D9D575D66786CB3B6F9D04A633"; observer.Reset(); @@ -310,12 +330,50 @@ UNIT_TEST(DownloadChunks) urls.push_back(TEST_URL_BIG_FILE); urls.push_back(TEST_URL_BIG_FILE); urls.push_back(TEST_URL_BIG_FILE); + FILESIZE = 47684; - { // 3 threads - scoped_ptr request(HttpRequest::GetChunks(urls, writer, 5, onFinish, onProgress, 2048)); + { // 3 threads - succeeded + scoped_ptr request(HttpRequest::GetChunks(urls, writer, FILESIZE, onFinish, onProgress, 2048)); // wait until download is finished QCoreApplication::exec(); observer.TestOk(); + TEST_EQUAL(buffer.size(), FILESIZE, ()); TEST_EQUAL(sha2::digest256(buffer), SHA256, (buffer)); } + + observer.Reset(); + writer.Seek(0); + buffer.clear(); + + urls.clear(); + urls.push_back(TEST_URL_BIG_FILE); + urls.push_back(TEST_URL1); + urls.push_back(TEST_URL_404); + FILESIZE = 47684; + + { // 3 threads with only one valid url - succeeded + scoped_ptr request(HttpRequest::GetChunks(urls, writer, FILESIZE, onFinish, onProgress, 2048)); + // wait until download is finished + QCoreApplication::exec(); + observer.TestOk(); + TEST_EQUAL(buffer.size(), FILESIZE, ()); + TEST_EQUAL(sha2::digest256(buffer), SHA256, (buffer)); + } + + observer.Reset(); + writer.Seek(0); + buffer.clear(); + + urls.clear(); + urls.push_back(TEST_URL_BIG_FILE); + urls.push_back(TEST_URL_BIG_FILE); + FILESIZE = 12345; + + { // 2 threads and all points to file with invalid size - fail + scoped_ptr request(HttpRequest::GetChunks(urls, writer, FILESIZE, onFinish, onProgress, 2048)); + // wait until download is finished + QCoreApplication::exec(); + observer.TestFailed(); + TEST_EQUAL(buffer.size(), 0, ()); + } }