diff --git a/platform/http_request.cpp b/platform/http_request.cpp index ce2a9895e9..323ab3156d 100644 --- a/platform/http_request.cpp +++ b/platform/http_request.cpp @@ -6,7 +6,7 @@ #include "../base/thread.hpp" #endif -#include "../coding/writer.hpp" +#include "../coding/file_writer.hpp" class HttpThread; @@ -23,10 +23,17 @@ HttpThread * CreateNativeHttpThread(string const & url, void DeleteNativeHttpThread(HttpThread * request); ////////////////////////////////////////////////////////////////////////////////////////// -HttpRequest::HttpRequest(Writer & writer, CallbackT onFinish, CallbackT onProgress) - : m_status(EInProgress), m_progress(make_pair(0, -1)), m_writer(writer), +HttpRequest::HttpRequest(string const & filePath, CallbackT onFinish, CallbackT onProgress) + : m_status(EInProgress), m_progress(make_pair(0, -1)), m_onFinish(onFinish), m_onProgress(onProgress) { + if (filePath.empty()) + m_writer.reset(new MemWriter(m_data)); + else + { + m_data = filePath; + m_writer.reset(new FileWriter(filePath, FileWriter::OP_WRITE_EXISTING)); + } } HttpRequest::~HttpRequest() @@ -39,28 +46,29 @@ class SimpleHttpRequest : public HttpRequest, public IHttpThreadCallback virtual void OnWrite(int64_t, void const * buffer, size_t size) { - m_writer.Write(buffer, size); + m_writer->Write(buffer, size); m_progress.first += size; if (m_onProgress) m_onProgress(*this); } - virtual void OnFinish(long httpCode, int64_t begRange, int64_t endRange) + virtual void OnFinish(long httpCode, int64_t, int64_t) { m_status = (httpCode == 200) ? ECompleted : EFailed; + m_writer.reset(); m_onFinish(*this); } public: - SimpleHttpRequest(string const & url, Writer & writer, CallbackT onFinish, CallbackT onProgress) - : HttpRequest(writer, onFinish, onProgress) + SimpleHttpRequest(string const & url, string const & filePath, CallbackT onFinish, CallbackT onProgress) + : HttpRequest(filePath, onFinish, onProgress) { m_thread = CreateNativeHttpThread(url, *this); } - SimpleHttpRequest(string const & url, Writer & writer, string const & postData, + SimpleHttpRequest(string const & url, string const & filePath, string const & postData, CallbackT onFinish, CallbackT onProgress) - : HttpRequest(writer, onFinish, onProgress) + : HttpRequest(filePath, onFinish, onProgress) { m_thread = CreateNativeHttpThread(url, *this, 0, -1, -1, postData); } @@ -71,15 +79,16 @@ public: } }; -HttpRequest * HttpRequest::Get(string const & url, Writer & writer, CallbackT onFinish, CallbackT onProgress) +HttpRequest * HttpRequest::Get(string const & url, string const & filePath, + CallbackT onFinish, CallbackT onProgress) { - return new SimpleHttpRequest(url, writer, onFinish, onProgress); + return new SimpleHttpRequest(url, filePath, onFinish, onProgress); } -HttpRequest * HttpRequest::Post(string const & url, Writer & writer, string const & postData, +HttpRequest * HttpRequest::Post(string const & url, string const & filePath, string const & postData, CallbackT onFinish, CallbackT onProgress) { - return new SimpleHttpRequest(url, writer, postData, onFinish, onProgress); + return new SimpleHttpRequest(url, filePath, postData, onFinish, onProgress); } @@ -118,8 +127,8 @@ class ChunksHttpRequest : public HttpRequest, public IHttpThreadCallback static threads::ThreadID const id = threads::GetCurrentThreadID(); ASSERT_EQUAL(id, threads::GetCurrentThreadID(), ("OnWrite called from different threads")); #endif - m_writer.Seek(offset); - m_writer.Write(buffer, size); + m_writer->Seek(offset); + m_writer->Write(buffer, size); } virtual void OnFinish(long httpCode, int64_t begRange, int64_t endRange) @@ -149,13 +158,16 @@ class ChunksHttpRequest : public HttpRequest, public IHttpThreadCallback m_status = ECompleted; if (m_status != EInProgress) + { + m_writer.reset(); m_onFinish(*this); + } } public: - ChunksHttpRequest(vector const & urls, Writer & writer, int64_t fileSize, + ChunksHttpRequest(vector const & urls, string const & filePath, int64_t fileSize, CallbackT onFinish, CallbackT onProgress, int64_t chunkSize) - : HttpRequest(writer, onFinish, onProgress), m_strategy(urls, fileSize, chunkSize) + : HttpRequest(filePath, onFinish, onProgress), m_strategy(urls, fileSize, chunkSize) { ASSERT(!urls.empty(), ("Urls list shouldn't be empty")); // store expected file size for future checks @@ -170,10 +182,10 @@ public: } }; -HttpRequest * HttpRequest::GetChunks(vector const & urls, Writer & writer, int64_t fileSize, +HttpRequest * HttpRequest::GetChunks(vector const & urls, string const & filePath, int64_t fileSize, CallbackT onFinish, CallbackT onProgress, int64_t chunkSize) { - return new ChunksHttpRequest(urls, writer, fileSize, onFinish, onProgress, chunkSize); + return new ChunksHttpRequest(urls, filePath, fileSize, onFinish, onProgress, chunkSize); } } // namespace downloader diff --git a/platform/http_request.hpp b/platform/http_request.hpp index 7e07554ddb..f46961449a 100644 --- a/platform/http_request.hpp +++ b/platform/http_request.hpp @@ -4,6 +4,7 @@ #include "../std/string.hpp" #include "../std/vector.hpp" #include "../std/utility.hpp" +#include "../std/scoped_ptr.hpp" class Writer; @@ -28,24 +29,28 @@ public: protected: StatusT m_status; ProgressT m_progress; - Writer & m_writer; CallbackT m_onFinish; CallbackT m_onProgress; + string m_data; + scoped_ptr m_writer; - explicit HttpRequest(Writer & writer, CallbackT onFinish, CallbackT onProgress); + explicit HttpRequest(string const & filePath, CallbackT onFinish, CallbackT onProgress); public: virtual ~HttpRequest() = 0; StatusT Status() const { return m_status; } ProgressT Progress() const { return m_progress; } + /// Retrieve either file path or downloaded data + string const & Data() const { return m_data; } - static HttpRequest * Get(string const & url, Writer & writer, CallbackT onFinish, + /// @param[in] filePath if empty, request will be saved to memory, see Data() + static HttpRequest * Get(string const & url, string const & filePath, CallbackT onFinish, CallbackT onProgress = CallbackT()); /// Content-type for request is always "application/json" - static HttpRequest * Post(string const & url, Writer & writer, string const & postData, + static HttpRequest * Post(string const & url, string const & filePath, string const & postData, CallbackT onFinish, CallbackT onProgress = CallbackT()); - static HttpRequest * GetChunks(vector const & urls, Writer & writer, int64_t fileSize, + static HttpRequest * GetChunks(vector const & urls, string const & filePath, int64_t fileSize, CallbackT onFinish, CallbackT onProgress = CallbackT(), int64_t chunkSize = 512 * 1024); }; diff --git a/platform/platform_tests/downloader_test.cpp b/platform/platform_tests/downloader_test.cpp index b3b0ebea67..7011df167d 100644 --- a/platform/platform_tests/downloader_test.cpp +++ b/platform/platform_tests/downloader_test.cpp @@ -2,7 +2,8 @@ #include "../../base/logging.hpp" -#include "../../coding/writer.hpp" +#include "../../coding/file_reader.hpp" +#include "../../coding/file_writer.hpp" #include "../../coding/sha2.hpp" #include "../http_request.hpp" @@ -77,6 +78,7 @@ struct CancelDownload { void OnProgress(HttpRequest & request) { + TEST_GREATER(request.Data().size(), 0, ()); delete &request; QCoreApplication::quit(); } @@ -86,89 +88,134 @@ struct CancelDownload } }; +struct DeleteOnFinish +{ + void OnProgress(HttpRequest & request) + { + TEST_GREATER(request.Data().size(), 0, ()); + } + void OnFinish(HttpRequest & request) + { + delete &request; + QCoreApplication::quit(); + } +}; + UNIT_TEST(DownloaderSimpleGet) { DownloadObserver observer; - string buffer; - MemWriter writer(buffer); HttpRequest::CallbackT onFinish = bind(&DownloadObserver::OnDownloadFinish, &observer, _1); HttpRequest::CallbackT onProgress = bind(&DownloadObserver::OnDownloadProgress, &observer, _1); { // simple success case - scoped_ptr request(HttpRequest::Get(TEST_URL1, writer, onFinish, onProgress)); + scoped_ptr request(HttpRequest::Get(TEST_URL1, string(), onFinish, onProgress)); // wait until download is finished QCoreApplication::exec(); observer.TestOk(); - TEST_EQUAL(buffer, "Test1", (buffer)); + TEST_EQUAL(request->Data(), "Test1", ()); } - buffer.clear(); - writer.Seek(0); observer.Reset(); { // permanent redirect success case - scoped_ptr request(HttpRequest::Get(TEST_URL_PERMANENT, writer, onFinish, onProgress)); + scoped_ptr request(HttpRequest::Get(TEST_URL_PERMANENT, string(), onFinish, onProgress)); QCoreApplication::exec(); observer.TestOk(); - TEST_EQUAL(buffer, "Test1", (buffer)); + TEST_EQUAL(request->Data(), "Test1", ()); } - buffer.clear(); - writer.Seek(0); observer.Reset(); { // fail case 404 - scoped_ptr request(HttpRequest::Get(TEST_URL_404, writer, onFinish, onProgress)); + scoped_ptr request(HttpRequest::Get(TEST_URL_404, string(), onFinish, onProgress)); QCoreApplication::exec(); observer.TestFailed(); - TEST_EQUAL(buffer.size(), 0, (buffer)); + TEST_EQUAL(request->Data().size(), 0, (request->Data())); } - buffer.clear(); - writer.Seek(0); observer.Reset(); { // fail case not existing host - scoped_ptr request(HttpRequest::Get(TEST_URL_INVALID_HOST, writer, onFinish, onProgress)); + scoped_ptr request(HttpRequest::Get(TEST_URL_INVALID_HOST, string(), onFinish, onProgress)); QCoreApplication::exec(); observer.TestFailed(); - TEST_EQUAL(buffer.size(), 0, (buffer)); + TEST_EQUAL(request->Data().size(), 0, (request->Data())); } - buffer.clear(); - writer.Seek(0); - { + { // cancel download in the middle of the progress CancelDownload canceler; /// should be deleted in canceler - HttpRequest::Get(TEST_URL_BIG_FILE, writer, + HttpRequest::Get(TEST_URL_BIG_FILE, string(), bind(&CancelDownload::OnFinish, &canceler, _1), bind(&CancelDownload::OnProgress, &canceler, _1)); QCoreApplication::exec(); - TEST_GREATER(buffer.size(), 0, ()); } - buffer.clear(); - writer.Seek(0); observer.Reset(); { // https success case - scoped_ptr request(HttpRequest::Get(TEST_URL_HTTPS, writer, onFinish, onProgress)); + scoped_ptr request(HttpRequest::Get(TEST_URL_HTTPS, string(), onFinish, onProgress)); // wait until download is finished QCoreApplication::exec(); observer.TestOk(); - TEST_GREATER(buffer.size(), 0, (buffer)); + TEST_GREATER(request->Data().size(), 0, ()); + } + + string const FILENAME = "some_downloader_test_file"; + observer.Reset(); + { // download file success case + scoped_ptr request(HttpRequest::Get(TEST_URL1, FILENAME, onFinish, onProgress)); + // wait until download is finished + QCoreApplication::exec(); + observer.TestOk(); + { + FileReader f(FILENAME); + string s; + f.ReadAsString(s); + TEST_EQUAL(s, "Test1", ()); + } + TEST_EQUAL(request->Data(), FILENAME, (request->Data())); + FileWriter::DeleteFileX(FILENAME); + } + + observer.Reset(); + { // download file error case + scoped_ptr request(HttpRequest::Get(TEST_URL_404, FILENAME, onFinish, onProgress)); + // wait until download is finished + QCoreApplication::exec(); + observer.TestFailed(); + { + FileReader f(FILENAME); + TEST_EQUAL(f.Size(), 0, ()); + } + TEST_EQUAL(request->Data(), FILENAME, (request->Data())); + FileWriter::DeleteFileX(FILENAME); + } + + { // Delete request at the end of successful file download + DeleteOnFinish deleter; + /// should be deleted in deleter on finish + HttpRequest::Get(TEST_URL1, FILENAME, + bind(&DeleteOnFinish::OnFinish, &deleter, _1), + bind(&DeleteOnFinish::OnProgress, &deleter, _1)); + QCoreApplication::exec(); + { + FileReader f(FILENAME); + string s; + f.ReadAsString(s); + TEST_EQUAL(s, "Test1", ()); + } + FileWriter::DeleteFileX(FILENAME); } } UNIT_TEST(DownloaderSimplePost) { DownloadObserver observer; - string buffer; - MemWriter writer(buffer); HttpRequest::CallbackT onFinish = bind(&DownloadObserver::OnDownloadFinish, &observer, _1); HttpRequest::CallbackT onProgress = bind(&DownloadObserver::OnDownloadProgress, &observer, _1); { // simple success case string const postData = "{\"jsonKey\":\"jsonValue\"}"; - scoped_ptr request(HttpRequest::Post(TEST_URL_POST, writer, postData, onFinish, onProgress)); + scoped_ptr request(HttpRequest::Post(TEST_URL_POST, string(), postData, onFinish, onProgress)); // wait until download is finished QCoreApplication::exec(); observer.TestOk(); - TEST_EQUAL(buffer, postData, (buffer)); + TEST_EQUAL(request->Data(), postData, ()); } } @@ -277,14 +324,11 @@ UNIT_TEST(ChunksDownloadStrategyFAIL) strategy.ChunkFinished(false, beg2, end2); TEST_EQUAL(strategy.NextChunk(s2, beg2, end2), ChunksDownloadStrategy::EDownloadFailed, ()); - } UNIT_TEST(DownloadChunks) { DownloadObserver observer; - string buffer; - MemWriter writer(buffer); HttpRequest::CallbackT onFinish = bind(&DownloadObserver::OnDownloadFinish, &observer, _1); HttpRequest::CallbackT onProgress = bind(&DownloadObserver::OnDownloadProgress, &observer, _1); @@ -294,17 +338,15 @@ UNIT_TEST(DownloadChunks) int64_t FILESIZE = 5; { // should use only one thread - scoped_ptr request(HttpRequest::GetChunks(urls, writer, FILESIZE, onFinish, onProgress)); + scoped_ptr request(HttpRequest::GetChunks(urls, string(), FILESIZE, onFinish, onProgress)); // wait until download is finished QCoreApplication::exec(); observer.TestOk(); - TEST_EQUAL(buffer.size(), FILESIZE, ()); - TEST_EQUAL(buffer, "Test1", (buffer)); + TEST_EQUAL(request->Data().size(), FILESIZE, ()); + TEST_EQUAL(request->Data(), "Test1", ()); } observer.Reset(); - writer.Seek(0); - buffer.clear(); urls.clear(); urls.push_back(TEST_URL_BIG_FILE); @@ -313,18 +355,16 @@ UNIT_TEST(DownloadChunks) FILESIZE = 5; { // 3 threads - fail, because of invalid size - scoped_ptr request(HttpRequest::GetChunks(urls, writer, FILESIZE, onFinish, onProgress, 2048)); + scoped_ptr request(HttpRequest::GetChunks(urls, string(), FILESIZE, onFinish, onProgress, 2048)); // wait until download is finished QCoreApplication::exec(); observer.TestFailed(); - TEST_EQUAL(buffer.size(), 0, ()); + TEST_EQUAL(request->Data().size(), 0, ()); } string const SHA256 = "EE6AE6A2A3619B2F4A397326BEC32583DE2196D9D575D66786CB3B6F9D04A633"; observer.Reset(); - writer.Seek(0); - buffer.clear(); urls.clear(); urls.push_back(TEST_URL_BIG_FILE); @@ -333,17 +373,15 @@ UNIT_TEST(DownloadChunks) FILESIZE = 47684; { // 3 threads - succeeded - scoped_ptr request(HttpRequest::GetChunks(urls, writer, FILESIZE, onFinish, onProgress, 2048)); + scoped_ptr request(HttpRequest::GetChunks(urls, string(), FILESIZE, onFinish, onProgress, 2048)); // wait until download is finished QCoreApplication::exec(); observer.TestOk(); - TEST_EQUAL(buffer.size(), FILESIZE, ()); - TEST_EQUAL(sha2::digest256(buffer), SHA256, (buffer)); + TEST_EQUAL(request->Data().size(), FILESIZE, ()); + TEST_EQUAL(sha2::digest256(request->Data()), SHA256, ()); } observer.Reset(); - writer.Seek(0); - buffer.clear(); urls.clear(); urls.push_back(TEST_URL_BIG_FILE); @@ -352,17 +390,15 @@ UNIT_TEST(DownloadChunks) FILESIZE = 47684; { // 3 threads with only one valid url - succeeded - scoped_ptr request(HttpRequest::GetChunks(urls, writer, FILESIZE, onFinish, onProgress, 2048)); + scoped_ptr request(HttpRequest::GetChunks(urls, string(), FILESIZE, onFinish, onProgress, 2048)); // wait until download is finished QCoreApplication::exec(); observer.TestOk(); - TEST_EQUAL(buffer.size(), FILESIZE, ()); - TEST_EQUAL(sha2::digest256(buffer), SHA256, (buffer)); + TEST_EQUAL(request->Data().size(), FILESIZE, ()); + TEST_EQUAL(sha2::digest256(request->Data()), SHA256, ()); } observer.Reset(); - writer.Seek(0); - buffer.clear(); urls.clear(); urls.push_back(TEST_URL_BIG_FILE); @@ -370,10 +406,34 @@ UNIT_TEST(DownloadChunks) FILESIZE = 12345; { // 2 threads and all points to file with invalid size - fail - scoped_ptr request(HttpRequest::GetChunks(urls, writer, FILESIZE, onFinish, onProgress, 2048)); + scoped_ptr request(HttpRequest::GetChunks(urls, string(), FILESIZE, onFinish, onProgress, 2048)); // wait until download is finished QCoreApplication::exec(); observer.TestFailed(); - TEST_EQUAL(buffer.size(), 0, ()); + TEST_EQUAL(request->Data().size(), 0, ()); + } + + observer.Reset(); + + urls.clear(); + urls.push_back(TEST_URL_BIG_FILE); + urls.push_back(TEST_URL_BIG_FILE); + FILESIZE = 47684; + string const FILENAME = "some_downloader_test_file"; + + { // 3 threads - download to file - succeeded + scoped_ptr request(HttpRequest::GetChunks(urls, FILENAME, FILESIZE, onFinish, onProgress, 10000)); + // wait until download is finished + QCoreApplication::exec(); + observer.TestOk(); + TEST_EQUAL(request->Data(), FILENAME, ()); + { + FileReader f(FILENAME); + TEST_EQUAL(f.Size(), FILESIZE, ()); + string s; + f.ReadAsString(s); + TEST_EQUAL(sha2::digest256(s), SHA256, ()); + } + FileWriter::DeleteFileX(FILENAME); } }