From a38ccdfe6b4b7f4fc516924b945712d447a690b8 Mon Sep 17 00:00:00 2001 From: Alex Zolotarev Date: Wed, 9 Nov 2011 17:34:19 +0300 Subject: [PATCH] [downloader] Changed behavior - now only HttpRequest::GetFile() uses files for storage --- platform/http_request.cpp | 75 ++++--- platform/http_request.hpp | 20 +- platform/platform_tests/downloader_test.cpp | 232 +++++++++++++------- 3 files changed, 197 insertions(+), 130 deletions(-) diff --git a/platform/http_request.cpp b/platform/http_request.cpp index 323ab3156d..63174a80a8 100644 --- a/platform/http_request.cpp +++ b/platform/http_request.cpp @@ -8,6 +8,8 @@ #include "../coding/file_writer.hpp" +#include "../std/scoped_ptr.hpp" + class HttpThread; namespace downloader @@ -15,38 +17,34 @@ namespace downloader /// @return 0 if creation failed HttpThread * CreateNativeHttpThread(string const & url, - IHttpThreadCallback & callback, - int64_t begRange = 0, - int64_t endRange = -1, - int64_t expectedSize = -1, - string const & postBody = string()); + IHttpThreadCallback & callback, + int64_t begRange = 0, + int64_t endRange = -1, + int64_t expectedSize = -1, + string const & postBody = string()); void DeleteNativeHttpThread(HttpThread * request); ////////////////////////////////////////////////////////////////////////////////////////// -HttpRequest::HttpRequest(string const & filePath, CallbackT onFinish, CallbackT onProgress) +HttpRequest::HttpRequest(CallbackT onFinish, CallbackT onProgress) : m_status(EInProgress), m_progress(make_pair(0, -1)), m_onFinish(onFinish), m_onProgress(onProgress) { - if (filePath.empty()) - m_writer.reset(new MemWriter(m_data)); - else - { - m_data = filePath; - m_writer.reset(new FileWriter(filePath, FileWriter::OP_WRITE_EXISTING)); - } } HttpRequest::~HttpRequest() { } ////////////////////////////////////////////////////////////////////////////////////////////////////////// -class SimpleHttpRequest : public HttpRequest, public IHttpThreadCallback +class MemoryHttpRequest : public HttpRequest, public IHttpThreadCallback { HttpThread * m_thread; + string m_downloadedData; + MemWriter m_writer; + virtual void OnWrite(int64_t, void const * buffer, size_t size) { - m_writer->Write(buffer, size); + m_writer.Write(buffer, size); m_progress.first += size; if (m_onProgress) m_onProgress(*this); @@ -55,50 +53,55 @@ class SimpleHttpRequest : public HttpRequest, public IHttpThreadCallback virtual void OnFinish(long httpCode, int64_t, int64_t) { m_status = (httpCode == 200) ? ECompleted : EFailed; - m_writer.reset(); m_onFinish(*this); } public: - SimpleHttpRequest(string const & url, string const & filePath, CallbackT onFinish, CallbackT onProgress) - : HttpRequest(filePath, onFinish, onProgress) + MemoryHttpRequest(string const & url, CallbackT onFinish, CallbackT onProgress) + : HttpRequest(onFinish, onProgress), m_writer(m_downloadedData) { m_thread = CreateNativeHttpThread(url, *this); } - SimpleHttpRequest(string const & url, string const & filePath, string const & postData, + MemoryHttpRequest(string const & url, string const & postData, CallbackT onFinish, CallbackT onProgress) - : HttpRequest(filePath, onFinish, onProgress) + : HttpRequest(onFinish, onProgress), m_writer(m_downloadedData) { m_thread = CreateNativeHttpThread(url, *this, 0, -1, -1, postData); } - virtual ~SimpleHttpRequest() + virtual ~MemoryHttpRequest() { DeleteNativeHttpThread(m_thread); } + + virtual string const & Data() const + { + return m_downloadedData; + } }; -HttpRequest * HttpRequest::Get(string const & url, string const & filePath, - CallbackT onFinish, CallbackT onProgress) +HttpRequest * HttpRequest::Get(string const & url, CallbackT onFinish, CallbackT onProgress) { - return new SimpleHttpRequest(url, filePath, onFinish, onProgress); + return new MemoryHttpRequest(url, onFinish, onProgress); } -HttpRequest * HttpRequest::Post(string const & url, string const & filePath, string const & postData, +HttpRequest * HttpRequest::PostJson(string const & url, string const & postData, CallbackT onFinish, CallbackT onProgress) { - return new SimpleHttpRequest(url, filePath, postData, onFinish, onProgress); + return new MemoryHttpRequest(url, postData, onFinish, onProgress); } - //////////////////////////////////////////////////////////////////////////////////////////////// -class ChunksHttpRequest : public HttpRequest, public IHttpThreadCallback +class FileHttpRequest : public HttpRequest, public IHttpThreadCallback { ChunksDownloadStrategy m_strategy; typedef list > ThreadsContainerT; ThreadsContainerT m_threads; + string m_filePath; + scoped_ptr m_writer; + ChunksDownloadStrategy::ResultT StartThreads() { string url; @@ -165,9 +168,10 @@ class ChunksHttpRequest : public HttpRequest, public IHttpThreadCallback } public: - ChunksHttpRequest(vector const & urls, string const & filePath, int64_t fileSize, + FileHttpRequest(vector const & urls, string const & filePath, int64_t fileSize, CallbackT onFinish, CallbackT onProgress, int64_t chunkSize) - : HttpRequest(filePath, onFinish, onProgress), m_strategy(urls, fileSize, chunkSize) + : HttpRequest(onFinish, onProgress), m_strategy(urls, fileSize, chunkSize), + m_filePath(filePath), m_writer(new FileWriter(filePath, FileWriter::OP_WRITE_EXISTING)) { ASSERT(!urls.empty(), ("Urls list shouldn't be empty")); // store expected file size for future checks @@ -175,17 +179,22 @@ public: StartThreads(); } - virtual ~ChunksHttpRequest() + virtual ~FileHttpRequest() { for (ThreadsContainerT::iterator it = m_threads.begin(); it != m_threads.end(); ++it) DeleteNativeHttpThread(it->first); } + + virtual string const & Data() const + { + return m_filePath; + } }; -HttpRequest * HttpRequest::GetChunks(vector const & urls, string const & filePath, int64_t fileSize, +HttpRequest * HttpRequest::GetFile(vector const & urls, string const & filePath, int64_t fileSize, CallbackT onFinish, CallbackT onProgress, int64_t chunkSize) { - return new ChunksHttpRequest(urls, filePath, fileSize, onFinish, onProgress, chunkSize); + return new FileHttpRequest(urls, filePath, fileSize, onFinish, onProgress, chunkSize); } } // namespace downloader diff --git a/platform/http_request.hpp b/platform/http_request.hpp index f46961449a..8323911de3 100644 --- a/platform/http_request.hpp +++ b/platform/http_request.hpp @@ -4,9 +4,6 @@ #include "../std/string.hpp" #include "../std/vector.hpp" #include "../std/utility.hpp" -#include "../std/scoped_ptr.hpp" - -class Writer; namespace downloader { @@ -31,26 +28,25 @@ protected: ProgressT m_progress; CallbackT m_onFinish; CallbackT m_onProgress; - string m_data; - scoped_ptr m_writer; - explicit HttpRequest(string const & filePath, CallbackT onFinish, CallbackT onProgress); + explicit HttpRequest(CallbackT onFinish, CallbackT onProgress); public: virtual ~HttpRequest() = 0; StatusT Status() const { return m_status; } ProgressT Progress() const { return m_progress; } - /// Retrieve either file path or downloaded data - string const & Data() const { return m_data; } + /// Either file path (for chunks) or downloaded data + virtual string const & Data() const = 0; - /// @param[in] filePath if empty, request will be saved to memory, see Data() - static HttpRequest * Get(string const & url, string const & filePath, CallbackT onFinish, + /// Response saved to memory buffer and retrieved with Data() + static HttpRequest * Get(string const & url, CallbackT onFinish, CallbackT onProgress = CallbackT()); /// Content-type for request is always "application/json" - static HttpRequest * Post(string const & url, string const & filePath, string const & postData, + static HttpRequest * PostJson(string const & url, string const & postData, CallbackT onFinish, CallbackT onProgress = CallbackT()); - static HttpRequest * GetChunks(vector const & urls, string const & filePath, int64_t fileSize, + static HttpRequest * GetFile(vector const & urls, string const & filePath, + int64_t projectedFileSize, CallbackT onFinish, CallbackT onProgress = CallbackT(), int64_t chunkSize = 512 * 1024); }; diff --git a/platform/platform_tests/downloader_test.cpp b/platform/platform_tests/downloader_test.cpp index 7011df167d..d2a032af48 100644 --- a/platform/platform_tests/downloader_test.cpp +++ b/platform/platform_tests/downloader_test.cpp @@ -1,13 +1,15 @@ #include "../../testing/testing.hpp" #include "../../base/logging.hpp" +#include "../../base/std_serialization.hpp" -#include "../../coding/file_reader.hpp" -#include "../../coding/file_writer.hpp" +#include "../../coding/file_reader_stream.hpp" +#include "../../coding/file_writer_stream.hpp" #include "../../coding/sha2.hpp" #include "../http_request.hpp" #include "../chunks_download_strategy.hpp" +#include "../platform.hpp" #include "../../std/scoped_ptr.hpp" #include "../../std/bind.hpp" @@ -107,7 +109,7 @@ UNIT_TEST(DownloaderSimpleGet) HttpRequest::CallbackT onFinish = bind(&DownloadObserver::OnDownloadFinish, &observer, _1); HttpRequest::CallbackT onProgress = bind(&DownloadObserver::OnDownloadProgress, &observer, _1); { // simple success case - scoped_ptr request(HttpRequest::Get(TEST_URL1, string(), onFinish, onProgress)); + scoped_ptr request(HttpRequest::Get(TEST_URL1, onFinish, onProgress)); // wait until download is finished QCoreApplication::exec(); observer.TestOk(); @@ -116,7 +118,7 @@ UNIT_TEST(DownloaderSimpleGet) observer.Reset(); { // permanent redirect success case - scoped_ptr request(HttpRequest::Get(TEST_URL_PERMANENT, string(), onFinish, onProgress)); + scoped_ptr request(HttpRequest::Get(TEST_URL_PERMANENT, onFinish, onProgress)); QCoreApplication::exec(); observer.TestOk(); TEST_EQUAL(request->Data(), "Test1", ()); @@ -124,7 +126,7 @@ UNIT_TEST(DownloaderSimpleGet) observer.Reset(); { // fail case 404 - scoped_ptr request(HttpRequest::Get(TEST_URL_404, string(), onFinish, onProgress)); + scoped_ptr request(HttpRequest::Get(TEST_URL_404, onFinish, onProgress)); QCoreApplication::exec(); observer.TestFailed(); TEST_EQUAL(request->Data().size(), 0, (request->Data())); @@ -132,7 +134,7 @@ UNIT_TEST(DownloaderSimpleGet) observer.Reset(); { // fail case not existing host - scoped_ptr request(HttpRequest::Get(TEST_URL_INVALID_HOST, string(), onFinish, onProgress)); + scoped_ptr request(HttpRequest::Get(TEST_URL_INVALID_HOST, onFinish, onProgress)); QCoreApplication::exec(); observer.TestFailed(); TEST_EQUAL(request->Data().size(), 0, (request->Data())); @@ -141,7 +143,7 @@ UNIT_TEST(DownloaderSimpleGet) { // cancel download in the middle of the progress CancelDownload canceler; /// should be deleted in canceler - HttpRequest::Get(TEST_URL_BIG_FILE, string(), + HttpRequest::Get(TEST_URL_BIG_FILE, bind(&CancelDownload::OnFinish, &canceler, _1), bind(&CancelDownload::OnProgress, &canceler, _1)); QCoreApplication::exec(); @@ -149,58 +151,20 @@ UNIT_TEST(DownloaderSimpleGet) observer.Reset(); { // https success case - scoped_ptr request(HttpRequest::Get(TEST_URL_HTTPS, string(), onFinish, onProgress)); + scoped_ptr request(HttpRequest::Get(TEST_URL_HTTPS, onFinish, onProgress)); // wait until download is finished QCoreApplication::exec(); observer.TestOk(); TEST_GREATER(request->Data().size(), 0, ()); } - string const FILENAME = "some_downloader_test_file"; - observer.Reset(); - { // download file success case - scoped_ptr request(HttpRequest::Get(TEST_URL1, FILENAME, onFinish, onProgress)); - // wait until download is finished - QCoreApplication::exec(); - observer.TestOk(); - { - FileReader f(FILENAME); - string s; - f.ReadAsString(s); - TEST_EQUAL(s, "Test1", ()); - } - TEST_EQUAL(request->Data(), FILENAME, (request->Data())); - FileWriter::DeleteFileX(FILENAME); - } - - observer.Reset(); - { // download file error case - scoped_ptr request(HttpRequest::Get(TEST_URL_404, FILENAME, onFinish, onProgress)); - // wait until download is finished - QCoreApplication::exec(); - observer.TestFailed(); - { - FileReader f(FILENAME); - TEST_EQUAL(f.Size(), 0, ()); - } - TEST_EQUAL(request->Data(), FILENAME, (request->Data())); - FileWriter::DeleteFileX(FILENAME); - } - - { // Delete request at the end of successful file download + { // Delete request at the end of successful download DeleteOnFinish deleter; /// should be deleted in deleter on finish - HttpRequest::Get(TEST_URL1, FILENAME, + HttpRequest::Get(TEST_URL1, bind(&DeleteOnFinish::OnFinish, &deleter, _1), bind(&DeleteOnFinish::OnProgress, &deleter, _1)); QCoreApplication::exec(); - { - FileReader f(FILENAME); - string s; - f.ReadAsString(s); - TEST_EQUAL(s, "Test1", ()); - } - FileWriter::DeleteFileX(FILENAME); } } @@ -211,7 +175,7 @@ UNIT_TEST(DownloaderSimplePost) HttpRequest::CallbackT onProgress = bind(&DownloadObserver::OnDownloadProgress, &observer, _1); { // simple success case string const postData = "{\"jsonKey\":\"jsonValue\"}"; - scoped_ptr request(HttpRequest::Post(TEST_URL_POST, string(), postData, onFinish, onProgress)); + scoped_ptr request(HttpRequest::PostJson(TEST_URL_POST, postData, onFinish, onProgress)); // wait until download is finished QCoreApplication::exec(); observer.TestOk(); @@ -326,8 +290,24 @@ UNIT_TEST(ChunksDownloadStrategyFAIL) TEST_EQUAL(strategy.NextChunk(s2, beg2, end2), ChunksDownloadStrategy::EDownloadFailed, ()); } +bool ReadFileAsString(string const & file, string & outStr) +{ + try + { + FileReader f(file); + f.ReadAsString(outStr); + } + catch (FileReader::Exception const &) + { + return false; + } + return true; +} + UNIT_TEST(DownloadChunks) { + string const FILENAME = "some_downloader_test_file"; + DownloadObserver observer; HttpRequest::CallbackT onFinish = bind(&DownloadObserver::OnDownloadFinish, &observer, _1); HttpRequest::CallbackT onProgress = bind(&DownloadObserver::OnDownloadProgress, &observer, _1); @@ -338,12 +318,16 @@ UNIT_TEST(DownloadChunks) int64_t FILESIZE = 5; { // should use only one thread - scoped_ptr request(HttpRequest::GetChunks(urls, string(), FILESIZE, onFinish, onProgress)); + scoped_ptr request(HttpRequest::GetFile(urls, FILENAME, FILESIZE, + onFinish, onProgress)); // wait until download is finished QCoreApplication::exec(); observer.TestOk(); - TEST_EQUAL(request->Data().size(), FILESIZE, ()); - TEST_EQUAL(request->Data(), "Test1", ()); + TEST_EQUAL(request->Data(), FILENAME, ()); + string s; + TEST(ReadFileAsString(FILENAME, s), ()); + TEST_EQUAL(s, "Test1", ()); + FileWriter::DeleteFileX(FILENAME); } observer.Reset(); @@ -355,11 +339,12 @@ UNIT_TEST(DownloadChunks) FILESIZE = 5; { // 3 threads - fail, because of invalid size - scoped_ptr request(HttpRequest::GetChunks(urls, string(), FILESIZE, onFinish, onProgress, 2048)); + scoped_ptr request(HttpRequest::GetFile(urls, FILENAME, FILESIZE, + onFinish, onProgress, 2048)); // wait until download is finished QCoreApplication::exec(); observer.TestFailed(); - TEST_EQUAL(request->Data().size(), 0, ()); + FileWriter::DeleteFileX(FILENAME); } string const SHA256 = "EE6AE6A2A3619B2F4A397326BEC32583DE2196D9D575D66786CB3B6F9D04A633"; @@ -373,12 +358,15 @@ UNIT_TEST(DownloadChunks) FILESIZE = 47684; { // 3 threads - succeeded - scoped_ptr request(HttpRequest::GetChunks(urls, string(), FILESIZE, onFinish, onProgress, 2048)); + scoped_ptr request(HttpRequest::GetFile(urls, FILENAME, FILESIZE, + onFinish, onProgress, 2048)); // wait until download is finished QCoreApplication::exec(); observer.TestOk(); - TEST_EQUAL(request->Data().size(), FILESIZE, ()); - TEST_EQUAL(sha2::digest256(request->Data()), SHA256, ()); + string s; + TEST(ReadFileAsString(FILENAME, s), ()); + TEST_EQUAL(sha2::digest256(s), SHA256, ()); + FileWriter::DeleteFileX(FILENAME); } observer.Reset(); @@ -390,12 +378,14 @@ UNIT_TEST(DownloadChunks) FILESIZE = 47684; { // 3 threads with only one valid url - succeeded - scoped_ptr request(HttpRequest::GetChunks(urls, string(), FILESIZE, onFinish, onProgress, 2048)); + scoped_ptr request(HttpRequest::GetFile(urls, FILENAME, FILESIZE, onFinish, onProgress, 2048)); // wait until download is finished QCoreApplication::exec(); observer.TestOk(); - TEST_EQUAL(request->Data().size(), FILESIZE, ()); - TEST_EQUAL(sha2::digest256(request->Data()), SHA256, ()); + string s; + TEST(ReadFileAsString(FILENAME, s), ()); + TEST_EQUAL(sha2::digest256(s), SHA256, ()); + FileWriter::DeleteFileX(FILENAME); } observer.Reset(); @@ -406,34 +396,106 @@ UNIT_TEST(DownloadChunks) FILESIZE = 12345; { // 2 threads and all points to file with invalid size - fail - scoped_ptr request(HttpRequest::GetChunks(urls, string(), FILESIZE, onFinish, onProgress, 2048)); + scoped_ptr request(HttpRequest::GetFile(urls, FILENAME, FILESIZE, onFinish, onProgress, 2048)); // wait until download is finished QCoreApplication::exec(); observer.TestFailed(); - TEST_EQUAL(request->Data().size(), 0, ()); - } - - observer.Reset(); - - urls.clear(); - urls.push_back(TEST_URL_BIG_FILE); - urls.push_back(TEST_URL_BIG_FILE); - FILESIZE = 47684; - string const FILENAME = "some_downloader_test_file"; - - { // 3 threads - download to file - succeeded - scoped_ptr request(HttpRequest::GetChunks(urls, FILENAME, FILESIZE, onFinish, onProgress, 10000)); - // wait until download is finished - QCoreApplication::exec(); - observer.TestOk(); - TEST_EQUAL(request->Data(), FILENAME, ()); - { - FileReader f(FILENAME); - TEST_EQUAL(f.Size(), FILESIZE, ()); - string s; - f.ReadAsString(s); - TEST_EQUAL(sha2::digest256(s), SHA256, ()); - } + FileWriter::DeleteFileX(FILENAME); + } +} + + +int64_t FILESIZE = 47684; +int64_t const beg1 = 123, end1 = 1230, beg2 = 44000, end2 = 47683; + +struct ResumeChecker +{ + size_t m_counter; + ResumeChecker() : m_counter(0) {} + void OnProgress(HttpRequest & request) + { + if (m_counter == 0) + { + TEST_EQUAL(request.Progress(), make_pair(FILESIZE - beg2, FILESIZE), ()); + } + else if (m_counter == 1) + { + TEST_EQUAL(request.Progress(), make_pair(FILESIZE, FILESIZE), ()); + } + else + { + TEST(false, ("Progress should be called exactly 2 times")); + } + ++m_counter; + } + void OnFinish(HttpRequest & request) + { + TEST_EQUAL(request.Status(), HttpRequest::ECompleted, ()); + QCoreApplication::exit(); + } +}; + +UNIT_TEST(DownloadResumeChunks) +{ + string const FILENAME = "some_test_filename_12345"; + string const RESUME_FILENAME = FILENAME + ".resume"; + string const SHA256 = "EE6AE6A2A3619B2F4A397326BEC32583DE2196D9D575D66786CB3B6F9D04A633"; + + vector urls; + urls.push_back(TEST_URL_BIG_FILE); + + // 1st step - download full file + { + DownloadObserver observer; + + scoped_ptr request(HttpRequest::GetFile(urls, FILENAME, FILESIZE, + bind(&DownloadObserver::OnDownloadFinish, &observer, _1), + bind(&DownloadObserver::OnDownloadProgress, &observer, _1))); + QCoreApplication::exec(); + observer.TestOk(); + string s; + TEST(ReadFileAsString(FILENAME, s), ()); + TEST_EQUAL(sha2::digest256(s), SHA256, ()); + } + // 2nd step - mark some file blocks as not downloaded + { + FileWriter f(FILENAME, FileWriter::OP_WRITE_EXISTING); + f.Seek(beg1); + char b1[end1 - beg1]; + for (size_t i = 0; i < ARRAY_SIZE(b1); ++i) b1[i] = 0; + f.Write(b1, ARRAY_SIZE(b1)); + + f.Seek(beg2); + char b2[end2 - beg2]; + for (size_t i = 0; i < ARRAY_SIZE(b2); ++i) b2[i] = 0; + f.Write(b2, ARRAY_SIZE(b2)); + + vector > originalRanges; + originalRanges.push_back(make_pair(beg1, end1)); + originalRanges.push_back(make_pair(beg2, end2)); + { + FileWriterStream fws(RESUME_FILENAME); + fws << originalRanges; + } + // check if ranges are stored correctly + vector > loadedRanges; + { + FileReaderStream frs(RESUME_FILENAME); + frs >> loadedRanges; + } + TEST_EQUAL(originalRanges, loadedRanges, ()); + } + // 3rd step - check that resume works + { + ResumeChecker checker; + scoped_ptr request(HttpRequest::GetFile(urls, FILENAME, FILESIZE, + bind(&ResumeChecker::OnFinish, &checker, _1), + bind(&ResumeChecker::OnProgress, &checker, _1))); + QCoreApplication::exec(); + string s; + TEST(ReadFileAsString(FILENAME, s), ()); + TEST_EQUAL(sha2::digest256(s), SHA256, ()); + TEST(!GetPlatform().IsFileExists(RESUME_FILENAME), ()); FileWriter::DeleteFileX(FILENAME); } }