gpesquero/organicmaps
1
Fork 0
forked from organicmaps/organicmaps
Code Pull requests Activity

[downloader] Changed behavior - now only HttpRequest::GetFile() uses files for storage

Browse source
This commit is contained in:
Alex Zolotarev 2011-11-09 17:34:19 +03:00 • committed by Alex Zolotarev
parent b214f6823c
commit a38ccdfe6b
3 changed files with 197 additions and 130 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

75
platform/http_request.cpp
View file

@ -8,6 +8,8 @@
#include "../coding/file_writer.hpp"
#include "../std/scoped_ptr.hpp"
class HttpThread;
namespace downloader
@ -15,38 +17,34 @@ namespace downloader
/// @return 0 if creation failed
HttpThread * CreateNativeHttpThread(string const & url,
IHttpThreadCallback & callback,
int64_t begRange = 0,
int64_t endRange = -1,
int64_t expectedSize = -1,
string const & postBody = string());
IHttpThreadCallback & callback,
int64_t begRange = 0,
int64_t endRange = -1,
int64_t expectedSize = -1,
string const & postBody = string());
void DeleteNativeHttpThread(HttpThread * request);
//////////////////////////////////////////////////////////////////////////////////////////
HttpRequest::HttpRequest(string const & filePath, CallbackT onFinish, CallbackT onProgress)
HttpRequest::HttpRequest(CallbackT onFinish, CallbackT onProgress)
: m_status(EInProgress), m_progress(make_pair(0, -1)),
m_onFinish(onFinish), m_onProgress(onProgress)
{
if (filePath.empty())
m_writer.reset(new MemWriter<string>(m_data));
else
{
m_data = filePath;
m_writer.reset(new FileWriter(filePath, FileWriter::OP_WRITE_EXISTING));
}
}
HttpRequest::~HttpRequest()
{
}
//////////////////////////////////////////////////////////////////////////////////////////////////////////
class SimpleHttpRequest : public HttpRequest, public IHttpThreadCallback
class MemoryHttpRequest : public HttpRequest, public IHttpThreadCallback
{
HttpThread * m_thread;
string m_downloadedData;
MemWriter<string> m_writer;
virtual void OnWrite(int64_t, void const * buffer, size_t size)
{
m_writer->Write(buffer, size);
m_writer.Write(buffer, size);
m_progress.first += size;
if (m_onProgress)
m_onProgress(*this);
@ -55,50 +53,55 @@ class SimpleHttpRequest : public HttpRequest, public IHttpThreadCallback
virtual void OnFinish(long httpCode, int64_t, int64_t)
{
m_status = (httpCode == 200) ? ECompleted : EFailed;
m_writer.reset();
m_onFinish(*this);
}
public:
SimpleHttpRequest(string const & url, string const & filePath, CallbackT onFinish, CallbackT onProgress)
: HttpRequest(filePath, onFinish, onProgress)
MemoryHttpRequest(string const & url, CallbackT onFinish, CallbackT onProgress)
: HttpRequest(onFinish, onProgress), m_writer(m_downloadedData)
{
m_thread = CreateNativeHttpThread(url, *this);
}
SimpleHttpRequest(string const & url, string const & filePath, string const & postData,
MemoryHttpRequest(string const & url, string const & postData,
CallbackT onFinish, CallbackT onProgress)
: HttpRequest(filePath, onFinish, onProgress)
: HttpRequest(onFinish, onProgress), m_writer(m_downloadedData)
{
m_thread = CreateNativeHttpThread(url, *this, 0, -1, -1, postData);
}
virtual ~SimpleHttpRequest()
virtual ~MemoryHttpRequest()
{
DeleteNativeHttpThread(m_thread);
}
virtual string const & Data() const
{
return m_downloadedData;
}
};
HttpRequest * HttpRequest::Get(string const & url, string const & filePath,
CallbackT onFinish, CallbackT onProgress)
HttpRequest * HttpRequest::Get(string const & url, CallbackT onFinish, CallbackT onProgress)
{
return new SimpleHttpRequest(url, filePath, onFinish, onProgress);
return new MemoryHttpRequest(url, onFinish, onProgress);
}
HttpRequest * HttpRequest::Post(string const & url, string const & filePath, string const & postData,
HttpRequest * HttpRequest::PostJson(string const & url, string const & postData,
CallbackT onFinish, CallbackT onProgress)
{
return new SimpleHttpRequest(url, filePath, postData, onFinish, onProgress);
return new MemoryHttpRequest(url, postData, onFinish, onProgress);
}
////////////////////////////////////////////////////////////////////////////////////////////////
class ChunksHttpRequest : public HttpRequest, public IHttpThreadCallback
class FileHttpRequest : public HttpRequest, public IHttpThreadCallback
{
ChunksDownloadStrategy m_strategy;
typedef list<pair<HttpThread *, int64_t> > ThreadsContainerT;
ThreadsContainerT m_threads;
string m_filePath;
scoped_ptr<FileWriter> m_writer;
ChunksDownloadStrategy::ResultT StartThreads()
{
string url;
@ -165,9 +168,10 @@ class ChunksHttpRequest : public HttpRequest, public IHttpThreadCallback
}
public:
ChunksHttpRequest(vector<string> const & urls, string const & filePath, int64_t fileSize,
FileHttpRequest(vector<string> const & urls, string const & filePath, int64_t fileSize,
CallbackT onFinish, CallbackT onProgress, int64_t chunkSize)
: HttpRequest(filePath, onFinish, onProgress), m_strategy(urls, fileSize, chunkSize)
: HttpRequest(onFinish, onProgress), m_strategy(urls, fileSize, chunkSize),
m_filePath(filePath), m_writer(new FileWriter(filePath, FileWriter::OP_WRITE_EXISTING))
{
ASSERT(!urls.empty(), ("Urls list shouldn't be empty"));
// store expected file size for future checks
@ -175,17 +179,22 @@ public:
StartThreads();
}
virtual ~ChunksHttpRequest()
virtual ~FileHttpRequest()
{
for (ThreadsContainerT::iterator it = m_threads.begin(); it != m_threads.end(); ++it)
DeleteNativeHttpThread(it->first);
}
virtual string const & Data() const
{
return m_filePath;
}
};
HttpRequest * HttpRequest::GetChunks(vector<string> const & urls, string const & filePath, int64_t fileSize,
HttpRequest * HttpRequest::GetFile(vector<string> const & urls, string const & filePath, int64_t fileSize,
CallbackT onFinish, CallbackT onProgress, int64_t chunkSize)
{
return new ChunksHttpRequest(urls, filePath, fileSize, onFinish, onProgress, chunkSize);
return new FileHttpRequest(urls, filePath, fileSize, onFinish, onProgress, chunkSize);
}
} // namespace downloader

20
platform/http_request.hpp
View file

@ -4,9 +4,6 @@
#include "../std/string.hpp"
#include "../std/vector.hpp"
#include "../std/utility.hpp"
#include "../std/scoped_ptr.hpp"
class Writer;
namespace downloader
{
@ -31,26 +28,25 @@ protected:
ProgressT m_progress;
CallbackT m_onFinish;
CallbackT m_onProgress;
string m_data;
scoped_ptr<Writer> m_writer;
explicit HttpRequest(string const & filePath, CallbackT onFinish, CallbackT onProgress);
explicit HttpRequest(CallbackT onFinish, CallbackT onProgress);
public:
virtual ~HttpRequest() = 0;
StatusT Status() const { return m_status; }
ProgressT Progress() const { return m_progress; }
/// Retrieve either file path or downloaded data
string const & Data() const { return m_data; }
/// Either file path (for chunks) or downloaded data
virtual string const & Data() const = 0;
/// @param[in] filePath if empty, request will be saved to memory, see Data()
static HttpRequest * Get(string const & url, string const & filePath, CallbackT onFinish,
/// Response saved to memory buffer and retrieved with Data()
static HttpRequest * Get(string const & url, CallbackT onFinish,
CallbackT onProgress = CallbackT());
/// Content-type for request is always "application/json"
static HttpRequest * Post(string const & url, string const & filePath, string const & postData,
static HttpRequest * PostJson(string const & url, string const & postData,
CallbackT onFinish, CallbackT onProgress = CallbackT());
static HttpRequest * GetChunks(vector<string> const & urls, string const & filePath, int64_t fileSize,
static HttpRequest * GetFile(vector<string> const & urls, string const & filePath,
int64_t projectedFileSize,
CallbackT onFinish, CallbackT onProgress = CallbackT(),
int64_t chunkSize = 512 * 1024);
};

232
platform/platform_tests/downloader_test.cpp
View file

@ -1,13 +1,15 @@
#include "../../testing/testing.hpp"
#include "../../base/logging.hpp"
#include "../../base/std_serialization.hpp"
#include "../../coding/file_reader.hpp"
#include "../../coding/file_writer.hpp"
#include "../../coding/file_reader_stream.hpp"
#include "../../coding/file_writer_stream.hpp"
#include "../../coding/sha2.hpp"
#include "../http_request.hpp"
#include "../chunks_download_strategy.hpp"
#include "../platform.hpp"
#include "../../std/scoped_ptr.hpp"
#include "../../std/bind.hpp"
@ -107,7 +109,7 @@ UNIT_TEST(DownloaderSimpleGet)
HttpRequest::CallbackT onFinish = bind(&DownloadObserver::OnDownloadFinish, &observer, _1);
HttpRequest::CallbackT onProgress = bind(&DownloadObserver::OnDownloadProgress, &observer, _1);
{ // simple success case
scoped_ptr<HttpRequest> request(HttpRequest::Get(TEST_URL1, string(), onFinish, onProgress));
scoped_ptr<HttpRequest> request(HttpRequest::Get(TEST_URL1, onFinish, onProgress));
// wait until download is finished
QCoreApplication::exec();
observer.TestOk();
@ -116,7 +118,7 @@ UNIT_TEST(DownloaderSimpleGet)
observer.Reset();
{ // permanent redirect success case
scoped_ptr<HttpRequest> request(HttpRequest::Get(TEST_URL_PERMANENT, string(), onFinish, onProgress));
scoped_ptr<HttpRequest> request(HttpRequest::Get(TEST_URL_PERMANENT, onFinish, onProgress));
QCoreApplication::exec();
observer.TestOk();
TEST_EQUAL(request->Data(), "Test1", ());
@ -124,7 +126,7 @@ UNIT_TEST(DownloaderSimpleGet)
observer.Reset();
{ // fail case 404
scoped_ptr<HttpRequest> request(HttpRequest::Get(TEST_URL_404, string(), onFinish, onProgress));
scoped_ptr<HttpRequest> request(HttpRequest::Get(TEST_URL_404, onFinish, onProgress));
QCoreApplication::exec();
observer.TestFailed();
TEST_EQUAL(request->Data().size(), 0, (request->Data()));
@ -132,7 +134,7 @@ UNIT_TEST(DownloaderSimpleGet)
observer.Reset();
{ // fail case not existing host
scoped_ptr<HttpRequest> request(HttpRequest::Get(TEST_URL_INVALID_HOST, string(), onFinish, onProgress));
scoped_ptr<HttpRequest> request(HttpRequest::Get(TEST_URL_INVALID_HOST, onFinish, onProgress));
QCoreApplication::exec();
observer.TestFailed();
TEST_EQUAL(request->Data().size(), 0, (request->Data()));
@ -141,7 +143,7 @@ UNIT_TEST(DownloaderSimpleGet)
{ // cancel download in the middle of the progress
CancelDownload canceler;
/// should be deleted in canceler
HttpRequest::Get(TEST_URL_BIG_FILE, string(),
HttpRequest::Get(TEST_URL_BIG_FILE,
bind(&CancelDownload::OnFinish, &canceler, _1),
bind(&CancelDownload::OnProgress, &canceler, _1));
QCoreApplication::exec();
@ -149,58 +151,20 @@ UNIT_TEST(DownloaderSimpleGet)
observer.Reset();
{ // https success case
scoped_ptr<HttpRequest> request(HttpRequest::Get(TEST_URL_HTTPS, string(), onFinish, onProgress));
scoped_ptr<HttpRequest> request(HttpRequest::Get(TEST_URL_HTTPS, onFinish, onProgress));
// wait until download is finished
QCoreApplication::exec();
observer.TestOk();
TEST_GREATER(request->Data().size(), 0, ());
}
string const FILENAME = "some_downloader_test_file";
observer.Reset();
{ // download file success case
scoped_ptr<HttpRequest> request(HttpRequest::Get(TEST_URL1, FILENAME, onFinish, onProgress));
// wait until download is finished
QCoreApplication::exec();
observer.TestOk();
{
FileReader f(FILENAME);
string s;
f.ReadAsString(s);
TEST_EQUAL(s, "Test1", ());
}
TEST_EQUAL(request->Data(), FILENAME, (request->Data()));
FileWriter::DeleteFileX(FILENAME);
}
observer.Reset();
{ // download file error case
scoped_ptr<HttpRequest> request(HttpRequest::Get(TEST_URL_404, FILENAME, onFinish, onProgress));
// wait until download is finished
QCoreApplication::exec();
observer.TestFailed();
{
FileReader f(FILENAME);
TEST_EQUAL(f.Size(), 0, ());
}
TEST_EQUAL(request->Data(), FILENAME, (request->Data()));
FileWriter::DeleteFileX(FILENAME);
}
{ // Delete request at the end of successful file download
{ // Delete request at the end of successful download
DeleteOnFinish deleter;
/// should be deleted in deleter on finish
HttpRequest::Get(TEST_URL1, FILENAME,
HttpRequest::Get(TEST_URL1,
bind(&DeleteOnFinish::OnFinish, &deleter, _1),
bind(&DeleteOnFinish::OnProgress, &deleter, _1));
QCoreApplication::exec();
{
FileReader f(FILENAME);
string s;
f.ReadAsString(s);
TEST_EQUAL(s, "Test1", ());
}
FileWriter::DeleteFileX(FILENAME);
}
}
@ -211,7 +175,7 @@ UNIT_TEST(DownloaderSimplePost)
HttpRequest::CallbackT onProgress = bind(&DownloadObserver::OnDownloadProgress, &observer, _1);
{ // simple success case
string const postData = "{\"jsonKey\":\"jsonValue\"}";
scoped_ptr<HttpRequest> request(HttpRequest::Post(TEST_URL_POST, string(), postData, onFinish, onProgress));
scoped_ptr<HttpRequest> request(HttpRequest::PostJson(TEST_URL_POST, postData, onFinish, onProgress));
// wait until download is finished
QCoreApplication::exec();
observer.TestOk();
@ -326,8 +290,24 @@ UNIT_TEST(ChunksDownloadStrategyFAIL)
TEST_EQUAL(strategy.NextChunk(s2, beg2, end2), ChunksDownloadStrategy::EDownloadFailed, ());
}
bool ReadFileAsString(string const & file, string & outStr)
{
try
{
FileReader f(file);
f.ReadAsString(outStr);
}
catch (FileReader::Exception const &)
{
return false;
}
return true;
}
UNIT_TEST(DownloadChunks)
{
string const FILENAME = "some_downloader_test_file";
DownloadObserver observer;
HttpRequest::CallbackT onFinish = bind(&DownloadObserver::OnDownloadFinish, &observer, _1);
HttpRequest::CallbackT onProgress = bind(&DownloadObserver::OnDownloadProgress, &observer, _1);
@ -338,12 +318,16 @@ UNIT_TEST(DownloadChunks)
int64_t FILESIZE = 5;
{ // should use only one thread
scoped_ptr<HttpRequest> request(HttpRequest::GetChunks(urls, string(), FILESIZE, onFinish, onProgress));
scoped_ptr<HttpRequest> request(HttpRequest::GetFile(urls, FILENAME, FILESIZE,
onFinish, onProgress));
// wait until download is finished
QCoreApplication::exec();
observer.TestOk();
TEST_EQUAL(request->Data().size(), FILESIZE, ());
TEST_EQUAL(request->Data(), "Test1", ());
TEST_EQUAL(request->Data(), FILENAME, ());
string s;
TEST(ReadFileAsString(FILENAME, s), ());
TEST_EQUAL(s, "Test1", ());
FileWriter::DeleteFileX(FILENAME);
}
observer.Reset();
@ -355,11 +339,12 @@ UNIT_TEST(DownloadChunks)
FILESIZE = 5;
{ // 3 threads - fail, because of invalid size
scoped_ptr<HttpRequest> request(HttpRequest::GetChunks(urls, string(), FILESIZE, onFinish, onProgress, 2048));
scoped_ptr<HttpRequest> request(HttpRequest::GetFile(urls, FILENAME, FILESIZE,
onFinish, onProgress, 2048));
// wait until download is finished
QCoreApplication::exec();
observer.TestFailed();
TEST_EQUAL(request->Data().size(), 0, ());
FileWriter::DeleteFileX(FILENAME);
}
string const SHA256 = "EE6AE6A2A3619B2F4A397326BEC32583DE2196D9D575D66786CB3B6F9D04A633";
@ -373,12 +358,15 @@ UNIT_TEST(DownloadChunks)
FILESIZE = 47684;
{ // 3 threads - succeeded
scoped_ptr<HttpRequest> request(HttpRequest::GetChunks(urls, string(), FILESIZE, onFinish, onProgress, 2048));
scoped_ptr<HttpRequest> request(HttpRequest::GetFile(urls, FILENAME, FILESIZE,
onFinish, onProgress, 2048));
// wait until download is finished
QCoreApplication::exec();
observer.TestOk();
TEST_EQUAL(request->Data().size(), FILESIZE, ());
TEST_EQUAL(sha2::digest256(request->Data()), SHA256, ());
string s;
TEST(ReadFileAsString(FILENAME, s), ());
TEST_EQUAL(sha2::digest256(s), SHA256, ());
FileWriter::DeleteFileX(FILENAME);
}
observer.Reset();
@ -390,12 +378,14 @@ UNIT_TEST(DownloadChunks)
FILESIZE = 47684;
{ // 3 threads with only one valid url - succeeded
scoped_ptr<HttpRequest> request(HttpRequest::GetChunks(urls, string(), FILESIZE, onFinish, onProgress, 2048));
scoped_ptr<HttpRequest> request(HttpRequest::GetFile(urls, FILENAME, FILESIZE, onFinish, onProgress, 2048));
// wait until download is finished
QCoreApplication::exec();
observer.TestOk();
TEST_EQUAL(request->Data().size(), FILESIZE, ());
TEST_EQUAL(sha2::digest256(request->Data()), SHA256, ());
string s;
TEST(ReadFileAsString(FILENAME, s), ());
TEST_EQUAL(sha2::digest256(s), SHA256, ());
FileWriter::DeleteFileX(FILENAME);
}
observer.Reset();
@ -406,34 +396,106 @@ UNIT_TEST(DownloadChunks)
FILESIZE = 12345;
{ // 2 threads and all points to file with invalid size - fail
scoped_ptr<HttpRequest> request(HttpRequest::GetChunks(urls, string(), FILESIZE, onFinish, onProgress, 2048));
scoped_ptr<HttpRequest> request(HttpRequest::GetFile(urls, FILENAME, FILESIZE, onFinish, onProgress, 2048));
// wait until download is finished
QCoreApplication::exec();
observer.TestFailed();
TEST_EQUAL(request->Data().size(), 0, ());
}
observer.Reset();
urls.clear();
urls.push_back(TEST_URL_BIG_FILE);
urls.push_back(TEST_URL_BIG_FILE);
FILESIZE = 47684;
string const FILENAME = "some_downloader_test_file";
{ // 3 threads - download to file - succeeded
scoped_ptr<HttpRequest> request(HttpRequest::GetChunks(urls, FILENAME, FILESIZE, onFinish, onProgress, 10000));
// wait until download is finished
QCoreApplication::exec();
observer.TestOk();
TEST_EQUAL(request->Data(), FILENAME, ());
{
FileReader f(FILENAME);
TEST_EQUAL(f.Size(), FILESIZE, ());
string s;
f.ReadAsString(s);
TEST_EQUAL(sha2::digest256(s), SHA256, ());
}
FileWriter::DeleteFileX(FILENAME);
}
}
int64_t FILESIZE = 47684;
int64_t const beg1 = 123, end1 = 1230, beg2 = 44000, end2 = 47683;
struct ResumeChecker
{
size_t m_counter;
ResumeChecker() : m_counter(0) {}
void OnProgress(HttpRequest & request)
{
if (m_counter == 0)
{
TEST_EQUAL(request.Progress(), make_pair(FILESIZE - beg2, FILESIZE), ());
}
else if (m_counter == 1)
{
TEST_EQUAL(request.Progress(), make_pair(FILESIZE, FILESIZE), ());
}
else
{
TEST(false, ("Progress should be called exactly 2 times"));
}
++m_counter;
}
void OnFinish(HttpRequest & request)
{
TEST_EQUAL(request.Status(), HttpRequest::ECompleted, ());
QCoreApplication::exit();
}
};
UNIT_TEST(DownloadResumeChunks)
{
string const FILENAME = "some_test_filename_12345";
string const RESUME_FILENAME = FILENAME + ".resume";
string const SHA256 = "EE6AE6A2A3619B2F4A397326BEC32583DE2196D9D575D66786CB3B6F9D04A633";
vector<string> urls;
urls.push_back(TEST_URL_BIG_FILE);
// 1st step - download full file
{
DownloadObserver observer;
scoped_ptr<HttpRequest> request(HttpRequest::GetFile(urls, FILENAME, FILESIZE,
bind(&DownloadObserver::OnDownloadFinish, &observer, _1),
bind(&DownloadObserver::OnDownloadProgress, &observer, _1)));
QCoreApplication::exec();
observer.TestOk();
string s;
TEST(ReadFileAsString(FILENAME, s), ());
TEST_EQUAL(sha2::digest256(s), SHA256, ());
}
// 2nd step - mark some file blocks as not downloaded
{
FileWriter f(FILENAME, FileWriter::OP_WRITE_EXISTING);
f.Seek(beg1);
char b1[end1 - beg1];
for (size_t i = 0; i < ARRAY_SIZE(b1); ++i) b1[i] = 0;
f.Write(b1, ARRAY_SIZE(b1));
f.Seek(beg2);
char b2[end2 - beg2];
for (size_t i = 0; i < ARRAY_SIZE(b2); ++i) b2[i] = 0;
f.Write(b2, ARRAY_SIZE(b2));
vector<pair<int64_t, int64_t> > originalRanges;
originalRanges.push_back(make_pair(beg1, end1));
originalRanges.push_back(make_pair(beg2, end2));
{
FileWriterStream fws(RESUME_FILENAME);
fws << originalRanges;
}
// check if ranges are stored correctly
vector<pair<int64_t, int64_t> > loadedRanges;
{
FileReaderStream frs(RESUME_FILENAME);
frs >> loadedRanges;
}
TEST_EQUAL(originalRanges, loadedRanges, ());
}
// 3rd step - check that resume works
{
ResumeChecker checker;
scoped_ptr<HttpRequest> request(HttpRequest::GetFile(urls, FILENAME, FILESIZE,
bind(&ResumeChecker::OnFinish, &checker, _1),
bind(&ResumeChecker::OnProgress, &checker, _1)));
QCoreApplication::exec();
string s;
TEST(ReadFileAsString(FILENAME, s), ());
TEST_EQUAL(sha2::digest256(s), SHA256, ());
TEST(!GetPlatform().IsFileExists(RESUME_FILENAME), ());
FileWriter::DeleteFileX(FILENAME);
}
}