From 21e37fcd9a29abe140c706f6910ba52025c65d53 Mon Sep 17 00:00:00 2001 From: Alex Zolotarev Date: Wed, 8 Dec 2010 04:57:20 +0000 Subject: [PATCH] Added download resume support --- iphone/Maps/Platform/IPhoneDownload.h | 6 +- iphone/Maps/Platform/IPhoneDownload.mm | 49 +++++++++++--- iphone/Maps/Platform/IPhoneDownloadManager.mm | 8 ++- map/storage.cpp | 5 +- platform/download_manager.hpp | 5 +- platform/platform_tests/download_test.cpp | 64 ++++++++++++++++++- platform/qt_download.cpp | 19 +++--- platform/qt_download.hpp | 5 +- platform/qt_download_manager.cpp | 5 +- platform/qt_download_manager.hpp | 3 +- 10 files changed, 138 insertions(+), 31 deletions(-) diff --git a/iphone/Maps/Platform/IPhoneDownload.h b/iphone/Maps/Platform/IPhoneDownload.h index 04347b4b7f..bac2714e68 100644 --- a/iphone/Maps/Platform/IPhoneDownload.h +++ b/iphone/Maps/Platform/IPhoneDownload.h @@ -19,9 +19,11 @@ } - (void) dealloc; -- (string const &) Url; +- (std::string const &) Url; - (BOOL) StartDownloadWithUrl: (char const *)originalUrl andFile: (char const *)file - andFinishFunc: (TDownloadFinishedFunction &)finishFunc andProgressFunc: (TDownloadProgressFunction &)progressFunc; + andFinishFunc: (TDownloadFinishedFunction &)finishFunc + andProgressFunc: (TDownloadProgressFunction &)progressFunc + andUseResume: (BOOL)resume; // Added because release from manager doesn't destroy it immediately... - (void) Cancel; @end diff --git a/iphone/Maps/Platform/IPhoneDownload.mm b/iphone/Maps/Platform/IPhoneDownload.mm index a245c3d0a0..b7d4a3103c 100644 --- a/iphone/Maps/Platform/IPhoneDownload.mm +++ b/iphone/Maps/Platform/IPhoneDownload.mm @@ -21,7 +21,7 @@ - (void) dealloc { - NSLog(@"~IPhoneDownload() for url: %s", m_url.c_str()); +// NSLog(@"~IPhoneDownload() for url: %s", m_url.c_str()); if (m_connection) { [m_connection cancel]; @@ -39,14 +39,15 @@ - (BOOL) StartDownloadWithUrl: (char const *)originalUrl andFile: (char const *)file andFinishFunc: (TDownloadFinishedFunction &)finishFunc andProgressFunc: (TDownloadProgressFunction &)progressFunc + andUseResume: (BOOL)resume { m_finishObserver = finishFunc; m_progressObserver = progressFunc; // try to create file first - string tmpFile = file; + std::string tmpFile = file; tmpFile += DOWNLOADING_FILE_EXTENSION; - m_file = fopen(tmpFile.c_str(), "wb"); + m_file = fopen(tmpFile.c_str(), resume ? "ab" : "wb"); if (m_file == 0) { NSLog(@"Error opening %s file for download: %s", tmpFile.c_str(), strerror(errno)); @@ -60,15 +61,22 @@ m_url = originalUrl; // Create the request. - NSURLRequest * request = [NSURLRequest requestWithURL:[NSURL URLWithString: [NSString stringWithUTF8String:m_url.c_str()]] + NSMutableURLRequest * request = [NSMutableURLRequest requestWithURL:[NSURL URLWithString: [NSString stringWithUTF8String:m_url.c_str()]] cachePolicy:NSURLRequestReloadIgnoringLocalCacheData timeoutInterval:TIMEOUT_IN_SECONDS]; + long long fileSize = ftello(m_file); + if (resume && fileSize > 0) + { + NSString * val = [[NSString alloc] initWithFormat: @"bytes=%qi-", fileSize]; + [request addValue:val forHTTPHeaderField:@"Range"]; + [val release]; + } // create the connection with the request and start loading the data m_connection = [[NSURLConnection alloc] initWithRequest:request delegate:self]; // [request release]; if (m_connection == 0) { - NSLog(@"Can't create connection for url %s", originalUrl); - // notify observer about error and exit + NSLog(@"Can't create connection for url %s", originalUrl); + // notify observer about error and exit if (m_finishObserver) m_finishObserver(originalUrl, false); return NO; @@ -82,11 +90,34 @@ // This method is called when the server has determined that it // has enough information to create the NSURLResponse. - // It can be called multiple times, for example in the case of a - // redirect, so each time we reset the data. + // check if this is OK (not a 404 or the like) + if ([response respondsToSelector:@selector(statusCode)]) + { + NSInteger statusCode = [(NSHTTPURLResponse *)response statusCode]; + if (statusCode < 200 || statusCode > 299) + { + NSLog(@"Received HTTP error code %d, canceling download", statusCode); + // deleting file + fclose(m_file); + m_file = 0; + remove((m_requestedFileName + DOWNLOADING_FILE_EXTENSION).c_str()); + // notify user + if (m_finishObserver) + m_finishObserver(m_url.c_str(), false); + // and selfdestruct... + GetDownloadManager().CancelDownload(m_url.c_str()); + return; + } + } - fseeko(m_file, 0, SEEK_SET); m_projectedFileSize = [response expectedContentLength]; + // if server doesn't support resume, make sure we're downloading file from scratch + if (m_projectedFileSize < 0) + { + fclose(m_file); + m_file = fopen((m_requestedFileName + DOWNLOADING_FILE_EXTENSION).c_str(), "wb"); + } + NSLog(@"Projected file size: %qi", m_projectedFileSize); } - (void) connection: (NSURLConnection *)connection didReceiveData: (NSData *)data diff --git a/iphone/Maps/Platform/IPhoneDownloadManager.mm b/iphone/Maps/Platform/IPhoneDownloadManager.mm index 092ca37a51..b03f7480c1 100644 --- a/iphone/Maps/Platform/IPhoneDownloadManager.mm +++ b/iphone/Maps/Platform/IPhoneDownloadManager.mm @@ -24,7 +24,7 @@ public: } virtual void DownloadFile(char const * url, char const * fileName, - TDownloadFinishedFunction finishFunc, TDownloadProgressFunction progressFunc) + TDownloadFinishedFunction finishFunc, TDownloadProgressFunction progressFunc, bool resume) { // check if download is already active for (NSUInteger i = 0; i < [activeDownloads count]; ++i) @@ -38,7 +38,11 @@ public: } IPhoneDownload * download = [[IPhoneDownload alloc] init]; - if ([download StartDownloadWithUrl:url andFile:fileName andFinishFunc:finishFunc andProgressFunc:progressFunc]) + if ([download StartDownloadWithUrl:url + andFile:fileName + andFinishFunc:finishFunc + andProgressFunc:progressFunc + andUseResume:resume]) { // save download in array to cancel it later if necessary [activeDownloads addObject:download]; diff --git a/map/storage.cpp b/map/storage.cpp index 91af6032e7..4edf19f65a 100644 --- a/map/storage.cpp +++ b/map/storage.cpp @@ -105,7 +105,7 @@ namespace mapinfo UPDATE_FULL_URL, (GetPlatform().WritablePathForFile(UPDATE_CHECK_FILE)).c_str(), boost::bind(&Storage::OnUpdateDownloadFinished, this, _1, _2), - TDownloadProgressFunction()); + TDownloadProgressFunction(), false); return true; } @@ -268,7 +268,8 @@ namespace mapinfo it->first.c_str(), (GetPlatform().WritablePathForFile(mapinfo::FileNameFromUrl(it->first))).c_str(), boost::bind(&Storage::OnMapDownloadFinished, this, _1, _2), - boost::bind(&Storage::OnMapDownloadProgress, this, _1, _2)); + boost::bind(&Storage::OnMapDownloadProgress, this, _1, _2), + true); // enabled resume support by default // notify GUI - new status for country, "Downloading" if (m_observerChange) m_observerChange(index); diff --git a/platform/download_manager.hpp b/platform/download_manager.hpp index 3bd4a95b29..ce89f7e810 100644 --- a/platform/download_manager.hpp +++ b/platform/download_manager.hpp @@ -8,7 +8,7 @@ /// Appended to all downloading files and removed after successful download #define DOWNLOADING_FILE_EXTENSION ".downloading" -typedef std::pair TDownloadProgress; +typedef std::pair TDownloadProgress; typedef boost::function TDownloadProgressFunction; typedef boost::function TDownloadFinishedFunction; @@ -19,7 +19,8 @@ class DownloadManager public: virtual ~DownloadManager() {} virtual void DownloadFile(char const * url, char const * fileName, - TDownloadFinishedFunction finish, TDownloadProgressFunction progress) = 0; + TDownloadFinishedFunction finish, TDownloadProgressFunction progress, + bool useResume = false) = 0; /// @note Doesn't notifies clients on canceling! virtual void CancelDownload(char const * url) = 0; virtual void CancelAllDownloads() = 0; diff --git a/platform/platform_tests/download_test.cpp b/platform/platform_tests/download_test.cpp index 050171e697..750b902f0f 100644 --- a/platform/platform_tests/download_test.cpp +++ b/platform/platform_tests/download_test.cpp @@ -73,7 +73,10 @@ struct DlObserver void OnDownloadProgress(char const * url, TDownloadProgress progress) { m_progressUrl = url; - TEST_NOT_EQUAL( progress.second, -1, ("Your hosting doesn't support total file size", GetHostFromUrl(url)) ); + if (progress.second < 0) + { + cerr << "Your hosting doesn't support total file size or invalid resume range was given for " << url << endl; + } // for big file - cancel downloading after 40Kb were transferred if (progress.first > 40 * 1024) gMgr.CancelDownload(url); @@ -204,3 +207,62 @@ UNIT_TEST(LongDownloadCanceling) FileWriter::DeleteFile(TEST_FILE_NAME1); } */ +UNIT_TEST(DownloadResume) +{ + size_t const NUM = 1; + + string const fileContent("aLeX"); + + { + FileWriter f(TEST_FILE_NAME1 DOWNLOADING_FILE_EXTENSION); + f.Write(fileContent.c_str(), fileContent.size()); + } + + DlObserver observer1; + gMgr.DownloadFile(TEST_FILE_URL1, TEST_FILE_NAME1, + boost::bind(&DlObserver::OnDownloadFinished, &observer1, _1, _2), + boost::bind(&DlObserver::OnDownloadProgress, &observer1, _1, _2), + false); + WAIT_FOR_ASYNC_DOWNLOAD; + TEST_EQUAL( observer1.m_result[0], true, () ); + + DlObserver observer2; + gMgr.DownloadFile(TEST_FILE_URL1, TEST_FILE_NAME2, + boost::bind(&DlObserver::OnDownloadFinished, &observer2, _1, _2), + boost::bind(&DlObserver::OnDownloadProgress, &observer2, _1, _2), + false); + WAIT_FOR_ASYNC_DOWNLOAD; + TEST_EQUAL( observer2.m_result[0], true, () ); + + uint64_t size1 = 4, size2 = 5; + TEST( GetPlatform().GetFileSize(TEST_FILE_NAME1, size1), ()); + TEST( GetPlatform().GetFileSize(TEST_FILE_NAME2, size2), ()); + + TEST_EQUAL(size1, size2, ("Sizes should be equal - no resume was enabled")); + + // resume should append content to 1st file + { + FileWriter f(TEST_FILE_NAME1 DOWNLOADING_FILE_EXTENSION); + f.Write(fileContent.c_str(), fileContent.size()); + } + DlObserver observer3; + gMgr.DownloadFile(TEST_FILE_URL1, TEST_FILE_NAME1, + boost::bind(&DlObserver::OnDownloadFinished, &observer3, _1, _2), + boost::bind(&DlObserver::OnDownloadProgress, &observer3, _1, _2), + true); + WAIT_FOR_ASYNC_DOWNLOAD; + TEST_EQUAL( observer3.m_result[0], true, () ); + + TEST( GetPlatform().GetFileSize(TEST_FILE_NAME1, size1), ()); + TEST_EQUAL( size1, size2, () ); + + { + string str; + ReadFileToString(TEST_FILE_NAME1, str); + TEST_EQUAL(fileContent, str.substr(0, fileContent.size()), ()); + TEST_NOT_EQUAL(fileContent, str, ()); + } + + FileWriter::DeleteFile(TEST_FILE_NAME1); + FileWriter::DeleteFile(TEST_FILE_NAME2); +} diff --git a/platform/qt_download.cpp b/platform/qt_download.cpp index 3e7dd9b2c6..4fb22be926 100644 --- a/platform/qt_download.cpp +++ b/platform/qt_download.cpp @@ -5,7 +5,8 @@ #include "../base/assert.hpp" QtDownload::QtDownload(QtDownloadManager & manager, char const * url, - char const * fileName, TDownloadFinishedFunction & finish, TDownloadProgressFunction & progress) + char const * fileName, TDownloadFinishedFunction & finish, + TDownloadProgressFunction & progress, bool useResume) : QObject(&manager), m_currentUrl(url), m_reply(0), m_file(0), m_httpRequestAborted(false), m_finish(finish), m_progress(progress) { @@ -13,11 +14,8 @@ QtDownload::QtDownload(QtDownloadManager & manager, char const * url, QString tmpFileName(fileName); tmpFileName += DOWNLOADING_FILE_EXTENSION; - // @TODO implement resume download - QFile::remove(tmpFileName); - m_file = new QFile(tmpFileName); - if (!m_file->open(QIODevice::WriteOnly)) + if (!m_file->open(useResume ? QIODevice::Append : QIODevice::WriteOnly)) { QString const err = m_file->errorString(); LOG(LERROR, ("Can't open file while downloading", qPrintable(tmpFileName), qPrintable(err))); @@ -50,16 +48,21 @@ QtDownload::~QtDownload() } void QtDownload::StartDownload(QtDownloadManager & manager, char const * url, - char const * fileName, TDownloadFinishedFunction & finish, TDownloadProgressFunction & progress) + char const * fileName, TDownloadFinishedFunction & finish, + TDownloadProgressFunction & progress, bool useResume) { ASSERT(url && fileName, ()); // manager is responsible for auto deleting - new QtDownload(manager, url, fileName, finish, progress); + new QtDownload(manager, url, fileName, finish, progress, useResume); } void QtDownload::StartRequest() { - m_reply = static_cast(parent())->NetAccessManager().get(QNetworkRequest(m_currentUrl)); + QNetworkRequest httpRequest(m_currentUrl); + qint64 fileSize = m_file->size(); + if (fileSize > 0) // need resume + httpRequest.setRawHeader("Range", QString("bytes=%1-").arg(fileSize).toAscii()); + m_reply = static_cast(parent())->NetAccessManager().get(httpRequest); connect(m_reply, SIGNAL(finished()), this, SLOT(OnHttpFinished())); connect(m_reply, SIGNAL(readyRead()), this, SLOT(OnHttpReadyRead())); connect(m_reply, SIGNAL(downloadProgress(qint64, qint64)), diff --git a/platform/qt_download.hpp b/platform/qt_download.hpp index d8efafd7a5..40e4ba28ca 100644 --- a/platform/qt_download.hpp +++ b/platform/qt_download.hpp @@ -19,7 +19,8 @@ private: TDownloadProgressFunction m_progress; QtDownload(QtDownloadManager & manager, char const * url, char const * fileName, - TDownloadFinishedFunction & finish, TDownloadProgressFunction & progress); + TDownloadFinishedFunction & finish, TDownloadProgressFunction & progress, + bool useResume); void StartRequest(); private slots: @@ -31,6 +32,6 @@ public: /// Created instance is automanaged as a manager's child /// and can be manipulated later by manager->findChild(url); static void StartDownload(QtDownloadManager & manager, char const * url, char const * fileName, - TDownloadFinishedFunction & finish, TDownloadProgressFunction & progress); + TDownloadFinishedFunction & finish, TDownloadProgressFunction & progress, bool useResume); virtual ~QtDownload(); }; diff --git a/platform/qt_download_manager.cpp b/platform/qt_download_manager.cpp index 42bda7c86b..6635096646 100644 --- a/platform/qt_download_manager.cpp +++ b/platform/qt_download_manager.cpp @@ -4,11 +4,12 @@ #include "../base/assert.hpp" void QtDownloadManager::DownloadFile(char const * url, char const * fileName, - TDownloadFinishedFunction finish, TDownloadProgressFunction progress) + TDownloadFinishedFunction finish, TDownloadProgressFunction progress, + bool useResume) { QList downloads = findChildren(url); if (downloads.empty()) - QtDownload::StartDownload(*this, url, fileName, finish, progress); + QtDownload::StartDownload(*this, url, fileName, finish, progress, useResume); else { ASSERT(false, ("Download with the same url is already in progress!")); diff --git a/platform/qt_download_manager.hpp b/platform/qt_download_manager.hpp index 3f04160342..b58b6ef4c6 100644 --- a/platform/qt_download_manager.hpp +++ b/platform/qt_download_manager.hpp @@ -10,7 +10,8 @@ class QtDownloadManager : public QObject, public DownloadManager public: virtual void DownloadFile(char const * url, char const * fileName, - TDownloadFinishedFunction finish, TDownloadProgressFunction progress); + TDownloadFinishedFunction finish, TDownloadProgressFunction progress, + bool useResume); virtual void CancelDownload(char const * url); virtual void CancelAllDownloads();