[mac][downloader] Added but not activated new implementation, still without chunks

platform/http_request.cpp

@@ -0,0 +1,68 @@
+#include "http_request.hpp"
+
+#ifdef DEBUG
+  #include "../base/thread.hpp"
+#endif
+
+#include "../coding/writer.hpp"
+
+namespace downloader
+{
+
+/// @return 0 if creation failed
+HttpRequestImpl * CreateNativeHttpRequest(string const & url,
+                                          IHttpRequestImplCallback & callback,
+                                          int64_t begRange = 0,
+                                          int64_t endRange = -1,
+                                          string const & postBody = string());
+void DeleteNativeHttpRequest(HttpRequestImpl * request);
+
+
+HttpRequest::HttpRequest(Writer & writer, CallbackT onFinish, CallbackT onProgress)
+  : m_status(EInProgress), m_progress(make_pair(-1, -1)), m_writer(writer),
+    m_onFinish(onFinish), m_onProgress(onProgress)
+{
+}
+
+HttpRequest::~HttpRequest()
+{
+  for (ThreadsContainerT::iterator it = m_threads.begin(); it != m_threads.end(); ++it)
+     DeleteNativeHttpRequest(*it);
+}
+
+void HttpRequest::OnWrite(int64_t offset, void const * buffer, size_t size)
+{
+#ifdef DEBUG
+  static threads::ThreadID id = threads::GetCurrentThreadID();
+  ASSERT_EQUAL(id, threads::GetCurrentThreadID(), ("OnWrite called from different threads"));
+#endif
+  m_writer.Seek(offset);
+  m_writer.Write(buffer, size);
+  m_progress.first += size;
+  if (m_onProgress)
+    m_onProgress(*this);
+}
+
+void HttpRequest::OnFinish(long httpCode, int64_t begRange, int64_t endRange)
+{
+  m_status = (httpCode == 200) ? ECompleted : EFailed;
+  ASSERT(m_onFinish, ());
+  m_onFinish(*this);
+}
+
+HttpRequest * HttpRequest::Get(string const & url, Writer & writer, CallbackT onFinish, CallbackT onProgress)
+{
+  HttpRequest * self = new HttpRequest(writer, onFinish, onProgress);
+  self->m_threads.push_back(CreateNativeHttpRequest(url, *self));
+  return self;
+}
+
+HttpRequest * HttpRequest::Post(string const & url, Writer & writer, string const & postData,
+                                CallbackT onFinish, CallbackT onProgress)
+{
+  HttpRequest * self = new HttpRequest(writer, onFinish, onProgress);
+  self->m_threads.push_back(CreateNativeHttpRequest(url, *self, 0, -1, postData));
+  return self;
+}
+
+}

platform/http_request.hpp

@@ -0,0 +1,63 @@
+#pragma once
+
+#include "../std/function.hpp"
+#include "../std/string.hpp"
+#include "../std/list.hpp"
+#include "../std/utility.hpp"
+
+#include "http_request_impl_callback.hpp"
+
+class Writer;
+class HttpRequestImpl;
+
+namespace downloader
+{
+
+/// Request will be canceled on delete
+class HttpRequest : public IHttpRequestImplCallback
+{
+public:
+  enum StatusT
+  {
+    EInProgress,
+    ECompleted,
+    EFailed
+  };
+
+  /// <current, total>, total can be -1 if size is unknown
+  typedef pair<int64_t, int64_t> ProgressT;
+
+  typedef function<void(HttpRequest &)> CallbackT;
+
+private:
+  StatusT m_status;
+  ProgressT m_progress;
+  Writer & m_writer;
+  CallbackT m_onFinish;
+  CallbackT m_onProgress;
+
+  typedef list<HttpRequestImpl *> ThreadsContainerT;
+  ThreadsContainerT m_threads;
+
+  explicit HttpRequest(Writer & writer, CallbackT onFinish, CallbackT onProgress);
+
+  /// @name Callbacks for internal native downloading threads
+  //@{
+  virtual void OnWrite(int64_t offset, void const * buffer, size_t size);
+  virtual void OnFinish(long httpCode, int64_t begRange, int64_t endRange);
+  //@}
+
+public:
+  virtual ~HttpRequest();
+
+  StatusT Status() const { return m_status; }
+  ProgressT Progress() const { return m_progress; }
+
+  static HttpRequest * Get(string const & url, Writer & writer, CallbackT onFinish,
+                           CallbackT onProgress = CallbackT());
+  /// Content-type is always "application/json"
+  static HttpRequest * Post(string const & url, Writer & writer, string const & postData,
+                            CallbackT onFinish, CallbackT onProgress = CallbackT());
+};
+
+} // namespace downloader

platform/http_request_impl_apple.h

@@ -0,0 +1,22 @@
+#pragma once
+
+#import <Foundation/Foundation.h>
+
+#include "../std/string.hpp"
+
+namespace downloader { class IHttpRequestImplCallback; }
+
+@interface HttpRequestImpl : NSObject
+{
+  downloader::IHttpRequestImplCallback * m_callback;
+  NSURLConnection * m_connection;
+  int64_t m_begRange, m_endRange;
+  int64_t m_downloadedBytes;
+}
+
+- (id) initWith:(string const &)url callback:(downloader::IHttpRequestImplCallback &)cb
+       begRange:(int64_t)beg endRange:(int64_t)end contentType:(string const &)ct
+       postBody:(string const &)pb;
+
+- (void) cancel;
+@end

platform/http_request_impl_apple.mm

@@ -0,0 +1,160 @@
+#import "http_request_impl_apple.h"
+
+#include "http_request_impl_callback.hpp"
+#include "platform.hpp"
+
+#include "../base/logging.hpp"
+
+#define TIMEOUT_IN_SECONDS 15.0
+
+@implementation HttpRequestImpl
+
+- (void) dealloc
+{
+  LOG(LDEBUG, ("ID:", [self hash], "Connection is destroyed"));
+  [m_connection cancel];
+  [m_connection release];
+  [super dealloc];
+}
+
+- (void) cancel
+{
+  [m_connection cancel];
+}
+
+- (id) initWith:(string const &)url callback:(downloader::IHttpRequestImplCallback &)cb
+       begRange:(int64_t)beg endRange:(int64_t)end postBody:(string const &)pb
+{
+  self = [super init];
+
+	m_callback = &cb;
+	m_begRange = beg;
+	m_endRange = end;
+	m_downloadedBytes = 0;
+
+	NSMutableURLRequest * request = [NSMutableURLRequest requestWithURL:
+			[NSURL URLWithString:[NSString stringWithUTF8String:url.c_str()]]
+			cachePolicy:NSURLRequestReloadIgnoringLocalCacheData timeoutInterval:TIMEOUT_IN_SECONDS];
+
+	if (beg > 0)
+	{
+		NSString * val;
+		if (end > 0)
+		{
+			LOG(LDEBUG, (url, "downloading range [", beg, ",", end, "]"));
+			val = [[NSString alloc] initWithFormat: @"bytes=%qi-%qi", beg, end];
+		}
+		else
+		{
+			LOG(LDEBUG, (url, "resuming download from position", beg));
+			val = [[NSString alloc] initWithFormat: @"bytes=%qi-", beg];
+		}
+		[request addValue:val forHTTPHeaderField:@"Range"];
+		[val release];
+	}
+
+	if (!pb.empty())
+	{
+		NSData * postData = [NSData dataWithBytes:pb.data() length:pb.size()];
+		[request setHTTPBody:postData];
+		[request setHTTPMethod:@"POST"];
+		[request addValue:@"application/json" forHTTPHeaderField:@"Content-Type"];
+	}
+	// set user-agent with unique client id only for mapswithme requests
+	if (url.find("mapswithme.com") != string::npos)
+	{
+		static string const uid = GetPlatform().UniqueClientId();
+		[request addValue:[NSString stringWithUTF8String: uid.c_str()] forHTTPHeaderField:@"User-Agent"];
+	}
+
+	// create the connection with the request and start loading the data
+	m_connection = [[NSURLConnection alloc] initWithRequest:request delegate:self];
+
+  if (m_connection == 0)
+  {
+    LOG(LERROR, ("Can't create connection for", url));
+    [self release];
+    return nil;
+  }
+  else
+    LOG(LDEBUG, ("ID:", [self hash], "Starting connection to", url));
+
+  return self;
+}
+
+- (void) connection: (NSURLConnection *)connection didReceiveResponse: (NSURLResponse *)response
+{
+	// This method is called when the server has determined that it
+	// has enough information to create the NSURLResponse.
+
+  // check if this is OK (not a 404 or the like)
+  if ([response isKindOfClass:[NSHTTPURLResponse class]])
+  {
+    NSInteger const statusCode = [(NSHTTPURLResponse *)response statusCode];
+    LOG(LDEBUG, ("Got response with status code", statusCode));
+#ifdef DEBUG
+//    NSDictionary * fields = [(NSHTTPURLResponse *)response allHeaderFields];
+//    for (id key in fields)
+//      NSLog(@"%@: %@", key, [fields objectForKey:key]);
+#endif
+    if (statusCode < 200 || statusCode > 299)
+    {
+      LOG(LWARNING, ("Received HTTP error, canceling download", statusCode));
+      [m_connection cancel];
+      m_callback->OnFinish(statusCode, m_begRange, m_endRange);
+      return;
+    }
+
+    int64_t const expectedLength = [response expectedContentLength];
+    if (expectedLength < 0)
+      LOG(LDEBUG, ("Server doesn't support HTTP Range"));
+    else
+      LOG(LDEBUG, ("Expected content length", expectedLength));
+  }
+  else
+  { // in theory, we should never be here
+    LOG(LWARNING, ("Invalid non-http response, aborting request"));
+    [m_connection cancel];
+    m_callback->OnFinish(-1, m_begRange, m_endRange);
+  }
+}
+
+- (void) connection:(NSURLConnection *)connection didReceiveData:(NSData *)data
+{
+  int64_t const length = [data length];
+  m_downloadedBytes += length;
+  m_callback->OnWrite(m_begRange + m_downloadedBytes - length, [data bytes], length);
+}
+
+- (void) connection:(NSURLConnection *)connection didFailWithError:(NSError *)error
+{
+  LOG(LWARNING, ("Connection failed", [[error localizedDescription] cStringUsingEncoding:NSUTF8StringEncoding]));
+  m_callback->OnFinish([error code], m_begRange, m_endRange);
+}
+
+- (void) connectionDidFinishLoading:(NSURLConnection *)connection
+{
+  m_callback->OnFinish(200, m_begRange, m_endRange);
+}
+
+@end
+
+///////////////////////////////////////////////////////////////////////////////////////
+namespace downloader
+{
+HttpRequestImpl * CreateNativeHttpRequest(string const & url,
+                                          downloader::IHttpRequestImplCallback & cb,
+                                          int64_t beg,
+                                          int64_t end,
+                                          string const & pb)
+{
+  return [[HttpRequestImpl alloc] initWith:url callback:cb begRange:beg endRange:end postBody:pb];
+}
+
+void DeleteNativeHttpRequest(HttpRequestImpl * request)
+{
+  [request cancel];
+  [request release];
+}
+
+} // namespace downloader

platform/http_request_impl_callback.hpp

@@ -0,0 +1,13 @@
+#pragma once
+
+namespace downloader
+{
+
+class IHttpRequestImplCallback
+{
+public:
+  virtual void OnWrite(int64_t offset, void const * buffer, size_t size) = 0;
+  virtual void OnFinish(long httpCode, int64_t begRange, int64_t endRange) = 0;
+};
+
+} // namespace downloader

platform/platform.pro

@@ -39,9 +39,11 @@ include($$ROOT_DIR/common.pri)
 }
 
 macx*|iphone* {
-  HEADERS += apple_download.h
+  HEADERS += apple_download.h \
+             http_request_impl_apple.h
   OBJECTIVE_SOURCES += apple_download.mm \
-                       apple_download_manager.mm
+                       apple_download_manager.mm \
+                       http_request_impl_apple.mm
 }
 
 win32*|linux* {
@@ -64,6 +66,8 @@ HEADERS += \
     video_timer.hpp \
     languages.hpp \
     url_generator.hpp \
+    http_request.hpp \
+    http_request_impl_callback.hpp
 
 SOURCES += \
     preferred_languages.cpp \
@@ -71,3 +75,4 @@ SOURCES += \
     video_timer.cpp \
     languages.cpp \
     url_generator.cpp \
+    http_request.cpp

platform/platform_tests/downloader_test.cpp

@@ -0,0 +1,171 @@
+#include "../../testing/testing.hpp"
+
+#include "../../base/logging.hpp"
+
+#include "../../coding/writer.hpp"
+
+#include "../../platform/http_request.hpp"
+
+#include "../../std/scoped_ptr.hpp"
+#include "../../std/bind.hpp"
+
+#include <QCoreApplication>
+
+#define TEST_URL1 "http://melnichek.ath.cx:34568/unit_tests/1.txt"
+#define TEST_URL_404 "http://melnichek.ath.cx:34568/unit_tests/notexisting_unittest"
+#define TEST_URL_PERMANENT "http://melnichek.ath.cx:34568/unit_tests/permanent"
+#define TEST_URL_INVALID_HOST "http://melnichek12345.ath.cx"
+#define TEST_URL_BIG_FILE "http://melnichek.ath.cx:34568/unit_tests/47kb.file"
+#define TEST_URL_HTTPS "https://github.com"
+#define TEST_URL_POST "http://melnichek.ath.cx:34568/unit_tests/post.php"
+
+using namespace downloader;
+
+
+class DownloadObserver
+{
+  bool m_progressWasCalled;
+  HttpRequest::StatusT * m_status;
+
+public:
+  DownloadObserver() : m_status(0)
+  {
+    Reset();
+    my::g_LogLevel = LDEBUG;
+  }
+
+  void Reset()
+  {
+    m_progressWasCalled = false;
+    if (m_status)
+      delete m_status;
+    m_status = 0;
+  }
+
+  void TestOk()
+  {
+    TEST(m_progressWasCalled, ("Download progress wasn't called"));
+    TEST_NOT_EQUAL(m_status, 0, ());
+    TEST_EQUAL(*m_status, HttpRequest::ECompleted, ());
+  }
+
+  void TestFailed()
+  {
+    TEST_NOT_EQUAL(m_status, 0, ());
+    TEST_EQUAL(*m_status, HttpRequest::EFailed, ());
+  }
+
+  void OnDownloadProgress(HttpRequest & request)
+  {
+    m_progressWasCalled = true;
+    TEST_EQUAL(request.Status(), HttpRequest::EInProgress, ());
+  }
+
+  void OnDownloadFinish(HttpRequest & request)
+  {
+    TEST_EQUAL(m_status, 0, ());
+    m_status = new HttpRequest::StatusT(request.Status());
+    TEST(*m_status == HttpRequest::EFailed || *m_status == HttpRequest::ECompleted, ());
+    QCoreApplication::quit();
+  }
+
+};
+
+struct CancelDownload
+{
+  void OnProgress(HttpRequest & request)
+  {
+    delete &request;
+    QCoreApplication::quit();
+  }
+  void OnFinish(HttpRequest &)
+  {
+    TEST(false, ("Should be never called"));
+  }
+};
+
+UNIT_TEST(DownloaderSimpleGet)
+{
+  DownloadObserver observer;
+  string buffer;
+  MemWriter<string> writer(buffer);
+  HttpRequest::CallbackT onFinish = bind(&DownloadObserver::OnDownloadFinish, &observer, _1);
+  HttpRequest::CallbackT onProgress = bind(&DownloadObserver::OnDownloadProgress, &observer, _1);
+  { // simple success case
+    scoped_ptr<HttpRequest> request(HttpRequest::Get(TEST_URL1, writer, onFinish, onProgress));
+    // wait until download is finished
+    QCoreApplication::exec();
+    observer.TestOk();
+    TEST_EQUAL(buffer, "Test1", (buffer));
+  }
+
+  buffer.clear();
+  writer.Seek(0);
+  observer.Reset();
+  { // permanent redirect success case
+    scoped_ptr<HttpRequest> request(HttpRequest::Get(TEST_URL_PERMANENT, writer, onFinish, onProgress));
+    QCoreApplication::exec();
+    observer.TestOk();
+    TEST_EQUAL(buffer, "Test1", (buffer));
+  }
+
+  buffer.clear();
+  writer.Seek(0);
+  observer.Reset();
+  { // fail case 404
+    scoped_ptr<HttpRequest> request(HttpRequest::Get(TEST_URL_404, writer, onFinish, onProgress));
+    QCoreApplication::exec();
+    observer.TestFailed();
+    TEST_EQUAL(buffer.size(), 0, (buffer));
+  }
+
+  buffer.clear();
+  writer.Seek(0);
+  observer.Reset();
+  { // fail case not existing host
+    scoped_ptr<HttpRequest> request(HttpRequest::Get(TEST_URL_INVALID_HOST, writer, onFinish, onProgress));
+    QCoreApplication::exec();
+    observer.TestFailed();
+    TEST_EQUAL(buffer.size(), 0, (buffer));
+  }
+
+  buffer.clear();
+  writer.Seek(0);
+  {
+    CancelDownload canceler;
+    /// should be deleted in canceler
+    HttpRequest::Get(TEST_URL_BIG_FILE, writer,
+        bind(&CancelDownload::OnFinish, &canceler, _1),
+        bind(&CancelDownload::OnProgress, &canceler, _1));
+    QCoreApplication::exec();
+    TEST_GREATER(buffer.size(), 0, ());
+  }
+
+  buffer.clear();
+  writer.Seek(0);
+  observer.Reset();
+  { // https success case
+    scoped_ptr<HttpRequest> request(HttpRequest::Get(TEST_URL_HTTPS, writer, onFinish, onProgress));
+    // wait until download is finished
+    QCoreApplication::exec();
+    observer.TestOk();
+    TEST_GREATER(buffer.size(), 0, (buffer));
+  }
+}
+
+UNIT_TEST(DownloaderSimplePost)
+{
+  DownloadObserver observer;
+  string buffer;
+  MemWriter<string> writer(buffer);
+  HttpRequest::CallbackT onFinish = bind(&DownloadObserver::OnDownloadFinish, &observer, _1);
+  HttpRequest::CallbackT onProgress = bind(&DownloadObserver::OnDownloadProgress, &observer, _1);
+  { // simple success case
+    string const postData = "{\"jsonKey\":\"jsonValue\"}";
+    scoped_ptr<HttpRequest> request(HttpRequest::Post(TEST_URL_POST, writer, postData, onFinish, onProgress));
+    // wait until download is finished
+    QCoreApplication::exec();
+    observer.TestOk();
+    TEST_EQUAL(buffer, postData, (buffer));
+  }
+}

platform/platform_tests/platform_tests.pro

@@ -26,11 +26,13 @@ win32*|linux* {
   QT *= network
 }
 
+#    download_test.cpp \
+
 SOURCES += \
     ../../testing/testingmain.cpp \
     platform_test.cpp \
-    download_test.cpp \
     jansson_test.cpp \
     concurrent_runner_test.cpp \
     language_test.cpp \
     url_generator_test.cpp \
+    downloader_test.cpp \