diff --git a/3party/Alohalytics/examples/android/build.gradle b/3party/Alohalytics/examples/android/build.gradle index 6077dd1f4c..884aaf38c1 100644 --- a/3party/Alohalytics/examples/android/build.gradle +++ b/3party/Alohalytics/examples/android/build.gradle @@ -22,12 +22,12 @@ dependencies { android { compileSdkVersion 21 - buildToolsVersion '21.1.1' + buildToolsVersion '22' defaultConfig { applicationId 'org.alohalytics.demoapp' minSdkVersion 9 - targetSdkVersion 21 + targetSdkVersion 22 versionCode 1 versionName '1.0' @@ -35,7 +35,7 @@ android { moduleName 'alohalytics' stl 'c++_static' cFlags '-frtti -fexceptions' - ldLibs 'log', 'atomic' + ldLibs 'log', 'atomic', 'z' } } diff --git a/3party/Alohalytics/examples/android/src/androidTest/java/org/alohalytics/test/HttpTransportTest.java b/3party/Alohalytics/examples/android/src/androidTest/java/org/alohalytics/test/HttpTransportTest.java index 860be618a2..e5b13218ba 100644 --- a/3party/Alohalytics/examples/android/src/androidTest/java/org/alohalytics/test/HttpTransportTest.java +++ b/3party/Alohalytics/examples/android/src/androidTest/java/org/alohalytics/test/HttpTransportTest.java @@ -39,8 +39,15 @@ public class HttpTransportTest extends InstrumentationTestCase { HttpTransport.TIMEOUT_IN_MILLISECONDS = 3000; } + public static final String CACHE_DIR = "/data/data/org.alohalytics.demoapp/cache/"; + + @Override + protected void setUp() { + new File(CACHE_DIR).mkdirs(); + } + private String getFullWritablePathForFile(String fileName) { - return getInstrumentation().getContext().getCacheDir().getAbsolutePath() + "/" + fileName; + return CACHE_DIR + fileName; } public void testGetIntoMemory() throws Exception { @@ -72,9 +79,8 @@ public class HttpTransportTest extends InstrumentationTestCase { final HttpTransport.Params r = HttpTransport.run(p); assertEquals(200, r.httpResponseCode); final String receivedBody = new String(r.data); - // Server mirrors our content which we have gzipped. - assertTrue(receivedBody, receivedBody.contains("\"Content-Encoding\": \"gzip\"")); - assertTrue(receivedBody, receivedBody.contains("data:application/octet-stream;base64,H4sIAAAAAAAAAPNIzcnJ11EIzy/KSVEEANDDSuwNAAAA")); + // Server mirrors our content. + assertTrue(receivedBody, receivedBody.contains(postBody)); } public void testPostMissingContentType() throws Exception { @@ -132,8 +138,7 @@ public class HttpTransportTest extends InstrumentationTestCase { // TODO(AlexZ): Think about using data field in the future for error pages (404 etc) //assertNull(r.data); final String receivedBody = Util.ReadFileAsUtf8String(p.outputFilePath); - assertTrue(receivedBody, receivedBody.contains("\"Content-Encoding\": \"gzip\"")); - assertTrue(receivedBody, receivedBody.contains("data:application/octet-stream;base64,H4sIAAAAAAAAAMsoKSmw0tfPAFJJmXl6+UXp+gX5xSUAYum2hhcAAAA=")); + assertTrue(receivedBody, receivedBody.contains("\"data\": \"http://httpbin.org/post\"")); } finally { (new File(p.outputFilePath)).delete(); } diff --git a/3party/Alohalytics/examples/android/src/androidTest/java/org/alohalytics/test/UtilTest.java b/3party/Alohalytics/examples/android/src/androidTest/java/org/alohalytics/test/UtilTest.java index b329e83549..85564f7990 100644 --- a/3party/Alohalytics/examples/android/src/androidTest/java/org/alohalytics/test/UtilTest.java +++ b/3party/Alohalytics/examples/android/src/androidTest/java/org/alohalytics/test/UtilTest.java @@ -31,8 +31,13 @@ import java.io.FileNotFoundException; public class UtilTest extends InstrumentationTestCase { + @Override + protected void setUp() { + new File(org.alohalytics.test.HttpTransportTest.CACHE_DIR).mkdirs(); + } + private String getFullWritablePathForFile(String fileName) { - return getInstrumentation().getContext().getCacheDir().getAbsolutePath() + "/" + fileName; + return org.alohalytics.test.HttpTransportTest.CACHE_DIR + fileName; } public void testReadAndWriteStringToFile() throws Exception { diff --git a/3party/Alohalytics/examples/server/gunzip.cc b/3party/Alohalytics/examples/server/gunzip.cc index da91432125..92785c05c8 100644 --- a/3party/Alohalytics/examples/server/gunzip.cc +++ b/3party/Alohalytics/examples/server/gunzip.cc @@ -53,7 +53,7 @@ int main(int argc, char** argv) { std::cout << ptr->ToString() << std::endl; } } catch (const std::exception& ex) { - std::cerr << "Exception: " << ex.what() << std::endl; + std::cerr << "Exception: " << ex.what() << " in file " << argv[1] << std::endl; return -1; } return 0; diff --git a/3party/Alohalytics/src/android/java/org/alohalytics/HttpTransport.java b/3party/Alohalytics/src/android/java/org/alohalytics/HttpTransport.java index 76f7c1e347..c05792901b 100644 --- a/3party/Alohalytics/src/android/java/org/alohalytics/HttpTransport.java +++ b/3party/Alohalytics/src/android/java/org/alohalytics/HttpTransport.java @@ -36,7 +36,6 @@ import java.io.IOException; import java.io.OutputStream; import java.net.HttpURLConnection; import java.net.URL; -import java.util.zip.GZIPOutputStream; public class HttpTransport { @@ -52,8 +51,8 @@ public class HttpTransport { Log.d(TAG, "Connecting to " + p.url); try { connection = (HttpURLConnection) new URL(p.url).openConnection(); // NullPointerException, MalformedUrlException, IOException - // TODO(AlexZ): Customize redirects following in the future implementation for safer transfers. - connection.setInstanceFollowRedirects(true); + // We DO NOT follow any redirects for safer data transfer. + connection.setInstanceFollowRedirects(false); connection.setConnectTimeout(TIMEOUT_IN_MILLISECONDS); connection.setReadTimeout(TIMEOUT_IN_MILLISECONDS); connection.setUseCaches(false); @@ -66,27 +65,20 @@ public class HttpTransport { throw new NullPointerException("Please set Content-Type for POST requests."); } connection.setRequestProperty("Content-Type", p.contentType); + if (p.contentEncoding != null) { + connection.setRequestProperty("Content-Encoding", p.contentEncoding); + } connection.setDoOutput(true); if (p.data != null) { - // Use gzip compression for memory-only transfers. - // TODO(AlexZ): Move compression to the lower file-level (file storage queue) to save device space. - final ByteArrayOutputStream bos = new ByteArrayOutputStream(); - final GZIPOutputStream zos = new GZIPOutputStream(bos); - try { - zos.write(p.data); - } finally { - zos.close(); - } - connection.setFixedLengthStreamingMode(bos.size()); - connection.setRequestProperty("Content-Encoding", "gzip"); + connection.setFixedLengthStreamingMode(p.data.length); final OutputStream os = connection.getOutputStream(); try { - os.write(bos.toByteArray()); + os.write(p.data); } finally { os.close(); } if (p.debugMode) - Log.d(TAG, "Sent POST with gzipped content of size " + bos.size()); + Log.d(TAG, "Sent POST with content of size " + p.data.length); } else { final File file = new File(p.inputFilePath); assert (file.length() == (int) file.length()); @@ -110,6 +102,7 @@ public class HttpTransport { Log.d(TAG, "Received HTTP " + p.httpResponseCode + " from server."); p.receivedUrl = connection.getURL().toString(); p.contentType = connection.getContentType(); + p.contentEncoding = connection.getContentEncoding(); // This implementation receives any data only if we have HTTP::OK (200). if (p.httpResponseCode == HttpURLConnection.HTTP_OK) { OutputStream ostream; @@ -158,8 +151,11 @@ public class HttpTransport { // Can be different from url in case of redirects. public String receivedUrl = null; // SHOULD be specified for any POST request (any request where we send data to the server). - // On return, contains received Content-Type + // On return, contains received Content-Type or null. public String contentType = null; + // Can be specified for any POST request (any request where we send data to the server). + // On return, contains received Content-Encoding or null. + public String contentEncoding = null; // GET if null and inputFilePath is null. // Sent in POST otherwise. public byte[] data = null; diff --git a/3party/Alohalytics/src/android/jni/jni_alohalytics.cc b/3party/Alohalytics/src/android/jni/jni_alohalytics.cc index bcc08e56f7..c60af2edb9 100644 --- a/3party/Alohalytics/src/android/jni/jni_alohalytics.cc +++ b/3party/Alohalytics/src/android/jni/jni_alohalytics.cc @@ -267,6 +267,16 @@ bool HTTPClientPlatformWrapper::RunHTTPRequest() { CLEAR_AND_RETURN_FALSE_ON_EXCEPTION } + const static jfieldID contentEncodingField = + env->GetFieldID(g_httpParamsClass, "contentEncoding", "Ljava/lang/String;"); + if (!content_encoding_.empty()) { + const auto jniContentEncoding = MakePointerScopeGuard(env->NewStringUTF(content_encoding_.c_str()), deleteLocalRef); + CLEAR_AND_RETURN_FALSE_ON_EXCEPTION + + env->SetObjectField(httpParamsObject.get(), contentEncodingField, jniContentEncoding.get()); + CLEAR_AND_RETURN_FALSE_ON_EXCEPTION + } + if (!user_agent_.empty()) { const static jfieldID userAgentField = env->GetFieldID(g_httpParamsClass, "userAgent", "Ljava/lang/String;"); @@ -335,7 +345,15 @@ bool HTTPClientPlatformWrapper::RunHTTPRequest() { static_cast(env->GetObjectField(response, contentTypeField)), deleteLocalRef); CLEAR_AND_RETURN_FALSE_ON_EXCEPTION if (jniContentType) { - content_type_ = std::move(ToStdString(env, jniContentType.get())); + content_type_received_ = std::move(ToStdString(env, jniContentType.get())); + } + + // contentEncodingField is already cached above. + const auto jniContentEncoding = MakePointerScopeGuard( + static_cast(env->GetObjectField(response, contentEncodingField)), deleteLocalRef); + CLEAR_AND_RETURN_FALSE_ON_EXCEPTION + if (jniContentEncoding) { + content_encoding_received_ = std::move(ToStdString(env, jniContentEncoding.get())); } // dataField is already cached above. diff --git a/3party/Alohalytics/src/apple/http_client_apple.mm b/3party/Alohalytics/src/apple/http_client_apple.mm index 0f1c9b654a..93f1092de2 100644 --- a/3party/Alohalytics/src/apple/http_client_apple.mm +++ b/3party/Alohalytics/src/apple/http_client_apple.mm @@ -38,7 +38,6 @@ SOFTWARE. #include "../http_client.h" #include "../logger.h" -#include "../gzip_wrapper.h" #define TIMEOUT_IN_SECONDS 30.0 @@ -53,14 +52,13 @@ bool HTTPClientPlatformWrapper::RunHTTPRequest() { if (!content_type_.empty()) [request setValue:[NSString stringWithUTF8String:content_type_.c_str()] forHTTPHeaderField:@"Content-Type"]; + if (!content_encoding_.empty()) + [request setValue:[NSString stringWithUTF8String:content_encoding_.c_str()] forHTTPHeaderField:@"Content-Encoding"]; if (!user_agent_.empty()) [request setValue:[NSString stringWithUTF8String:user_agent_.c_str()] forHTTPHeaderField:@"User-Agent"]; if (!post_body_.empty()) { - // TODO(AlexZ): Compress data in file queue impl, before calling this method, to use less disk space in offline. - const std::string compressed = Gzip(post_body_); - request.HTTPBody = [NSData dataWithBytes:compressed.data() length:compressed.size()]; - [request setValue:@"gzip" forHTTPHeaderField:@"Content-Encoding"]; + request.HTTPBody = [NSData dataWithBytes:post_body_.data() length:post_body_.size()]; request.HTTPMethod = @"POST"; } else if (!post_file_.empty()) { NSError * err = nil; @@ -85,6 +83,14 @@ bool HTTPClientPlatformWrapper::RunHTTPRequest() { if (response) { error_code_ = static_cast(response.statusCode); url_received_ = [response.URL.absoluteString UTF8String]; + NSString * content = [response.allHeaderFields objectForKey:@"Content-Type"]; + if (content) { + content_type_received_ = [content UTF8String]; + } + NSString * encoding = [response.allHeaderFields objectForKey:@"Content-Encoding"]; + if (encoding) { + content_encoding_received_ = [encoding UTF8String]; + } if (url_data) { if (received_file_.empty()) { server_response_.assign(reinterpret_cast(url_data.bytes), url_data.length); diff --git a/3party/Alohalytics/src/cpp/alohalytics.cc b/3party/Alohalytics/src/cpp/alohalytics.cc index e630a7958e..ac3f48836f 100644 --- a/3party/Alohalytics/src/cpp/alohalytics.cc +++ b/3party/Alohalytics/src/cpp/alohalytics.cc @@ -31,6 +31,7 @@ #include "../http_client.h" #include "../logger.h" #include "../event_base.h" +#include "../gzip_wrapper.h" #include "../cereal/include/archives/binary.hpp" #include "../cereal/include/types/string.hpp" @@ -55,13 +56,15 @@ Stats::Stats() : message_queue_(*this) {} bool Stats::UploadBuffer(const std::string& url, std::string&& buffer, bool debug_mode) { HTTPClientPlatformWrapper request(url); request.set_debug_mode(debug_mode); - request.set_post_body(std::move(buffer), "application/alohalytics-binary-blob"); try { + // TODO(AlexZ): Refactor FileStorageQueue to automatically append ID and gzip files, so we don't need + // temporary memory buffer any more and files take less space. + request.set_post_body(alohalytics::Gzip(buffer), "application/alohalytics-binary-blob", "gzip"); return request.RunHTTPRequest() && 200 == request.error_code() && !request.was_redirected(); } catch (const std::exception& ex) { if (debug_mode) { - ALOG("Exception while trying to upload file", ex.what()); + ALOG("Exception while trying to UploadBuffer", ex.what()); } } return false; @@ -81,6 +84,8 @@ void Stats::OnMessage(const std::string& message, size_t dropped_events) { static const size_t kMaxEventsInMemory = 2048; if (container.size() > kMaxEventsInMemory) { container.pop_front(); + LOG_IF_DEBUG("Warning: maximum numbers of events in memory (", kMaxEventsInMemory, + ") was reached and the oldest one was dropped."); } } } diff --git a/3party/Alohalytics/src/gzip_wrapper.h b/3party/Alohalytics/src/gzip_wrapper.h index 6254e26f67..381c40e2c5 100644 --- a/3party/Alohalytics/src/gzip_wrapper.h +++ b/3party/Alohalytics/src/gzip_wrapper.h @@ -21,6 +21,7 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. *******************************************************************************/ +#include // std::memset #include #include #include @@ -32,25 +33,19 @@ namespace alohalytics { static constexpr size_t kGzipBufferSize = 32768; struct GzipErrorException : public std::exception { - int err_; std::string msg_; - GzipErrorException(int err, const char* msg) : err_(err), msg_(msg ? msg : "") {} + GzipErrorException(int err, const char* msg) { + msg_ = std::string("ERROR ") + std::to_string(err) + " while gzipping with zlib. " + (msg ? msg : ""); + } virtual char const* what() const noexcept { - return ("ERROR " + std::to_string(err_) + " while gzipping with zlib. " + msg_).c_str(); + return msg_.c_str(); } }; -struct GunzipErrorException : public std::exception { - int err_; - std::string msg_; - GunzipErrorException(int err, const char* msg) : err_(err), msg_(msg ? msg : "") {} - virtual char const* what() const noexcept { - return ("ERROR " + std::to_string(err_) + " while gunzipping with zlib. " + msg_).c_str(); - } -}; - -inline std::string Gzip(const std::string& data_to_compress) throw(GzipErrorException) { - z_stream z = {}; +// Throws GzipErrorException on any gzip processing error. +inline std::string Gzip(const std::string& data_to_compress) { + z_stream z; + std::memset(&z, 0, sizeof(z)); int res = ::deflateInit2(&z, Z_BEST_COMPRESSION, Z_DEFLATED, 15 + 16, 8, Z_DEFAULT_STRATEGY); if (Z_OK == res) { z.next_in = const_cast(reinterpret_cast(data_to_compress.data())); @@ -75,8 +70,20 @@ inline std::string Gzip(const std::string& data_to_compress) throw(GzipErrorExce throw GzipErrorException(res, z.msg); } -inline std::string Gunzip(const std::string& data_to_decompress) throw(GzipErrorException) { - z_stream z = {}; +struct GunzipErrorException : public std::exception { + std::string msg_; + GunzipErrorException(int err, const char* msg) { + msg_ = std::string("ERROR ") + std::to_string(err) + " while gzipping with zlib. " + (msg ? msg : ""); + } + virtual char const* what() const noexcept { + return msg_.c_str(); + } +}; + +// Throws GunzipErrorException on any gunzip processing error. +inline std::string Gunzip(const std::string& data_to_decompress) { + z_stream z; + std::memset(&z, 0, sizeof(z)); int res = ::inflateInit2(&z, 16 + MAX_WBITS); if (Z_OK == res) { z.next_in = const_cast(reinterpret_cast(data_to_decompress.data())); diff --git a/3party/Alohalytics/src/http_client.h b/3party/Alohalytics/src/http_client.h index 6cf5ddd57c..8fa1545779 100644 --- a/3party/Alohalytics/src/http_client.h +++ b/3party/Alohalytics/src/http_client.h @@ -47,6 +47,9 @@ class HTTPClientPlatformWrapper { // Data we received from the server if output_file_ wasn't initialized. std::string server_response_; std::string content_type_; + std::string content_type_received_; + std::string content_encoding_; + std::string content_encoding_received_; std::string user_agent_; std::string post_body_; bool debug_mode_ = false; @@ -84,19 +87,23 @@ class HTTPClientPlatformWrapper { return *this; } // This method is mutually exclusive with set_post_file(). - HTTPClientPlatformWrapper& set_post_body(const std::string& post_body, const std::string& content_type) { + HTTPClientPlatformWrapper& set_post_body(const std::string& post_body, const std::string& content_type, + const std::string& content_encoding = "") { post_body_ = post_body; content_type_ = content_type; + content_encoding_ = content_encoding; // TODO (dkorolev) replace with exceptions as discussed offline. assert(post_file_.empty()); return *this; } // Move version to avoid string copying. // This method is mutually exclusive with set_post_file(). - HTTPClientPlatformWrapper& set_post_body(std::string&& post_body, const std::string& content_type) { - post_body_ = post_body; + HTTPClientPlatformWrapper& set_post_body(std::string&& post_body, const std::string& content_type, + const std::string& content_encoding = "") { + post_body_ = std::move(post_body); post_file_.clear(); content_type_ = content_type; + content_encoding_ = content_encoding; return *this; } diff --git a/3party/Alohalytics/src/posix/http_client_curl.cc b/3party/Alohalytics/src/posix/http_client_curl.cc index a68d48f596..de5d5460db 100644 --- a/3party/Alohalytics/src/posix/http_client_curl.cc +++ b/3party/Alohalytics/src/posix/http_client_curl.cc @@ -54,11 +54,15 @@ struct ScopedTmpFileDeleter { std::string RunCurl(const std::string& cmd) { FILE* pipe = ::popen(cmd.c_str(), "r"); assert(pipe); - std::array s; + std::array arr; std::string result; - while (nullptr != ::fgets(s.data(), s.size(), pipe)) { - result += s.data(); - } + size_t read; + do { + read = ::fread(arr.data(), 1, arr.size(), pipe); + if (read > 0) { + result.append(arr.data(), read); + } + } while (read == arr.size()); const int err = ::pclose(pipe); if (err) { throw std::runtime_error("Error " + std::to_string(err) + " while calling " + cmd); @@ -66,12 +70,16 @@ std::string RunCurl(const std::string& cmd) { return result; } +// TODO(AlexZ): Add support for content_type_received_ and content_encoding_received_. bool HTTPClientPlatformWrapper::RunHTTPRequest() { // Last 3 chars in server's response will be http status code static constexpr size_t kCurlHttpCodeSize = 3; std::string cmd = "curl --max-redirs 0 -s -w '%{http_code}' "; if (!content_type_.empty()) { - cmd += "-H 'Content-Type: application/json' "; + cmd += "-H 'Content-Type: " + content_type_ + "' "; + } + if (!content_encoding_.empty()) { + cmd += "-H 'Content-Encoding: " + content_encoding_ + "' "; } ScopedTmpFileDeleter deleter; @@ -102,6 +110,7 @@ bool HTTPClientPlatformWrapper::RunHTTPRequest() { cmd += url_requested_; try { + // TODO(AlexZ): Do not store data in memory if received_file_ was specified. server_response_ = RunCurl(cmd); error_code_ = -1; std::string & s = server_response_;