gpesquero/organicmaps
1
Fork 0
forked from organicmaps/organicmaps
Code Pull requests Activity

[alohalytics] Statistics bug fixes and improvements.

Browse source
This commit is contained in:
Alex Zolotarev 2015-03-19 20:12:20 +03:00
parent 7e8f0172e8
commit c054b3af6a
11 changed files with 118 additions and 60 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
3party/Alohalytics/examples/android/build.gradle
View file

@ -22,12 +22,12 @@ dependencies {
android {
compileSdkVersion 21
buildToolsVersion '21.1.1'
buildToolsVersion '22'
defaultConfig {
applicationId 'org.alohalytics.demoapp'
minSdkVersion 9
targetSdkVersion 21
targetSdkVersion 22
versionCode 1
versionName '1.0'
@ -35,7 +35,7 @@ android {
moduleName 'alohalytics'
stl 'c++_static'
cFlags '-frtti -fexceptions'
ldLibs 'log', 'atomic'
ldLibs 'log', 'atomic', 'z'
}
}

17
3party/Alohalytics/examples/android/src/androidTest/java/org/alohalytics/test/HttpTransportTest.java
View file

@ -39,8 +39,15 @@ public class HttpTransportTest extends InstrumentationTestCase {
HttpTransport.TIMEOUT_IN_MILLISECONDS = 3000;
}
public static final String CACHE_DIR = "/data/data/org.alohalytics.demoapp/cache/";
@Override
protected void setUp() {
new File(CACHE_DIR).mkdirs();
}
private String getFullWritablePathForFile(String fileName) {
return getInstrumentation().getContext().getCacheDir().getAbsolutePath() + "/" + fileName;
return CACHE_DIR + fileName;
}
public void testGetIntoMemory() throws Exception {
@ -72,9 +79,8 @@ public class HttpTransportTest extends InstrumentationTestCase {
final HttpTransport.Params r = HttpTransport.run(p);
assertEquals(200, r.httpResponseCode);
final String receivedBody = new String(r.data);
// Server mirrors our content which we have gzipped.
assertTrue(receivedBody, receivedBody.contains("\"Content-Encoding\": \"gzip\""));
assertTrue(receivedBody, receivedBody.contains("data:application/octet-stream;base64,H4sIAAAAAAAAAPNIzcnJ11EIzy/KSVEEANDDSuwNAAAA"));
// Server mirrors our content.
assertTrue(receivedBody, receivedBody.contains(postBody));
}
public void testPostMissingContentType() throws Exception {
@ -132,8 +138,7 @@ public class HttpTransportTest extends InstrumentationTestCase {
// TODO(AlexZ): Think about using data field in the future for error pages (404 etc)
//assertNull(r.data);
final String receivedBody = Util.ReadFileAsUtf8String(p.outputFilePath);
assertTrue(receivedBody, receivedBody.contains("\"Content-Encoding\": \"gzip\""));
assertTrue(receivedBody, receivedBody.contains("data:application/octet-stream;base64,H4sIAAAAAAAAAMsoKSmw0tfPAFJJmXl6+UXp+gX5xSUAYum2hhcAAAA="));
assertTrue(receivedBody, receivedBody.contains("\"data\": \"http://httpbin.org/post\""));
} finally {
(new File(p.outputFilePath)).delete();
}

7
3party/Alohalytics/examples/android/src/androidTest/java/org/alohalytics/test/UtilTest.java
View file

@ -31,8 +31,13 @@ import java.io.FileNotFoundException;
public class UtilTest extends InstrumentationTestCase {
@Override
protected void setUp() {
new File(org.alohalytics.test.HttpTransportTest.CACHE_DIR).mkdirs();
}
private String getFullWritablePathForFile(String fileName) {
return getInstrumentation().getContext().getCacheDir().getAbsolutePath() + "/" + fileName;
return org.alohalytics.test.HttpTransportTest.CACHE_DIR + fileName;
}
public void testReadAndWriteStringToFile() throws Exception {

2
3party/Alohalytics/examples/server/gunzip.cc
View file

@ -53,7 +53,7 @@ int main(int argc, char** argv) {
std::cout << ptr->ToString() << std::endl;
}
} catch (const std::exception& ex) {
std::cerr << "Exception: " << ex.what() << std::endl;
std::cerr << "Exception: " << ex.what() << " in file " << argv[1] << std::endl;
return -1;
}
return 0;

30
3party/Alohalytics/src/android/java/org/alohalytics/HttpTransport.java
View file

@ -36,7 +36,6 @@ import java.io.IOException;
import java.io.OutputStream;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.zip.GZIPOutputStream;
public class HttpTransport {
@ -52,8 +51,8 @@ public class HttpTransport {
Log.d(TAG, "Connecting to " + p.url);
try {
connection = (HttpURLConnection) new URL(p.url).openConnection(); // NullPointerException, MalformedUrlException, IOException
// TODO(AlexZ): Customize redirects following in the future implementation for safer transfers.
connection.setInstanceFollowRedirects(true);
// We DO NOT follow any redirects for safer data transfer.
connection.setInstanceFollowRedirects(false);
connection.setConnectTimeout(TIMEOUT_IN_MILLISECONDS);
connection.setReadTimeout(TIMEOUT_IN_MILLISECONDS);
connection.setUseCaches(false);
@ -66,27 +65,20 @@ public class HttpTransport {
throw new NullPointerException("Please set Content-Type for POST requests.");
}
connection.setRequestProperty("Content-Type", p.contentType);
if (p.contentEncoding != null) {
connection.setRequestProperty("Content-Encoding", p.contentEncoding);
}
connection.setDoOutput(true);
if (p.data != null) {
// Use gzip compression for memory-only transfers.
// TODO(AlexZ): Move compression to the lower file-level (file storage queue) to save device space.
final ByteArrayOutputStream bos = new ByteArrayOutputStream();
final GZIPOutputStream zos = new GZIPOutputStream(bos);
try {
zos.write(p.data);
} finally {
zos.close();
}
connection.setFixedLengthStreamingMode(bos.size());
connection.setRequestProperty("Content-Encoding", "gzip");
connection.setFixedLengthStreamingMode(p.data.length);
final OutputStream os = connection.getOutputStream();
try {
os.write(bos.toByteArray());
os.write(p.data);
} finally {
os.close();
}
if (p.debugMode)
Log.d(TAG, "Sent POST with gzipped content of size " + bos.size());
Log.d(TAG, "Sent POST with content of size " + p.data.length);
} else {
final File file = new File(p.inputFilePath);
assert (file.length() == (int) file.length());
@ -110,6 +102,7 @@ public class HttpTransport {
Log.d(TAG, "Received HTTP " + p.httpResponseCode + " from server.");
p.receivedUrl = connection.getURL().toString();
p.contentType = connection.getContentType();
p.contentEncoding = connection.getContentEncoding();
// This implementation receives any data only if we have HTTP::OK (200).
if (p.httpResponseCode == HttpURLConnection.HTTP_OK) {
OutputStream ostream;
@ -158,8 +151,11 @@ public class HttpTransport {
// Can be different from url in case of redirects.
public String receivedUrl = null;
// SHOULD be specified for any POST request (any request where we send data to the server).
// On return, contains received Content-Type
// On return, contains received Content-Type or null.
public String contentType = null;
// Can be specified for any POST request (any request where we send data to the server).
// On return, contains received Content-Encoding or null.
public String contentEncoding = null;
// GET if null and inputFilePath is null.
// Sent in POST otherwise.
public byte[] data = null;

20
3party/Alohalytics/src/android/jni/jni_alohalytics.cc
View file

@ -267,6 +267,16 @@ bool HTTPClientPlatformWrapper::RunHTTPRequest() {
CLEAR_AND_RETURN_FALSE_ON_EXCEPTION
}
const static jfieldID contentEncodingField =
env->GetFieldID(g_httpParamsClass, "contentEncoding", "Ljava/lang/String;");
if (!content_encoding_.empty()) {
const auto jniContentEncoding = MakePointerScopeGuard(env->NewStringUTF(content_encoding_.c_str()), deleteLocalRef);
CLEAR_AND_RETURN_FALSE_ON_EXCEPTION
env->SetObjectField(httpParamsObject.get(), contentEncodingField, jniContentEncoding.get());
CLEAR_AND_RETURN_FALSE_ON_EXCEPTION
}
if (!user_agent_.empty()) {
const static jfieldID userAgentField =
env->GetFieldID(g_httpParamsClass, "userAgent", "Ljava/lang/String;");
@ -335,7 +345,15 @@ bool HTTPClientPlatformWrapper::RunHTTPRequest() {
static_cast<jstring>(env->GetObjectField(response, contentTypeField)), deleteLocalRef);
CLEAR_AND_RETURN_FALSE_ON_EXCEPTION
if (jniContentType) {
content_type_ = std::move(ToStdString(env, jniContentType.get()));
content_type_received_ = std::move(ToStdString(env, jniContentType.get()));
}
// contentEncodingField is already cached above.
const auto jniContentEncoding = MakePointerScopeGuard(
static_cast<jstring>(env->GetObjectField(response, contentEncodingField)), deleteLocalRef);
CLEAR_AND_RETURN_FALSE_ON_EXCEPTION
if (jniContentEncoding) {
content_encoding_received_ = std::move(ToStdString(env, jniContentEncoding.get()));
}
// dataField is already cached above.

16
3party/Alohalytics/src/apple/http_client_apple.mm
View file

@ -38,7 +38,6 @@ SOFTWARE.
#include "../http_client.h"
#include "../logger.h"
#include "../gzip_wrapper.h"
#define TIMEOUT_IN_SECONDS 30.0
@ -53,14 +52,13 @@ bool HTTPClientPlatformWrapper::RunHTTPRequest() {
if (!content_type_.empty())
[request setValue:[NSString stringWithUTF8String:content_type_.c_str()] forHTTPHeaderField:@"Content-Type"];
if (!content_encoding_.empty())
[request setValue:[NSString stringWithUTF8String:content_encoding_.c_str()] forHTTPHeaderField:@"Content-Encoding"];
if (!user_agent_.empty())
[request setValue:[NSString stringWithUTF8String:user_agent_.c_str()] forHTTPHeaderField:@"User-Agent"];
if (!post_body_.empty()) {
// TODO(AlexZ): Compress data in file queue impl, before calling this method, to use less disk space in offline.
const std::string compressed = Gzip(post_body_);
request.HTTPBody = [NSData dataWithBytes:compressed.data() length:compressed.size()];
[request setValue:@"gzip" forHTTPHeaderField:@"Content-Encoding"];
request.HTTPBody = [NSData dataWithBytes:post_body_.data() length:post_body_.size()];
request.HTTPMethod = @"POST";
} else if (!post_file_.empty()) {
NSError * err = nil;
@ -85,6 +83,14 @@ bool HTTPClientPlatformWrapper::RunHTTPRequest() {
if (response) {
error_code_ = static_cast<int>(response.statusCode);
url_received_ = [response.URL.absoluteString UTF8String];
NSString * content = [response.allHeaderFields objectForKey:@"Content-Type"];
if (content) {
content_type_received_ = [content UTF8String];
}
NSString * encoding = [response.allHeaderFields objectForKey:@"Content-Encoding"];
if (encoding) {
content_encoding_received_ = [encoding UTF8String];
}
if (url_data) {
if (received_file_.empty()) {
server_response_.assign(reinterpret_cast<char const *>(url_data.bytes), url_data.length);

9
3party/Alohalytics/src/cpp/alohalytics.cc
View file

@ -31,6 +31,7 @@
#include "../http_client.h"
#include "../logger.h"
#include "../event_base.h"
#include "../gzip_wrapper.h"
#include "../cereal/include/archives/binary.hpp"
#include "../cereal/include/types/string.hpp"
@ -55,13 +56,15 @@ Stats::Stats() : message_queue_(*this) {}
bool Stats::UploadBuffer(const std::string& url, std::string&& buffer, bool debug_mode) {
HTTPClientPlatformWrapper request(url);
request.set_debug_mode(debug_mode);
request.set_post_body(std::move(buffer), "application/alohalytics-binary-blob");
try {
// TODO(AlexZ): Refactor FileStorageQueue to automatically append ID and gzip files, so we don't need
// temporary memory buffer any more and files take less space.
request.set_post_body(alohalytics::Gzip(buffer), "application/alohalytics-binary-blob", "gzip");
return request.RunHTTPRequest() && 200 == request.error_code() && !request.was_redirected();
} catch (const std::exception& ex) {
if (debug_mode) {
ALOG("Exception while trying to upload file", ex.what());
ALOG("Exception while trying to UploadBuffer", ex.what());
}
}
return false;
@ -81,6 +84,8 @@ void Stats::OnMessage(const std::string& message, size_t dropped_events) {
static const size_t kMaxEventsInMemory = 2048;
if (container.size() > kMaxEventsInMemory) {
container.pop_front();
LOG_IF_DEBUG("Warning: maximum numbers of events in memory (", kMaxEventsInMemory,
") was reached and the oldest one was dropped.");
}
}
}

39
3party/Alohalytics/src/gzip_wrapper.h
View file

@ -21,6 +21,7 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*******************************************************************************/
#include <cstring> // std::memset
#include <string>
#include <vector>
#include <zlib.h>
@ -32,25 +33,19 @@ namespace alohalytics {
static constexpr size_t kGzipBufferSize = 32768;
struct GzipErrorException : public std::exception {
int err_;
std::string msg_;
GzipErrorException(int err, const char* msg) : err_(err), msg_(msg ? msg : "") {}
GzipErrorException(int err, const char* msg) {
msg_ = std::string("ERROR ") + std::to_string(err) + " while gzipping with zlib. " + (msg ? msg : "");
}
virtual char const* what() const noexcept {
return ("ERROR " + std::to_string(err_) + " while gzipping with zlib. " + msg_).c_str();
return msg_.c_str();
}
};
struct GunzipErrorException : public std::exception {
int err_;
std::string msg_;
GunzipErrorException(int err, const char* msg) : err_(err), msg_(msg ? msg : "") {}
virtual char const* what() const noexcept {
return ("ERROR " + std::to_string(err_) + " while gunzipping with zlib. " + msg_).c_str();
}
};
inline std::string Gzip(const std::string& data_to_compress) throw(GzipErrorException) {
z_stream z = {};
// Throws GzipErrorException on any gzip processing error.
inline std::string Gzip(const std::string& data_to_compress) {
z_stream z;
std::memset(&z, 0, sizeof(z));
int res = ::deflateInit2(&z, Z_BEST_COMPRESSION, Z_DEFLATED, 15 + 16, 8, Z_DEFAULT_STRATEGY);
if (Z_OK == res) {
z.next_in = const_cast<Bytef*>(reinterpret_cast<const Bytef*>(data_to_compress.data()));
@ -75,8 +70,20 @@ inline std::string Gzip(const std::string& data_to_compress) throw(GzipErrorExce
throw GzipErrorException(res, z.msg);
}
inline std::string Gunzip(const std::string& data_to_decompress) throw(GzipErrorException) {
z_stream z = {};
struct GunzipErrorException : public std::exception {
std::string msg_;
GunzipErrorException(int err, const char* msg) {
msg_ = std::string("ERROR ") + std::to_string(err) + " while gzipping with zlib. " + (msg ? msg : "");
}
virtual char const* what() const noexcept {
return msg_.c_str();
}
};
// Throws GunzipErrorException on any gunzip processing error.
inline std::string Gunzip(const std::string& data_to_decompress) {
z_stream z;
std::memset(&z, 0, sizeof(z));
int res = ::inflateInit2(&z, 16 + MAX_WBITS);
if (Z_OK == res) {
z.next_in = const_cast<Bytef*>(reinterpret_cast<const Bytef*>(data_to_decompress.data()));

13
3party/Alohalytics/src/http_client.h
View file

@ -47,6 +47,9 @@ class HTTPClientPlatformWrapper {
// Data we received from the server if output_file_ wasn't initialized.
std::string server_response_;
std::string content_type_;
std::string content_type_received_;
std::string content_encoding_;
std::string content_encoding_received_;
std::string user_agent_;
std::string post_body_;
bool debug_mode_ = false;
@ -84,19 +87,23 @@ class HTTPClientPlatformWrapper {
return *this;
}
// This method is mutually exclusive with set_post_file().
HTTPClientPlatformWrapper& set_post_body(const std::string& post_body, const std::string& content_type) {
HTTPClientPlatformWrapper& set_post_body(const std::string& post_body, const std::string& content_type,
const std::string& content_encoding = "") {
post_body_ = post_body;
content_type_ = content_type;
content_encoding_ = content_encoding;
// TODO (dkorolev) replace with exceptions as discussed offline.
assert(post_file_.empty());
return *this;
}
// Move version to avoid string copying.
// This method is mutually exclusive with set_post_file().
HTTPClientPlatformWrapper& set_post_body(std::string&& post_body, const std::string& content_type) {
post_body_ = post_body;
HTTPClientPlatformWrapper& set_post_body(std::string&& post_body, const std::string& content_type,
const std::string& content_encoding = "") {
post_body_ = std::move(post_body);
post_file_.clear();
content_type_ = content_type;
content_encoding_ = content_encoding;
return *this;
}

19
3party/Alohalytics/src/posix/http_client_curl.cc
View file

@ -54,11 +54,15 @@ struct ScopedTmpFileDeleter {
std::string RunCurl(const std::string& cmd) {
FILE* pipe = ::popen(cmd.c_str(), "r");
assert(pipe);
std::array<char, 8 * 1024> s;
std::array<char, 8 * 1024> arr;
std::string result;
while (nullptr != ::fgets(s.data(), s.size(), pipe)) {
result += s.data();
}
size_t read;
do {
read = ::fread(arr.data(), 1, arr.size(), pipe);
if (read > 0) {
result.append(arr.data(), read);
}
} while (read == arr.size());
const int err = ::pclose(pipe);
if (err) {
throw std::runtime_error("Error " + std::to_string(err) + " while calling " + cmd);
@ -66,12 +70,16 @@ std::string RunCurl(const std::string& cmd) {
return result;
}
// TODO(AlexZ): Add support for content_type_received_ and content_encoding_received_.
bool HTTPClientPlatformWrapper::RunHTTPRequest() {
// Last 3 chars in server's response will be http status code
static constexpr size_t kCurlHttpCodeSize = 3;
std::string cmd = "curl --max-redirs 0 -s -w '%{http_code}' ";
if (!content_type_.empty()) {
cmd += "-H 'Content-Type: application/json' ";
cmd += "-H 'Content-Type: " + content_type_ + "' ";
}
if (!content_encoding_.empty()) {
cmd += "-H 'Content-Encoding: " + content_encoding_ + "' ";
}
ScopedTmpFileDeleter deleter;
@ -102,6 +110,7 @@ bool HTTPClientPlatformWrapper::RunHTTPRequest() {
cmd += url_requested_;
try {
// TODO(AlexZ): Do not store data in memory if received_file_ was specified.
server_response_ = RunCurl(cmd);
error_code_ = -1;
std::string & s = server_response_;