diff --git a/3party/jansson/myjansson.cpp b/3party/jansson/myjansson.cpp index c97401e2b0..3e1b0d3f75 100644 --- a/3party/jansson/myjansson.cpp +++ b/3party/jansson/myjansson.cpp @@ -98,6 +98,11 @@ void ToJSONObject(json_t & root, string const & field, int value) json_object_set_new(&root, field.c_str(), json_integer(value)); } +void ToJSONObject(json_t & root, std::string const & field, json_int_t value) +{ + json_object_set_new(&root, field.c_str(), json_integer(value)); +} + void FromJSON(json_t * root, string & result) { if (!json_is_string(root)) diff --git a/3party/jansson/myjansson.hpp b/3party/jansson/myjansson.hpp index ae4dff67bd..3945675e8a 100644 --- a/3party/jansson/myjansson.hpp +++ b/3party/jansson/myjansson.hpp @@ -59,6 +59,7 @@ void FromJSONObjectOptionalField(json_t * root, std::string const & field, json_ void ToJSONObject(json_t & root, std::string const & field, double value); void ToJSONObject(json_t & root, std::string const & field, int value); +void ToJSONObject(json_t & root, std::string const & field, json_int_t value); void FromJSON(json_t * root, std::string & result); inline my::JSONPtr ToJSON(std::string const & s) { return my::NewJSONString(s); } @@ -106,6 +107,11 @@ void FromJSONObjectOptionalField(json_t * root, std::string const & field, std:: FromJSON(json_array_get(arr, i), result[i]); } +struct JSONFreeDeleter +{ + void operator()(char * buffer) const { free(buffer); } +}; + namespace strings { void FromJSONObject(json_t * root, std::string const & field, UniString & result); diff --git a/configure.sh b/configure.sh index be30855db0..93a7bd0a91 100755 --- a/configure.sh +++ b/configure.sh @@ -59,6 +59,8 @@ else #define TRACKING_HISTORICAL_HOST "" #define TRACKING_HISTORICAL_PORT 0 #define TRAFFIC_DATA_BASE_URL "" +#define LOCAL_ADS_SERVER_URL "" +#define LOCAL_ADS_STATISTICS_SERVER_URL "" ' > "$PRIVATE_HEADER" echo 'ext { spropStoreFile = "../tools/android/debug.keystore" diff --git a/local_ads/CMakeLists.txt b/local_ads/CMakeLists.txt index 4b54b099a1..ab9ee2eacf 100644 --- a/local_ads/CMakeLists.txt +++ b/local_ads/CMakeLists.txt @@ -5,6 +5,7 @@ set( campaign.hpp campaign_serialization.cpp campaign_serialization.hpp + config.hpp event.cpp event.hpp file_helpers.cpp diff --git a/local_ads/config.hpp b/local_ads/config.hpp new file mode 100644 index 0000000000..89f86284c9 --- /dev/null +++ b/local_ads/config.hpp @@ -0,0 +1,4 @@ +#pragma once + +// Use local ads servers for development. +#define DEV_LOCAL_ADS_SERVER diff --git a/local_ads/local_ads.pro b/local_ads/local_ads.pro index bb685f2cba..b8ccaa7c60 100644 --- a/local_ads/local_ads.pro +++ b/local_ads/local_ads.pro @@ -17,6 +17,7 @@ SOURCES += \ HEADERS += \ campaign.hpp \ campaign_serialization.hpp \ + config.hpp \ event.hpp \ file_helpers.hpp \ icons_info.hpp \ diff --git a/local_ads/statistics.cpp b/local_ads/statistics.cpp index d1514eacf1..95ffc1e8f6 100644 --- a/local_ads/statistics.cpp +++ b/local_ads/statistics.cpp @@ -1,4 +1,5 @@ #include "local_ads/statistics.hpp" +#include "local_ads/config.hpp" #include "local_ads/file_helpers.hpp" #include "platform/http_client.hpp" @@ -19,6 +20,8 @@ #include #include +#include "private.h" + namespace { std::string const kStatisticsFolderName = "local_ads_stats"; @@ -31,8 +34,7 @@ auto constexpr kSendingTimeout = std::chrono::hours(1); int64_t constexpr kEventMaxLifetimeInSeconds = 24 * 183 * 3600; // About half of year. auto constexpr kDeletionPeriod = std::chrono::hours(24); -// TODO: set correct address -std::string const kStatisticsServer = ""; +std::string const kStatisticsServer = LOCAL_ADS_STATISTICS_SERVER_URL; void WriteMetadata(FileWriter & writer, std::string const & countryId, int64_t mwmVersion, local_ads::Timestamp const & ts) @@ -198,7 +200,8 @@ bool Statistics::RequestEvents(std::list & events, bool & needToSend) return false; using namespace std::chrono; - needToSend = isTimeout || (steady_clock::now() > (m_lastSending + kSendingTimeout)); + needToSend = m_isFirstSending || isTimeout || + (steady_clock::now() > (m_lastSending + kSendingTimeout)); events = std::move(m_events); m_events.clear(); @@ -381,21 +384,41 @@ void Statistics::ProcessEvents(std::list & events) void Statistics::SendToServer() { + std::string userId; + ServerSerializer serializer; + { + std::lock_guard lock(m_mutex); + userId = m_userId; + serializer = m_serverSerializer; + } + for (auto it = m_metadataCache.begin(); it != m_metadataCache.end();) { std::string const url = MakeRemoteURL(m_userId, it->first.first, it->first.second); if (url.empty()) return; - std::vector bytes = SerializeForServer(ReadEvents(it->second.m_fileName)); - if (bytes.empty()) + std::list events = ReadEvents(it->second.m_fileName); + if (events.empty()) { ++it; continue; } + std::string contentType = "application/octet-stream"; + std::string contentEncoding = ""; + std::vector bytes = serializer != nullptr + ? serializer(events, userId, contentType, contentEncoding) + : SerializeForServer(events); + ASSERT(!bytes.empty(), ()); + platform::HttpClient request(url); - request.SetBodyData(std::string(bytes.begin(), bytes.end()), "application/octet-stream"); +#ifdef DEV_LOCAL_ADS_SERVER + request.LoadHeaders(true); + request.SetRawHeader("Host", "localads-statistics.maps.me"); +#endif + request.SetBodyData(std::string(bytes.begin(), bytes.end()), contentType, "POST", + contentEncoding); if (request.RunHttpRequest() && request.ErrorCode() == 200) { FileWriter::DeleteFileX(it->second.m_fileName); @@ -403,16 +426,18 @@ void Statistics::SendToServer() } else { + LOG(LWARNING, + ("Sending statistics failed:", request.ErrorCode(), it->first.first, it->first.second)); ++it; } } m_lastSending = std::chrono::steady_clock::now(); + m_isFirstSending = false; } std::vector Statistics::SerializeForServer(std::list const & events) const { - if (events.empty()) - return {}; + ASSERT(!events.empty(), ()); // TODO: implement serialization return std::vector{1, 2, 3, 4, 5}; @@ -529,4 +554,10 @@ void Statistics::CleanupAfterTesting() if (GetPlatform().IsFileExistsByFullPath(statsFolder)) GetPlatform().RmDirRecursively(statsFolder); } + +void Statistics::SetCustomServerSerializer(ServerSerializer && serializer) +{ + std::lock_guard lock(m_mutex); + m_serverSerializer = std::move(serializer); +} } // namespace local_ads diff --git a/local_ads/statistics.hpp b/local_ads/statistics.hpp index 06f6100132..c9b41a8c42 100644 --- a/local_ads/statistics.hpp +++ b/local_ads/statistics.hpp @@ -6,6 +6,7 @@ #include #include +#include #include #include #include @@ -14,6 +15,10 @@ namespace local_ads { +using ServerSerializer = + std::function(std::list const & events, std::string const & userId, + std::string & contentType, std::string & contentEncoding)>; + class Statistics final { public: @@ -35,6 +40,8 @@ public: void SetUserId(std::string const & userId); + void SetCustomServerSerializer(ServerSerializer && serializer); + void RegisterEvent(Event && event); void RegisterEvents(std::list && events); @@ -73,8 +80,10 @@ private: }; std::map m_metadataCache; Timestamp m_lastSending; + bool m_isFirstSending = true; std::string m_userId; + ServerSerializer m_serverSerializer; bool m_isRunning = false; std::list m_events; diff --git a/map/framework.cpp b/map/framework.cpp index e343564caa..4abd9a1a4d 100644 --- a/map/framework.cpp +++ b/map/framework.cpp @@ -416,10 +416,7 @@ Framework::Framework(FrameworkParams const & params) , m_lastReportedCountry(kInvalidCountryId) { if (!params.m_disableLocalAds) - { m_localAdsManager.Startup(); - m_localAdsManager.GetStatistics().SetUserId(GetPlatform().UniqueClientId()); - } m_startBackgroundTime = my::Timer::LocalTime(); diff --git a/map/local_ads_manager.cpp b/map/local_ads_manager.cpp index c00c1eaed0..d52054296b 100644 --- a/map/local_ads_manager.cpp +++ b/map/local_ads_manager.cpp @@ -1,6 +1,7 @@ #include "map/local_ads_manager.hpp" #include "local_ads/campaign_serialization.hpp" +#include "local_ads/config.hpp" #include "local_ads/file_helpers.hpp" #include "local_ads/icons_info.hpp" @@ -13,10 +14,23 @@ #include "platform/platform.hpp" #include "coding/file_name_utils.hpp" +#include "coding/url_encode.hpp" +#include "coding/zlib.hpp" + +#include "3party/jansson/myjansson.hpp" + +#include "private.h" + +#include +#include + +// First implementation of local ads statistics serialization +// is deflated JSON. +#define TEMPORARY_LOCAL_ADS_JSON_SERIALIZATION namespace { -std::string const kServerUrl = "";//"http://172.27.15.68"; +std::string const kServerUrl = LOCAL_ADS_SERVER_URL; std::string const kCampaignFile = "local_ads_campaigns.dat"; std::string const kLocalAdsSymbolsFile = "local_ads_symbols.txt"; @@ -48,11 +62,17 @@ std::chrono::steady_clock::time_point Now() { return std::chrono::steady_clock::now(); } - + std::string MakeRemoteURL(MwmSet::MwmId const & mwmId) { - // TODO: build correct URL after server completion. - return {}; // kServerUrl + "/campaigns.data"; + if (kServerUrl.empty() || !mwmId.IsAlive()) + return {}; + + std::ostringstream ss; + ss << kServerUrl << "/"; + ss << mwmId.GetInfo()->GetVersion() << "/"; + ss << UrlEncode(mwmId.GetInfo()->GetCountryName()) << ".ads"; + return ss.str(); } std::vector DownloadCampaign(MwmSet::MwmId const & mwmId) @@ -86,6 +106,41 @@ df::CustomSymbols ParseCampaign(std::vector const & rawData, MwmSet::Mw return symbols; } + +#ifdef TEMPORARY_LOCAL_ADS_JSON_SERIALIZATION +std::vector SerializeLocalAdsToJSON(std::list const & events, + std::string const & userId, std::string & contentType, + std::string & contentEncoding) +{ + using namespace std::chrono; + ASSERT(!events.empty(), ()); + auto root = my::NewJSONObject(); + ToJSONObject(*root, "userId", userId); + ToJSONObject(*root, "countryId", events.front().m_countryId); + ToJSONObject(*root, "mwmVersion", events.front().m_mwmVersion); + auto eventsNode = my::NewJSONArray(); + for (auto const & event : events) + { + auto eventNode = my::NewJSONObject(); + auto s = duration_cast(event.m_timestamp.time_since_epoch()).count(); + ToJSONObject(*eventNode, "type", static_cast(event.m_type)); + ToJSONObject(*eventNode, "timestamp", static_cast(s)); + ToJSONObject(*eventNode, "featureId", static_cast(event.m_featureId)); + ToJSONObject(*eventNode, "zoomLevel", event.m_zoomLevel); + ToJSONObject(*eventNode, "latitude", event.m_latitude); + ToJSONObject(*eventNode, "longitude", event.m_longitude); + ToJSONObject(*eventNode, "accuracyInMeters", event.m_accuracyInMeters); + json_array_append_new(eventsNode.get(), eventNode.release()); + } + json_object_set_new(root.get(), "events", eventsNode.release()); + std::unique_ptr buffer( + json_dumps(root.get(), JSON_COMPACT | JSON_ENSURE_ASCII)); + std::vector result; + coding::ZLib::Deflate(buffer.get(), strlen(buffer.get()), coding::ZLib::Level::BestCompression, + std::back_inserter(result)); + return result; +} +#endif } // namespace LocalAdsManager::LocalAdsManager(GetMwmsByRectFn const & getMwmsByRectFn, @@ -95,6 +150,13 @@ LocalAdsManager::LocalAdsManager(GetMwmsByRectFn const & getMwmsByRectFn, { CHECK(m_getMwmsByRectFn != nullptr, ()); CHECK(m_getMwmIdByNameFn != nullptr, ()); + + m_statistics.SetUserId(GetPlatform().UniqueClientId()); + +#ifdef TEMPORARY_LOCAL_ADS_JSON_SERIALIZATION + using namespace std::placeholders; + m_statistics.SetCustomServerSerializer(std::bind(&SerializeLocalAdsToJSON, _1, _2, _3, _4)); +#endif } LocalAdsManager::~LocalAdsManager() diff --git a/platform/http_client.hpp b/platform/http_client.hpp index bf7b12cd95..c4bb22d4cd 100644 --- a/platform/http_client.hpp +++ b/platform/http_client.hpp @@ -59,13 +59,14 @@ public: template HttpClient & SetBodyData(StringT && body_data, string const & content_type, string const & http_method = "POST", - string const & content_encoding = "") + string const & content_encoding = {}) { m_bodyData = forward(body_data); m_inputFile.clear(); m_headers.emplace("Content-Type", content_type); m_httpMethod = http_method; - m_headers.emplace("Content-Encoding", content_encoding); + if (!content_encoding.empty()) + m_headers.emplace("Content-Encoding", content_encoding); return *this; } // HTTP Basic Auth. diff --git a/search/CMakeLists.txt b/search/CMakeLists.txt index 5c3cac49a5..e4d5108b4c 100644 --- a/search/CMakeLists.txt +++ b/search/CMakeLists.txt @@ -23,6 +23,8 @@ set( emitter.hpp engine.cpp engine.hpp + everywhere_search_callback.cpp + everywhere_search_callback.hpp everywhere_search_params.hpp feature_loader.cpp feature_loader.hpp diff --git a/search/search_quality/sample.cpp b/search/search_quality/sample.cpp index b923ad1191..2d0c532fe8 100644 --- a/search/search_quality/sample.cpp +++ b/search/search_quality/sample.cpp @@ -46,11 +46,6 @@ bool Equal(std::vector lhs, std::vector rhs) sort(rhs.begin(), rhs.end()); return lhs == rhs; } - -struct FreeDeletor -{ - void operator()(char * buffer) const { free(buffer); } -}; } // namespace namespace search @@ -159,7 +154,7 @@ void Sample::SerializeToJSON(std::vector const & samples, std::string & auto array = my::NewJSONArray(); for (auto const & sample : samples) json_array_append_new(array.get(), sample.SerializeToJSON().release()); - std::unique_ptr buffer( + std::unique_ptr buffer( json_dumps(array.get(), JSON_COMPACT | JSON_ENSURE_ASCII)); jsonStr.assign(buffer.get()); } diff --git a/xcode/local_ads/local_ads.xcodeproj/project.pbxproj b/xcode/local_ads/local_ads.xcodeproj/project.pbxproj index dce6a2ef9e..6ecd012a94 100644 --- a/xcode/local_ads/local_ads.xcodeproj/project.pbxproj +++ b/xcode/local_ads/local_ads.xcodeproj/project.pbxproj @@ -17,6 +17,7 @@ 455C5DB01E97EBC300DBFE48 /* file_helpers.hpp in Headers */ = {isa = PBXBuildFile; fileRef = 455C5DAA1E97EBC300DBFE48 /* file_helpers.hpp */; }; 455C5DB11E97EBC300DBFE48 /* statistics.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 455C5DAB1E97EBC300DBFE48 /* statistics.cpp */; }; 455C5DB21E97EBC300DBFE48 /* statistics.hpp in Headers */ = {isa = PBXBuildFile; fileRef = 455C5DAC1E97EBC300DBFE48 /* statistics.hpp */; }; + 456FAB731EA5E9F0000E4CDC /* config.hpp in Headers */ = {isa = PBXBuildFile; fileRef = 456FAB721EA5E9F0000E4CDC /* config.hpp */; }; 4580DAC61E9D2C3D00E8BCDE /* libgeometry.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 4580DAC51E9D2C3D00E8BCDE /* libgeometry.a */; }; 45812AB51E9781D500D7D3B3 /* libplatform_tests_support.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 45812AB41E9781D500D7D3B3 /* libplatform_tests_support.a */; }; 45FFD6571E965E0600DB854E /* campaign_serialization.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 45FFD6541E965E0600DB854E /* campaign_serialization.cpp */; }; @@ -43,6 +44,7 @@ 455C5DAA1E97EBC300DBFE48 /* file_helpers.hpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; path = file_helpers.hpp; sourceTree = ""; }; 455C5DAB1E97EBC300DBFE48 /* statistics.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = statistics.cpp; sourceTree = ""; }; 455C5DAC1E97EBC300DBFE48 /* statistics.hpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; path = statistics.hpp; sourceTree = ""; }; + 456FAB721EA5E9F0000E4CDC /* config.hpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; path = config.hpp; sourceTree = ""; }; 4580DAC51E9D2C3D00E8BCDE /* libgeometry.a */ = {isa = PBXFileReference; lastKnownFileType = archive.ar; name = libgeometry.a; path = "../../../omim-build/xcode/Debug-iphonesimulator/libgeometry.a"; sourceTree = ""; }; 45812AB41E9781D500D7D3B3 /* libplatform_tests_support.a */ = {isa = PBXFileReference; lastKnownFileType = archive.ar; name = libplatform_tests_support.a; path = "../../../omim-build/xcode/Debug-iphonesimulator/libplatform_tests_support.a"; sourceTree = ""; }; 45FFD6461E965DBB00DB854E /* liblocal_ads.a */ = {isa = PBXFileReference; explicitFileType = archive.ar; includeInIndex = 0; path = liblocal_ads.a; sourceTree = BUILT_PRODUCTS_DIR; }; @@ -111,6 +113,7 @@ 45FFD6481E965DBB00DB854E /* local_ads */ = { isa = PBXGroup; children = ( + 456FAB721EA5E9F0000E4CDC /* config.hpp */, 45FFD6541E965E0600DB854E /* campaign_serialization.cpp */, 45FFD6551E965E0600DB854E /* campaign_serialization.hpp */, 45FFD6561E965E0600DB854E /* campaign.hpp */, @@ -166,6 +169,7 @@ 455C5DAE1E97EBC300DBFE48 /* event.hpp in Headers */, 45FFD6581E965E0600DB854E /* campaign_serialization.hpp in Headers */, 455C5DB21E97EBC300DBFE48 /* statistics.hpp in Headers */, + 456FAB731EA5E9F0000E4CDC /* config.hpp in Headers */, ); runOnlyForDeploymentPostprocessing = 0; };