From 45d6ad8e87f2d939672a65a68996b62fac313d3b Mon Sep 17 00:00:00 2001
From: Arsentiy Milchakov <milcars@mapswithme.com>
Date: Tue, 26 Feb 2019 19:08:09 +0300
Subject: [PATCH] [ugc][statistics] check for zombie reviews

---
 map/framework.cpp | 11 ++++++++++
 ugc/api.cpp       |  5 +++++
 ugc/api.hpp       |  2 ++
 ugc/storage.cpp   | 52 +++++++++++++++++++++++++++++++++++++++++++++++
 ugc/storage.hpp   |  1 +
 5 files changed, 71 insertions(+)

diff --git a/map/framework.cpp b/map/framework.cpp
index b424a022f2..5ae05a2de5 100644
--- a/map/framework.cpp
+++ b/map/framework.cpp
@@ -1440,6 +1440,17 @@ void Framework::InitUGC()
   ASSERT(!m_ugcApi.get(), ("InitUGC() must be called only once."));
 
   m_ugcApi = make_unique<ugc::Api>(m_model.GetDataSource(), [this](size_t numberOfUnsynchronized) {
+
+    bool ugcStorageValidationExecuted = false;
+    if (!settings::Get("WasUgcStorageValidationExecuted", ugcStorageValidationExecuted))
+      ugcStorageValidationExecuted = false;
+
+    if (!ugcStorageValidationExecuted)
+    {
+      m_ugcApi->ValidateStorage();
+      settings::Set("WasUgcStorageValidationExecuted", true);
+    }
+
     if (numberOfUnsynchronized == 0)
       return;
 
diff --git a/ugc/api.cpp b/ugc/api.cpp
index 172bda4bc2..55dff7e4e0 100644
--- a/ugc/api.cpp
+++ b/ugc/api.cpp
@@ -59,6 +59,11 @@ Loader & Api::GetLoader()
   return m_loader;
 }
 
+void Api::ValidateStorage()
+{
+  m_thread.Push([this] { m_storage.Validate(); });
+}
+
 void Api::GetUGCImpl(FeatureID const & id, UGCCallbackUnsafe const & callback)
 {
   CHECK(callback, ());
diff --git a/ugc/api.hpp b/ugc/api.hpp
index eebaa7cea4..45bbcd190e 100644
--- a/ugc/api.hpp
+++ b/ugc/api.hpp
@@ -44,6 +44,8 @@ public:
 
   Loader & GetLoader();
 
+  void ValidateStorage();
+
 private:
   void GetUGCImpl(FeatureID const & id, UGCCallbackUnsafe const & callback);
   Storage::SettingResult SetUGCUpdateImpl(FeatureID const & id, UGCUpdate const & ugc);
diff --git a/ugc/storage.cpp b/ugc/storage.cpp
index d60fa379bd..65c700fed7 100644
--- a/ugc/storage.cpp
+++ b/ugc/storage.cpp
@@ -19,7 +19,9 @@
 #include "coding/internal/file_data.hpp"
 #include "coding/point_coding.hpp"
 
+#include "base/logging.hpp"
 #include "base/stl_helpers.hpp"
+#include "base/string_utils.hpp"
 
 #include <algorithm>
 #include <map>
@@ -212,6 +214,51 @@ ugc::Storage::SettingResult SetGenericUGCUpdate(UGCUpdate const & ugc,
   return SaveIndexes(indexes) ? ugc::Storage::SettingResult::Success
                               : ugc::Storage::SettingResult::WritingError;
 }
+
+void FindZombieObjects(size_t indexesCount)
+{
+  auto const ugcFilePath = GetUGCFilePath();
+  vector<uint8_t> ugcFileContent;
+  try
+  {
+    FileReader r(ugcFilePath);
+    ugcFileContent = r.ReadAsBytes();
+  }
+  catch (FileReader::Exception const & exception)
+  {
+    ugcFileContent.clear();
+  }
+
+  uint32_t ugcCount = 0;
+
+  MemReaderWithExceptions r(ugcFileContent.data(), ugcFileContent.size());
+  NonOwningReaderSource source(r);
+  ugc::UGCUpdate unused;
+  try
+  {
+    while (source.Size() != 0)
+    {
+      ++ugcCount;
+      ugc::Deserialize(source, unused);
+    }
+  }
+  catch (RootException const & e)
+  {
+    auto const error = "Cannot deserialize ugc.update.bin file during zombie objects search";
+    LOG(LERROR, (error));
+    alohalytics::Stats::Instance().LogEvent("UGC_File_error", {{"error", error}});
+    return;
+  }
+
+  if (indexesCount == ugcCount)
+    return;
+
+  auto const zombieCount = ugcCount - indexesCount;
+  LOG(LERROR, ("Zombie objects are detected. ", zombieCount, "zombie objects are found."));
+
+  alohalytics::Stats::Instance().LogEvent(
+      "UGC_File_error", {{"error", "zombie: " + strings::to_string(zombieCount)}});
+}
 }  // namespace
 
 namespace ugc
@@ -503,6 +550,11 @@ bool Storage::HasUGCForPlace(uint32_t bestType, m2::PointD const & point) const
   return FindIndex(bestType, point) != m_indexes.cend();
 }
 
+void Storage::Validate() const
+{
+  FindZombieObjects(m_indexes.size());
+}
+
 void Storage::MarkAllAsSynchronized()
 {
   if (m_indexes.empty())
diff --git a/ugc/storage.hpp b/ugc/storage.hpp
index b9c367288f..0d89e46567 100644
--- a/ugc/storage.hpp
+++ b/ugc/storage.hpp
@@ -41,6 +41,7 @@ public:
   void Load();
   size_t GetNumberOfUnsynchronized() const;
   bool HasUGCForPlace(uint32_t bestType, m2::PointD const & point) const;
+  void Validate() const;
 
   /// Testing
   UpdateIndexes & GetIndexesForTesting() { return m_indexes; }