From f2db795928d5181948f19995d9bc6ded72debb30 Mon Sep 17 00:00:00 2001
From: LaGrunge <l.fedorov@corp.mail.ru>
Date: Thu, 27 Jun 2019 20:53:11 +0300
Subject: [PATCH] [generator] make ids hex strings

---
 base/string_utils.cpp                 | 35 ++++++++++---------
 base/string_utils.hpp                 | 48 ++++++++++++++++++++-------
 generator/geo_objects/geo_objects.cpp |  4 +--
 generator/key_value_storage.cpp       | 28 ++++++++++++----
 generator/key_value_storage.hpp       |  2 ++
 generator/regions/regions.cpp         | 13 ++++----
 generator/streets/streets_builder.cpp |  4 +--
 7 files changed, 89 insertions(+), 45 deletions(-)

diff --git a/base/string_utils.cpp b/base/string_utils.cpp
index 63226660ab..0607b4a978 100644
--- a/base/string_utils.cpp
+++ b/base/string_utils.cpp
@@ -1,5 +1,5 @@
-#include "base/assert.hpp"
 #include "base/string_utils.hpp"
+#include "base/assert.hpp"
 
 #include "std/target_os.hpp"
 
@@ -34,10 +34,7 @@ SimpleDelimiter::SimpleDelimiter(char const * delims)
     m_delims.push_back(utf8::unchecked::next(it));
 }
 
-SimpleDelimiter::SimpleDelimiter(char delim)
-{
-  m_delims.push_back(delim);
-}
+SimpleDelimiter::SimpleDelimiter(char delim) { m_delims.push_back(delim); }
 
 bool SimpleDelimiter::operator()(UniChar c) const
 {
@@ -71,7 +68,7 @@ bool IntegerCheck(char const * start, char const * stop, T x, TResult & out)
 bool to_int(char const * start, int & i, int base /*= 10*/)
 {
   char * stop;
-  errno = 0; // Library functions do not reset it.
+  errno = 0;  // Library functions do not reset it.
   long const v = strtol(start, &stop, base);
   return IntegerCheck(start, stop, v, i);
 }
@@ -79,18 +76,18 @@ bool to_int(char const * start, int & i, int base /*= 10*/)
 bool to_uint(char const * start, unsigned int & i, int base /*= 10*/)
 {
   char * stop;
-  errno = 0; // Library functions do not reset it.
+  errno = 0;  // Library functions do not reset it.
   unsigned long const v = strtoul(start, &stop, base);
   return IntegerCheck(start, stop, v, i);
 }
 
-bool to_uint64(char const * s, uint64_t & i)
+bool to_uint64(char const * s, uint64_t & i, int base /*= 10*/)
 {
   char * stop;
 #ifdef OMIM_OS_WINDOWS_NATIVE
-  i = _strtoui64(s, &stop, 10);
+  i = _strtoui64(s, &stop, base);
 #else
-  i = strtoull(s, &stop, 10);
+  i = strtoull(s, &stop, base);
 #endif
   return *stop == 0 && s != stop;
 }
@@ -219,7 +216,7 @@ char ascii_to_lower(char in)
     return (in + diff);
   return in;
 }
-}
+}  // namespace
 
 void AsciiToLower(std::string & s) { transform(s.begin(), s.end(), s.begin(), &ascii_to_lower); }
 void Trim(std::string & s) { boost::trim(s); }
@@ -286,9 +283,15 @@ bool StartsWith(UniString const & s, UniString const & p)
   return StartsWith(s.begin(), s.end(), p.begin(), p.end());
 }
 
-bool StartsWith(std::string const & s1, char const * s2) { return (s1.compare(0, strlen(s2), s2) == 0); }
+bool StartsWith(std::string const & s1, char const * s2)
+{
+  return (s1.compare(0, strlen(s2), s2) == 0);
+}
 
-bool StartsWith(std::string const & s1, std::string const & s2) { return (s1.compare(0, s2.length(), s2) == 0); }
+bool StartsWith(std::string const & s1, std::string const & s2)
+{
+  return (s1.compare(0, s2.length(), s2) == 0);
+}
 
 bool EndsWith(UniString const & s1, UniString const & s2)
 {
@@ -366,13 +369,15 @@ bool IsHTML(std::string const & utf8)
 
 bool AlmostEqual(std::string const & str1, std::string const & str2, size_t mismatchedCount)
 {
-  std::pair<std::string::const_iterator, std::string::const_iterator> mis(str1.begin(), str2.begin());
+  std::pair<std::string::const_iterator, std::string::const_iterator> mis(str1.begin(),
+                                                                          str2.begin());
   auto const str1End = str1.end();
   auto const str2End = str2.end();
 
   for (size_t i = 0; i <= mismatchedCount; ++i)
   {
-    auto const end = mis.first + std::min(distance(mis.first, str1End), distance(mis.second, str2End));
+    auto const end =
+        mis.first + std::min(distance(mis.first, str1End), distance(mis.second, str2End));
     mis = mismatch(mis.first, end, mis.second);
     if (mis.first == str1End && mis.second == str2End)
       return true;
diff --git a/base/string_utils.hpp b/base/string_utils.hpp
index 47456ad975..1c3173b353 100644
--- a/base/string_utils.hpp
+++ b/base/string_utils.hpp
@@ -342,7 +342,7 @@ void Tokenize(std::string const & str, char const * delims, TFunctor && f)
   }
 }
 
-template <template <typename ...> class Collection = std::vector>
+template <template <typename...> class Collection = std::vector>
 Collection<std::string> Tokenize(std::string const & str, char const * delims)
 {
   Collection<std::string> c;
@@ -361,7 +361,7 @@ void ParseCSVRow(std::string const & s, char const delimiter, std::vector<std::s
 UniChar LastUniChar(std::string const & s);
 
 template <class T, size_t N, class TT>
-bool IsInArray(T(&arr)[N], TT const & t)
+bool IsInArray(T (&arr)[N], TT const & t)
 {
   for (size_t i = 0; i < N; ++i)
     if (arr[i] == t)
@@ -373,7 +373,7 @@ bool IsInArray(T(&arr)[N], TT const & t)
 //@{
 WARN_UNUSED_RESULT bool to_int(char const * s, int & i, int base = 10);
 WARN_UNUSED_RESULT bool to_uint(char const * s, unsigned int & i, int base = 10);
-WARN_UNUSED_RESULT bool to_uint64(char const * s, uint64_t & i);
+WARN_UNUSED_RESULT bool to_uint64(char const * s, uint64_t & i, int base = 10);
 WARN_UNUSED_RESULT bool to_int64(char const * s, int64_t & i);
 WARN_UNUSED_RESULT bool to_float(char const * s, float & f);
 WARN_UNUSED_RESULT bool to_double(char const * s, double & d);
@@ -384,16 +384,31 @@ WARN_UNUSED_RESULT inline bool is_number(std::string const & s)
   return to_int64(s.c_str(), dummy);
 }
 
-WARN_UNUSED_RESULT inline bool to_int(std::string const & s, int & i, int base = 10) { return to_int(s.c_str(), i, base); }
+WARN_UNUSED_RESULT inline bool to_int(std::string const & s, int & i, int base = 10)
+{
+  return to_int(s.c_str(), i, base);
+}
 WARN_UNUSED_RESULT inline bool to_uint(std::string const & s, unsigned int & i, int base = 10)
 {
   return to_uint(s.c_str(), i, base);
 }
 
-WARN_UNUSED_RESULT inline bool to_uint64(std::string const & s, uint64_t & i) { return to_uint64(s.c_str(), i); }
-WARN_UNUSED_RESULT inline bool to_int64(std::string const & s, int64_t & i) { return to_int64(s.c_str(), i); }
-WARN_UNUSED_RESULT inline bool to_float(std::string const & s, float & f) { return to_float(s.c_str(), f); }
-WARN_UNUSED_RESULT inline bool to_double(std::string const & s, double & d) { return to_double(s.c_str(), d); }
+WARN_UNUSED_RESULT inline bool to_uint64(std::string const & s, uint64_t & i, int base = 10)
+{
+  return to_uint64(s.c_str(), i, base);
+}
+WARN_UNUSED_RESULT inline bool to_int64(std::string const & s, int64_t & i)
+{
+  return to_int64(s.c_str(), i);
+}
+WARN_UNUSED_RESULT inline bool to_float(std::string const & s, float & f)
+{
+  return to_float(s.c_str(), f);
+}
+WARN_UNUSED_RESULT inline bool to_double(std::string const & s, double & d)
+{
+  return to_double(s.c_str(), d);
+}
 //@}
 
 /// @name From numeric to string.
@@ -409,8 +424,14 @@ std::string to_string(T t)
 }
 
 WARN_UNUSED_RESULT inline bool to_any(std::string const & s, int & i) { return to_int(s, i); }
-WARN_UNUSED_RESULT inline bool to_any(std::string const & s, unsigned int & i) { return to_uint(s, i); }
-WARN_UNUSED_RESULT inline bool to_any(std::string const & s, uint64_t & i) { return to_uint64(s, i); }
+WARN_UNUSED_RESULT inline bool to_any(std::string const & s, unsigned int & i)
+{
+  return to_uint(s, i);
+}
+WARN_UNUSED_RESULT inline bool to_any(std::string const & s, uint64_t & i)
+{
+  return to_uint64(s, i);
+}
 WARN_UNUSED_RESULT inline bool to_any(std::string const & s, int64_t & i) { return to_int64(s, i); }
 WARN_UNUSED_RESULT inline bool to_any(std::string const & s, float & f) { return to_float(s, f); }
 WARN_UNUSED_RESULT inline bool to_any(std::string const & s, double & d) { return to_double(s, d); }
@@ -476,7 +497,7 @@ std::string to_string_unsigned(T i)
   char * beg = to_string_digits(end, i);
   return std::string(beg, end - beg);
 }
-}
+}  // namespace impl
 
 inline std::string to_string(int32_t i) { return impl::to_string_signed(i); }
 inline std::string to_string(int64_t i) { return impl::to_string_signed(i); }
@@ -484,7 +505,10 @@ inline std::string to_string(uint64_t i) { return impl::to_string_unsigned(i); }
 /// Use this function to get string with fixed count of
 /// "Digits after comma".
 std::string to_string_dac(double d, int dac);
-inline std::string to_string_with_digits_after_comma(double d, int dac) { return to_string_dac(d, dac); }
+inline std::string to_string_with_digits_after_comma(double d, int dac)
+{
+  return to_string_dac(d, dac);
+}
 //@}
 
 template <typename IterT1, typename IterT2>
diff --git a/generator/geo_objects/geo_objects.cpp b/generator/geo_objects/geo_objects.cpp
index b9c18938bf..1292aed488 100644
--- a/generator/geo_objects/geo_objects.cpp
+++ b/generator/geo_objects/geo_objects.cpp
@@ -83,7 +83,7 @@ base::JSONPtr AddAddress(FeatureBuilder const & fb, KeyValue const & regionKeyVa
   else
     ToJSONObject(*address, "building", base::NewJSONNull());
 
-  ToJSONObject(*properties, "dref", regionKeyValue.first);
+  ToJSONObject(*properties, "dref", KeyValueStorage::Serialize(regionKeyValue.first));
   // auto locales = json_object_get(result.get(), "locales");
   // auto en = json_object_get(result.get(), "en");
   // todo(maksimandrianov): Add en locales.
@@ -214,7 +214,7 @@ void BuildGeoObjectsWithoutAddresses(KeyValueStorage & geoObjectsKv,
     if (!house)
       return;
 
-    auto const id = static_cast<int64_t>(fb.GetMostGenericOsmId().GetEncodedId());
+    auto const id = fb.GetMostGenericOsmId().GetEncodedId();
     auto jsonValue = MakeGeoObjectValueWithoutAddress(fb, *house);
 
     std::lock_guard<std::mutex> lock(updateMutex);
diff --git a/generator/key_value_storage.cpp b/generator/key_value_storage.cpp
index 273d8381d8..0aa8ee2865 100644
--- a/generator/key_value_storage.cpp
+++ b/generator/key_value_storage.cpp
@@ -1,10 +1,12 @@
 #include "generator/key_value_storage.hpp"
+#include <boost/algorithm/string.hpp>
+
+#include <algorithm>
+#include <cstring>
+#include <iomanip>
 #include "base/exception.hpp"
 #include "base/logging.hpp"
 #include "coding/reader.hpp"
-
-#include <cstring>
-
 namespace generator
 {
 KeyValueStorage::KeyValueStorage(std::string const & path, size_t cacheValuesCountLimit,
@@ -60,14 +62,16 @@ bool KeyValueStorage::ParseKeyValueLine(std::string const & line, std::streamoff
     return false;
   }
 
-  int64_t id;
-  if (!strings::to_int64(line.substr(0, pos), id))
+  std::string idStr = line.substr(0, pos);
+  boost::to_lower(idStr);
+  uint64_t id = 0;
+  if (!strings::to_uint64(idStr, id, 16))
   {
     LOG(LWARNING, ("Cannot parse id", line.substr(0, pos), "in line", lineNumber));
     return false;
   }
 
-  key = static_cast<uint64_t>(id);
+  key = id;
   value = line.c_str() + pos + 1;
   return true;
 }
@@ -85,7 +89,8 @@ void KeyValueStorage::Insert(uint64_t key, JsonValue && value)
 
   auto const & emplaceIterator = emplaceResult.first;
   auto const & result = boost::get<std::string>(emplaceIterator->second);
-  m_storage << static_cast<int64_t>(key) << " " << result << "\n";
+
+  m_storage << Serialize(key) << " " << result << "\n";
 }
 
 std::shared_ptr<JsonValue> KeyValueStorage::Find(uint64_t key) const
@@ -104,5 +109,14 @@ std::shared_ptr<JsonValue> KeyValueStorage::Find(uint64_t key) const
   return json;
 }
 
+std::string KeyValueStorage::Serialize(uint64_t number)
+{
+  std::stringstream stream;
+  stream << std::setw(16) << std::setfill('0') << std::hex << number;
+  std::string result = stream.str();
+  boost::to_upper(result);
+  return result;
+}
+
 size_t KeyValueStorage::Size() const { return m_values.size(); }
 }  // namespace generator
diff --git a/generator/key_value_storage.hpp b/generator/key_value_storage.hpp
index 427b278e1e..ec2c12cb98 100644
--- a/generator/key_value_storage.hpp
+++ b/generator/key_value_storage.hpp
@@ -62,6 +62,8 @@ public:
     return base::DumpToString(ptr, JSON_COMPACT | JSON_REAL_PRECISION(kDefaultPrecision));
   }
 
+  static std::string Serialize(uint64_t number);
+
 private:
   using Value = boost::variant<std::shared_ptr<JsonValue>, std::string>;
 
diff --git a/generator/regions/regions.cpp b/generator/regions/regions.cpp
index 18a008ba65..9677ca10cb 100644
--- a/generator/regions/regions.cpp
+++ b/generator/regions/regions.cpp
@@ -103,9 +103,8 @@ private:
     ToJSONObject(*geometry, "coordinates", coordinates);
 
     auto address = base::NewJSONObject();
-    boost::optional<int64_t> pid;
-
     Localizator localizator;
+    boost::optional<std::string> dref;
 
     for (auto const & p : path)
     {
@@ -127,8 +126,8 @@ private:
             region.GetTranslatedOrTransliteratedName(StringUtf8Multilang::GetLangIndex(language))};
       });
 
-      if (!pid && region.GetId() != main.GetId())
-        pid = static_cast<int64_t>(region.GetId().GetEncodedId());
+      if (!dref && region.GetId() != main.GetId())
+        dref = KeyValueStorage::Serialize(region.GetId().GetEncodedId());
     }
 
     auto properties = base::NewJSONObject();
@@ -136,8 +135,8 @@ private:
     ToJSONObject(*properties, "rank", main.GetRank());
     ToJSONObject(*properties, "address", address);
     ToJSONObject(*properties, "locales", localizator.BuildLocales());
-    if (pid)
-      ToJSONObject(*properties, "dref", *pid);
+    if (dref)
+      ToJSONObject(*properties, "dref", *dref);
     else
       ToJSONObject(*properties, "dref", base::NewJSONNull());
 
@@ -184,7 +183,7 @@ private:
 
         if (regionCountryEmplace.second)
         {
-          m_regionsKv << static_cast<int64_t>(objectId.GetEncodedId()) << " "
+          m_regionsKv << KeyValueStorage::Serialize(objectId.GetEncodedId()) << " "
                       << KeyValueStorage::Serialize(BuildRegionValue(path)) << "\n";
           ++countryObjectCount;
         }
diff --git a/generator/streets/streets_builder.cpp b/generator/streets/streets_builder.cpp
index cb34f78bd4..80c7416e86 100644
--- a/generator/streets/streets_builder.cpp
+++ b/generator/streets/streets_builder.cpp
@@ -58,7 +58,7 @@ void StreetsBuilder::SaveRegionStreetsKv(std::ostream & streamStreetsKv, uint64_
     auto const & bbox = street.second.GetBbox();
     auto const & pin = street.second.GetOrChoosePin();
 
-    auto const id = static_cast<int64_t>(pin.m_osmId.GetEncodedId());
+    auto const id = KeyValueStorage::Serialize(pin.m_osmId.GetEncodedId());
     auto const & value =
         MakeStreetValue(regionId, *regionObject, street.first, bbox, pin.m_position);
     streamStreetsKv << id << " " << KeyValueStorage::Serialize(value) << "\n";
@@ -175,7 +175,7 @@ base::JSONPtr StreetsBuilder::MakeStreetValue(uint64_t regionId, JsonValue const
   auto properties = base::NewJSONObject();
   ToJSONObject(*properties, "address", std::move(address));
   ToJSONObject(*properties, "name", streetName);
-  ToJSONObject(*properties, "dref", regionId);
+  ToJSONObject(*properties, "dref", KeyValueStorage::Serialize(regionId));
   ToJSONObject(*streetObject, "properties", std::move(properties));
 
   auto const & leftBottom = MercatorBounds::ToLatLon(bbox.LeftBottom());