Using elias_fano for keeping city roads.

generator/city_roads_generator.cpp

@@ -19,8 +19,7 @@
 
 #include "defines.hpp"
 
-#include "3party/succinct/bit_vector.hpp"
-#include "3party/succinct/rs_bit_vector.hpp"
+#include "3party/succinct/elias_fano.hpp"
 
 using namespace generator;
 using namespace std;
@@ -80,26 +79,33 @@ void SerializeCityRoads(string const & dataPath, vector<uint64_t> && cityRoadFea
   auto const startOffset = w.Pos();
   header.Serialize(w);
 
-  size_t const maxFid = *max_element(cityRoadFeatureIds.cbegin(), cityRoadFeatureIds.cend());
-  succinct::bit_vector_builder builder(maxFid);
+  std::sort(cityRoadFeatureIds.begin(), cityRoadFeatureIds.end());
+  CHECK(std::adjacent_find(cityRoadFeatureIds.cbegin(), cityRoadFeatureIds.cend()) ==
+            cityRoadFeatureIds.cend(),
+        ("City road feaute ids should be unique."));
+  succinct::elias_fano::elias_fano_builder builder(cityRoadFeatureIds.back(),
+                                                   cityRoadFeatureIds.size());
   for (auto fid : cityRoadFeatureIds)
-    builder.set(fid, true /* road feature id */);
+    builder.push_back(fid);
 
   coding::FreezeVisitor<Writer> visitor(w);
-  succinct::rs_bit_vector(&builder).map(visitor);
+  succinct::elias_fano(&builder).map(visitor);
+
   auto const endOffset = w.Pos();
   header.m_dataSize = static_cast<uint32_t>(endOffset - startOffset - sizeof(CityRoadsHeader));
 
   w.Seek(startOffset);
   header.Serialize(w);
   w.Seek(endOffset);
+
+  LOG(LINFO, ("city_roads section is built in", dataPath, ". Serialized", cityRoadFeatureIds.size(),
+      "road feature ids in cities. Size:", endOffset - startOffset, "bytes."));
 }
 
 bool BuildCityRoads(string const & dataPath, OsmIdToBoundariesTable & table)
 {
   LOG(LDEBUG, ("BuildCityRoads(", dataPath, ");"));
   vector<uint64_t> cityRoadFeatureIds;
-  size_t cityRoadFeatureIdsSz = 0;
   try
   {
     // @TODO(bykoianko) The generation city roads section process is based on two stages now:
@@ -112,7 +118,6 @@ bool BuildCityRoads(string const & dataPath, OsmIdToBoundariesTable & table)
     // on the first step. And then to try using the real geometry should be used for generating city
     // road features. But there's a chance that it takes to long time.
     CalcRoadFeatureIds(dataPath, table, cityRoadFeatureIds);
-    cityRoadFeatureIdsSz = cityRoadFeatureIds.size();
     SerializeCityRoads(dataPath, move(cityRoadFeatureIds));
   }
   catch (Reader::Exception const & e)
@@ -120,8 +125,6 @@ bool BuildCityRoads(string const & dataPath, OsmIdToBoundariesTable & table)
     LOG(LERROR, ("Error while building section city_roads in", dataPath, ". Message:", e.Msg()));
     return false;
   }
-  LOG(LINFO, ("city_roads section is built in", dataPath, ". Serialized", cityRoadFeatureIdsSz,
-              "road feature ids in cities."));
   return true;
 }
 }  // namespace routing

generator/generator_tests/city_roads_tests.cpp

@@ -117,13 +117,19 @@ UNIT_TEST(CityRoadsGenerationTest_CommonCase)
   TestCityRoadsBuilding(vector<uint64_t>({100, 203, 204, 1008, 1009}));
 }
 
-UNIT_TEST(CityRoadsGenerationTest_BigNumbers)
+UNIT_TEST(CityRoadsGenerationTest_SortedIds1)
 {
   TestCityRoadsBuilding(
       vector<uint64_t>({1000,  1203,  11004,  11008, 11009, 11010, 11011, 11012, 11013, 11014, 11015,
                         11016, 11017, 11018, 11019, 11020, 11021, 11022, 11023, 11024, 11025}));
 }
 
+UNIT_TEST(CityRoadsGenerationTest_SortedIds2)
+{
+  TestCityRoadsBuilding(
+      vector<uint64_t>({75000,  75001,  75004,  250000, 250001, 330003, 330007}));
+}
+
 UNIT_TEST(CityRoadsGenerationTest_UnsortedIds)
 {
   TestCityRoadsBuilding(vector<uint64_t>({100, 1, 101, 2, 204, 1008, 1009}));
@@ -132,7 +138,31 @@ UNIT_TEST(CityRoadsGenerationTest_UnsortedIds)
 UNIT_TEST(CityRoadsGenerationTest_UnsortedIds2)
 {
   TestCityRoadsBuilding(
-      vector<uint64_t>({1000,  1203, 1, 11004,  11, 11009, 11010, 1011, 11012, 11013, 11, 4, 11015,
+      vector<uint64_t>({1000,  1203, 1, 11004,  11, 11009, 11010, 1011, 11012, 11013, 4, 11015,
                         11016, 11017, 11018, 11019, 11020, 11021, 11022, 11023, 11024, 11025, 2}));
 }
+
+UNIT_TEST(CityRoadsGenerationTest_UnsortedIds3)
+{
+  TestCityRoadsBuilding(vector<uint64_t>(
+      {181998, 354195, 470394, 356217, 331537, 272789, 449031, 420305, 139273, 482371, 85866,
+       142591, 105206, 217360, 380898, 390284, 96547, 110547, 201338, 428964, 246086, 29141,
+       179975, 493052, 53822, 238723, 316810, 349592, 154010, 107966, 113307, 97285, 145351,
+       1153, 433747, 3176, 294890, 52537, 412384, 67264, 102626, 129329, 49219, 289549,
+       68559, 364318, 211462, 170032, 59900, 257505, 164691, 75922, 209439, 308889, 143329,
+       140010, 17175, 385681, 147374, 362296, 483109, 257221, 78957, 246540, 111001, 150408,
+       399397, 285220, 260539, 201792, 378034, 349308, 68275, 278584, 14869, 71593, 34209,
+       146363, 177111, 319287, 25550, 39549, 130341, 225177, 175089, 458144, 108978, 289265,
+       482825, 167725, 113023, 11551, 315969, 402715, 408055, 392033, 440099, 295901, 228495,
+       297924, 89638, 214496, 207133, 362012, 397374, 424077, 343967, 84297, 230518, 159067,
+       6210, 331991, 354649, 52253, 326650, 370671, 4187, 103637, 305287, 434759, 311923,
+       180429, 442122, 157044, 145067, 415419, 237154, 404737, 269198, 308605, 57594, 310628,
+       418737, 359989, 36231, 7505, 226473, 436781, 173066, 97001, 59617, 171771, 335309,
+       477484, 183747, 64957, 155749, 383375, 333286, 116341, 134386, 447463, 141022, 193133,
+       271221, 169748, 474166, 60912, 66253, 50231, 98297, 321309, 386693, 456121, 247835,
+       372693, 365330, 20209, 55571, 82275, 2165, 242495, 388715, 317264, 164407, 490188,
+       12846, 210451, 484847, 28868, 162385, 129045, 463485, 92956, 337331, 338627, 100319,
+       182452, 303265, 73616, 262562, 62935, 294606, 466803, 215791, 468825, 76934, 18187,
+       194429, 32913}));
+}
 }  // namespace

routing/city_roads_loader.hpp

@@ -9,7 +9,7 @@
 #include <memory>
 #include <vector>
 
-#include "3party/succinct/rs_bit_vector.hpp"
+#include "3party/succinct/elias_fano.hpp"
 
 class DataSource;
 
@@ -51,6 +51,6 @@ public:
 
 private:
   std::unique_ptr<CopiedMemoryRegion> m_cityRoadsRegion;
-  succinct::rs_bit_vector m_cityRoads;
+  succinct::elias_fano m_cityRoads;
 };
 }  // namespace routing