From 99e7ec203c22b5acdf4f6be4ea4a23687f64e320 Mon Sep 17 00:00:00 2001 From: Vladimir Byko-Ianko Date: Tue, 11 Sep 2018 10:39:17 +0300 Subject: [PATCH] Using elias_fano for keeping city roads. --- generator/city_roads_generator.cpp | 23 +++++++------ .../generator_tests/city_roads_tests.cpp | 34 +++++++++++++++++-- routing/city_roads_loader.hpp | 4 +-- 3 files changed, 47 insertions(+), 14 deletions(-) diff --git a/generator/city_roads_generator.cpp b/generator/city_roads_generator.cpp index 122113b888..3fb2cdcb53 100644 --- a/generator/city_roads_generator.cpp +++ b/generator/city_roads_generator.cpp @@ -19,8 +19,7 @@ #include "defines.hpp" -#include "3party/succinct/bit_vector.hpp" -#include "3party/succinct/rs_bit_vector.hpp" +#include "3party/succinct/elias_fano.hpp" using namespace generator; using namespace std; @@ -80,26 +79,33 @@ void SerializeCityRoads(string const & dataPath, vector && cityRoadFea auto const startOffset = w.Pos(); header.Serialize(w); - size_t const maxFid = *max_element(cityRoadFeatureIds.cbegin(), cityRoadFeatureIds.cend()); - succinct::bit_vector_builder builder(maxFid); + std::sort(cityRoadFeatureIds.begin(), cityRoadFeatureIds.end()); + CHECK(std::adjacent_find(cityRoadFeatureIds.cbegin(), cityRoadFeatureIds.cend()) == + cityRoadFeatureIds.cend(), + ("City road feaute ids should be unique.")); + succinct::elias_fano::elias_fano_builder builder(cityRoadFeatureIds.back(), + cityRoadFeatureIds.size()); for (auto fid : cityRoadFeatureIds) - builder.set(fid, true /* road feature id */); + builder.push_back(fid); coding::FreezeVisitor visitor(w); - succinct::rs_bit_vector(&builder).map(visitor); + succinct::elias_fano(&builder).map(visitor); + auto const endOffset = w.Pos(); header.m_dataSize = static_cast(endOffset - startOffset - sizeof(CityRoadsHeader)); w.Seek(startOffset); header.Serialize(w); w.Seek(endOffset); + + LOG(LINFO, ("city_roads section is built in", dataPath, ". Serialized", cityRoadFeatureIds.size(), + "road feature ids in cities. Size:", endOffset - startOffset, "bytes.")); } bool BuildCityRoads(string const & dataPath, OsmIdToBoundariesTable & table) { LOG(LDEBUG, ("BuildCityRoads(", dataPath, ");")); vector cityRoadFeatureIds; - size_t cityRoadFeatureIdsSz = 0; try { // @TODO(bykoianko) The generation city roads section process is based on two stages now: @@ -112,7 +118,6 @@ bool BuildCityRoads(string const & dataPath, OsmIdToBoundariesTable & table) // on the first step. And then to try using the real geometry should be used for generating city // road features. But there's a chance that it takes to long time. CalcRoadFeatureIds(dataPath, table, cityRoadFeatureIds); - cityRoadFeatureIdsSz = cityRoadFeatureIds.size(); SerializeCityRoads(dataPath, move(cityRoadFeatureIds)); } catch (Reader::Exception const & e) @@ -120,8 +125,6 @@ bool BuildCityRoads(string const & dataPath, OsmIdToBoundariesTable & table) LOG(LERROR, ("Error while building section city_roads in", dataPath, ". Message:", e.Msg())); return false; } - LOG(LINFO, ("city_roads section is built in", dataPath, ". Serialized", cityRoadFeatureIdsSz, - "road feature ids in cities.")); return true; } } // namespace routing diff --git a/generator/generator_tests/city_roads_tests.cpp b/generator/generator_tests/city_roads_tests.cpp index 634dc64410..4f01e951c6 100644 --- a/generator/generator_tests/city_roads_tests.cpp +++ b/generator/generator_tests/city_roads_tests.cpp @@ -117,13 +117,19 @@ UNIT_TEST(CityRoadsGenerationTest_CommonCase) TestCityRoadsBuilding(vector({100, 203, 204, 1008, 1009})); } -UNIT_TEST(CityRoadsGenerationTest_BigNumbers) +UNIT_TEST(CityRoadsGenerationTest_SortedIds1) { TestCityRoadsBuilding( vector({1000, 1203, 11004, 11008, 11009, 11010, 11011, 11012, 11013, 11014, 11015, 11016, 11017, 11018, 11019, 11020, 11021, 11022, 11023, 11024, 11025})); } +UNIT_TEST(CityRoadsGenerationTest_SortedIds2) +{ + TestCityRoadsBuilding( + vector({75000, 75001, 75004, 250000, 250001, 330003, 330007})); +} + UNIT_TEST(CityRoadsGenerationTest_UnsortedIds) { TestCityRoadsBuilding(vector({100, 1, 101, 2, 204, 1008, 1009})); @@ -132,7 +138,31 @@ UNIT_TEST(CityRoadsGenerationTest_UnsortedIds) UNIT_TEST(CityRoadsGenerationTest_UnsortedIds2) { TestCityRoadsBuilding( - vector({1000, 1203, 1, 11004, 11, 11009, 11010, 1011, 11012, 11013, 11, 4, 11015, + vector({1000, 1203, 1, 11004, 11, 11009, 11010, 1011, 11012, 11013, 4, 11015, 11016, 11017, 11018, 11019, 11020, 11021, 11022, 11023, 11024, 11025, 2})); } + +UNIT_TEST(CityRoadsGenerationTest_UnsortedIds3) +{ + TestCityRoadsBuilding(vector( + {181998, 354195, 470394, 356217, 331537, 272789, 449031, 420305, 139273, 482371, 85866, + 142591, 105206, 217360, 380898, 390284, 96547, 110547, 201338, 428964, 246086, 29141, + 179975, 493052, 53822, 238723, 316810, 349592, 154010, 107966, 113307, 97285, 145351, + 1153, 433747, 3176, 294890, 52537, 412384, 67264, 102626, 129329, 49219, 289549, + 68559, 364318, 211462, 170032, 59900, 257505, 164691, 75922, 209439, 308889, 143329, + 140010, 17175, 385681, 147374, 362296, 483109, 257221, 78957, 246540, 111001, 150408, + 399397, 285220, 260539, 201792, 378034, 349308, 68275, 278584, 14869, 71593, 34209, + 146363, 177111, 319287, 25550, 39549, 130341, 225177, 175089, 458144, 108978, 289265, + 482825, 167725, 113023, 11551, 315969, 402715, 408055, 392033, 440099, 295901, 228495, + 297924, 89638, 214496, 207133, 362012, 397374, 424077, 343967, 84297, 230518, 159067, + 6210, 331991, 354649, 52253, 326650, 370671, 4187, 103637, 305287, 434759, 311923, + 180429, 442122, 157044, 145067, 415419, 237154, 404737, 269198, 308605, 57594, 310628, + 418737, 359989, 36231, 7505, 226473, 436781, 173066, 97001, 59617, 171771, 335309, + 477484, 183747, 64957, 155749, 383375, 333286, 116341, 134386, 447463, 141022, 193133, + 271221, 169748, 474166, 60912, 66253, 50231, 98297, 321309, 386693, 456121, 247835, + 372693, 365330, 20209, 55571, 82275, 2165, 242495, 388715, 317264, 164407, 490188, + 12846, 210451, 484847, 28868, 162385, 129045, 463485, 92956, 337331, 338627, 100319, + 182452, 303265, 73616, 262562, 62935, 294606, 466803, 215791, 468825, 76934, 18187, + 194429, 32913})); +} } // namespace diff --git a/routing/city_roads_loader.hpp b/routing/city_roads_loader.hpp index b3907d34f5..3ff959ec04 100644 --- a/routing/city_roads_loader.hpp +++ b/routing/city_roads_loader.hpp @@ -9,7 +9,7 @@ #include #include -#include "3party/succinct/rs_bit_vector.hpp" +#include "3party/succinct/elias_fano.hpp" class DataSource; @@ -51,6 +51,6 @@ public: private: std::unique_ptr m_cityRoadsRegion; - succinct::rs_bit_vector m_cityRoads; + succinct::elias_fano m_cityRoads; }; } // namespace routing