From 3a984d01109f3f52ab26766a764ed2a7b754e276 Mon Sep 17 00:00:00 2001 From: Vladimir Byko-Ianko Date: Thu, 6 Sep 2018 13:11:22 +0300 Subject: [PATCH] Using rs_bit_vector intead of cbv. --- coding/memory_region.hpp | 2 + generator/city_roads_generator.cpp | 27 ++++++--- .../generator_tests/city_roads_tests.cpp | 45 ++++++--------- indexer/altitude_loader.hpp | 6 ++ routing/CMakeLists.txt | 3 +- routing/city_roads_loader.cpp | 42 ++++++++++++++ routing/city_roads_loader.hpp | 55 +++++++++++++++++++ routing/city_roads_serialization.hpp | 50 ----------------- .../routing/routing.xcodeproj/project.pbxproj | 12 ++-- 9 files changed, 151 insertions(+), 91 deletions(-) create mode 100644 routing/city_roads_loader.cpp create mode 100644 routing/city_roads_loader.hpp delete mode 100644 routing/city_roads_serialization.hpp diff --git a/coding/memory_region.hpp b/coding/memory_region.hpp index 29b92d7759..f6c88c8036 100644 --- a/coding/memory_region.hpp +++ b/coding/memory_region.hpp @@ -5,6 +5,8 @@ #include "base/macros.hpp" #include "std/cstdint.hpp" +#include "std/utility.hpp" +#include "std/vector.hpp" class MemoryRegion { diff --git a/generator/city_roads_generator.cpp b/generator/city_roads_generator.cpp index 5b1fe0f0ed..122113b888 100644 --- a/generator/city_roads_generator.cpp +++ b/generator/city_roads_generator.cpp @@ -2,14 +2,14 @@ #include "generator/cities_boundaries_checker.hpp" -#include "routing/city_roads_serialization.hpp" +#include "routing/city_roads_loader.hpp" #include "routing/routing_helpers.hpp" #include "indexer/feature.hpp" #include "indexer/feature_data.cpp" #include "indexer/feature_processor.hpp" -#include "coding/compressed_bit_vector.hpp" +#include "coding/succinct_mapper.hpp" #include "base/assert.hpp" #include "base/geo_object_id.hpp" @@ -19,6 +19,9 @@ #include "defines.hpp" +#include "3party/succinct/bit_vector.hpp" +#include "3party/succinct/rs_bit_vector.hpp" + using namespace generator; using namespace std; @@ -71,17 +74,25 @@ void SerializeCityRoads(string const & dataPath, vector && cityRoadFea if (cityRoadFeatureIds.empty()) return; - sort(cityRoadFeatureIds.begin(), cityRoadFeatureIds.end()); - FilesContainerW cont(dataPath, FileWriter::OP_WRITE_EXISTING); FileWriter w = cont.GetWriter(CITY_ROADS_FILE_TAG); CityRoadsHeader header; + auto const startOffset = w.Pos(); header.Serialize(w); - auto const cbv = coding::CompressedBitVectorBuilder::FromBitPositions(move(cityRoadFeatureIds)); - CHECK(cbv, ()); + size_t const maxFid = *max_element(cityRoadFeatureIds.cbegin(), cityRoadFeatureIds.cend()); + succinct::bit_vector_builder builder(maxFid); + for (auto fid : cityRoadFeatureIds) + builder.set(fid, true /* road feature id */); - CityRoadsSerializer::Serialize(*cbv, w); + coding::FreezeVisitor visitor(w); + succinct::rs_bit_vector(&builder).map(visitor); + auto const endOffset = w.Pos(); + header.m_dataSize = static_cast(endOffset - startOffset - sizeof(CityRoadsHeader)); + + w.Seek(startOffset); + header.Serialize(w); + w.Seek(endOffset); } bool BuildCityRoads(string const & dataPath, OsmIdToBoundariesTable & table) @@ -94,7 +105,7 @@ bool BuildCityRoads(string const & dataPath, OsmIdToBoundariesTable & table) // @TODO(bykoianko) The generation city roads section process is based on two stages now: // * dumping cities boundaries on feature generation step // * calculating feature ids and building section when feature ids are available - // As a result of dumping cities boundaries instansed of indexer::CityBoundary objects + // As a result of dumping cities boundaries instances of indexer::CityBoundary objects // are generated and dumped. These objects are used for generating city roads section. // Using real geometry of cities boundaries should be considered for generating city road // features. That mean that the real geometry of cities boundaries should be dumped diff --git a/generator/generator_tests/city_roads_tests.cpp b/generator/generator_tests/city_roads_tests.cpp index 9311d795f4..634dc64410 100644 --- a/generator/generator_tests/city_roads_tests.cpp +++ b/generator/generator_tests/city_roads_tests.cpp @@ -4,7 +4,9 @@ #include "generator/city_roads_generator.hpp" -#include "routing/city_roads_serialization.hpp" +#include "routing/city_roads_loader.hpp" + +#include "indexer/data_source.hpp" #include "platform/platform_tests_support/scoped_dir.hpp" #include "platform/platform_tests_support/scoped_file.hpp" @@ -13,7 +15,6 @@ #include "platform/local_country_file.hpp" #include "platform/platform.hpp" -#include "coding/compressed_bit_vector.hpp" #include "coding/file_container.hpp" #include "coding/file_name_utils.hpp" #include "coding/reader.hpp" @@ -23,6 +24,7 @@ #include #include +#include #include #include #include @@ -47,28 +49,14 @@ void BuildEmptyMwm(LocalCountryFile & country) generator::tests_support::TestMwmBuilder builder(country, feature::DataHeader::country); } -unique_ptr LoadCityRoads(string const & mwmFilePath) +unique_ptr LoadCityRoads(LocalCountryFile const & country) { - FilesContainerR const cont(mwmFilePath); - if (!cont.IsExist(CITY_ROADS_FILE_TAG)) - return nullptr; + FrozenDataSource dataSource; + auto const regResult = dataSource.RegisterMap(country); + TEST_EQUAL(regResult.second, MwmSet::RegResult::Success, ()); + auto const & mwmId = regResult.first; - try - { - FilesContainerR::TReader const reader = cont.GetReader(CITY_ROADS_FILE_TAG); - ReaderSource src(reader); - - CityRoadsHeader header; - header.Deserialize(src); - TEST_EQUAL(header.m_version, 0, ()); - - return CompressedBitVectorBuilder::DeserializeFromSource(src); - } - catch (Reader::OpenException const & e) - { - TEST(false, ("Error while reading", CITY_ROADS_FILE_TAG, "section.", e.Msg())); - return nullptr; - } + return make_unique(dataSource, mwmId); } /// \brief Builds mwm with city_roads section, read the section and compare original feature ids @@ -83,6 +71,7 @@ void TestCityRoadsBuilding(vector && cityRoadFeatureIds) LocalCountryFile country(my::JoinPath(writableDir, kTestDir), CountryFile(kTestMwm), 0 /* version */); ScopedDir const scopedDir(kTestDir); + string const mwmRelativePath = my::JoinPath(kTestDir, kTestMwm + DATA_FILE_EXTENSION); ScopedFile const scopedMwm(mwmRelativePath, ScopedFile::Mode::Create); BuildEmptyMwm(country); @@ -92,24 +81,24 @@ void TestCityRoadsBuilding(vector && cityRoadFeatureIds) vector originalCityRoadFeatureIds = cityRoadFeatureIds; SerializeCityRoads(mwmFullPath, move(cityRoadFeatureIds)); - // Loading city_roads section. - auto const loadedCityRoadFeatureIds = LoadCityRoads(mwmFullPath); + auto const loader = LoadCityRoads(country); + TEST(loader, ()); // Comparing loading form mwm and expected feature ids. if (originalCityRoadFeatureIds.empty()) { - TEST(!loadedCityRoadFeatureIds, ()); + TEST(!loader->HasCityRoads(), ()); return; } - TEST(loadedCityRoadFeatureIds, ()); sort(originalCityRoadFeatureIds.begin(), originalCityRoadFeatureIds.end()); size_t const kMaxRoadFeatureId = originalCityRoadFeatureIds.back(); - for (uint64_t fid = 0; fid < kMaxRoadFeatureId; ++fid) + CHECK_LESS(kMaxRoadFeatureId, numeric_limits::max(), ()); + for (uint32_t fid = 0; fid < kMaxRoadFeatureId; ++fid) { bool const isCityRoad = binary_search(originalCityRoadFeatureIds.cbegin(), originalCityRoadFeatureIds.cend(), fid); - TEST_EQUAL(loadedCityRoadFeatureIds->GetBit(fid), isCityRoad, (fid)); + TEST_EQUAL(loader->IsCityRoad(fid), isCityRoad, (fid)); } } diff --git a/indexer/altitude_loader.hpp b/indexer/altitude_loader.hpp index 0f673b4593..0617c5620d 100644 --- a/indexer/altitude_loader.hpp +++ b/indexer/altitude_loader.hpp @@ -14,6 +14,12 @@ class DataSource; namespace feature { +// @TODO(bykoianko) |m_altitudeAvailability| and |m_featureTable| are saved without +// taking into account endianness. It should be fixed. The plan is +// * to use one bit form AltitudeHeader::m_version for keeping information about endianness. (Zero +// should be used for big endian.) +// * to check the endianness of the reader and the bit while reading and to use an appropriate +// methods for reading. class AltitudeLoader { public: diff --git a/routing/CMakeLists.txt b/routing/CMakeLists.txt index 67170c58ea..0014bc0e99 100644 --- a/routing/CMakeLists.txt +++ b/routing/CMakeLists.txt @@ -20,7 +20,8 @@ set( checkpoint_predictor.hpp checkpoints.cpp checkpoints.hpp - city_roads_serialization.hpp + city_roads_loader.cpp + city_roads_loader.hpp coding.hpp cross_mwm_connector.cpp cross_mwm_connector.hpp diff --git a/routing/city_roads_loader.cpp b/routing/city_roads_loader.cpp new file mode 100644 index 0000000000..7eb3b84a84 --- /dev/null +++ b/routing/city_roads_loader.cpp @@ -0,0 +1,42 @@ +#include "routing/city_roads_loader.hpp" + +#include "indexer/data_source.hpp" + +#include "coding/reader.hpp" +#include "coding/succinct_mapper.hpp" + +#include + +namespace routing +{ +CityRoadsLoader::CityRoadsLoader(DataSource const & dataSource, MwmSet::MwmId const & mwmId) +{ + MwmSet::MwmHandle m_handle(dataSource.GetMwmHandleById(mwmId)); + if (!m_handle.IsAlive()) + return; + + auto const & mwmValue = *m_handle.GetValue(); + if (!mwmValue.m_cont.IsExist(CITY_ROADS_FILE_TAG)) + return; + + try + { + auto reader = std::make_unique(mwmValue.m_cont.GetReader(CITY_ROADS_FILE_TAG)); + ReaderSource src(*reader); + + CityRoadsHeader header; + header.Deserialize(src); + + std::vector data(header.m_dataSize); + src.Read(data.data(), data.size()); + m_cityRoadsRegion = std::make_unique(std::move(data)); + coding::MapVisitor visitor(m_cityRoadsRegion->ImmutableData()); + m_cityRoads.map(visitor); + } + catch (Reader::OpenException const & e) + { + LOG(LERROR, ("File", mwmValue.GetCountryFileName(), "Error while reading", CITY_ROADS_FILE_TAG, + "section.", e.Msg())); + } +} +} // namespace routing diff --git a/routing/city_roads_loader.hpp b/routing/city_roads_loader.hpp new file mode 100644 index 0000000000..98b07f09a0 --- /dev/null +++ b/routing/city_roads_loader.hpp @@ -0,0 +1,55 @@ +#pragma once + +#include "indexer/mwm_set.hpp" + +#include "coding/memory_region.hpp" +#include "coding/writer.hpp" + +#include +#include +#include + +#include "3party/succinct/rs_bit_vector.hpp" + +class DataSource; + +namespace routing +{ +struct CityRoadsHeader +{ + template + void Serialize(Sink & sink) const + { + WriteToSink(sink, m_version); + WriteToSink(sink, m_endianness); + WriteToSink(sink, m_dataSize); + } + + template + void Deserialize(Source & src) + { + m_version = ReadPrimitiveFromSource(src); + m_endianness = ReadPrimitiveFromSource(src); + m_dataSize = ReadPrimitiveFromSource(src); + } + + uint16_t m_version = 0; + uint16_t m_endianness = 0; + uint32_t m_dataSize = 0; +}; + +static_assert(sizeof(CityRoadsHeader) == 8, "Wrong header size of city_roads section."); + +class CityRoadsLoader +{ +public: + CityRoadsLoader(DataSource const & dataSource, MwmSet::MwmId const & mwmId); + + bool HasCityRoads() const { return m_cityRoads.size() > 0; } + bool IsCityRoad(uint32_t fid) const { return m_cityRoads[fid]; } + +private: + std::unique_ptr m_cityRoadsRegion; + succinct::rs_bit_vector m_cityRoads; +}; +} // namespace routing diff --git a/routing/city_roads_serialization.hpp b/routing/city_roads_serialization.hpp deleted file mode 100644 index 22d72a1f26..0000000000 --- a/routing/city_roads_serialization.hpp +++ /dev/null @@ -1,50 +0,0 @@ -#pragma once - -#include "coding/compressed_bit_vector.hpp" -#include "coding/reader.hpp" -#include "coding/write_to_sink.hpp" - -#include -#include -#include - -namespace routing -{ -struct CityRoadsHeader -{ - template - void Serialize(Sink & sink) const - { - WriteToSink(sink, m_version); - WriteToSink(sink, m_reserved); - } - - template - void Deserialize(Source & src) - { - m_version = ReadPrimitiveFromSource(src); - m_reserved = ReadPrimitiveFromSource(src); - } - - uint16_t m_version = 0; - uint16_t m_reserved = 0; -}; - -static_assert(sizeof(CityRoadsHeader) == 4, "Wrong header size of city_roads section."); - -class CityRoadsSerializer -{ -public: - template - static void Serialize(coding::CompressedBitVector const & cityRoadFeatureIds, Sink & sink) - { - cityRoadFeatureIds.Serialize(sink); - } - - template - static std::unique_ptr Deserialize(Source & src) - { - return coding::CompressedBitVectorBuilder::DeserializeFromSource(src); - } -}; -} // namespace routing diff --git a/xcode/routing/routing.xcodeproj/project.pbxproj b/xcode/routing/routing.xcodeproj/project.pbxproj index aae0529ad6..7bee841833 100644 --- a/xcode/routing/routing.xcodeproj/project.pbxproj +++ b/xcode/routing/routing.xcodeproj/project.pbxproj @@ -81,6 +81,8 @@ 44E5574A2136EEC900B01439 /* speed_camera_ser_des.hpp in Headers */ = {isa = PBXBuildFile; fileRef = 44E557492136EEC800B01439 /* speed_camera_ser_des.hpp */; }; 44E5574C2136EED000B01439 /* speed_camera_ser_des.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 44E5574B2136EED000B01439 /* speed_camera_ser_des.cpp */; }; 44F45B282136B069001B1618 /* speed_cameras_tests.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 44F45B272136B069001B1618 /* speed_cameras_tests.cpp */; }; + 56085C3C21413222004159B4 /* city_roads_loader.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 56085C3A21413221004159B4 /* city_roads_loader.cpp */; }; + 56085C3D21413222004159B4 /* city_roads_loader.hpp in Headers */ = {isa = PBXBuildFile; fileRef = 56085C3B21413222004159B4 /* city_roads_loader.hpp */; }; 56099E291CC7C97D00A7772A /* loaded_path_segment.hpp in Headers */ = {isa = PBXBuildFile; fileRef = 56099E251CC7C97D00A7772A /* loaded_path_segment.hpp */; }; 56099E2A1CC7C97D00A7772A /* routing_result_graph.hpp in Headers */ = {isa = PBXBuildFile; fileRef = 56099E261CC7C97D00A7772A /* routing_result_graph.hpp */; }; 56099E2B1CC7C97D00A7772A /* turn_candidate.hpp in Headers */ = {isa = PBXBuildFile; fileRef = 56099E271CC7C97D00A7772A /* turn_candidate.hpp */; }; @@ -89,7 +91,6 @@ 56290B87206A3232003892E0 /* routing_algorithm.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 56290B85206A3231003892E0 /* routing_algorithm.cpp */; }; 56290B88206A3232003892E0 /* routing_algorithm.hpp in Headers */ = {isa = PBXBuildFile; fileRef = 56290B86206A3231003892E0 /* routing_algorithm.hpp */; }; 562BDE2020D14860008EFF6F /* routing_callbacks.hpp in Headers */ = {isa = PBXBuildFile; fileRef = 562BDE1F20D14860008EFF6F /* routing_callbacks.hpp */; }; - 56332AD52134243800B85DF8 /* city_roads_serialization.hpp in Headers */ = {isa = PBXBuildFile; fileRef = 56332AD42134243800B85DF8 /* city_roads_serialization.hpp */; }; 56555E561D897C90009D786D /* libalohalitics.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 6742ACE61C68A23B009CB89E /* libalohalitics.a */; }; 56555E581D897C9D009D786D /* liboauthcpp.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 6742ACFA1C68A2D7009CB89E /* liboauthcpp.a */; }; 56555E591D897D28009D786D /* testingmain.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 6742ACDE1C68A13F009CB89E /* testingmain.cpp */; }; @@ -362,6 +363,8 @@ 44E557492136EEC800B01439 /* speed_camera_ser_des.hpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; path = speed_camera_ser_des.hpp; sourceTree = ""; }; 44E5574B2136EED000B01439 /* speed_camera_ser_des.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = speed_camera_ser_des.cpp; sourceTree = ""; }; 44F45B272136B069001B1618 /* speed_cameras_tests.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = speed_cameras_tests.cpp; sourceTree = ""; }; + 56085C3A21413221004159B4 /* city_roads_loader.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = city_roads_loader.cpp; sourceTree = ""; }; + 56085C3B21413222004159B4 /* city_roads_loader.hpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; path = city_roads_loader.hpp; sourceTree = ""; }; 56099E251CC7C97D00A7772A /* loaded_path_segment.hpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; path = loaded_path_segment.hpp; sourceTree = ""; }; 56099E261CC7C97D00A7772A /* routing_result_graph.hpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; path = routing_result_graph.hpp; sourceTree = ""; }; 56099E271CC7C97D00A7772A /* turn_candidate.hpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; path = turn_candidate.hpp; sourceTree = ""; }; @@ -370,7 +373,6 @@ 56290B85206A3231003892E0 /* routing_algorithm.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = routing_algorithm.cpp; sourceTree = ""; }; 56290B86206A3231003892E0 /* routing_algorithm.hpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; path = routing_algorithm.hpp; sourceTree = ""; }; 562BDE1F20D14860008EFF6F /* routing_callbacks.hpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; path = routing_callbacks.hpp; sourceTree = ""; }; - 56332AD42134243800B85DF8 /* city_roads_serialization.hpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; path = city_roads_serialization.hpp; sourceTree = ""; }; 5661A5CD20DE51C500C6B1D1 /* tools.hpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; path = tools.hpp; sourceTree = ""; }; 567059591F3AF96D0062672D /* checkpoint_predictor_test.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = checkpoint_predictor_test.cpp; sourceTree = ""; }; 5670595B1F3AF97F0062672D /* checkpoint_predictor.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = checkpoint_predictor.cpp; sourceTree = ""; }; @@ -783,9 +785,10 @@ 675343FA1A3F640D00A0A8C3 /* routing */ = { isa = PBXGroup; children = ( + 56085C3A21413221004159B4 /* city_roads_loader.cpp */, + 56085C3B21413222004159B4 /* city_roads_loader.hpp */, 44E5574B2136EED000B01439 /* speed_camera_ser_des.cpp */, 44E557492136EEC800B01439 /* speed_camera_ser_des.hpp */, - 56332AD42134243800B85DF8 /* city_roads_serialization.hpp */, 562BDE1F20D14860008EFF6F /* routing_callbacks.hpp */, 56FA20461FBF23A90045DE78 /* cross_mwm_ids.hpp */, 40BEC0801F99FFD600E06CA4 /* transit_info.hpp */, @@ -938,6 +941,7 @@ buildActionMask = 2147483647; files = ( 0C5FEC631DDE192A0017688C /* joint_index.hpp in Headers */, + 56085C3D21413222004159B4 /* city_roads_loader.hpp in Headers */, 67C79BA21E2CEE1400C40034 /* restriction_loader.hpp in Headers */, 674F9BCB1B0A580E00704FFA /* async_router.hpp in Headers */, 0C08AA391DF8329B004195DD /* routing_exceptions.hpp in Headers */, @@ -989,7 +993,6 @@ 56EA2FD51D8FD8590083F01A /* routing_helpers.hpp in Headers */, 0C15B8021F02A61B0058E253 /* checkpoints.hpp in Headers */, 0C5F5D211E798B0400307B98 /* cross_mwm_connector_serialization.hpp in Headers */, - 56332AD52134243800B85DF8 /* city_roads_serialization.hpp in Headers */, 0C0DF9221DE898B70055A22F /* index_graph_starter.hpp in Headers */, 56290B88206A3232003892E0 /* routing_algorithm.hpp in Headers */, 40A111CE1F2F6776005E6AD5 /* route_weight.hpp in Headers */, @@ -1255,6 +1258,7 @@ 671F58BD1B874EC80032311E /* followed_polyline.cpp in Sources */, 670EE55D1B6001E7001E8064 /* routing_session.cpp in Sources */, 40C645161F8D167F002E05A0 /* fake_ending.cpp in Sources */, + 56085C3C21413222004159B4 /* city_roads_loader.cpp in Sources */, 56099E331CC9247E00A7772A /* bicycle_directions.cpp in Sources */, 0C08AA341DF83223004195DD /* index_graph_serialization.cpp in Sources */, 5694CECA1EBA25F7004576D3 /* road_access_serialization.cpp in Sources */,