pastk/organicmaps
1
Fork 0
forked from organicmaps/organicmaps
Code Pull requests Activity

Using rs_bit_vector intead of cbv.

Browse source
This commit is contained in:
Vladimir Byko-Ianko 2018-09-06 13:11:22 +03:00 • committed by mpimenov
parent 3a7f976870
commit 3a984d0110
9 changed files with 151 additions and 91 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
coding/memory_region.hpp
View file

@ -5,6 +5,8 @@
#include "base/macros.hpp"
#include "std/cstdint.hpp"
#include "std/utility.hpp"
#include "std/vector.hpp"
class MemoryRegion
{

27
generator/city_roads_generator.cpp
View file

@ -2,14 +2,14 @@
#include "generator/cities_boundaries_checker.hpp"
#include "routing/city_roads_serialization.hpp"
#include "routing/city_roads_loader.hpp"
#include "routing/routing_helpers.hpp"
#include "indexer/feature.hpp"
#include "indexer/feature_data.cpp"
#include "indexer/feature_processor.hpp"
#include "coding/compressed_bit_vector.hpp"
#include "coding/succinct_mapper.hpp"
#include "base/assert.hpp"
#include "base/geo_object_id.hpp"
@ -19,6 +19,9 @@
#include "defines.hpp"
#include "3party/succinct/bit_vector.hpp"
#include "3party/succinct/rs_bit_vector.hpp"
using namespace generator;
using namespace std;
@ -71,17 +74,25 @@ void SerializeCityRoads(string const & dataPath, vector<uint64_t> && cityRoadFea
if (cityRoadFeatureIds.empty())
return;
sort(cityRoadFeatureIds.begin(), cityRoadFeatureIds.end());
FilesContainerW cont(dataPath, FileWriter::OP_WRITE_EXISTING);
FileWriter w = cont.GetWriter(CITY_ROADS_FILE_TAG);
CityRoadsHeader header;
auto const startOffset = w.Pos();
header.Serialize(w);
auto const cbv = coding::CompressedBitVectorBuilder::FromBitPositions(move(cityRoadFeatureIds));
CHECK(cbv, ());
size_t const maxFid = *max_element(cityRoadFeatureIds.cbegin(), cityRoadFeatureIds.cend());
succinct::bit_vector_builder builder(maxFid);
for (auto fid : cityRoadFeatureIds)
builder.set(fid, true /* road feature id */);
CityRoadsSerializer::Serialize(*cbv, w);
coding::FreezeVisitor<Writer> visitor(w);
succinct::rs_bit_vector(&builder).map(visitor);
auto const endOffset = w.Pos();
header.m_dataSize = static_cast<uint32_t>(endOffset - startOffset - sizeof(CityRoadsHeader));
w.Seek(startOffset);
header.Serialize(w);
w.Seek(endOffset);
}
bool BuildCityRoads(string const & dataPath, OsmIdToBoundariesTable & table)
@ -94,7 +105,7 @@ bool BuildCityRoads(string const & dataPath, OsmIdToBoundariesTable & table)
// @TODO(bykoianko) The generation city roads section process is based on two stages now:
// * dumping cities boundaries on feature generation step
// * calculating feature ids and building section when feature ids are available
// As a result of dumping cities boundaries instansed of indexer::CityBoundary objects
// As a result of dumping cities boundaries instances of indexer::CityBoundary objects
// are generated and dumped. These objects are used for generating city roads section.
// Using real geometry of cities boundaries should be considered for generating city road
// features. That mean that the real geometry of cities boundaries should be dumped

45
generator/generator_tests/city_roads_tests.cpp
View file

@ -4,7 +4,9 @@
#include "generator/city_roads_generator.hpp"
#include "routing/city_roads_serialization.hpp"
#include "routing/city_roads_loader.hpp"
#include "indexer/data_source.hpp"
#include "platform/platform_tests_support/scoped_dir.hpp"
#include "platform/platform_tests_support/scoped_file.hpp"
@ -13,7 +15,6 @@
#include "platform/local_country_file.hpp"
#include "platform/platform.hpp"
#include "coding/compressed_bit_vector.hpp"
#include "coding/file_container.hpp"
#include "coding/file_name_utils.hpp"
#include "coding/reader.hpp"
@ -23,6 +24,7 @@
#include <algorithm>
#include <cstdint>
#include <limits>
#include <string>
#include <utility>
#include <vector>
@ -47,28 +49,14 @@ void BuildEmptyMwm(LocalCountryFile & country)
generator::tests_support::TestMwmBuilder builder(country, feature::DataHeader::country);
}
unique_ptr<CompressedBitVector> LoadCityRoads(string const & mwmFilePath)
unique_ptr<CityRoadsLoader> LoadCityRoads(LocalCountryFile const & country)
{
FilesContainerR const cont(mwmFilePath);
if (!cont.IsExist(CITY_ROADS_FILE_TAG))
return nullptr;
FrozenDataSource dataSource;
auto const regResult = dataSource.RegisterMap(country);
TEST_EQUAL(regResult.second, MwmSet::RegResult::Success, ());
auto const & mwmId = regResult.first;
try
{
FilesContainerR::TReader const reader = cont.GetReader(CITY_ROADS_FILE_TAG);
ReaderSource<FilesContainerR::TReader> src(reader);
CityRoadsHeader header;
header.Deserialize(src);
TEST_EQUAL(header.m_version, 0, ());
return CompressedBitVectorBuilder::DeserializeFromSource(src);
}
catch (Reader::OpenException const & e)
{
TEST(false, ("Error while reading", CITY_ROADS_FILE_TAG, "section.", e.Msg()));
return nullptr;
}
return make_unique<CityRoadsLoader>(dataSource, mwmId);
}
/// \brief Builds mwm with city_roads section, read the section and compare original feature ids
@ -83,6 +71,7 @@ void TestCityRoadsBuilding(vector<uint64_t> && cityRoadFeatureIds)
LocalCountryFile country(my::JoinPath(writableDir, kTestDir), CountryFile(kTestMwm),
0 /* version */);
ScopedDir const scopedDir(kTestDir);
string const mwmRelativePath = my::JoinPath(kTestDir, kTestMwm + DATA_FILE_EXTENSION);
ScopedFile const scopedMwm(mwmRelativePath, ScopedFile::Mode::Create);
BuildEmptyMwm(country);
@ -92,24 +81,24 @@ void TestCityRoadsBuilding(vector<uint64_t> && cityRoadFeatureIds)
vector<uint64_t> originalCityRoadFeatureIds = cityRoadFeatureIds;
SerializeCityRoads(mwmFullPath, move(cityRoadFeatureIds));
// Loading city_roads section.
auto const loadedCityRoadFeatureIds = LoadCityRoads(mwmFullPath);
auto const loader = LoadCityRoads(country);
TEST(loader, ());
// Comparing loading form mwm and expected feature ids.
if (originalCityRoadFeatureIds.empty())
{
TEST(!loadedCityRoadFeatureIds, ());
TEST(!loader->HasCityRoads(), ());
return;
}
TEST(loadedCityRoadFeatureIds, ());
sort(originalCityRoadFeatureIds.begin(), originalCityRoadFeatureIds.end());
size_t const kMaxRoadFeatureId = originalCityRoadFeatureIds.back();
for (uint64_t fid = 0; fid < kMaxRoadFeatureId; ++fid)
CHECK_LESS(kMaxRoadFeatureId, numeric_limits<uint32_t>::max(), ());
for (uint32_t fid = 0; fid < kMaxRoadFeatureId; ++fid)
{
bool const isCityRoad =
binary_search(originalCityRoadFeatureIds.cbegin(), originalCityRoadFeatureIds.cend(), fid);
TEST_EQUAL(loadedCityRoadFeatureIds->GetBit(fid), isCityRoad, (fid));
TEST_EQUAL(loader->IsCityRoad(fid), isCityRoad, (fid));
}
}

6
indexer/altitude_loader.hpp
View file

@ -14,6 +14,12 @@ class DataSource;
namespace feature
{
// @TODO(bykoianko) |m_altitudeAvailability| and |m_featureTable| are saved without
// taking into account endianness. It should be fixed. The plan is
// * to use one bit form AltitudeHeader::m_version for keeping information about endianness. (Zero
// should be used for big endian.)
// * to check the endianness of the reader and the bit while reading and to use an appropriate
// methods for reading.
class AltitudeLoader
{
public:

3
routing/CMakeLists.txt
View file

@ -20,7 +20,8 @@ set(
checkpoint_predictor.hpp
checkpoints.cpp
checkpoints.hpp
city_roads_serialization.hpp
city_roads_loader.cpp
city_roads_loader.hpp
coding.hpp
cross_mwm_connector.cpp
cross_mwm_connector.hpp

42
routing/city_roads_loader.cpp Normal file
View file

@ -0,0 +1,42 @@
#include "routing/city_roads_loader.hpp"
#include "indexer/data_source.hpp"
#include "coding/reader.hpp"
#include "coding/succinct_mapper.hpp"
#include <utility>
namespace routing
{
CityRoadsLoader::CityRoadsLoader(DataSource const & dataSource, MwmSet::MwmId const & mwmId)
{
MwmSet::MwmHandle m_handle(dataSource.GetMwmHandleById(mwmId));
if (!m_handle.IsAlive())
return;
auto const & mwmValue = *m_handle.GetValue<MwmValue>();
if (!mwmValue.m_cont.IsExist(CITY_ROADS_FILE_TAG))
return;
try
{
auto reader = std::make_unique<FilesContainerR::TReader>(mwmValue.m_cont.GetReader(CITY_ROADS_FILE_TAG));
ReaderSource<FilesContainerR::TReader> src(*reader);
CityRoadsHeader header;
header.Deserialize(src);
std::vector<uint8_t> data(header.m_dataSize);
src.Read(data.data(), data.size());
m_cityRoadsRegion = std::make_unique<CopiedMemoryRegion>(std::move(data));
coding::MapVisitor visitor(m_cityRoadsRegion->ImmutableData());
m_cityRoads.map(visitor);
}
catch (Reader::OpenException const & e)
{
LOG(LERROR, ("File", mwmValue.GetCountryFileName(), "Error while reading", CITY_ROADS_FILE_TAG,
"section.", e.Msg()));
}
}
} // namespace routing

55
routing/city_roads_loader.hpp Normal file
View file

@ -0,0 +1,55 @@
#pragma once
#include "indexer/mwm_set.hpp"
#include "coding/memory_region.hpp"
#include "coding/writer.hpp"
#include <cstdint>
#include <memory>
#include <vector>
#include "3party/succinct/rs_bit_vector.hpp"
class DataSource;
namespace routing
{
struct CityRoadsHeader
{
template <class Sink>
void Serialize(Sink & sink) const
{
WriteToSink(sink, m_version);
WriteToSink(sink, m_endianness);
WriteToSink(sink, m_dataSize);
}
template <class Source>
void Deserialize(Source & src)
{
m_version = ReadPrimitiveFromSource<uint16_t>(src);
m_endianness = ReadPrimitiveFromSource<uint16_t>(src);
m_dataSize = ReadPrimitiveFromSource<uint32_t>(src);
}
uint16_t m_version = 0;
uint16_t m_endianness = 0;
uint32_t m_dataSize = 0;
};
static_assert(sizeof(CityRoadsHeader) == 8, "Wrong header size of city_roads section.");
class CityRoadsLoader
{
public:
CityRoadsLoader(DataSource const & dataSource, MwmSet::MwmId const & mwmId);
bool HasCityRoads() const { return m_cityRoads.size() > 0; }
bool IsCityRoad(uint32_t fid) const { return m_cityRoads[fid]; }
private:
std::unique_ptr<CopiedMemoryRegion> m_cityRoadsRegion;
succinct::rs_bit_vector m_cityRoads;
};
} // namespace routing

50
routing/city_roads_serialization.hpp
View file

@ -1,50 +0,0 @@
#pragma once
#include "coding/compressed_bit_vector.hpp"
#include "coding/reader.hpp"
#include "coding/write_to_sink.hpp"
#include <cstdint>
#include <memory>
#include <vector>
namespace routing
{
struct CityRoadsHeader
{
template <class Sink>
void Serialize(Sink & sink) const
{
WriteToSink(sink, m_version);
WriteToSink(sink, m_reserved);
}
template <class Source>
void Deserialize(Source & src)
{
m_version = ReadPrimitiveFromSource<uint16_t>(src);
m_reserved = ReadPrimitiveFromSource<uint16_t>(src);
}
uint16_t m_version = 0;
uint16_t m_reserved = 0;
};
static_assert(sizeof(CityRoadsHeader) == 4, "Wrong header size of city_roads section.");
class CityRoadsSerializer
{
public:
template <class Sink>
static void Serialize(coding::CompressedBitVector const & cityRoadFeatureIds, Sink & sink)
{
cityRoadFeatureIds.Serialize(sink);
}
template <class Source>
static std::unique_ptr<coding::CompressedBitVector> Deserialize(Source & src)
{
return coding::CompressedBitVectorBuilder::DeserializeFromSource(src);
}
};
} // namespace routing

12
xcode/routing/routing.xcodeproj/project.pbxproj
View file

@ -81,6 +81,8 @@
44E5574A2136EEC900B01439 /* speed_camera_ser_des.hpp in Headers */ = {isa = PBXBuildFile; fileRef = 44E557492136EEC800B01439 /* speed_camera_ser_des.hpp */; };
44E5574C2136EED000B01439 /* speed_camera_ser_des.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 44E5574B2136EED000B01439 /* speed_camera_ser_des.cpp */; };
44F45B282136B069001B1618 /* speed_cameras_tests.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 44F45B272136B069001B1618 /* speed_cameras_tests.cpp */; };
56085C3C21413222004159B4 /* city_roads_loader.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 56085C3A21413221004159B4 /* city_roads_loader.cpp */; };
56085C3D21413222004159B4 /* city_roads_loader.hpp in Headers */ = {isa = PBXBuildFile; fileRef = 56085C3B21413222004159B4 /* city_roads_loader.hpp */; };
56099E291CC7C97D00A7772A /* loaded_path_segment.hpp in Headers */ = {isa = PBXBuildFile; fileRef = 56099E251CC7C97D00A7772A /* loaded_path_segment.hpp */; };
56099E2A1CC7C97D00A7772A /* routing_result_graph.hpp in Headers */ = {isa = PBXBuildFile; fileRef = 56099E261CC7C97D00A7772A /* routing_result_graph.hpp */; };
56099E2B1CC7C97D00A7772A /* turn_candidate.hpp in Headers */ = {isa = PBXBuildFile; fileRef = 56099E271CC7C97D00A7772A /* turn_candidate.hpp */; };
@ -89,7 +91,6 @@
56290B87206A3232003892E0 /* routing_algorithm.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 56290B85206A3231003892E0 /* routing_algorithm.cpp */; };
56290B88206A3232003892E0 /* routing_algorithm.hpp in Headers */ = {isa = PBXBuildFile; fileRef = 56290B86206A3231003892E0 /* routing_algorithm.hpp */; };
562BDE2020D14860008EFF6F /* routing_callbacks.hpp in Headers */ = {isa = PBXBuildFile; fileRef = 562BDE1F20D14860008EFF6F /* routing_callbacks.hpp */; };
56332AD52134243800B85DF8 /* city_roads_serialization.hpp in Headers */ = {isa = PBXBuildFile; fileRef = 56332AD42134243800B85DF8 /* city_roads_serialization.hpp */; };
56555E561D897C90009D786D /* libalohalitics.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 6742ACE61C68A23B009CB89E /* libalohalitics.a */; };
56555E581D897C9D009D786D /* liboauthcpp.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 6742ACFA1C68A2D7009CB89E /* liboauthcpp.a */; };
56555E591D897D28009D786D /* testingmain.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 6742ACDE1C68A13F009CB89E /* testingmain.cpp */; };
@ -362,6 +363,8 @@
44E557492136EEC800B01439 /* speed_camera_ser_des.hpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; path = speed_camera_ser_des.hpp; sourceTree = "<group>"; };
44E5574B2136EED000B01439 /* speed_camera_ser_des.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = speed_camera_ser_des.cpp; sourceTree = "<group>"; };
44F45B272136B069001B1618 /* speed_cameras_tests.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = speed_cameras_tests.cpp; sourceTree = "<group>"; };
56085C3A21413221004159B4 /* city_roads_loader.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = city_roads_loader.cpp; sourceTree = "<group>"; };
56085C3B21413222004159B4 /* city_roads_loader.hpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; path = city_roads_loader.hpp; sourceTree = "<group>"; };
56099E251CC7C97D00A7772A /* loaded_path_segment.hpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; path = loaded_path_segment.hpp; sourceTree = "<group>"; };
56099E261CC7C97D00A7772A /* routing_result_graph.hpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; path = routing_result_graph.hpp; sourceTree = "<group>"; };
56099E271CC7C97D00A7772A /* turn_candidate.hpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; path = turn_candidate.hpp; sourceTree = "<group>"; };
@ -370,7 +373,6 @@
56290B85206A3231003892E0 /* routing_algorithm.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = routing_algorithm.cpp; sourceTree = "<group>"; };
56290B86206A3231003892E0 /* routing_algorithm.hpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; path = routing_algorithm.hpp; sourceTree = "<group>"; };
562BDE1F20D14860008EFF6F /* routing_callbacks.hpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; path = routing_callbacks.hpp; sourceTree = "<group>"; };
56332AD42134243800B85DF8 /* city_roads_serialization.hpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; path = city_roads_serialization.hpp; sourceTree = "<group>"; };
5661A5CD20DE51C500C6B1D1 /* tools.hpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; path = tools.hpp; sourceTree = "<group>"; };
567059591F3AF96D0062672D /* checkpoint_predictor_test.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = checkpoint_predictor_test.cpp; sourceTree = "<group>"; };
5670595B1F3AF97F0062672D /* checkpoint_predictor.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = checkpoint_predictor.cpp; sourceTree = "<group>"; };
@ -783,9 +785,10 @@
675343FA1A3F640D00A0A8C3 /* routing */ = {
isa = PBXGroup;
children = (
56085C3A21413221004159B4 /* city_roads_loader.cpp */,
56085C3B21413222004159B4 /* city_roads_loader.hpp */,
44E5574B2136EED000B01439 /* speed_camera_ser_des.cpp */,
44E557492136EEC800B01439 /* speed_camera_ser_des.hpp */,
56332AD42134243800B85DF8 /* city_roads_serialization.hpp */,
562BDE1F20D14860008EFF6F /* routing_callbacks.hpp */,
56FA20461FBF23A90045DE78 /* cross_mwm_ids.hpp */,
40BEC0801F99FFD600E06CA4 /* transit_info.hpp */,
@ -938,6 +941,7 @@
buildActionMask = 2147483647;
files = (
0C5FEC631DDE192A0017688C /* joint_index.hpp in Headers */,
56085C3D21413222004159B4 /* city_roads_loader.hpp in Headers */,
67C79BA21E2CEE1400C40034 /* restriction_loader.hpp in Headers */,
674F9BCB1B0A580E00704FFA /* async_router.hpp in Headers */,
0C08AA391DF8329B004195DD /* routing_exceptions.hpp in Headers */,
@ -989,7 +993,6 @@
56EA2FD51D8FD8590083F01A /* routing_helpers.hpp in Headers */,
0C15B8021F02A61B0058E253 /* checkpoints.hpp in Headers */,
0C5F5D211E798B0400307B98 /* cross_mwm_connector_serialization.hpp in Headers */,
56332AD52134243800B85DF8 /* city_roads_serialization.hpp in Headers */,
0C0DF9221DE898B70055A22F /* index_graph_starter.hpp in Headers */,
56290B88206A3232003892E0 /* routing_algorithm.hpp in Headers */,
40A111CE1F2F6776005E6AD5 /* route_weight.hpp in Headers */,
@ -1255,6 +1258,7 @@
671F58BD1B874EC80032311E /* followed_polyline.cpp in Sources */,
670EE55D1B6001E7001E8064 /* routing_session.cpp in Sources */,
40C645161F8D167F002E05A0 /* fake_ending.cpp in Sources */,
56085C3C21413222004159B4 /* city_roads_loader.cpp in Sources */,
56099E331CC9247E00A7772A /* bicycle_directions.cpp in Sources */,
0C08AA341DF83223004195DD /* index_graph_serialization.cpp in Sources */,
5694CECA1EBA25F7004576D3 /* road_access_serialization.cpp in Sources */,