RedAuburn/organicmaps
1
Fork 0
forked from organicmaps/organicmaps
Code Pull requests Activity

Writing altitude section without intermediate files and using MemoryRegion for reading it.

Browse source
This commit is contained in:
Vladimir Byko-Ianko 2016-07-23 21:38:16 +03:00
parent 9879d1808a
commit 979f48bfaf
12 changed files with 123 additions and 137 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

157
generator/altitude_generator.cpp
View file

@ -13,6 +13,7 @@
#include "coding/file_name_utils.hpp"
#include "coding/read_write_utils.hpp"
#include "coding/reader.hpp"
#include "coding/succinct_mapper.hpp"
#include "coding/varint.hpp"
#include "geometry/latlon.hpp"
@ -42,7 +43,7 @@ namespace
{
using namespace routing;
AltitudeHeader::TAltitudeSectionVersion constexpr kAltitudeSectionVersion = 1;
AltitudeHeader::TAltitudeSectionVersion constexpr kAltitudeSectionVersion = 0;
class SrtmGetter : public IAltitudeGetter
{
@ -72,13 +73,16 @@ public:
TFeatureAltitudes const & GetFeatureAltitudes() const { return m_featureAltitudes; }
vector<bool> const & GetAltitudeAvailability() const { return m_altitudeAvailability; }
succinct::bit_vector_builder & GetAltitudeAvailabilityBuilder()
{
return m_altitudeAvailabilityBuilder;
}
TAltitude GetMinAltitude() const { return m_minAltitude; }
void operator()(FeatureType const & f, uint32_t const & id)
{
if (id != m_altitudeAvailability.size())
if (id != m_altitudeAvailabilityBuilder.size())
{
LOG(LERROR, ("There's a gap in feature id order."));
return;
@ -87,7 +91,7 @@ public:
bool hasAltitude = false;
MY_SCOPE_GUARD(removeTmpDir, [&] ()
{
m_altitudeAvailability.push_back(hasAltitude);
m_altitudeAvailabilityBuilder.push_back(hasAltitude);
});
if (!routing::IsRoad(feature::TypesHolder(f)))
@ -134,120 +138,89 @@ public:
return !m_featureAltitudes.empty();
}
void SortFeatureAltitudes()
bool IsFeatureAltitudesSorted()
{
sort(m_featureAltitudes.begin(), m_featureAltitudes.end(), my::LessBy(&Processor::TFeatureAltitude::first));
return is_sorted(m_featureAltitudes.begin(), m_featureAltitudes.end(),
my::LessBy(&Processor::TFeatureAltitude::first));
}
private:
IAltitudeGetter & m_altitudeGetter;
TFeatureAltitudes m_featureAltitudes;
vector<bool> m_altitudeAvailability;
succinct::bit_vector_builder m_altitudeAvailabilityBuilder;
TAltitude m_minAltitude;
};
uint32_t GetFileSize(string const & filePath)
{
uint64_t size;
if (!my::GetFileSize(filePath, size))
{
LOG(LERROR, (filePath, "Unable to get file size"));
return 0;
}
LOG(LINFO, (filePath, "size =", size, "bytes"));
return size;
}
void MoveFileToAltitudeSection(string const & filePath, uint32_t fileSize, FileWriter & w)
{
{
ReaderSource<FileReader> r = FileReader(filePath);
w.Write(&fileSize, sizeof(fileSize));
rw::ReadAndWrite(r, w);
LOG(LINFO, (filePath, "size is", fileSize));
}
FileWriter::DeleteFileX(filePath);
}
} // namespace
static_assert(sizeof(AltitudeHeader) == 16, "Wrong header size of altitude section.");
namespace routing
{
void BuildRoadAltitudes(string const & baseDir, string const & countryName, IAltitudeGetter & altitudeGetter)
void BuildRoadAltitudes(string const & mwmPath, IAltitudeGetter & altitudeGetter)
{
try
{
// Preparing altitude information.
string const mwmPath = my::JoinFoldersToPath(baseDir, countryName + DATA_FILE_EXTENSION);
Processor processor(altitudeGetter);
feature::ForEachFromDat(mwmPath, processor);
if (!processor.HasAltitudeInfo())
{
LOG(LINFO, ("No altitude information for road features of mwm", countryName));
LOG(LINFO, ("No altitude information for road features of mwm:", mwmPath));
return;
}
processor.SortFeatureAltitudes();
Processor::TFeatureAltitudes const & featureAltitudes = processor.GetFeatureAltitudes();
// Writing compressed bit vector with features which have altitude information.
succinct::rs_bit_vector altitudeAvailability(processor.GetAltitudeAvailability());
string const altitudeAvailabilityPath = my::JoinFoldersToPath(baseDir, "altitude_availability.bitvector");
LOG(LINFO, ("altitudeAvailability succinct::mapper::size_of(altitudeAvailability) =", succinct::mapper::size_of(altitudeAvailability)));
succinct::mapper::freeze(altitudeAvailability, altitudeAvailabilityPath.c_str());
// Writing feature altitude information to a file and memorizing the offsets.
string const altitudeInfoPath = my::JoinFoldersToPath(baseDir, "altitude_info");
vector<uint32_t> offsets;
TAltitude const minAltitude = processor.GetMinAltitude();
offsets.reserve(featureAltitudes.size());
{
FileWriter altitudeInfoW(altitudeInfoPath);
for (auto const & a : featureAltitudes)
{
offsets.push_back(altitudeInfoW.Pos());
// Feature altitude serializing.
a.second.Serialize(minAltitude, altitudeInfoW);
}
}
LOG(LINFO, ("Altitude was written for", featureAltitudes.size(), "features."));
// Writing feature altitude offsets.
CHECK(is_sorted(offsets.begin(), offsets.end()), ());
CHECK(adjacent_find(offsets.begin(), offsets.end()) == offsets.end(), ());
LOG(LINFO, ("Max altitude info offset =", offsets.back(), "number of offsets = =", offsets.size()));
succinct::elias_fano::elias_fano_builder builder(offsets.back(), offsets.size());
for (uint32_t offset : offsets)
builder.push_back(offset);
succinct::elias_fano featureTable(&builder);
string const featuresTablePath = my::JoinFoldersToPath(baseDir, "altitude_offsets.elias_fano");
succinct::mapper::freeze(featureTable, featuresTablePath.c_str());
uint32_t const altitudeAvailabilitySize = GetFileSize(altitudeAvailabilityPath);
uint32_t const altitudeInfoSize = GetFileSize(altitudeInfoPath);
uint32_t const featuresTableSize = GetFileSize(featuresTablePath);
CHECK(processor.IsFeatureAltitudesSorted(), ());
FilesContainerW cont(mwmPath, FileWriter::OP_WRITE_EXISTING);
FileWriter w = cont.GetWriter(ALTITUDES_FILE_TAG);
// Writing section with altitude information.
// Writing altitude section header.
TAltitudeSectionOffset const headerSize = AltitudeHeader::GetHeaderSize();
TAltitudeSectionOffset const featuresTableOffset = headerSize +
sizeof(altitudeAvailabilitySize) + altitudeAvailabilitySize;
TAltitudeSectionOffset const altitudeInfoOffset = featuresTableOffset +
sizeof(featuresTableSize) + featuresTableSize;
AltitudeHeader header(kAltitudeSectionVersion, processor.GetMinAltitude(),
altitudeInfoOffset + sizeof(TAltitudeSectionOffset) /* for altitude info size */);
AltitudeHeader header;
header.minAltitude = processor.GetMinAltitude();
int64_t const startOffset = w.Pos();
header.Serialize(w);
{
// Altitude availability serialization.
coding::FreezeVisitor<Writer> visitor(w);
succinct::bit_vector_builder & builder = processor.GetAltitudeAvailabilityBuilder();
succinct::rs_bit_vector(&builder).map(visitor);
}
header.featureTableOffset = w.Pos() - startOffset;
// Copying parts of altitude sections to mwm.
MoveFileToAltitudeSection(altitudeAvailabilityPath, altitudeAvailabilitySize, w);
MoveFileToAltitudeSection(featuresTablePath, featuresTableSize, w);
MoveFileToAltitudeSection(altitudeInfoPath, altitudeInfoSize, w);
vector<uint32_t> offsets;
vector<uint8_t> deltas;
{
// Altitude info serialization to memory.
MemWriter<vector<uint8_t>> writer(deltas);
Processor::TFeatureAltitudes const & featureAltitudes = processor.GetFeatureAltitudes();
for (auto const & a : featureAltitudes)
{
offsets.push_back(writer.Pos());
a.second.Serialize(header.minAltitude, writer);
}
}
{
// Altitude offsets serialization.
CHECK(is_sorted(offsets.begin(), offsets.end()), ());
CHECK(adjacent_find(offsets.begin(), offsets.end()) == offsets.end(), ());
succinct::elias_fano::elias_fano_builder builder(offsets.back(), offsets.size());
for (uint32_t offset : offsets)
builder.push_back(offset);
coding::FreezeVisitor<Writer> visitor(w);
succinct::elias_fano(&builder).map(visitor);
}
// Writing altitude info.
header.altitudeInfoOffset = w.Pos() - startOffset;
w.Write(deltas.data(), deltas.size());
header.endOffset = w.Pos() - startOffset;
// Rewriting header info.
header.version = kAltitudeSectionVersion;
int64_t const endOffset = w.Pos();
w.Seek(startOffset);
header.Serialize(w);
w.Seek(endOffset);
}
catch (RootException const & e)
{
@ -255,10 +228,10 @@ void BuildRoadAltitudes(string const & baseDir, string const & countryName, IAlt
}
}
void BuildRoadAltitudes(string const & srtmPath, string const & baseDir, string const & countryName)
void BuildRoadAltitudes(string const & mwmPath, string const & srtmPath)
{
LOG(LINFO, ("srtmPath =", srtmPath, "baseDir =", baseDir, "countryName =", countryName));
LOG(LINFO, ("mwmPath =", mwmPath, "srtmPath =", srtmPath));
SrtmGetter srtmGetter(srtmPath);
BuildRoadAltitudes(baseDir, countryName, srtmGetter);
BuildRoadAltitudes(mwmPath, srtmGetter);
}
} // namespace routing

16
generator/altitude_generator.hpp
View file

@ -14,8 +14,16 @@ public:
virtual feature::TAltitude GetAltitude(m2::PointD const & p) = 0;
};
void BuildRoadAltitudes(string const & baseDir, string const & countryName,
IAltitudeGetter & altitudeGetter);
void BuildRoadAltitudes(string const & srtmPath, string const & baseDir,
string const & countryName);
/// \brief Adds altitude section to mwm. It has the following format:
/// File offset (bytes) Field name Field size (bytes)
/// 0 version 2
/// 2 min altitude 2
/// 4 feature table offset 4
/// 8 altitude info offset 4
/// 12 end of section 4
/// 16 altitude availability feat. table offset - 16
/// feat. table offset feature table alt. info offset - f. table offset
/// alt. info offset altitude info alt. info offset - end of section
void BuildRoadAltitudes(string const & mwmPath, IAltitudeGetter & altitudeGetter);
void BuildRoadAltitudes(string const & mwmPath, string const & srtmPath);
} // namespace routing

4
generator/generator_tests/altitude_test.cpp
View file

@ -180,7 +180,8 @@ void TestAltitudeSection(vector<TRounded3DGeom> const & roadFeatures)
MockAltitudeGetter altitudeGetter(move(altitudes));
// Adding altitude section to mwm.
BuildRoadAltitudes(testDirFullPath, kTestMwm, altitudeGetter);
string const mwmPath = my::JoinFoldersToPath(testDirFullPath, kTestMwm + DATA_FILE_EXTENSION);
BuildRoadAltitudes(mwmPath, altitudeGetter);
// Reading from mwm and testing altitue information.
Index index;
@ -190,7 +191,6 @@ void TestAltitudeSection(vector<TRounded3DGeom> const & roadFeatures)
MwmSet::MwmHandle mwmHandle = index.GetMwmHandleById(regResult.first);
CHECK(mwmHandle.IsAlive(), ());
string const mwmPath = my::JoinFoldersToPath(testDirFullPath, kTestMwm + DATA_FILE_EXTENSION);
ReadAndTestAltitudeInfo(*mwmHandle.GetValue<MwmValue>(), mwmPath, altitudeGetter);
}
} // namespace

5
generator/generator_tool/generator_tool.cpp
View file

@ -250,7 +250,10 @@ int main(int argc, char ** argv)
feature::DumpFeatureNames(datFile, FLAGS_dump_feature_names);
if (!FLAGS_srtm_path.empty())
routing::BuildRoadAltitudes(FLAGS_srtm_path, path, FLAGS_output);
{
string const mwmPath = my::JoinFoldersToPath(path, FLAGS_output + DATA_FILE_EXTENSION);
routing::BuildRoadAltitudes(FLAGS_srtm_path, mwmPath);
}
if (FLAGS_unpack_mwm)
UnpackMwm(datFile);

33
indexer/altitude_loader.cpp
View file

@ -1,6 +1,7 @@
#include "indexer/altitude_loader.hpp"
#include "coding/reader.hpp"
#include "coding/succinct_mapper.hpp"
#include "base/logging.hpp"
#include "base/stl_helpers.hpp"
@ -12,18 +13,15 @@
namespace
{
void ReadBuffer(ReaderSource<FilesContainerR::TReader> & src, vector<char> & buf)
template<class TCont>
void Map(size_t dataSize, ReaderSource<FilesContainerR::TReader> & src,
TCont & cont, unique_ptr<CopiedMemoryRegion> & region)
{
uint32_t bufSz = 0;
src.Read(&bufSz, sizeof(bufSz));
if (bufSz > src.Size() + src.Pos())
{
ASSERT(false, ());
return;
}
buf.clear();
buf.resize(bufSz);
src.Read(buf.data(), bufSz);
vector<uint8_t> data(dataSize);
src.Read(data.data(), data.size());
region = make_unique<CopiedMemoryRegion>(move(data));
coding::MapVisitor visitor(region->ImmutableData());
cont.map(visitor);
}
} // namespace
@ -43,13 +41,8 @@ AltitudeLoader::AltitudeLoader(MwmValue const & mwmValue)
ReaderSource<FilesContainerR::TReader> src(*m_reader);
m_header.Deserialize(src);
// Reading src_bit_vector with altitude availability information.
ReadBuffer(src, m_altitudeAvailabilitBuf);
succinct::mapper::map(m_altitudeAvailability, m_altitudeAvailabilitBuf.data());
// Reading table with altitude offsets for features.
ReadBuffer(src, m_featureTableBuf);
succinct::mapper::map(m_featureTable, m_featureTableBuf.data());
Map(m_header.GetAltitudeAvailabilitySize(), src, m_altitudeAvailability, m_altitudeAvailabilityRegion);
Map(m_header.GetFeatureTableSize(), src, m_featureTable, m_featureTableRegion);
}
catch (Reader::OpenException const & e)
{
@ -60,12 +53,12 @@ AltitudeLoader::AltitudeLoader(MwmValue const & mwmValue)
bool AltitudeLoader::IsAvailable() const
{
return m_header.minAltitude != kInvalidAltitude && m_header.altitudeInfoOffset != 0;
return m_header.minAltitude != kInvalidAltitude;
}
TAltitudes const & AltitudeLoader::GetAltitudes(uint32_t featureId, size_t pointCount) const
{
if (m_header.altitudeInfoOffset == 0)
if (!IsAvailable())
{
// The version of mwm is less then version::Format::v8 or there's no altitude section in mwm.
return m_dummy;

8
indexer/altitude_loader.hpp
View file

@ -2,6 +2,8 @@
#include "indexer/feature_altitude.hpp"
#include "indexer/index.hpp"
#include "coding/memory_region.hpp"
#include "std/unique_ptr.hpp"
#include "std/vector.hpp"
@ -18,10 +20,12 @@ public:
bool IsAvailable() const;
private:
vector<char> m_altitudeAvailabilitBuf;
vector<char> m_featureTableBuf;
unique_ptr<CopiedMemoryRegion> m_altitudeAvailabilityRegion;
unique_ptr<CopiedMemoryRegion> m_featureTableRegion;
succinct::rs_bit_vector m_altitudeAvailability;
succinct::elias_fano m_featureTable;
unique_ptr<FilesContainerR::TReader> m_reader;
mutable map<uint32_t, TAltitudes> m_cache;
TAltitudes const m_dummy;

25
indexer/feature_altitude.hpp
View file

@ -1,7 +1,7 @@
#pragma once
#include "coding/varint.hpp"
#include "base/logging.hpp"
#include "base/assert.hpp"
#include "std/cstdint.hpp"
#include "std/limits.hpp"
@ -24,17 +24,14 @@ struct AltitudeHeader
Reset();
}
AltitudeHeader(TAltitudeSectionVersion v, TAltitude min, TAltitudeSectionOffset offset)
: version(v), minAltitude(min), altitudeInfoOffset(offset)
{
}
template <class TSink>
void Serialize(TSink & sink) const
{
sink.Write(&version, sizeof(version));
sink.Write(&minAltitude, sizeof(minAltitude));
sink.Write(&featureTableOffset, sizeof(featureTableOffset));
sink.Write(&altitudeInfoOffset, sizeof(altitudeInfoOffset));
sink.Write(&endOffset, sizeof(endOffset));
}
template <class TSource>
@ -42,24 +39,32 @@ struct AltitudeHeader
{
src.Read(&version, sizeof(version));
src.Read(&minAltitude, sizeof(minAltitude));
src.Read(&featureTableOffset, sizeof(featureTableOffset));
src.Read(&altitudeInfoOffset, sizeof(altitudeInfoOffset));
src.Read(&endOffset, sizeof(endOffset));
}
static size_t GetHeaderSize()
{
return sizeof(version) + sizeof(minAltitude) + sizeof(altitudeInfoOffset);
}
// Methods below return sizes of parts of altitude section in bytes.
static size_t GetHeaderSize() { return sizeof(AltitudeHeader); }
size_t GetAltitudeAvailabilitySize() const { return featureTableOffset - GetHeaderSize(); }
size_t GetFeatureTableSize() const { return altitudeInfoOffset - featureTableOffset; }
size_t GetAltitudeInfo() const { return endOffset - altitudeInfoOffset; }
void Reset()
{
version = 1;
minAltitude = kInvalidAltitude;
featureTableOffset = 0;
altitudeInfoOffset = 0;
endOffset = 0;
}
TAltitudeSectionVersion version;
TAltitude minAltitude;
TAltitudeSectionOffset featureTableOffset;
TAltitudeSectionOffset altitudeInfoOffset;
TAltitudeSectionOffset endOffset;
};
class Altitude

2
routing/bicycle_model.cpp
View file

@ -639,7 +639,7 @@ bool BicycleModel::IsOneWay(FeatureType const & f) const
}
// static
BicycleModel const & BicycleModel::DefaultInstance()
BicycleModel const & BicycleModel::AllLimitsInstance()
{
static BicycleModel const instance;
return instance;

2
routing/bicycle_model.hpp
View file

@ -17,7 +17,7 @@ public:
/// VehicleModel overrides:
bool IsOneWay(FeatureType const & f) const override;
static BicycleModel const & DefaultInstance();
static BicycleModel const & AllLimitsInstance();
protected:
RoadAvailability GetRoadAvailability(feature::TypesHolder const & types) const override;

2
routing/pedestrian_model.cpp
View file

@ -643,7 +643,7 @@ IVehicleModel::RoadAvailability PedestrianModel::GetRoadAvailability(feature::Ty
}
// static
PedestrianModel const & PedestrianModel::DefaultInstance()
PedestrianModel const & PedestrianModel::AllLimitsInstance()
{
static PedestrianModel const instance;
return instance;

2
routing/pedestrian_model.hpp
View file

@ -17,7 +17,7 @@ public:
/// VehicleModel overrides:
bool IsOneWay(FeatureType const &) const override { return false; }
static PedestrianModel const & DefaultInstance();
static PedestrianModel const & AllLimitsInstance();
protected:
RoadAvailability GetRoadAvailability(feature::TypesHolder const & types) const override;

4
routing/routing_helpers.hpp
View file

@ -11,7 +11,7 @@ template <class TTypes>
bool IsRoad(TTypes const & types)
{
return CarModel::Instance().HasRoadType(types) ||
PedestrianModel::DefaultInstance().HasRoadType(types) ||
BicycleModel::DefaultInstance().HasRoadType(types);
PedestrianModel::AllLimitsInstance().HasRoadType(types) ||
BicycleModel::AllLimitsInstance().HasRoadType(types);
}
} // namespace rouing