RedAuburn/organicmaps
1
Fork 0
forked from organicmaps/organicmaps
Code Pull requests Activity

Implementation index for altitude data with the help of succinct strutures.

Browse source
This commit is contained in:
Vladimir Byko-Ianko 2016-07-19 14:49:38 +03:00
parent ec73bce900
commit 3fa7071a1b
9 changed files with 330 additions and 99 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

191
generator/altitude_generator.cpp
View file

@ -3,6 +3,7 @@
#include "routing/routing_helpers.hpp"
#include "indexer/altitude_loader.hpp"
#include "indexer/feature.hpp"
#include "indexer/feature_altitude.hpp"
#include "indexer/feature_data.hpp"
@ -10,6 +11,8 @@
#include "coding/file_container.hpp"
#include "coding/file_name_utils.hpp"
#include "coding/read_write_utils.hpp"
#include "coding/reader.hpp"
#include "coding/varint.hpp"
#include "coding/internal/file_data.hpp"
@ -21,6 +24,10 @@
#include "defines.hpp"
#include "3party/succinct/elias_fano.hpp"
#include "3party/succinct/mapper.hpp"
#include "3party/succinct/rs_bit_vector.hpp"
#include "std/algorithm.hpp"
#include "std/type_traits.hpp"
#include "std/utility.hpp"
@ -30,29 +37,79 @@ using namespace feature;
namespace
{
TAltitudeSectionVersion constexpr kAltitudeSectionVersion = 1;
class Processor
{
public:
using TFeatureAltitude = pair<uint32_t, Altitudes>;
using TFeatureAltitude = pair<uint32_t, Altitude>;
using TFeatureAltitudes = vector<TFeatureAltitude>;
Processor(string const & srtmPath) : m_srtmManager(srtmPath) {}
Processor(string const & srtmPath) : m_srtmManager(srtmPath), m_minAltitude(kInvalidAltitude) {}
TFeatureAltitudes const & GetFeatureAltitudes() const { return m_featureAltitudes; }
vector<bool> const & GetAltitudeAvailability() const { return m_altitudeAvailability; }
TAltitude GetMinAltitude() const { return m_minAltitude; }
void operator()(FeatureType const & f, uint32_t const & id)
{
if (!routing::IsRoad(feature::TypesHolder(f)))
if (id != m_altitudeAvailability.size())
{
LOG(LERROR, ("There's a gap in feature id order."));
return;
}
f.ParseGeometry(FeatureType::BEST_GEOMETRY);
size_t const pointsCount = f.GetPointsCount();
if (pointsCount == 0)
return;
bool hasAltitude = false;
do
{
if (!routing::IsRoad(feature::TypesHolder(f)))
break;
Altitudes alts(m_srtmManager.GetHeight(MercatorBounds::ToLatLon(f.GetPoint(0))),
m_srtmManager.GetHeight(MercatorBounds::ToLatLon(f.GetPoint(pointsCount - 1))));
m_featureAltitudes.push_back(make_pair(id, alts));
f.ParseGeometry(FeatureType::BEST_GEOMETRY);
size_t const pointsCount = f.GetPointsCount();
if (pointsCount == 0)
break;
TAltitudes altitudes;
bool allPointsValidAltFlag = true;
TAltitude minFeatureAltitude = kInvalidAltitude;
for (size_t i = 0; i < pointsCount; ++i)
{
TAltitude const a = m_srtmManager.GetHeight(MercatorBounds::ToLatLon(f.GetPoint(i)));
if (a == kInvalidAltitude)
{
allPointsValidAltFlag = false;
break;
}
if (minFeatureAltitude == kInvalidAltitude)
minFeatureAltitude = a;
else
minFeatureAltitude = min(minFeatureAltitude, a);
altitudes.push_back(a);
}
if (!allPointsValidAltFlag)
break;
hasAltitude = true;
m_featureAltitudes.push_back(make_pair(id, Altitude(move(altitudes))));
if (m_minAltitude == kInvalidAltitude)
m_minAltitude = minFeatureAltitude;
else
m_minAltitude = min(minFeatureAltitude, m_minAltitude);
}
while(false);
m_altitudeAvailability.push_back(hasAltitude);
}
bool HasAltitudeInfo()
{
return !m_featureAltitudes.empty();
}
void SortFeatureAltitudes()
@ -63,32 +120,128 @@ public:
private:
generator::SrtmTileManager m_srtmManager;
TFeatureAltitudes m_featureAltitudes;
vector<bool> m_altitudeAvailability;
TAltitude m_minAltitude;
};
uint32_t GetFileSize(string const & filePath)
{
uint64_t size;
if (!my::GetFileSize(filePath, size))
{
LOG(LWARNING, ("altitudeAvailability", filePath, "size = 0"));
return 0;
}
LOG(LINFO, ("altitudeAvailability ", filePath, "size =", size, "bytes"));
return size;
}
void MoveFileToAltitudeSection(string const & filePath, uint32_t fileSize, FileWriter & w)
{
{
ReaderSource<FileReader> r = FileReader(filePath);
w.Write(&fileSize, sizeof(fileSize));
rw::ReadAndWrite(r, w);
LOG(LINFO, (filePath, "size is", fileSize));
}
FileWriter::DeleteFileX(filePath);
}
void SerializeHeader(TAltitudeSectionVersion version, TAltitude minAltitude,
TAltitudeSectionOffset altitudeInfoOffset, FileWriter & w)
{
w.Write(&version, sizeof(version));
w.Write(&minAltitude, sizeof(minAltitude));
LOG(LINFO, ("altitudeAvailability writing minAltitude =", minAltitude));
w.Write(&altitudeInfoOffset, sizeof(altitudeInfoOffset));
LOG(LINFO, ("altitudeAvailability writing altitudeInfoOffset =", altitudeInfoOffset));
}
} // namespace
namespace routing
{
void BuildRoadAltitudes(string const & srtmPath, string const & baseDir, string const & countryName)
{
LOG(LINFO, ("srtmPath =", srtmPath, "baseDir =", baseDir, "countryName =", countryName));
string const mwmPath = my::JoinFoldersToPath(baseDir, countryName + DATA_FILE_EXTENSION);
// Writing section with altitude information.
try
{
FilesContainerW cont(mwmPath, FileWriter::OP_WRITE_EXISTING);
FileWriter w = cont.GetWriter(ALTITUDE_FILE_TAG);
// Preparing altitude information.
LOG(LINFO, ("srtmPath =", srtmPath, "baseDir =", baseDir, "countryName =", countryName));
string const mwmPath = my::JoinFoldersToPath(baseDir, countryName + DATA_FILE_EXTENSION);
Processor processor(srtmPath);
feature::ForEachFromDat(mwmPath, processor);
processor.SortFeatureAltitudes();
Processor::TFeatureAltitudes const & featureAltitudes = processor.GetFeatureAltitudes();
for (auto const & a : featureAltitudes)
if (!processor.HasAltitudeInfo())
{
Altitude altitude(a.first /* feature id */, a.second /* feature altitudes */);
altitude.Serialize(w);
LOG(LINFO, ("No altitude information for road features of mwm", countryName));
return;
}
// Writing compressed bit vector with features which have altitude information.
succinct::rs_bit_vector altitudeAvailability(processor.GetAltitudeAvailability());
string const altitudeAvailabilityPath = my::JoinFoldersToPath(baseDir, "altitude_availability.bitvector");
LOG(LINFO, ("altitudeAvailability succinct::mapper::size_of(altitudeAvailability) =", succinct::mapper::size_of(altitudeAvailability)));
succinct::mapper::freeze(altitudeAvailability, altitudeAvailabilityPath.c_str());
// Writing feature altitude information to a file and memorizing the offsets.
string const altitudeInfoPath = my::JoinFoldersToPath(baseDir, "altitude_info");
vector<uint32_t> offsets;
TAltitude const minAltitude = processor.GetMinAltitude();
offsets.reserve(featureAltitudes.size());
{
FileWriter altitudeInfoW(altitudeInfoPath);
for (auto const & a : featureAltitudes)
{
offsets.push_back(altitudeInfoW.Pos());
// Feature altitude serializing.
a.second.Serialize(minAltitude, altitudeInfoW);
}
}
LOG(LINFO, ("Altitude was written for", featureAltitudes.size(), "features."));
// Writing feature altitude offsets.
CHECK(is_sorted(offsets.begin(), offsets.end()), ());
CHECK(adjacent_find(offsets.begin(), offsets.end()) == offsets.end(), ());
LOG(LINFO, ("Max altitude info offset =", offsets.back(), "offsets.size() =", offsets.size()));
succinct::elias_fano::elias_fano_builder builder(offsets.back(), offsets.size());
for (uint32_t offset : offsets)
builder.push_back(offset);
succinct::elias_fano featureTable(&builder);
string const featuresTablePath = my::JoinFoldersToPath(baseDir, "altitude_offsets.elias_fano");
succinct::mapper::freeze(featureTable, featuresTablePath.c_str());
uint32_t const altitudeAvailabilitySize = GetFileSize(altitudeAvailabilityPath);
uint32_t const altitudeInfoSize = GetFileSize(altitudeInfoPath);
uint32_t const featuresTableSize = GetFileSize(featuresTablePath);
FilesContainerW cont(mwmPath, FileWriter::OP_WRITE_EXISTING);
FileWriter w = cont.GetWriter(ALTITUDE_FILE_TAG);
// Writing section with altitude information.
// Writing altitude section header.
TAltitudeSectionOffset const headerSize = sizeof(kAltitudeSectionVersion) +
sizeof(minAltitude) + sizeof(TAltitudeSectionOffset);
TAltitudeSectionOffset const featuresTableOffset = headerSize +
sizeof(altitudeAvailabilitySize) + altitudeAvailabilitySize;
TAltitudeSectionOffset const altitudeInfoOffset = featuresTableOffset +
sizeof(featuresTableSize) + featuresTableSize;
SerializeHeader(kAltitudeSectionVersion, processor.GetMinAltitude(),
altitudeInfoOffset + sizeof(TAltitudeSectionOffset) /* for altitude info size */, w);
// Coping parts of altitude sections to mwm.
MoveFileToAltitudeSection(altitudeAvailabilityPath, altitudeAvailabilitySize, w);
MoveFileToAltitudeSection(featuresTablePath, featuresTableSize, w);
MoveFileToAltitudeSection(altitudeInfoPath, altitudeInfoSize, w);
}
catch (RootException const & e)
{
LOG(LERROR, ("An exception happend while creating", ALTITUDE_FILE_TAG, "section. ", e.what()));
}
}
} // namespace routing

101
indexer/altitude_loader.cpp
View file

@ -1,49 +1,110 @@
#include "indexer/altitude_loader.hpp"
#include "coding/reader.hpp"
#include "base/logging.hpp"
#include "base/stl_helpers.hpp"
#include "base/thread.hpp"
#include "defines.hpp"
#include "3party/succinct/mapper.hpp"
namespace
{
void ReadBuffer(ReaderSource<FilesContainerR::TReader> & rs, vector<char> & buf)
{
uint32_t bufSz = 0;
rs.Read(&bufSz, sizeof(bufSz));
if (bufSz > rs.Size() + rs.Pos())
{
ASSERT(false, ());
return;
}
buf.clear();
buf.resize(bufSz);
rs.Read(buf.data(), bufSz);
}
} // namespace
namespace feature
{
AltitudeLoader::AltitudeLoader(MwmValue const * mwmValue)
: reader(mwmValue->m_cont.GetReader(ALTITUDE_FILE_TAG)), m_altitudeInfoOffset(0), m_minAltitude(kInvalidAltitude)
{
if (!mwmValue || mwmValue->GetHeader().GetFormat() < version::Format::v8 )
return;
try
{
m_idx = make_unique<DDVector<TAltitudeIndexEntry, FilesContainerR::TReader>>
(mwmValue->m_cont.GetReader(ALTITUDE_FILE_TAG));
ReaderSource<FilesContainerR::TReader> rs(reader);
DeserializeHeader(rs);
// Reading rs_bit_vector with altitude availability information.
ReadBuffer(rs, m_altitudeAvailabilitBuf);
m_altitudeAvailability = make_unique<succinct::rs_bit_vector>();
succinct::mapper::map(*m_altitudeAvailability, m_altitudeAvailabilitBuf.data());
// Reading table with altitude ofsets for features.
ReadBuffer(rs, m_featureTableBuf);
m_featureTable = make_unique<succinct::elias_fano>();
succinct::mapper::map(*m_featureTable, m_featureTableBuf.data());
}
catch (Reader::OpenException const &)
catch (Reader::OpenException const & e)
{
LOG(LINFO, ("MWM does not contain", ALTITUDE_FILE_TAG, "section."));
m_altitudeInfoOffset = 0;
m_minAltitude = kInvalidAltitude;
LOG(LINFO, ("MWM does not contain", ALTITUDE_FILE_TAG, "section.", e.Msg()));
}
}
Altitudes AltitudeLoader::GetAltitudes(uint32_t featureId) const
void AltitudeLoader::DeserializeHeader(ReaderSource<FilesContainerR::TReader> & rs)
{
if (!m_idx || m_idx->size() == 0)
return Altitudes();
TAltitudeSectionVersion version;
rs.Read(&version, sizeof(version));
LOG(LINFO, ("Reading version =", version));
rs.Read(&m_minAltitude, sizeof(m_minAltitude));
LOG(LINFO, ("Reading m_minAltitude =", m_minAltitude));
rs.Read(&m_altitudeInfoOffset, sizeof(m_altitudeInfoOffset));
LOG(LINFO, ("Reading m_altitudeInfoOffset =", m_altitudeInfoOffset));
}
auto it = lower_bound(m_idx->begin(), m_idx->end(),
TAltitudeIndexEntry{static_cast<uint32_t>(featureId), 0, 0},
my::LessBy(&TAltitudeIndexEntry::featureId));
if (it == m_idx->end())
return Altitudes();
if (featureId != it->featureId)
TAltitudes AltitudeLoader::GetAltitude(uint32_t featureId, size_t pointCount) const
{
if (m_altitudeInfoOffset == 0)
{
ASSERT(false, ());
return Altitudes();
// The version of mwm is less then version::Format::v8 or there's no altitude section in mwm.
return TAltitudes();
}
if (it->beginAlt == kInvalidAltitude || it->endAlt == kInvalidAltitude)
return Altitudes();
if (!(*m_altitudeAvailability)[featureId])
{
LOG(LINFO, ("Feature featureId =", featureId, "does not contain any altitude information."));
return TAltitudes();
}
return Altitudes(it->beginAlt, it->endAlt);
uint64_t const r = m_altitudeAvailability->rank(featureId);
CHECK_LESS(r, m_altitudeAvailability->size(), (featureId));
uint64_t const offset = m_featureTable->select(r);
CHECK_LESS(offset, m_featureTable->size(), (featureId));
uint64_t const m_altitudeInfoOffsetInSection = m_altitudeInfoOffset + offset;
CHECK_LESS(m_altitudeInfoOffsetInSection, reader.Size(), ());
try
{
Altitude a;
ReaderSource<FilesContainerR::TReader> rs(reader);
rs.Skip(m_altitudeInfoOffsetInSection);
a.Deserialize(m_minAltitude, pointCount, rs);
// @TODO(bykoianko) Considers using move semantic for returned value here.
return a.GetAltitudes();
}
catch (Reader::OpenException const & e)
{
LOG(LERROR, ("Error while getting mwm data", e.Msg()));
return TAltitudes();
}
}
} // namespace feature

26
indexer/altitude_loader.hpp
View file

@ -2,24 +2,32 @@
#include "indexer/feature_altitude.hpp"
#include "indexer/index.hpp"
#include "coding/dd_vector.hpp"
#include "3party/succinct/rs_bit_vector.hpp"
#include "std/unique_ptr.hpp"
#include "std/vector.hpp"
namespace feature
{
using TAltitudeSectionVersion = uint16_t;
using TAltitudeSectionOffset = uint32_t;
class AltitudeLoader
{
public:
AltitudeLoader(MwmValue const * mwmValue);
Altitudes GetAltitudes(uint32_t featureId) const;
TAltitudes GetAltitude(uint32_t featureId, size_t pointCount) const;
private:
struct TAltitudeIndexEntry
{
uint32_t featureId;
feature::TAltitude beginAlt;
feature::TAltitude endAlt;
};
void DeserializeHeader(ReaderSource<FilesContainerR::TReader> & rs);
unique_ptr<DDVector<TAltitudeIndexEntry, FilesContainerR::TReader>> m_idx;
vector<char> m_altitudeAvailabilitBuf;
vector<char> m_featureTableBuf;
unique_ptr<succinct::rs_bit_vector> m_altitudeAvailability;
unique_ptr<succinct::elias_fano> m_featureTable;
FilesContainerR::TReader reader;
TAltitudeSectionOffset m_altitudeInfoOffset;
TAltitude m_minAltitude;
};
} // namespace feature

4
indexer/feature.hpp
View file

@ -362,10 +362,6 @@ private:
mutable points_t m_points, m_triangles;
mutable feature::Metadata m_metadata;
// @TODO |m_altitudes| should be exchanged with vector<TAltitude>.
// If the vector is empty no altitude information is available for this feature.
feature::Altitudes m_altitudes;
mutable bool m_header2Parsed = false;
mutable bool m_pointsParsed = false;
mutable bool m_trianglesParsed = false;

65
indexer/feature_altitude.hpp
View file

@ -11,44 +11,55 @@ using TAltitude = int16_t;
using TAltitudes = vector<feature::TAltitude>;
TAltitude constexpr kInvalidAltitude = numeric_limits<TAltitude>::min();
struct Altitudes
{
Altitudes() = default;
Altitudes(TAltitude b, TAltitude e) : begin(b), end(e) {}
TAltitude begin = kInvalidAltitude;
TAltitude end = kInvalidAltitude;
};
class Altitude
{
public:
Altitude() = default;
Altitude(uint32_t featureId, Altitudes const & altitudes)
: m_featureId(featureId), m_altitudes(altitudes)
{
}
Altitude(TAltitudes const & altitudes) : m_pointAlt(altitudes) {}
template <class TSink>
void Serialize(TSink & sink) const
void Serialize(TAltitude minAltitude, TSink & sink) const
{
sink.Write(&m_featureId, sizeof(uint32_t));
sink.Write(&m_altitudes.begin, sizeof(TAltitude));
sink.Write(&m_altitudes.end, sizeof(TAltitude));
CHECK(!m_pointAlt.empty(), ());
TAltitude prevPntAltitude = minAltitude;
for (size_t i = 0; i < m_pointAlt.size(); ++i)
{
WriteVarInt(sink, static_cast<int32_t>(static_cast<int32_t>(m_pointAlt[i] - prevPntAltitude)));
prevPntAltitude = m_pointAlt[i];
}
}
/// @TODO template <class TSource> void Deserialize(TSource & src) should be implement here.
/// But now for test purposes deserialization is done with DDVector construction.
template <class TSource>
void Deserialize(TAltitude minAltitude, size_t pointCount, TSource & src)
{
m_pointAlt.clear();
if (pointCount == 0)
{
ASSERT(false, ());
return;
}
uint32_t GetFeatureId() const { return m_featureId; }
Altitudes const & GetAltitudes() const { return m_altitudes; }
m_pointAlt.resize(pointCount);
TAltitude prevPntAltitude = minAltitude;
for (size_t i = 0; i < pointCount; ++i)
{
m_pointAlt[i] = static_cast<TAltitude>(ReadVarInt<int32_t>(src) + prevPntAltitude);
prevPntAltitude = m_pointAlt[i];
}
}
TAltitudes GetAltitudes() const
{
return m_pointAlt;
}
private:
/// @TODO Note. Feature id is located here because there's no index for altitudes.
/// There's only pairs sorted by feature id. Before merging to master some index has to be
/// implemented.
/// Don't forget to remove |m_featureId|.
uint32_t m_featureId = 0;
Altitudes m_altitudes;
/// \note |m_pointAlt| is a vector of feature point altitudes. There's two posibilities:
/// * |m_pointAlt| is empty. It means no altitude information defines.
/// * size of |m_pointAlt| is equal to feature point number. In that case every item of
/// |m_pointAlt| defines altitude in meters for every feature point. If altitude is not defined
/// for some feature point corresponding vector items are equel to |kInvalidAltitude|
TAltitudes m_pointAlt;
};
} // namespace feature

2
routing/bicycle_model.cpp
View file

@ -686,7 +686,7 @@ shared_ptr<IVehicleModel> BicycleModelFactory::GetVehicleModelForCountry(string
LOG(LDEBUG, ("Bicycle model was found:", country));
return itr->second;
}
LOG(LDEBUG, ("Bicycle model wasn't found, default model is used instead:", country));
LOG(LDEBUG, ("Bicycle model wasn't found, default bicycle model is used instead:", country));
return BicycleModelFactory::GetVehicleModel();
}
} // routing

28
routing/features_road_graph.cpp
View file

@ -263,30 +263,23 @@ void FeaturesRoadGraph::ExtractRoadInfo(FeatureID const & featureId, FeatureType
ri.m_speedKMPH = speedKMPH;
ft.ParseGeometry(FeatureType::BEST_GEOMETRY);
feature::Altitudes altitudes = value.altitudeLoader.GetAltitudes(featureId.m_index);
LOG(LINFO, ("Feature idx =", featureId.m_index, "altitudes.begin =", altitudes.begin,
"altitudes.end =", altitudes.end));
feature::TAltitudes altitudes = value.altitudeLoader.GetAltitude(featureId.m_index, ft.GetPointsCount());
// @TODO It's a temprarery solution until a vector of feature altitudes is saved in mwm.
bool const isAltidudeValid = altitudes.begin != feature::kInvalidAltitude &&
altitudes.end != feature::kInvalidAltitude;
feature::TAltitude pointAlt = altitudes.begin;
size_t const pointsCount = ft.GetPointsCount();
feature::TAltitude const diffAlt =
isAltidudeValid ? (altitudes.end - altitudes.begin) / pointsCount : 0;
if (altitudes.size() != pointsCount)
{
ASSERT(false, ("altitudes.size is different from ft.GetPointsCount()"));
altitudes.clear();
}
ri.m_junctions.clear();
ri.m_junctions.resize(pointsCount);
for (size_t i = 0; i < pointsCount; ++i)
{
if (!isAltidudeValid)
{
if (altitudes.empty())
ri.m_junctions[i] = Junction(ft.GetPoint(i), feature::kInvalidAltitude);
continue;
}
ri.m_junctions[i] = Junction(ft.GetPoint(i), pointAlt);
pointAlt += diffAlt;
else
ri.m_junctions[i] = Junction(ft.GetPoint(i), altitudes[i]);
}
}
@ -343,8 +336,7 @@ FeaturesRoadGraph::Value const & FeaturesRoadGraph::LockFeatureMwm(FeatureID con
if (mwmHandle.IsAlive())
mwmValue = mwmHandle.GetValue<MwmValue>();
Value value = {move(mwmHandle), feature::AltitudeLoader(mwmValue)};
Value value(move(mwmHandle), mwmValue);
return m_mwmLocks.insert(make_pair(move(mwmId), move(value))).first->second;
}
} // namespace routing

5
routing/features_road_graph.hpp
View file

@ -81,6 +81,11 @@ private:
struct Value
{
Value(MwmSet::MwmHandle && handle, MwmValue const * MwmValue)
: mwmHandle(move(handle)), altitudeLoader(MwmValue)
{
}
MwmSet::MwmHandle mwmHandle;
feature::AltitudeLoader altitudeLoader;
};

7
routing/routing_algorithm.cpp
View file

@ -20,7 +20,12 @@ inline double TimeBetweenSec(Junction const & j1, Junction const & j2, double sp
{
ASSERT(speedMPS > 0.0, ());
double const distanceM = MercatorBounds::DistanceOnEarth(j1.GetPoint(), j2.GetPoint());
double const altidudeDiffM = j2.GetAltitude() - j1.GetAltitude();
feature::TAltitude const j1Altitude = j1.GetAltitude();
feature::TAltitude const j2Altitude = j2.GetAltitude();
if (j1Altitude == feature::kInvalidAltitude || j2Altitude == feature::kInvalidAltitude)
return distanceM / speedMPS;
feature::TAltitude const altidudeDiffM = j2Altitude - j1Altitude;
return sqrt(distanceM * distanceM + altidudeDiffM * altidudeDiffM) / speedMPS;
}