kirylkaveryn/organicmaps
1
Fork 0
forked from organicmaps/organicmaps-tmp
Code Pull requests Activity

[generator] Collectors can work with files

Browse source
This commit is contained in:
Maksim Andrianov 2019-08-06 18:51:25 +03:00 • committed by LaGrunge
parent 05d01f8072
commit d800f51dc9
22 changed files with 319 additions and 169 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
coding/internal/file_data.cpp
View file

@ -291,6 +291,22 @@ bool WriteToTempAndRenameToFile(string const & dest, function<bool(string const
return true;
}
void AppendFileToFile(std::string const & fromFilename, std::string const & toFilename)
{
std::ifstream from;
from.exceptions(std::fstream::failbit | std::fstream::badbit);
from.open(fromFilename, std::ios::binary);
std::ofstream to;
to.exceptions(std::fstream::badbit);
to.open(toFilename, std::ios::binary | std::ios::app);
auto * buffer = from.rdbuf();
if (buffer->in_avail())
to << from.rdbuf();
}
bool CopyFileX(string const & fOld, string const & fNew)
{
try
@ -303,6 +319,7 @@ bool CopyFileX(string const & fOld, string const & fNew)
if (ifs.peek() == ifstream::traits_type::eof())
return true;
ofs << ifs.rdbuf();
ofs.flush();

2
coding/internal/file_data.hpp
View file

@ -70,6 +70,8 @@ bool WriteToTempAndRenameToFile(std::string const & dest,
std::function<bool(std::string const &)> const & write,
std::string const & tmp = "");
void AppendFileToFile(std::string const & fromFilename, std::string const & toFilename);
/// @return false if copy fails. DO NOT THROWS exceptions
bool CopyFileX(std::string const & fOld, std::string const & fNew);
bool IsEqualFiles(std::string const & firstFile, std::string const & secondFile);

30
generator/collector_addresses.cpp
View file

@ -5,14 +5,22 @@
#include "indexer/ftypes_matcher.hpp"
using namespace feature;
#include "platform/platform.hpp"
#include <fstream>
#include "coding/internal/file_data.hpp"
#include "base/assert.hpp"
using namespace feature;
namespace generator
{
CollectorAddresses::CollectorAddresses(std::string const & filename)
: CollectorInterface(filename) {}
: CollectorInterface(filename)
{
m_writer.exceptions(std::fstream::failbit | std::fstream::badbit);
m_writer.open(GetTmpFilename());
}
std::shared_ptr<CollectorInterface>
CollectorAddresses::Clone(std::shared_ptr<cache::IntermediateDataReader> const &) const
@ -25,15 +33,19 @@ void CollectorAddresses::CollectFeature(feature::FeatureBuilder const & feature,
std::string addr;
auto const & checker = ftypes::IsBuildingChecker::Instance();
if (checker(feature.GetTypes()) && feature.FormatFullAddress(addr))
m_stringStream << addr << "\n";
m_writer << addr << "\n";
}
void CollectorAddresses::Finish()
{
if (m_writer.is_open())
m_writer.close();
}
void CollectorAddresses::Save()
{
std::ofstream stream;
stream.exceptions(std::fstream::failbit | std::fstream::badbit);
stream.open(GetFilename());
stream << m_stringStream.str();
if (Platform::IsFileExistsByFullPath(GetTmpFilename()))
CHECK(base::CopyFileX(GetTmpFilename(), GetFilename()), ());
}
void CollectorAddresses::Merge(CollectorInterface const & collector)
@ -43,6 +55,6 @@ void CollectorAddresses::Merge(CollectorInterface const & collector)
void CollectorAddresses::MergeInto(CollectorAddresses & collector) const
{
collector.m_stringStream << m_stringStream.str();
base::AppendFileToFile(GetTmpFilename(), collector.GetTmpFilename());
}
} // namespace generator

5
generator/collector_addresses.hpp
View file

@ -2,8 +2,8 @@
#include "generator/collector_interface.hpp"
#include <fstream>
#include <memory>
#include <sstream>
#include <string>
namespace generator
@ -23,12 +23,13 @@ public:
Clone(std::shared_ptr<cache::IntermediateDataReader> const & = {}) const override;
void CollectFeature(feature::FeatureBuilder const & feature, OsmElement const &) override;
void Finish() override;
void Save() override;
void Merge(CollectorInterface const & collector) override;
void MergeInto(CollectorAddresses & collector) const override;
private:
std::stringstream m_stringStream;
std::ofstream m_writer;
};
} // namespace generator

72
generator/collector_camera.cpp
View file

@ -13,8 +13,12 @@
#include "platform/measurement_utils.hpp"
#include "platform/platform.hpp"
#include "coding/internal/file_data.hpp"
#include "coding/point_coding.hpp"
#include "coding/write_to_sink.hpp"
#include "coding/reader_writer_ops.hpp"
#include "geometry/latlon.hpp"
@ -27,7 +31,6 @@
using namespace feature;
namespace routing
{
size_t const CameraProcessor::kMaxSpeedSpeedStringLength = 32;
@ -47,8 +50,23 @@ CameraProcessor::CameraInfo::CameraInfo(OsmElement const & element)
, m_lat(element.m_lat)
{
auto const maxspeed = element.GetTag("maxspeed");
if (!maxspeed.empty())
m_speed = ValidateMaxSpeedString(maxspeed);
if (maxspeed.empty())
return;
auto const validatedMaxspeed = ValidateMaxSpeedString(maxspeed);
if (validatedMaxspeed.size() > kMaxSpeedSpeedStringLength ||
!strings::to_int(validatedMaxspeed.c_str(), m_speed))
{
LOG(LWARNING, ("Bad speed format of camera:", maxspeed, ", osmId:", element.m_id));
}
}
CameraProcessor::CameraProcessor(std::string const & filename)
: m_waysFilename(filename + ".roads_ids"), m_waysWriter(std::make_unique<FileWriter>(m_waysFilename)) {}
CameraProcessor::~CameraProcessor()
{
CHECK(Platform::RemoveFileIfExists(m_waysFilename), ());
}
void CameraProcessor::ForEachCamera(Fn && toDo) const
@ -63,29 +81,44 @@ void CameraProcessor::ForEachCamera(Fn && toDo) const
void CameraProcessor::ProcessWay(OsmElement const & element)
{
m_ways[element.m_id] = element.m_nodes;
m_waysWriter->Write(&element.m_id, sizeof(element.m_id));
rw::WriteVectorOfPOD(*m_waysWriter, element.m_nodes);
}
void CameraProcessor::FillCameraInWays()
{
for (auto const & way : m_ways)
FileReader reader(m_waysFilename);
ReaderSource<FileReader> src(reader);
auto const fileSize = reader.Size();
auto currPos = reader.GetOffset();
while (currPos < fileSize)
{
for (auto const & node : way.second)
uint64_t wayId;
std::vector<uint64_t> nodes;
src.Read(&wayId, sizeof(wayId));
rw::ReadVectorOfPOD(src, nodes);
for (auto const & node : nodes)
{
auto const itCamera = m_speedCameras.find(node);
if (itCamera == m_speedCameras.cend())
continue;
m_cameraToWays[itCamera->first].push_back(way.first);
m_cameraToWays[itCamera->first].push_back(wayId);
}
currPos = src.Pos();
}
}
void CameraProcessor::ProcessNode(OsmElement const & element)
{
CameraInfo camera(element);
CHECK_LESS(camera.m_speed.size(), kMaxSpeedSpeedStringLength, ());
m_speedCameras.emplace(element.m_id, std::move(camera));
if (camera.IsValid())
m_speedCameras.emplace(element.m_id, std::move(camera));
}
void CameraProcessor::Finish()
{
m_waysWriter.reset({});
}
void CameraProcessor::Merge(CameraProcessor const & cameraProcessor)
@ -93,12 +126,11 @@ void CameraProcessor::Merge(CameraProcessor const & cameraProcessor)
auto const & otherCameras = cameraProcessor.m_speedCameras;
m_speedCameras.insert(std::begin(otherCameras), std::end(otherCameras));
auto const & otherWays = cameraProcessor.m_ways;
m_ways.insert(std::begin(otherWays), std::end(otherWays));
base::AppendFileToFile(cameraProcessor.m_waysFilename, m_waysFilename);
}
CameraCollector::CameraCollector(std::string const & filename) :
generator::CollectorInterface(filename) {}
generator::CollectorInterface(filename), m_processor(GetTmpFilename()) {}
std::shared_ptr<generator::CollectorInterface>
CameraCollector::Clone(std::shared_ptr<generator::cache::IntermediateDataReader> const &) const
@ -130,13 +162,6 @@ void CameraCollector::CollectFeature(FeatureBuilder const & feature, OsmElement
void CameraCollector::Write(FileWriter & writer, CameraProcessor::CameraInfo const & camera,
std::vector<uint64_t> const & ways)
{
std::string maxSpeedStringKmPH = camera.m_speed;
int32_t maxSpeedKmPH = 0;
if (!strings::to_int(maxSpeedStringKmPH.c_str(), maxSpeedKmPH))
LOG(LWARNING, ("Bad speed format of camera:", maxSpeedStringKmPH, ", osmId:", camera.m_id));
CHECK_GREATER_OR_EQUAL(maxSpeedKmPH, 0, ());
uint32_t const lat =
DoubleToUint32(camera.m_lat, ms::LatLon::kMinLat, ms::LatLon::kMaxLat, kPointCoordBits);
WriteToSink(writer, lat);
@ -145,7 +170,7 @@ void CameraCollector::Write(FileWriter & writer, CameraProcessor::CameraInfo con
DoubleToUint32(camera.m_lon, ms::LatLon::kMinLon, ms::LatLon::kMaxLon, kPointCoordBits);
WriteToSink(writer, lon);
WriteToSink(writer, static_cast<uint32_t>(maxSpeedKmPH));
WriteToSink(writer, static_cast<uint32_t>(camera.m_speed));
auto const size = static_cast<uint32_t>(ways.size());
WriteToSink(writer, size);
@ -153,6 +178,11 @@ void CameraCollector::Write(FileWriter & writer, CameraProcessor::CameraInfo con
WriteToSink(writer, wayId);
}
void CameraCollector::Finish()
{
m_processor.Finish();
}
void CameraCollector::Save()
{
using namespace std::placeholders;
@ -170,6 +200,6 @@ void CameraCollector::Merge(generator::CollectorInterface const & collector)
void CameraCollector::MergeInto(CameraCollector & collector) const
{
collector.m_processor.Merge(this->m_processor);
collector.m_processor.Merge(m_processor);
}
} // namespace routing

13
generator/collector_camera.hpp
View file

@ -6,6 +6,7 @@
#include <cstdint>
#include <functional>
#include <fstream>
#include <memory>
#include <string>
#include <unordered_map>
@ -52,25 +53,32 @@ public:
{
CameraInfo(const OsmElement & element);
bool IsValid() const { return m_speed >= 0; }
uint64_t m_id = 0;
double m_lon = 0.0;
double m_lat = 0.0;
std::string m_speed;
int32_t m_speed = 0;
std::vector<uint64_t> m_ways;
};
static size_t const kMaxSpeedSpeedStringLength;
CameraProcessor(std::string const & filename);
~CameraProcessor();
void ForEachCamera(Fn && toDo) const;
void ProcessNode(OsmElement const & element);
void ProcessWay(OsmElement const & element);
void FillCameraInWays();
void Finish();
void Merge(CameraProcessor const & cameraProcessor);
private:
std::unordered_map<uint64_t, std::vector<uint64_t>> m_ways;
std::string m_waysFilename;
std::unique_ptr<FileWriter> m_waysWriter;
std::unordered_map<uint64_t, CameraInfo> m_speedCameras;
std::unordered_map<uint64_t, std::vector<uint64_t>> m_cameraToWays;
};
@ -88,6 +96,7 @@ public:
// We will process all nodes before ways because of o5m format:
// all nodes are first, then all ways, then all relations.
void CollectFeature(feature::FeatureBuilder const & feature, OsmElement const & element) override;
void Finish() override;
void Save() override;
void Merge(generator::CollectorInterface const & collector) override;

36
generator/collector_city_area.cpp
View file

@ -1,19 +1,27 @@
#include "generator/collector_city_boundary.hpp"
#include "generator/collector_city_area.hpp"
#include "generator/feature_generator.hpp"
#include "generator/intermediate_data.hpp"
#include "indexer/ftypes_matcher.hpp"
#include "platform/platform.hpp"
#include "coding/internal/file_data.hpp"
#include "base/assert.hpp"
#include <algorithm>
#include <iterator>
using namespace feature;
using namespace feature::serialization_policy;
namespace generator
{
CityAreaCollector::CityAreaCollector(std::string const & filename)
: CollectorInterface(filename) {}
: CollectorInterface(filename),
m_witer(std::make_unique<FeatureBuilderWriter<MaxAccuracy>>(GetTmpFilename())) {}
std::shared_ptr<CollectorInterface>
CityAreaCollector::Clone(std::shared_ptr<cache::IntermediateDataReader> const &) const
@ -23,18 +31,23 @@ CityAreaCollector::Clone(std::shared_ptr<cache::IntermediateDataReader> const &)
void CityAreaCollector::CollectFeature(FeatureBuilder const & feature, OsmElement const &)
{
if (feature.IsArea() && ftypes::IsCityTownOrVillage(feature.GetTypes()))
m_boundaries.emplace_back(feature);
if (!(feature.IsArea() && ftypes::IsCityTownOrVillage(feature.GetTypes())))
return;
auto copy = feature;
if (copy.PreSerialize())
m_witer->Write(copy);
}
void CityAreaCollector::Finish()
{
m_witer.reset({});
}
void CityAreaCollector::Save()
{
FeatureBuilderWriter<serialization_policy::MaxAccuracy> collector(GetFilename());
for (auto & boundary : m_boundaries)
{
if (boundary.PreSerialize())
collector.Write(boundary);
}
if (Platform::IsFileExistsByFullPath(GetTmpFilename()))
CHECK(base::CopyFileX(GetTmpFilename(), GetFilename()), ());
}
void CityAreaCollector::Merge(generator::CollectorInterface const & collector)
@ -44,7 +57,6 @@ void CityAreaCollector::Merge(generator::CollectorInterface const & collector)
void CityAreaCollector::MergeInto(CityAreaCollector & collector) const
{
std::copy(std::begin(m_boundaries), std::end(m_boundaries),
std::back_inserter(collector.m_boundaries));
base::AppendFileToFile(GetTmpFilename(), collector.GetTmpFilename());
}
} // namespace generator

4
generator/collector_city_area.hpp
View file

@ -4,7 +4,6 @@
#include "generator/feature_builder.hpp"
#include <memory>
#include <vector>
namespace generator
{
@ -23,12 +22,13 @@ public:
Clone(std::shared_ptr<cache::IntermediateDataReader> const & = {}) const override;
void CollectFeature(feature::FeatureBuilder const & feature, OsmElement const &) override;
void Finish() override;
void Save() override;
void Merge(generator::CollectorInterface const & collector) override;
void MergeInto(CityAreaCollector & collector) const override;
private:
std::vector<feature::FeatureBuilder> m_boundaries;
std::unique_ptr<feature::FeatureBuilderWriter<feature::serialization_policy::MaxAccuracy>> m_witer;
};
} // namespace generator

6
generator/collector_collection.cpp
View file

@ -35,6 +35,12 @@ void CollectorCollection::CollectFeature(FeatureBuilder const & feature, OsmElem
c->CollectFeature(feature, element);
}
void CollectorCollection::Finish()
{
for (auto & c : m_collection)
c->Finish();
}
void CollectorCollection::Save()
{
for (auto & c : m_collection)

1
generator/collector_collection.hpp
View file

@ -31,6 +31,7 @@ public:
void Collect(OsmElement const & element) override;
void CollectRelation(RelationElement const & element) override;
void CollectFeature(feature::FeatureBuilder const & feature, OsmElement const & element) override;
void Finish() override;
void Save() override;
void Merge(CollectorInterface const & collector) override;

19
generator/collector_interface.hpp
View file

@ -1,7 +1,12 @@
#pragma once
#include "base/assert.hpp"
#include "platform/platform.hpp"
#include "base/assert.hpp"
#include "base/logging.hpp"
#include <atomic>
#include <fstream>
#include <memory>
#include <string>
@ -48,8 +53,8 @@ class CollectorRegionInfo;
class CollectorInterface
{
public:
CollectorInterface(std::string const & filename = {}) : m_filename(filename) {}
virtual ~CollectorInterface() = default;
CollectorInterface(std::string const & filename = {}) : m_id(CreateId()), m_filename(filename) {}
virtual ~CollectorInterface() { CHECK(Platform::RemoveFileIfExists(GetTmpFilename()), ()); }
virtual std::shared_ptr<CollectorInterface>
Clone(std::shared_ptr<cache::IntermediateDataReader> const & = {}) const = 0;
@ -57,6 +62,7 @@ public:
virtual void Collect(OsmElement const &) {}
virtual void CollectRelation(RelationElement const &) {}
virtual void CollectFeature(feature::FeatureBuilder const &, OsmElement const &) {}
virtual void Finish() {}
virtual void Save() = 0;
virtual void Merge(CollectorInterface const &) = 0;
@ -72,11 +78,18 @@ public:
virtual void MergeInto(regions::CollectorRegionInfo &) const { FailIfMethodUnsupported(); }
virtual void MergeInto(CollectorCollection &) const { FailIfMethodUnsupported(); }
std::string GetTmpFilename() const { return m_filename + "." + std::to_string(m_id); }
std::string const & GetFilename() const { return m_filename; }
private:
void FailIfMethodUnsupported() const { CHECK(false, ("This method is unsupported.")); }
int CreateId()
{
static std::atomic_int id{0};
return id++;
}
int m_id;
std::string m_filename;
};
} // namespace generator

44
generator/collector_tag.cpp
View file

@ -3,51 +3,49 @@
#include "generator/intermediate_data.hpp"
#include "generator/osm_element.hpp"
#include "platform/platform.hpp"
#include "coding/internal/file_data.hpp"
#include "base/assert.hpp"
#include "base/geo_object_id.hpp"
#include "base/logging.hpp"
namespace generator
{
CollectorTag::CollectorTag(std::string const & filename, std::string const & tagKey,
Validator const & validator, bool ignoreIfNotOpen)
Validator const & validator)
: CollectorInterface(filename)
, m_tagKey(tagKey)
, m_validator(validator)
, m_ignoreIfNotOpen(ignoreIfNotOpen) {}
{
m_writer.exceptions(std::fstream::failbit | std::fstream::badbit);
m_writer.open(GetTmpFilename());
}
std::shared_ptr<CollectorInterface>
CollectorTag::Clone(std::shared_ptr<cache::IntermediateDataReader> const &) const
{
return std::make_shared<CollectorTag>(GetFilename(), m_tagKey, m_validator, m_ignoreIfNotOpen);
return std::make_shared<CollectorTag>(GetFilename(), m_tagKey, m_validator);
}
void CollectorTag::Collect(OsmElement const & el)
{
auto const tag = el.GetTag(m_tagKey);
if (!tag.empty() && m_validator(tag))
m_stream << GetGeoObjectId(el).GetEncodedId() << "\t" << tag << "\n";
m_writer << GetGeoObjectId(el).GetEncodedId() << "\t" << tag << "\n";
}
void CollectorTag::Finish()
{
if (m_writer.is_open())
m_writer.close();
}
void CollectorTag::Save()
{
std::ofstream stream;
stream.exceptions(std::fstream::failbit | std::fstream::badbit);
try
{
stream.open(GetFilename());
stream << m_stream.str();
}
catch (std::ios::failure const & e)
{
if (m_ignoreIfNotOpen)
{
LOG(LINFO, ("Сould not open file", GetFilename(), ". This was ignored."));
}
else
{
throw e;
}
}
if (Platform::IsFileExistsByFullPath(GetTmpFilename()))
CHECK(base::CopyFileX(GetTmpFilename(), GetFilename()), ());
}
void CollectorTag::Merge(CollectorInterface const & collector)
@ -57,6 +55,6 @@ void CollectorTag::Merge(CollectorInterface const & collector)
void CollectorTag::MergeInto(CollectorTag & collector) const
{
collector.m_stream << m_stream.str();
base::AppendFileToFile(GetTmpFilename(), collector.GetTmpFilename());
}
} // namespace generator

8
generator/collector_tag.hpp
View file

@ -5,7 +5,6 @@
#include <fstream>
#include <functional>
#include <memory>
#include <sstream>
#include <string>
struct OsmElement;
@ -29,21 +28,22 @@ public:
using Validator = std::function<bool(std::string const & tagValue)>;
explicit CollectorTag(std::string const & filename, std::string const & tagKey,
Validator const & validator, bool ignoreIfNotOpen = false);
Validator const & validator);
// CollectorInterface overrides:
std::shared_ptr<CollectorInterface>
Clone(std::shared_ptr<cache::IntermediateDataReader> const & = {}) const override;
void Collect(OsmElement const & el) override;
void Finish() override;
void Save() override;
void Merge(CollectorInterface const & collector) override;
void MergeInto(CollectorTag & collector) const override;
private:
std::stringstream m_stream;
std::ofstream m_writer;
std::string m_tagKey;
Validator m_validator;
bool m_ignoreIfNotOpen;
};
} // namespace generator

35
generator/maxspeeds_collector.cpp
View file

@ -4,14 +4,16 @@
#include "routing_common/maxspeed_conversion.hpp"
#include "coding/file_writer.hpp"
#include "platform/platform.hpp"
#include "coding/internal/file_data.hpp"
#include "base/assert.hpp"
#include "base/geo_object_id.hpp"
#include "base/logging.hpp"
#include "base/string_utils.hpp"
#include <algorithm>
#include <fstream>
#include <iterator>
#include <sstream>
@ -36,7 +38,11 @@ bool ParseMaxspeedAndWriteToStream(string const & maxspeed, SpeedInUnits & speed
namespace generator
{
MaxspeedsCollector::MaxspeedsCollector(string const & filename)
: CollectorInterface(filename) {}
: CollectorInterface(filename)
{
m_writer.exceptions(fstream::failbit | fstream::badbit);
m_writer.open(GetTmpFilename());
}
shared_ptr<CollectorInterface>
@ -65,7 +71,7 @@ void MaxspeedsCollector::CollectFeature(FeatureBuilder const &, OsmElement const
SpeedInUnits dummySpeed;
if (!ParseMaxspeedAndWriteToStream(t.m_value, dummySpeed, ss))
return;
m_data.push_back(ss.str());
m_writer << ss.str() << '\n';
return;
}
@ -108,21 +114,20 @@ void MaxspeedsCollector::CollectFeature(FeatureBuilder const &, OsmElement const
ss << "," << strings::to_string(maxspeedBackward.GetSpeed());
}
m_data.push_back(ss.str());
m_writer << ss.str() << '\n';
}
void MaxspeedsCollector::Finish()
{
if (m_writer.is_open())
m_writer.close();
}
void MaxspeedsCollector::Save()
{
LOG(LINFO, ("Saving maxspeed tag values to", GetFilename()));
ofstream stream;
stream.exceptions(fstream::failbit | fstream::badbit);
stream.open(GetFilename());
for (auto const & s : m_data)
stream << s << '\n';
LOG(LINFO, ("Wrote", m_data.size(), "maxspeed tags to", GetFilename()));
if (Platform::IsFileExistsByFullPath(GetTmpFilename()))
CHECK(CopyFileX(GetTmpFilename(), GetFilename()), ());
}
void MaxspeedsCollector::Merge(CollectorInterface const & collector)
@ -132,6 +137,6 @@ void MaxspeedsCollector::Merge(CollectorInterface const & collector)
void MaxspeedsCollector::MergeInto(MaxspeedsCollector & collector) const
{
copy(begin(m_data), end(m_data), back_inserter(collector.m_data));
base::AppendFileToFile(GetTmpFilename(), collector.GetTmpFilename());
}
} // namespace generator

5
generator/maxspeeds_collector.hpp
View file

@ -4,9 +4,9 @@
#include "generator/feature_builder.hpp"
#include "generator/osm_element.hpp"
#include <strstream>
#include <memory>
#include <string>
#include <vector>
namespace generator
{
@ -29,6 +29,7 @@ public:
Clone(std::shared_ptr<cache::IntermediateDataReader> const & = {}) const override;
void CollectFeature(feature::FeatureBuilder const &, OsmElement const & p) override;
void Finish() override;
void Save() override;
void Merge(CollectorInterface const & collector) override;
@ -57,6 +58,6 @@ private:
// with ParseMaxspeedTag() function. That means all macro like RU:urban or GE:rural
// are converted to an appropriate speed value and macro "none" and "walk" are converted
// to |kNoneMaxSpeed| and |kWalkMaxSpeed|.
std::vector<std::string> m_data;
std::ofstream m_writer;
};
} // namespace generator

2
generator/regions/collector_region_info.hpp
View file

@ -127,10 +127,10 @@ public:
CollectorRegionInfo(std::string const & filename);
// CollectorInterface overrides:
void Collect(OsmElement const & el) override;
std::shared_ptr<CollectorInterface>
Clone(std::shared_ptr<cache::IntermediateDataReader> const & = {}) const override;
void Collect(OsmElement const & el) override;
void Save() override;
void Merge(CollectorInterface const & collector) override;

42
generator/restriction_writer.cpp
View file

@ -7,6 +7,10 @@
#include "routing/restrictions_serialization.hpp"
#include "platform/platform.hpp"
#include "coding/internal/file_data.hpp"
#include "base/assert.hpp"
#include "base/geo_object_id.hpp"
#include "base/logging.hpp"
@ -92,17 +96,19 @@ std::string const RestrictionWriter::kNodeString = "node";
std::string const RestrictionWriter::kWayString = "way";
RestrictionWriter::RestrictionWriter(std::string const & filename,
generator::cache::IntermediateDataReader const & cache)
std::shared_ptr<generator::cache::IntermediateDataReader> const & cache)
: generator::CollectorInterface(filename)
, m_cache(cache)
{
m_stream << std::setprecision(20);
m_writer.exceptions(std::fstream::failbit | std::fstream::badbit);
m_writer.open(GetTmpFilename());
m_writer << std::setprecision(20);
}
std::shared_ptr<generator::CollectorInterface>
RestrictionWriter::Clone(std::shared_ptr<generator::cache::IntermediateDataReader> const & cache) const
{
return std::make_shared<RestrictionWriter>(GetFilename(), *cache);
{
return std::make_shared<RestrictionWriter>(GetFilename(), cache ? cache : m_cache);
}
//static
@ -177,38 +183,42 @@ void RestrictionWriter::CollectRelation(RelationElement const & relationElement)
: ViaType::Way;
auto const printHeader = [&]() {
m_stream << DebugPrint(type) << "," << DebugPrint(viaType) << ",";
m_writer << DebugPrint(type) << "," << DebugPrint(viaType) << ",";
};
if (viaType == ViaType::Way)
{
printHeader();
m_stream << fromOsmId << ",";
m_writer << fromOsmId << ",";
for (auto const & viaMember : via)
m_stream << viaMember.first << ",";
m_writer << viaMember.first << ",";
}
else
{
double y = 0.0;
double x = 0.0;
uint64_t const viaNodeOsmId = via.back().first;
if (!m_cache.GetNode(viaNodeOsmId, y, x))
if (!m_cache->GetNode(viaNodeOsmId, y, x))
return;
printHeader();
m_stream << x << "," << y << ",";
m_stream << fromOsmId << ",";
m_writer << x << "," << y << ",";
m_writer << fromOsmId << ",";
}
m_stream << toOsmId << '\n';
m_writer << toOsmId << '\n';
}
void RestrictionWriter::Finish()
{
if (m_writer.is_open())
m_writer.close();
}
void RestrictionWriter::Save()
{
std::ofstream stream;
stream.exceptions(std::fstream::failbit | std::fstream::badbit);
stream.open(GetFilename());
stream << m_stream.str();
if (Platform::IsFileExistsByFullPath(GetTmpFilename()))
CHECK(base::CopyFileX(GetTmpFilename(), GetFilename()), ());
}
void RestrictionWriter::Merge(generator::CollectorInterface const & collector)
@ -218,7 +228,7 @@ void RestrictionWriter::Merge(generator::CollectorInterface const & collector)
void RestrictionWriter::MergeInto(RestrictionWriter & collector) const
{
collector.m_stream << m_stream.str();
base::AppendFileToFile(GetTmpFilename(), collector.GetTmpFilename());
}
std::string DebugPrint(RestrictionWriter::ViaType const & type)

9
generator/restriction_writer.hpp
View file

@ -3,8 +3,8 @@
#include "generator/collector_interface.hpp"
#include "generator/intermediate_data.hpp"
#include <fstream>
#include <memory>
#include <sstream>
#include <string>
class RelationElement;
@ -35,13 +35,14 @@ public:
static std::string const kWayString;
RestrictionWriter(std::string const & filename,
generator::cache::IntermediateDataReader const & cache);
std::shared_ptr<generator::cache::IntermediateDataReader> const & cache);
// generator::CollectorInterface overrides:
std::shared_ptr<CollectorInterface>
Clone(std::shared_ptr<generator::cache::IntermediateDataReader> const & cache = {}) const override;
void CollectRelation(RelationElement const & relationElement) override;
void Finish() override;
void Save() override;
void Merge(generator::CollectorInterface const & collector) override;
@ -50,8 +51,8 @@ public:
static ViaType ConvertFromString(std::string const & str);
private:
std::stringstream m_stream;
generator::cache::IntermediateDataReader const & m_cache;
std::ofstream m_writer;
std::shared_ptr<generator::cache::IntermediateDataReader> m_cache;
};
std::string DebugPrint(RestrictionWriter::ViaType const & type);

117
generator/road_access_generator.cpp
View file

@ -14,21 +14,21 @@
#include "coding/file_container.hpp"
#include "coding/file_writer.hpp"
#include "coding/internal/file_data.hpp"
#include "base/assert.hpp"
#include "base/geo_object_id.hpp"
#include "base/logging.hpp"
#include "base/string_utils.hpp"
#include <initializer_list>
#include "defines.hpp"
#include <algorithm>
#include <initializer_list>
#include <unordered_map>
#include <utility>
#include "boost/optional.hpp"
#include <boost/optional.hpp>
#include "defines.hpp"
using namespace feature;
using namespace routing;
@ -289,7 +289,7 @@ RoadAccessTagProcessor::RoadAccessTagProcessor(VehicleType vehicleType)
}
}
void RoadAccessTagProcessor::Process(FeatureBuilder const & fb, OsmElement const & elem)
void RoadAccessTagProcessor::Process(OsmElement const & elem)
{
// We will process all nodes before ways because of o5m format:
// all nodes are first, then all ways, then all relations.
@ -300,21 +300,15 @@ void RoadAccessTagProcessor::Process(FeatureBuilder const & fb, OsmElement const
m_barriers.emplace(elem.m_id, accessType);
return;
}
if (elem.m_type != OsmElement::EntityType::Way)
return;
auto const accessType = GetAccessType(elem);
if (accessType != RoadAccess::Type::Yes)
m_wayToAccess.emplace(elem.m_id, accessType);
if (!routing::IsRoad(fb.GetTypes()))
return;
m_roads.emplace(elem.m_id, elem.m_nodes);
else if (elem.m_type == OsmElement::EntityType::Way)
{
auto const accessType = GetAccessType(elem);
if (accessType != RoadAccess::Type::Yes)
m_wayToAccess.emplace(elem.m_id, accessType);
}
}
void RoadAccessTagProcessor::Write(std::stringstream & stream)
void RoadAccessTagProcessor::WriteWayToAccess(std::ostream & stream)
{
// All feature tags.
for (auto const & i : m_wayToAccess)
@ -322,21 +316,21 @@ void RoadAccessTagProcessor::Write(std::stringstream & stream)
stream << ToString(m_vehicleType) << " " << ToString(i.second) << " " << i.first << " "
<< 0 /* wildcard segment Idx */ << endl;
}
}
// Barrier tags.
for (auto const & i : m_roads)
void RoadAccessTagProcessor::WriteBarrierTags(std::ostream & stream, uint64_t id,
std::vector<uint64_t> const & points)
{
for (size_t pointIdx = 0; pointIdx < points.size(); ++pointIdx)
{
for (size_t pointIdx = 0; pointIdx < i.second.size(); ++pointIdx)
{
auto const it = m_barriers.find(i.second[pointIdx]);
if (it == m_barriers.cend())
continue;
auto const it = m_barriers.find(points[pointIdx]);
if (it == m_barriers.cend())
continue;
RoadAccess::Type const roadAccessType = it->second;
// idx == 0 used as wildcard segment Idx, for nodes we store |pointIdx + 1| instead of |pointIdx|.
stream << ToString(m_vehicleType) << " " << ToString(roadAccessType) << " " << i.first << " "
<< pointIdx + 1 << endl;
}
RoadAccess::Type const roadAccessType = it->second;
// idx == 0 used as wildcard segment Idx, for nodes we store |pointIdx + 1| instead of |pointIdx|.
stream << ToString(m_vehicleType) << " " << ToString(roadAccessType) << " " << id << " "
<< pointIdx + 1 << endl;
}
}
@ -344,9 +338,8 @@ void RoadAccessTagProcessor::Merge(RoadAccessTagProcessor const & other)
{
CHECK_EQUAL(m_vehicleType, other.m_vehicleType, ());
m_barriers.insert(std::begin(other.m_barriers), std::end(other.m_barriers));
m_wayToAccess.insert(std::begin(other.m_wayToAccess), std::end(other.m_wayToAccess));
m_roads.insert(std::begin(other.m_roads), std::end(other.m_roads));
m_barriers.insert(begin(other.m_barriers), end(other.m_barriers));
m_wayToAccess.insert(begin(other.m_wayToAccess), end(other.m_wayToAccess));
}
bool RoadAccessTagProcessor::ShouldIgnoreBarrierWithoutAccess(OsmElement const & osmElement) const
@ -391,33 +384,65 @@ RoadAccess::Type RoadAccessTagProcessor::GetAccessType(OsmElement const & elem)
// RoadAccessWriter ------------------------------------------------------------
RoadAccessWriter::RoadAccessWriter(string const & filename)
: generator::CollectorInterface(filename)
, m_waysFilename(GetTmpFilename() + ".roads_ids")
, m_waysWriter(make_unique<FileWriter>(m_waysFilename))
{
for (size_t i = 0; i < static_cast<size_t>(VehicleType::Count); ++i)
m_tagProcessors.emplace_back(static_cast<VehicleType>(i));
}
std::shared_ptr<generator::CollectorInterface>
RoadAccessWriter::Clone(std::shared_ptr<generator::cache::IntermediateDataReader> const &) const
RoadAccessWriter::~RoadAccessWriter()
{
return std::make_shared<RoadAccessWriter>(GetFilename());
CHECK(Platform::RemoveFileIfExists(m_waysFilename), ());
}
shared_ptr<generator::CollectorInterface>
RoadAccessWriter::Clone(shared_ptr<generator::cache::IntermediateDataReader> const &) const
{
return make_shared<RoadAccessWriter>(GetFilename());
}
void RoadAccessWriter::CollectFeature(FeatureBuilder const & fb, OsmElement const & elem)
{
for (auto & p : m_tagProcessors)
p.Process(fb, elem);
p.Process(elem);
if (!routing::IsRoad(fb.GetTypes()))
return;
m_waysWriter->Write(&elem.m_id, sizeof(elem.m_id));
rw::WriteVectorOfPOD(*m_waysWriter, elem.m_nodes);
}
void RoadAccessWriter::Finish()
{
m_waysWriter.reset({});
}
void RoadAccessWriter::Save()
{
std::stringstream stream;
for (auto & p : m_tagProcessors)
p.Write(stream);
std::ofstream out;
out.exceptions(std::fstream::failbit | std::fstream::badbit);
ofstream out;
out.exceptions(fstream::failbit | fstream::badbit);
out.open(GetFilename());
out << stream.str();
for (auto & p : m_tagProcessors)
p.WriteWayToAccess(out);
FileReader reader(m_waysFilename);
ReaderSource<FileReader> src(reader);
auto const fileSize = reader.Size();
auto currPos = reader.GetOffset();
while (currPos < fileSize)
{
uint64_t wayId;
std::vector<uint64_t> nodes;
src.Read(&wayId, sizeof(wayId));
rw::ReadVectorOfPOD(src, nodes);
for (auto & p : m_tagProcessors)
p.WriteBarrierTags(out, wayId, nodes);
currPos = src.Pos();
}
}
void RoadAccessWriter::Merge(generator::CollectorInterface const & collector)
@ -432,6 +457,8 @@ void RoadAccessWriter::MergeInto(RoadAccessWriter & collector) const
for (size_t i = 0; i < otherProcessors.size(); ++i)
otherProcessors[i].Merge(m_tagProcessors[i]);
base::AppendFileToFile(m_waysFilename, collector.m_waysFilename);
}
// RoadAccessCollector ----------------------------------------------------------

12
generator/road_access_generator.hpp
View file

@ -43,8 +43,9 @@ public:
explicit RoadAccessTagProcessor(VehicleType vehicleType);
void Process(feature::FeatureBuilder const & fb, OsmElement const & elem);
void Write(std::stringstream & stream);
void Process(OsmElement const & elem);
void WriteWayToAccess(std::ostream & stream);
void WriteBarrierTags(std::ostream & stream, uint64_t id, std::vector<uint64_t> const & points);
void Merge(RoadAccessTagProcessor const & roadAccessTagProcessor);
private:
@ -65,7 +66,6 @@ private:
std::unordered_map<uint64_t, RoadAccess::Type> m_barriers;
std::unordered_map<uint64_t, RoadAccess::Type> m_wayToAccess;
std::unordered_map<uint64_t, std::vector<uint64_t>> m_roads;
};
class RoadAccessWriter : public generator::CollectorInterface
@ -74,17 +74,21 @@ public:
explicit RoadAccessWriter(std::string const & filename);
// CollectorInterface overrides:
~RoadAccessWriter() override;
std::shared_ptr<CollectorInterface>
Clone(std::shared_ptr<generator::cache::IntermediateDataReader> const & = {}) const override;
void CollectFeature(feature::FeatureBuilder const & fb, OsmElement const & elem) override;
void Finish() override;
void Save() override;
void Merge(generator::CollectorInterface const & collector) override;
void MergeInto(RoadAccessWriter & collector) const override;
private:
std::ofstream m_stream;
std::string m_waysFilename;
std::unique_ptr<FileWriter> m_waysWriter;
std::vector<RoadAccessTagProcessor> m_tagProcessors;
};

1
generator/translator.cpp
View file

@ -50,6 +50,7 @@ void Translator::Emit(OsmElement & element)
bool Translator::Finish()
{
m_collectors.Finish();
m_collectors.Save();
return m_emitter->Finish();
}

8
generator/translator_country.cpp
View file

@ -78,18 +78,18 @@ TranslatorCountry::TranslatorCountry(std::shared_ptr<EmitterInterface> emitter,
AddFilter(std::make_shared<FilterPlanet>());
AddFilter(std::make_shared<FilterElements>(base::JoinPath(GetPlatform().ResourcesDir(), SKIPPED_ELEMENTS_FILE)));
AddCollector(std::make_shared<CollectorTag>(info.m_idToWikidataFilename, "wikidata" /* tagKey */,
WikiDataValidator, true /* ignoreIfNotOpen */));
AddCollector(std::make_shared<feature::MetalinesBuilder>(info.GetIntermediateFileName(METALINES_FILENAME)));
// These are the four collector that collect additional information for the future building of routing section.
AddCollector(std::make_shared<MaxspeedsCollector>(info.GetIntermediateFileName(MAXSPEEDS_FILENAME)));
AddCollector(std::make_shared<routing::RestrictionWriter>(info.GetIntermediateFileName(RESTRICTIONS_FILENAME), cache));
AddCollector(std::make_shared<routing::RestrictionWriter>(info.GetIntermediateFileName(RESTRICTIONS_FILENAME),
std::shared_ptr<cache::IntermediateDataReader>(&cache, [](cache::IntermediateDataReader *) {})));
AddCollector(std::make_shared<routing::RoadAccessWriter>(info.GetIntermediateFileName(ROAD_ACCESS_FILENAME)));
AddCollector(std::make_shared<routing::CameraCollector>(info.GetIntermediateFileName(CAMERAS_TO_WAYS_FILENAME)));
if (info.m_genAddresses)
AddCollector(std::make_shared<CollectorAddresses>(info.GetAddressesFileName()));
if (!info.m_idToWikidataFilename.empty())
AddCollector(std::make_shared<CollectorTag>(info.m_idToWikidataFilename, "wikidata" /* tagKey */, WikiDataValidator));
}
void TranslatorCountry::Preprocess(OsmElement & element)