diff --git a/coding/internal/file_data.cpp b/coding/internal/file_data.cpp index 79a9bf3344..1375012eb5 100644 --- a/coding/internal/file_data.cpp +++ b/coding/internal/file_data.cpp @@ -291,6 +291,22 @@ bool WriteToTempAndRenameToFile(string const & dest, functionin_avail()) + to << from.rdbuf(); +} + + bool CopyFileX(string const & fOld, string const & fNew) { try @@ -303,6 +319,7 @@ bool CopyFileX(string const & fOld, string const & fNew) if (ifs.peek() == ifstream::traits_type::eof()) return true; + ofs << ifs.rdbuf(); ofs.flush(); diff --git a/coding/internal/file_data.hpp b/coding/internal/file_data.hpp index f46c2bea4a..daa8b60cfd 100644 --- a/coding/internal/file_data.hpp +++ b/coding/internal/file_data.hpp @@ -70,6 +70,8 @@ bool WriteToTempAndRenameToFile(std::string const & dest, std::function const & write, std::string const & tmp = ""); +void AppendFileToFile(std::string const & fromFilename, std::string const & toFilename); + /// @return false if copy fails. DO NOT THROWS exceptions bool CopyFileX(std::string const & fOld, std::string const & fNew); bool IsEqualFiles(std::string const & firstFile, std::string const & secondFile); diff --git a/generator/collector_addresses.cpp b/generator/collector_addresses.cpp index cd2ea7b0a0..d1d2e374f0 100644 --- a/generator/collector_addresses.cpp +++ b/generator/collector_addresses.cpp @@ -5,14 +5,22 @@ #include "indexer/ftypes_matcher.hpp" -using namespace feature; +#include "platform/platform.hpp" -#include +#include "coding/internal/file_data.hpp" + +#include "base/assert.hpp" + +using namespace feature; namespace generator { CollectorAddresses::CollectorAddresses(std::string const & filename) - : CollectorInterface(filename) {} + : CollectorInterface(filename) +{ + m_writer.exceptions(std::fstream::failbit | std::fstream::badbit); + m_writer.open(GetTmpFilename()); +} std::shared_ptr CollectorAddresses::Clone(std::shared_ptr const &) const @@ -25,15 +33,19 @@ void CollectorAddresses::CollectFeature(feature::FeatureBuilder const & feature, std::string addr; auto const & checker = ftypes::IsBuildingChecker::Instance(); if (checker(feature.GetTypes()) && feature.FormatFullAddress(addr)) - m_stringStream << addr << "\n"; + m_writer << addr << "\n"; +} + +void CollectorAddresses::Finish() +{ + if (m_writer.is_open()) + m_writer.close(); } void CollectorAddresses::Save() { - std::ofstream stream; - stream.exceptions(std::fstream::failbit | std::fstream::badbit); - stream.open(GetFilename()); - stream << m_stringStream.str(); + if (Platform::IsFileExistsByFullPath(GetTmpFilename())) + CHECK(base::CopyFileX(GetTmpFilename(), GetFilename()), ()); } void CollectorAddresses::Merge(CollectorInterface const & collector) @@ -43,6 +55,6 @@ void CollectorAddresses::Merge(CollectorInterface const & collector) void CollectorAddresses::MergeInto(CollectorAddresses & collector) const { - collector.m_stringStream << m_stringStream.str(); + base::AppendFileToFile(GetTmpFilename(), collector.GetTmpFilename()); } } // namespace generator diff --git a/generator/collector_addresses.hpp b/generator/collector_addresses.hpp index 87c72cf709..554438a1df 100644 --- a/generator/collector_addresses.hpp +++ b/generator/collector_addresses.hpp @@ -2,8 +2,8 @@ #include "generator/collector_interface.hpp" +#include #include -#include #include namespace generator @@ -23,12 +23,13 @@ public: Clone(std::shared_ptr const & = {}) const override; void CollectFeature(feature::FeatureBuilder const & feature, OsmElement const &) override; + void Finish() override; void Save() override; void Merge(CollectorInterface const & collector) override; void MergeInto(CollectorAddresses & collector) const override; private: - std::stringstream m_stringStream; + std::ofstream m_writer; }; } // namespace generator diff --git a/generator/collector_camera.cpp b/generator/collector_camera.cpp index 1669c029d7..5c9623d579 100644 --- a/generator/collector_camera.cpp +++ b/generator/collector_camera.cpp @@ -13,8 +13,12 @@ #include "platform/measurement_utils.hpp" +#include "platform/platform.hpp" + +#include "coding/internal/file_data.hpp" #include "coding/point_coding.hpp" #include "coding/write_to_sink.hpp" +#include "coding/reader_writer_ops.hpp" #include "geometry/latlon.hpp" @@ -27,7 +31,6 @@ using namespace feature; - namespace routing { size_t const CameraProcessor::kMaxSpeedSpeedStringLength = 32; @@ -47,8 +50,23 @@ CameraProcessor::CameraInfo::CameraInfo(OsmElement const & element) , m_lat(element.m_lat) { auto const maxspeed = element.GetTag("maxspeed"); - if (!maxspeed.empty()) - m_speed = ValidateMaxSpeedString(maxspeed); + if (maxspeed.empty()) + return; + + auto const validatedMaxspeed = ValidateMaxSpeedString(maxspeed); + if (validatedMaxspeed.size() > kMaxSpeedSpeedStringLength || + !strings::to_int(validatedMaxspeed.c_str(), m_speed)) + { + LOG(LWARNING, ("Bad speed format of camera:", maxspeed, ", osmId:", element.m_id)); + } +} + +CameraProcessor::CameraProcessor(std::string const & filename) + : m_waysFilename(filename + ".roads_ids"), m_waysWriter(std::make_unique(m_waysFilename)) {} + +CameraProcessor::~CameraProcessor() +{ + CHECK(Platform::RemoveFileIfExists(m_waysFilename), ()); } void CameraProcessor::ForEachCamera(Fn && toDo) const @@ -63,29 +81,44 @@ void CameraProcessor::ForEachCamera(Fn && toDo) const void CameraProcessor::ProcessWay(OsmElement const & element) { - m_ways[element.m_id] = element.m_nodes; + m_waysWriter->Write(&element.m_id, sizeof(element.m_id)); + rw::WriteVectorOfPOD(*m_waysWriter, element.m_nodes); } void CameraProcessor::FillCameraInWays() { - for (auto const & way : m_ways) + FileReader reader(m_waysFilename); + ReaderSource src(reader); + auto const fileSize = reader.Size(); + auto currPos = reader.GetOffset(); + while (currPos < fileSize) { - for (auto const & node : way.second) + uint64_t wayId; + std::vector nodes; + src.Read(&wayId, sizeof(wayId)); + rw::ReadVectorOfPOD(src, nodes); + for (auto const & node : nodes) { auto const itCamera = m_speedCameras.find(node); if (itCamera == m_speedCameras.cend()) continue; - m_cameraToWays[itCamera->first].push_back(way.first); + m_cameraToWays[itCamera->first].push_back(wayId); } + currPos = src.Pos(); } } void CameraProcessor::ProcessNode(OsmElement const & element) { CameraInfo camera(element); - CHECK_LESS(camera.m_speed.size(), kMaxSpeedSpeedStringLength, ()); - m_speedCameras.emplace(element.m_id, std::move(camera)); + if (camera.IsValid()) + m_speedCameras.emplace(element.m_id, std::move(camera)); +} + +void CameraProcessor::Finish() +{ + m_waysWriter.reset({}); } void CameraProcessor::Merge(CameraProcessor const & cameraProcessor) @@ -93,12 +126,11 @@ void CameraProcessor::Merge(CameraProcessor const & cameraProcessor) auto const & otherCameras = cameraProcessor.m_speedCameras; m_speedCameras.insert(std::begin(otherCameras), std::end(otherCameras)); - auto const & otherWays = cameraProcessor.m_ways; - m_ways.insert(std::begin(otherWays), std::end(otherWays)); + base::AppendFileToFile(cameraProcessor.m_waysFilename, m_waysFilename); } CameraCollector::CameraCollector(std::string const & filename) : - generator::CollectorInterface(filename) {} + generator::CollectorInterface(filename), m_processor(GetTmpFilename()) {} std::shared_ptr CameraCollector::Clone(std::shared_ptr const &) const @@ -130,13 +162,6 @@ void CameraCollector::CollectFeature(FeatureBuilder const & feature, OsmElement void CameraCollector::Write(FileWriter & writer, CameraProcessor::CameraInfo const & camera, std::vector const & ways) { - std::string maxSpeedStringKmPH = camera.m_speed; - int32_t maxSpeedKmPH = 0; - if (!strings::to_int(maxSpeedStringKmPH.c_str(), maxSpeedKmPH)) - LOG(LWARNING, ("Bad speed format of camera:", maxSpeedStringKmPH, ", osmId:", camera.m_id)); - - CHECK_GREATER_OR_EQUAL(maxSpeedKmPH, 0, ()); - uint32_t const lat = DoubleToUint32(camera.m_lat, ms::LatLon::kMinLat, ms::LatLon::kMaxLat, kPointCoordBits); WriteToSink(writer, lat); @@ -145,7 +170,7 @@ void CameraCollector::Write(FileWriter & writer, CameraProcessor::CameraInfo con DoubleToUint32(camera.m_lon, ms::LatLon::kMinLon, ms::LatLon::kMaxLon, kPointCoordBits); WriteToSink(writer, lon); - WriteToSink(writer, static_cast(maxSpeedKmPH)); + WriteToSink(writer, static_cast(camera.m_speed)); auto const size = static_cast(ways.size()); WriteToSink(writer, size); @@ -153,6 +178,11 @@ void CameraCollector::Write(FileWriter & writer, CameraProcessor::CameraInfo con WriteToSink(writer, wayId); } +void CameraCollector::Finish() +{ + m_processor.Finish(); +} + void CameraCollector::Save() { using namespace std::placeholders; @@ -170,6 +200,6 @@ void CameraCollector::Merge(generator::CollectorInterface const & collector) void CameraCollector::MergeInto(CameraCollector & collector) const { - collector.m_processor.Merge(this->m_processor); + collector.m_processor.Merge(m_processor); } } // namespace routing diff --git a/generator/collector_camera.hpp b/generator/collector_camera.hpp index 1b5447d117..d0c987c176 100644 --- a/generator/collector_camera.hpp +++ b/generator/collector_camera.hpp @@ -6,6 +6,7 @@ #include #include +#include #include #include #include @@ -52,25 +53,32 @@ public: { CameraInfo(const OsmElement & element); + bool IsValid() const { return m_speed >= 0; } + uint64_t m_id = 0; double m_lon = 0.0; double m_lat = 0.0; - std::string m_speed; + int32_t m_speed = 0; std::vector m_ways; }; static size_t const kMaxSpeedSpeedStringLength; + CameraProcessor(std::string const & filename); + ~CameraProcessor(); + void ForEachCamera(Fn && toDo) const; void ProcessNode(OsmElement const & element); void ProcessWay(OsmElement const & element); void FillCameraInWays(); + void Finish(); void Merge(CameraProcessor const & cameraProcessor); private: - std::unordered_map> m_ways; + std::string m_waysFilename; + std::unique_ptr m_waysWriter; std::unordered_map m_speedCameras; std::unordered_map> m_cameraToWays; }; @@ -88,6 +96,7 @@ public: // We will process all nodes before ways because of o5m format: // all nodes are first, then all ways, then all relations. void CollectFeature(feature::FeatureBuilder const & feature, OsmElement const & element) override; + void Finish() override; void Save() override; void Merge(generator::CollectorInterface const & collector) override; diff --git a/generator/collector_city_area.cpp b/generator/collector_city_area.cpp index c2d6a3e208..39c64d11ce 100644 --- a/generator/collector_city_area.cpp +++ b/generator/collector_city_area.cpp @@ -1,19 +1,27 @@ -#include "generator/collector_city_boundary.hpp" +#include "generator/collector_city_area.hpp" #include "generator/feature_generator.hpp" #include "generator/intermediate_data.hpp" #include "indexer/ftypes_matcher.hpp" +#include "platform/platform.hpp" + +#include "coding/internal/file_data.hpp" + +#include "base/assert.hpp" + #include #include using namespace feature; +using namespace feature::serialization_policy; namespace generator { CityAreaCollector::CityAreaCollector(std::string const & filename) - : CollectorInterface(filename) {} + : CollectorInterface(filename), + m_witer(std::make_unique>(GetTmpFilename())) {} std::shared_ptr CityAreaCollector::Clone(std::shared_ptr const &) const @@ -23,18 +31,23 @@ CityAreaCollector::Clone(std::shared_ptr const &) void CityAreaCollector::CollectFeature(FeatureBuilder const & feature, OsmElement const &) { - if (feature.IsArea() && ftypes::IsCityTownOrVillage(feature.GetTypes())) - m_boundaries.emplace_back(feature); + if (!(feature.IsArea() && ftypes::IsCityTownOrVillage(feature.GetTypes()))) + return; + + auto copy = feature; + if (copy.PreSerialize()) + m_witer->Write(copy); +} + +void CityAreaCollector::Finish() +{ + m_witer.reset({}); } void CityAreaCollector::Save() { - FeatureBuilderWriter collector(GetFilename()); - for (auto & boundary : m_boundaries) - { - if (boundary.PreSerialize()) - collector.Write(boundary); - } + if (Platform::IsFileExistsByFullPath(GetTmpFilename())) + CHECK(base::CopyFileX(GetTmpFilename(), GetFilename()), ()); } void CityAreaCollector::Merge(generator::CollectorInterface const & collector) @@ -44,7 +57,6 @@ void CityAreaCollector::Merge(generator::CollectorInterface const & collector) void CityAreaCollector::MergeInto(CityAreaCollector & collector) const { - std::copy(std::begin(m_boundaries), std::end(m_boundaries), - std::back_inserter(collector.m_boundaries)); + base::AppendFileToFile(GetTmpFilename(), collector.GetTmpFilename()); } } // namespace generator diff --git a/generator/collector_city_area.hpp b/generator/collector_city_area.hpp index 9519293ea2..aa66c740d3 100644 --- a/generator/collector_city_area.hpp +++ b/generator/collector_city_area.hpp @@ -4,7 +4,6 @@ #include "generator/feature_builder.hpp" #include -#include namespace generator { @@ -23,12 +22,13 @@ public: Clone(std::shared_ptr const & = {}) const override; void CollectFeature(feature::FeatureBuilder const & feature, OsmElement const &) override; + void Finish() override; void Save() override; void Merge(generator::CollectorInterface const & collector) override; void MergeInto(CityAreaCollector & collector) const override; private: - std::vector m_boundaries; + std::unique_ptr> m_witer; }; } // namespace generator diff --git a/generator/collector_collection.cpp b/generator/collector_collection.cpp index c59a4b7fb3..bb77078272 100644 --- a/generator/collector_collection.cpp +++ b/generator/collector_collection.cpp @@ -35,6 +35,12 @@ void CollectorCollection::CollectFeature(FeatureBuilder const & feature, OsmElem c->CollectFeature(feature, element); } +void CollectorCollection::Finish() +{ + for (auto & c : m_collection) + c->Finish(); +} + void CollectorCollection::Save() { for (auto & c : m_collection) diff --git a/generator/collector_collection.hpp b/generator/collector_collection.hpp index 498aa4f0ae..011f533cdd 100644 --- a/generator/collector_collection.hpp +++ b/generator/collector_collection.hpp @@ -31,6 +31,7 @@ public: void Collect(OsmElement const & element) override; void CollectRelation(RelationElement const & element) override; void CollectFeature(feature::FeatureBuilder const & feature, OsmElement const & element) override; + void Finish() override; void Save() override; void Merge(CollectorInterface const & collector) override; diff --git a/generator/collector_interface.hpp b/generator/collector_interface.hpp index fd9614dfa0..75836ad019 100644 --- a/generator/collector_interface.hpp +++ b/generator/collector_interface.hpp @@ -1,7 +1,12 @@ #pragma once -#include "base/assert.hpp" +#include "platform/platform.hpp" +#include "base/assert.hpp" +#include "base/logging.hpp" + +#include +#include #include #include @@ -48,8 +53,8 @@ class CollectorRegionInfo; class CollectorInterface { public: - CollectorInterface(std::string const & filename = {}) : m_filename(filename) {} - virtual ~CollectorInterface() = default; + CollectorInterface(std::string const & filename = {}) : m_id(CreateId()), m_filename(filename) {} + virtual ~CollectorInterface() { CHECK(Platform::RemoveFileIfExists(GetTmpFilename()), ()); } virtual std::shared_ptr Clone(std::shared_ptr const & = {}) const = 0; @@ -57,6 +62,7 @@ public: virtual void Collect(OsmElement const &) {} virtual void CollectRelation(RelationElement const &) {} virtual void CollectFeature(feature::FeatureBuilder const &, OsmElement const &) {} + virtual void Finish() {} virtual void Save() = 0; virtual void Merge(CollectorInterface const &) = 0; @@ -72,11 +78,18 @@ public: virtual void MergeInto(regions::CollectorRegionInfo &) const { FailIfMethodUnsupported(); } virtual void MergeInto(CollectorCollection &) const { FailIfMethodUnsupported(); } + std::string GetTmpFilename() const { return m_filename + "." + std::to_string(m_id); } std::string const & GetFilename() const { return m_filename; } private: void FailIfMethodUnsupported() const { CHECK(false, ("This method is unsupported.")); } + int CreateId() + { + static std::atomic_int id{0}; + return id++; + } + int m_id; std::string m_filename; }; } // namespace generator diff --git a/generator/collector_tag.cpp b/generator/collector_tag.cpp index f85241c451..b32119c643 100644 --- a/generator/collector_tag.cpp +++ b/generator/collector_tag.cpp @@ -3,51 +3,49 @@ #include "generator/intermediate_data.hpp" #include "generator/osm_element.hpp" +#include "platform/platform.hpp" + +#include "coding/internal/file_data.hpp" + +#include "base/assert.hpp" #include "base/geo_object_id.hpp" #include "base/logging.hpp" namespace generator { CollectorTag::CollectorTag(std::string const & filename, std::string const & tagKey, - Validator const & validator, bool ignoreIfNotOpen) + Validator const & validator) : CollectorInterface(filename) , m_tagKey(tagKey) , m_validator(validator) - , m_ignoreIfNotOpen(ignoreIfNotOpen) {} +{ + m_writer.exceptions(std::fstream::failbit | std::fstream::badbit); + m_writer.open(GetTmpFilename()); +} std::shared_ptr CollectorTag::Clone(std::shared_ptr const &) const { - return std::make_shared(GetFilename(), m_tagKey, m_validator, m_ignoreIfNotOpen); + return std::make_shared(GetFilename(), m_tagKey, m_validator); } void CollectorTag::Collect(OsmElement const & el) { auto const tag = el.GetTag(m_tagKey); if (!tag.empty() && m_validator(tag)) - m_stream << GetGeoObjectId(el).GetEncodedId() << "\t" << tag << "\n"; + m_writer << GetGeoObjectId(el).GetEncodedId() << "\t" << tag << "\n"; +} + +void CollectorTag::Finish() +{ + if (m_writer.is_open()) + m_writer.close(); } void CollectorTag::Save() { - std::ofstream stream; - stream.exceptions(std::fstream::failbit | std::fstream::badbit); - try - { - stream.open(GetFilename()); - stream << m_stream.str(); - } - catch (std::ios::failure const & e) - { - if (m_ignoreIfNotOpen) - { - LOG(LINFO, ("Сould not open file", GetFilename(), ". This was ignored.")); - } - else - { - throw e; - } - } + if (Platform::IsFileExistsByFullPath(GetTmpFilename())) + CHECK(base::CopyFileX(GetTmpFilename(), GetFilename()), ()); } void CollectorTag::Merge(CollectorInterface const & collector) @@ -57,6 +55,6 @@ void CollectorTag::Merge(CollectorInterface const & collector) void CollectorTag::MergeInto(CollectorTag & collector) const { - collector.m_stream << m_stream.str(); + base::AppendFileToFile(GetTmpFilename(), collector.GetTmpFilename()); } } // namespace generator diff --git a/generator/collector_tag.hpp b/generator/collector_tag.hpp index 57a735c04a..88eef2f841 100644 --- a/generator/collector_tag.hpp +++ b/generator/collector_tag.hpp @@ -5,7 +5,6 @@ #include #include #include -#include #include struct OsmElement; @@ -29,21 +28,22 @@ public: using Validator = std::function; explicit CollectorTag(std::string const & filename, std::string const & tagKey, - Validator const & validator, bool ignoreIfNotOpen = false); + Validator const & validator); // CollectorInterface overrides: std::shared_ptr Clone(std::shared_ptr const & = {}) const override; void Collect(OsmElement const & el) override; + void Finish() override; void Save() override; void Merge(CollectorInterface const & collector) override; void MergeInto(CollectorTag & collector) const override; + private: - std::stringstream m_stream; + std::ofstream m_writer; std::string m_tagKey; Validator m_validator; - bool m_ignoreIfNotOpen; }; } // namespace generator diff --git a/generator/maxspeeds_collector.cpp b/generator/maxspeeds_collector.cpp index abc2f5bec6..1bd76c6ade 100644 --- a/generator/maxspeeds_collector.cpp +++ b/generator/maxspeeds_collector.cpp @@ -4,14 +4,16 @@ #include "routing_common/maxspeed_conversion.hpp" -#include "coding/file_writer.hpp" +#include "platform/platform.hpp" +#include "coding/internal/file_data.hpp" + +#include "base/assert.hpp" #include "base/geo_object_id.hpp" #include "base/logging.hpp" #include "base/string_utils.hpp" #include -#include #include #include @@ -36,7 +38,11 @@ bool ParseMaxspeedAndWriteToStream(string const & maxspeed, SpeedInUnits & speed namespace generator { MaxspeedsCollector::MaxspeedsCollector(string const & filename) - : CollectorInterface(filename) {} + : CollectorInterface(filename) +{ + m_writer.exceptions(fstream::failbit | fstream::badbit); + m_writer.open(GetTmpFilename()); +} shared_ptr @@ -65,7 +71,7 @@ void MaxspeedsCollector::CollectFeature(FeatureBuilder const &, OsmElement const SpeedInUnits dummySpeed; if (!ParseMaxspeedAndWriteToStream(t.m_value, dummySpeed, ss)) return; - m_data.push_back(ss.str()); + m_writer << ss.str() << '\n'; return; } @@ -108,21 +114,20 @@ void MaxspeedsCollector::CollectFeature(FeatureBuilder const &, OsmElement const ss << "," << strings::to_string(maxspeedBackward.GetSpeed()); } - m_data.push_back(ss.str()); + m_writer << ss.str() << '\n'; +} + +void MaxspeedsCollector::Finish() +{ + if (m_writer.is_open()) + m_writer.close(); } void MaxspeedsCollector::Save() { LOG(LINFO, ("Saving maxspeed tag values to", GetFilename())); - - ofstream stream; - stream.exceptions(fstream::failbit | fstream::badbit); - stream.open(GetFilename()); - - for (auto const & s : m_data) - stream << s << '\n'; - - LOG(LINFO, ("Wrote", m_data.size(), "maxspeed tags to", GetFilename())); + if (Platform::IsFileExistsByFullPath(GetTmpFilename())) + CHECK(CopyFileX(GetTmpFilename(), GetFilename()), ()); } void MaxspeedsCollector::Merge(CollectorInterface const & collector) @@ -132,6 +137,6 @@ void MaxspeedsCollector::Merge(CollectorInterface const & collector) void MaxspeedsCollector::MergeInto(MaxspeedsCollector & collector) const { - copy(begin(m_data), end(m_data), back_inserter(collector.m_data)); + base::AppendFileToFile(GetTmpFilename(), collector.GetTmpFilename()); } } // namespace generator diff --git a/generator/maxspeeds_collector.hpp b/generator/maxspeeds_collector.hpp index 1171372945..4a959dc704 100644 --- a/generator/maxspeeds_collector.hpp +++ b/generator/maxspeeds_collector.hpp @@ -4,9 +4,9 @@ #include "generator/feature_builder.hpp" #include "generator/osm_element.hpp" +#include #include #include -#include namespace generator { @@ -29,6 +29,7 @@ public: Clone(std::shared_ptr const & = {}) const override; void CollectFeature(feature::FeatureBuilder const &, OsmElement const & p) override; + void Finish() override; void Save() override; void Merge(CollectorInterface const & collector) override; @@ -57,6 +58,6 @@ private: // with ParseMaxspeedTag() function. That means all macro like RU:urban or GE:rural // are converted to an appropriate speed value and macro "none" and "walk" are converted // to |kNoneMaxSpeed| and |kWalkMaxSpeed|. - std::vector m_data; + std::ofstream m_writer; }; } // namespace generator diff --git a/generator/regions/collector_region_info.hpp b/generator/regions/collector_region_info.hpp index 88368eece5..84a5261090 100644 --- a/generator/regions/collector_region_info.hpp +++ b/generator/regions/collector_region_info.hpp @@ -127,10 +127,10 @@ public: CollectorRegionInfo(std::string const & filename); // CollectorInterface overrides: - void Collect(OsmElement const & el) override; std::shared_ptr Clone(std::shared_ptr const & = {}) const override; + void Collect(OsmElement const & el) override; void Save() override; void Merge(CollectorInterface const & collector) override; diff --git a/generator/restriction_writer.cpp b/generator/restriction_writer.cpp index f859c5d9fb..0f9e2a0173 100644 --- a/generator/restriction_writer.cpp +++ b/generator/restriction_writer.cpp @@ -7,6 +7,10 @@ #include "routing/restrictions_serialization.hpp" +#include "platform/platform.hpp" + +#include "coding/internal/file_data.hpp" + #include "base/assert.hpp" #include "base/geo_object_id.hpp" #include "base/logging.hpp" @@ -92,17 +96,19 @@ std::string const RestrictionWriter::kNodeString = "node"; std::string const RestrictionWriter::kWayString = "way"; RestrictionWriter::RestrictionWriter(std::string const & filename, - generator::cache::IntermediateDataReader const & cache) + std::shared_ptr const & cache) : generator::CollectorInterface(filename) , m_cache(cache) { - m_stream << std::setprecision(20); + m_writer.exceptions(std::fstream::failbit | std::fstream::badbit); + m_writer.open(GetTmpFilename()); + m_writer << std::setprecision(20); } std::shared_ptr RestrictionWriter::Clone(std::shared_ptr const & cache) const -{ - return std::make_shared(GetFilename(), *cache); +{ + return std::make_shared(GetFilename(), cache ? cache : m_cache); } //static @@ -177,38 +183,42 @@ void RestrictionWriter::CollectRelation(RelationElement const & relationElement) : ViaType::Way; auto const printHeader = [&]() { - m_stream << DebugPrint(type) << "," << DebugPrint(viaType) << ","; + m_writer << DebugPrint(type) << "," << DebugPrint(viaType) << ","; }; if (viaType == ViaType::Way) { printHeader(); - m_stream << fromOsmId << ","; + m_writer << fromOsmId << ","; for (auto const & viaMember : via) - m_stream << viaMember.first << ","; + m_writer << viaMember.first << ","; } else { double y = 0.0; double x = 0.0; uint64_t const viaNodeOsmId = via.back().first; - if (!m_cache.GetNode(viaNodeOsmId, y, x)) + if (!m_cache->GetNode(viaNodeOsmId, y, x)) return; printHeader(); - m_stream << x << "," << y << ","; - m_stream << fromOsmId << ","; + m_writer << x << "," << y << ","; + m_writer << fromOsmId << ","; } - m_stream << toOsmId << '\n'; + m_writer << toOsmId << '\n'; +} + +void RestrictionWriter::Finish() +{ + if (m_writer.is_open()) + m_writer.close(); } void RestrictionWriter::Save() { - std::ofstream stream; - stream.exceptions(std::fstream::failbit | std::fstream::badbit); - stream.open(GetFilename()); - stream << m_stream.str(); + if (Platform::IsFileExistsByFullPath(GetTmpFilename())) + CHECK(base::CopyFileX(GetTmpFilename(), GetFilename()), ()); } void RestrictionWriter::Merge(generator::CollectorInterface const & collector) @@ -218,7 +228,7 @@ void RestrictionWriter::Merge(generator::CollectorInterface const & collector) void RestrictionWriter::MergeInto(RestrictionWriter & collector) const { - collector.m_stream << m_stream.str(); + base::AppendFileToFile(GetTmpFilename(), collector.GetTmpFilename()); } std::string DebugPrint(RestrictionWriter::ViaType const & type) diff --git a/generator/restriction_writer.hpp b/generator/restriction_writer.hpp index 1e2fe773b8..ae71784d7f 100644 --- a/generator/restriction_writer.hpp +++ b/generator/restriction_writer.hpp @@ -3,8 +3,8 @@ #include "generator/collector_interface.hpp" #include "generator/intermediate_data.hpp" +#include #include -#include #include class RelationElement; @@ -35,13 +35,14 @@ public: static std::string const kWayString; RestrictionWriter(std::string const & filename, - generator::cache::IntermediateDataReader const & cache); + std::shared_ptr const & cache); // generator::CollectorInterface overrides: std::shared_ptr Clone(std::shared_ptr const & cache = {}) const override; void CollectRelation(RelationElement const & relationElement) override; + void Finish() override; void Save() override; void Merge(generator::CollectorInterface const & collector) override; @@ -50,8 +51,8 @@ public: static ViaType ConvertFromString(std::string const & str); private: - std::stringstream m_stream; - generator::cache::IntermediateDataReader const & m_cache; + std::ofstream m_writer; + std::shared_ptr m_cache; }; std::string DebugPrint(RestrictionWriter::ViaType const & type); diff --git a/generator/road_access_generator.cpp b/generator/road_access_generator.cpp index 7629b7e80a..93d49cc439 100644 --- a/generator/road_access_generator.cpp +++ b/generator/road_access_generator.cpp @@ -14,21 +14,21 @@ #include "coding/file_container.hpp" #include "coding/file_writer.hpp" +#include "coding/internal/file_data.hpp" #include "base/assert.hpp" #include "base/geo_object_id.hpp" #include "base/logging.hpp" #include "base/string_utils.hpp" -#include - -#include "defines.hpp" - #include +#include #include #include -#include "boost/optional.hpp" +#include + +#include "defines.hpp" using namespace feature; using namespace routing; @@ -289,7 +289,7 @@ RoadAccessTagProcessor::RoadAccessTagProcessor(VehicleType vehicleType) } } -void RoadAccessTagProcessor::Process(FeatureBuilder const & fb, OsmElement const & elem) +void RoadAccessTagProcessor::Process(OsmElement const & elem) { // We will process all nodes before ways because of o5m format: // all nodes are first, then all ways, then all relations. @@ -300,21 +300,15 @@ void RoadAccessTagProcessor::Process(FeatureBuilder const & fb, OsmElement const m_barriers.emplace(elem.m_id, accessType); return; } - - if (elem.m_type != OsmElement::EntityType::Way) - return; - - auto const accessType = GetAccessType(elem); - if (accessType != RoadAccess::Type::Yes) - m_wayToAccess.emplace(elem.m_id, accessType); - - if (!routing::IsRoad(fb.GetTypes())) - return; - - m_roads.emplace(elem.m_id, elem.m_nodes); + else if (elem.m_type == OsmElement::EntityType::Way) + { + auto const accessType = GetAccessType(elem); + if (accessType != RoadAccess::Type::Yes) + m_wayToAccess.emplace(elem.m_id, accessType); + } } -void RoadAccessTagProcessor::Write(std::stringstream & stream) +void RoadAccessTagProcessor::WriteWayToAccess(std::ostream & stream) { // All feature tags. for (auto const & i : m_wayToAccess) @@ -322,21 +316,21 @@ void RoadAccessTagProcessor::Write(std::stringstream & stream) stream << ToString(m_vehicleType) << " " << ToString(i.second) << " " << i.first << " " << 0 /* wildcard segment Idx */ << endl; } +} - // Barrier tags. - for (auto const & i : m_roads) +void RoadAccessTagProcessor::WriteBarrierTags(std::ostream & stream, uint64_t id, + std::vector const & points) +{ + for (size_t pointIdx = 0; pointIdx < points.size(); ++pointIdx) { - for (size_t pointIdx = 0; pointIdx < i.second.size(); ++pointIdx) - { - auto const it = m_barriers.find(i.second[pointIdx]); - if (it == m_barriers.cend()) - continue; + auto const it = m_barriers.find(points[pointIdx]); + if (it == m_barriers.cend()) + continue; - RoadAccess::Type const roadAccessType = it->second; - // idx == 0 used as wildcard segment Idx, for nodes we store |pointIdx + 1| instead of |pointIdx|. - stream << ToString(m_vehicleType) << " " << ToString(roadAccessType) << " " << i.first << " " - << pointIdx + 1 << endl; - } + RoadAccess::Type const roadAccessType = it->second; + // idx == 0 used as wildcard segment Idx, for nodes we store |pointIdx + 1| instead of |pointIdx|. + stream << ToString(m_vehicleType) << " " << ToString(roadAccessType) << " " << id << " " + << pointIdx + 1 << endl; } } @@ -344,9 +338,8 @@ void RoadAccessTagProcessor::Merge(RoadAccessTagProcessor const & other) { CHECK_EQUAL(m_vehicleType, other.m_vehicleType, ()); - m_barriers.insert(std::begin(other.m_barriers), std::end(other.m_barriers)); - m_wayToAccess.insert(std::begin(other.m_wayToAccess), std::end(other.m_wayToAccess)); - m_roads.insert(std::begin(other.m_roads), std::end(other.m_roads)); + m_barriers.insert(begin(other.m_barriers), end(other.m_barriers)); + m_wayToAccess.insert(begin(other.m_wayToAccess), end(other.m_wayToAccess)); } bool RoadAccessTagProcessor::ShouldIgnoreBarrierWithoutAccess(OsmElement const & osmElement) const @@ -391,33 +384,65 @@ RoadAccess::Type RoadAccessTagProcessor::GetAccessType(OsmElement const & elem) // RoadAccessWriter ------------------------------------------------------------ RoadAccessWriter::RoadAccessWriter(string const & filename) : generator::CollectorInterface(filename) + , m_waysFilename(GetTmpFilename() + ".roads_ids") + , m_waysWriter(make_unique(m_waysFilename)) { for (size_t i = 0; i < static_cast(VehicleType::Count); ++i) m_tagProcessors.emplace_back(static_cast(i)); } -std::shared_ptr -RoadAccessWriter::Clone(std::shared_ptr const &) const +RoadAccessWriter::~RoadAccessWriter() { - return std::make_shared(GetFilename()); + CHECK(Platform::RemoveFileIfExists(m_waysFilename), ()); +} + +shared_ptr +RoadAccessWriter::Clone(shared_ptr const &) const +{ + return make_shared(GetFilename()); } void RoadAccessWriter::CollectFeature(FeatureBuilder const & fb, OsmElement const & elem) { for (auto & p : m_tagProcessors) - p.Process(fb, elem); + p.Process(elem); + + if (!routing::IsRoad(fb.GetTypes())) + return; + + m_waysWriter->Write(&elem.m_id, sizeof(elem.m_id)); + rw::WriteVectorOfPOD(*m_waysWriter, elem.m_nodes); +} + +void RoadAccessWriter::Finish() +{ + m_waysWriter.reset({}); } void RoadAccessWriter::Save() { - std::stringstream stream; - for (auto & p : m_tagProcessors) - p.Write(stream); - - std::ofstream out; - out.exceptions(std::fstream::failbit | std::fstream::badbit); + ofstream out; + out.exceptions(fstream::failbit | fstream::badbit); out.open(GetFilename()); - out << stream.str(); + + for (auto & p : m_tagProcessors) + p.WriteWayToAccess(out); + + FileReader reader(m_waysFilename); + ReaderSource src(reader); + auto const fileSize = reader.Size(); + auto currPos = reader.GetOffset(); + while (currPos < fileSize) + { + uint64_t wayId; + std::vector nodes; + src.Read(&wayId, sizeof(wayId)); + rw::ReadVectorOfPOD(src, nodes); + for (auto & p : m_tagProcessors) + p.WriteBarrierTags(out, wayId, nodes); + + currPos = src.Pos(); + } } void RoadAccessWriter::Merge(generator::CollectorInterface const & collector) @@ -432,6 +457,8 @@ void RoadAccessWriter::MergeInto(RoadAccessWriter & collector) const for (size_t i = 0; i < otherProcessors.size(); ++i) otherProcessors[i].Merge(m_tagProcessors[i]); + + base::AppendFileToFile(m_waysFilename, collector.m_waysFilename); } // RoadAccessCollector ---------------------------------------------------------- diff --git a/generator/road_access_generator.hpp b/generator/road_access_generator.hpp index e33e420a4a..48a27c2e2e 100644 --- a/generator/road_access_generator.hpp +++ b/generator/road_access_generator.hpp @@ -43,8 +43,9 @@ public: explicit RoadAccessTagProcessor(VehicleType vehicleType); - void Process(feature::FeatureBuilder const & fb, OsmElement const & elem); - void Write(std::stringstream & stream); + void Process(OsmElement const & elem); + void WriteWayToAccess(std::ostream & stream); + void WriteBarrierTags(std::ostream & stream, uint64_t id, std::vector const & points); void Merge(RoadAccessTagProcessor const & roadAccessTagProcessor); private: @@ -65,7 +66,6 @@ private: std::unordered_map m_barriers; std::unordered_map m_wayToAccess; - std::unordered_map> m_roads; }; class RoadAccessWriter : public generator::CollectorInterface @@ -74,17 +74,21 @@ public: explicit RoadAccessWriter(std::string const & filename); // CollectorInterface overrides: + ~RoadAccessWriter() override; + std::shared_ptr Clone(std::shared_ptr const & = {}) const override; void CollectFeature(feature::FeatureBuilder const & fb, OsmElement const & elem) override; + void Finish() override; void Save() override; void Merge(generator::CollectorInterface const & collector) override; void MergeInto(RoadAccessWriter & collector) const override; private: - std::ofstream m_stream; + std::string m_waysFilename; + std::unique_ptr m_waysWriter; std::vector m_tagProcessors; }; diff --git a/generator/translator.cpp b/generator/translator.cpp index 984569ba9f..776e13a82c 100644 --- a/generator/translator.cpp +++ b/generator/translator.cpp @@ -50,6 +50,7 @@ void Translator::Emit(OsmElement & element) bool Translator::Finish() { + m_collectors.Finish(); m_collectors.Save(); return m_emitter->Finish(); } diff --git a/generator/translator_country.cpp b/generator/translator_country.cpp index 13dd78ada3..bbbc33a68f 100644 --- a/generator/translator_country.cpp +++ b/generator/translator_country.cpp @@ -78,18 +78,18 @@ TranslatorCountry::TranslatorCountry(std::shared_ptr emitter, AddFilter(std::make_shared()); AddFilter(std::make_shared(base::JoinPath(GetPlatform().ResourcesDir(), SKIPPED_ELEMENTS_FILE))); - AddCollector(std::make_shared(info.m_idToWikidataFilename, "wikidata" /* tagKey */, - WikiDataValidator, true /* ignoreIfNotOpen */)); AddCollector(std::make_shared(info.GetIntermediateFileName(METALINES_FILENAME))); - // These are the four collector that collect additional information for the future building of routing section. AddCollector(std::make_shared(info.GetIntermediateFileName(MAXSPEEDS_FILENAME))); - AddCollector(std::make_shared(info.GetIntermediateFileName(RESTRICTIONS_FILENAME), cache)); + AddCollector(std::make_shared(info.GetIntermediateFileName(RESTRICTIONS_FILENAME), + std::shared_ptr(&cache, [](cache::IntermediateDataReader *) {}))); AddCollector(std::make_shared(info.GetIntermediateFileName(ROAD_ACCESS_FILENAME))); AddCollector(std::make_shared(info.GetIntermediateFileName(CAMERAS_TO_WAYS_FILENAME))); if (info.m_genAddresses) AddCollector(std::make_shared(info.GetAddressesFileName())); + if (!info.m_idToWikidataFilename.empty()) + AddCollector(std::make_shared(info.m_idToWikidataFilename, "wikidata" /* tagKey */, WikiDataValidator)); } void TranslatorCountry::Preprocess(OsmElement & element)