diff --git a/generator/generator_tool/generator_tool.cpp b/generator/generator_tool/generator_tool.cpp index 0e1b33f7c4..233d8f22e3 100644 --- a/generator/generator_tool/generator_tool.cpp +++ b/generator/generator_tool/generator_tool.cpp @@ -60,6 +60,8 @@ DEFINE_string(delete_section, "", "Delete specified section (defines.hpp) from c DEFINE_bool(fail_on_coasts, false, "Stop and exit with '255' code if some coastlines are not merged."); DEFINE_string(address_file_name, "", "Output file name for storing full addresses."); DEFINE_string(osrm_file_name, "", "Input osrm file to generate routing info"); +DEFINE_bool(make_routing, false, "Make routing info based on osrm file"); +DEFINE_bool(make_cross_section, false, "Make corss section in routing file for cross mwm routing"); DEFINE_string(osm_file_name, "", "Input osm area file"); DEFINE_string(osm_file_type, "xml", "Input osm area file type [xml, o5m]"); DEFINE_string(user_resource_path, "", "User defined resource path for classificator.txt and etc."); @@ -248,8 +250,11 @@ int main(int argc, char ** argv) if (FLAGS_check_mwm) check_model::ReadFeatures(datFile); - if (!FLAGS_osrm_file_name.empty()) + if (!FLAGS_osrm_file_name.empty() && FLAGS_make_routing) routing::BuildRoutingIndex(path, FLAGS_output, FLAGS_osrm_file_name); + if (!FLAGS_osrm_file_name.empty() && FLAGS_make_cross_section) + routing::BuildCrossRoutingIndex(path, FLAGS_output, FLAGS_osrm_file_name); + return 0; } diff --git a/generator/routing_generator.cpp b/generator/routing_generator.cpp index 7a96af05d0..6a17bbc426 100644 --- a/generator/routing_generator.cpp +++ b/generator/routing_generator.cpp @@ -34,55 +34,32 @@ namespace routing static double const EQUAL_POINT_RADIUS_M = 2.0; -void BuildRoutingIndex(string const & baseDir, string const & countryName, string const & osrmFile) +bool LoadIndexes(string const & mwmFile, string const & osrmFile, osrm::NodeDataVectorT & nodeData, gen::OsmID2FeatureID & osm2ft) { - classificator::Load(); - - string const mwmFile = baseDir + countryName + DATA_FILE_EXTENSION; - Index index; - m2::RectD rect; - if (!index.Add(mwmFile, rect)) + if (!osrm::LoadNodeDataFromFile(osrmFile + ".nodeData", nodeData)) { - LOG(LCRITICAL, ("MWM file not found")); - return; + LOG(LCRITICAL, ("Can't load node data")); + return false; } - - vector regionBorders; - routing::CrossRoutingContextWriter crossContext; - LOG(LINFO, ("Loading countries borders")); - borders::CountriesContainerT m_countries; - CHECK(borders::LoadCountriesList(baseDir, m_countries), - ("Error loading country polygons files")); - { - m_countries.ForEach([&](borders::CountryPolygons const & c) - { - if (c.m_name == countryName) - { - c.m_regions.ForEach([&](m2::RegionD const & region) - { - regionBorders.push_back(region); - }); - } - }); - } - - gen::OsmID2FeatureID osm2ft; { FileReader reader(mwmFile + OSM2FEATURE_FILE_EXTENSION); ReaderSource src(reader); osm2ft.Read(src); } + return true; +} - osrm::NodeDataVectorT nodeData; - if (!osrm::LoadNodeDataFromFile(osrmFile + ".nodeData", nodeData)) +void FindCrossNodes(osrm::NodeDataVectorT const & nodeData, gen::OsmID2FeatureID const & osm2ft, borders::CountriesContainerT const & m_countries, string const & countryName, routing::CrossRoutingContextWriter & crossContext) +{ + vector regionBorders; + m_countries.ForEach([&](borders::CountryPolygons const & c) { - LOG(LCRITICAL, ("Can't load node data")); - return; - } - - OsrmFtSegMappingBuilder mapping; - - uint32_t found = 0, all = 0, multiple = 0, equal = 0, moreThan1Seg = 0, stored = 0; + if (c.m_name == countryName) + c.m_regions.ForEach([®ionBorders](m2::RegionD const & region) + { + regionBorders.push_back(region); + }); + }); for (size_t nodeId = 0; nodeId < nodeData.size(); ++nodeId) { @@ -109,13 +86,15 @@ void BuildRoutingIndex(string const & baseDir, string const & countryName, strin break; if (intersection == m2::PointD::Zero()) continue; - else - // for old format compatibility - intersection = m2::PointD(MercatorBounds::XToLon(intersection.x), MercatorBounds::YToLat(intersection.y)); - if (outStart && !outEnd) + + // for old format compatibility + intersection = m2::PointD(MercatorBounds::XToLon(intersection.x), MercatorBounds::YToLat(intersection.y)); + if (!outStart && outEnd) + crossContext.addIngoingNode(nodeId, intersection); + else if (outStart && !outEnd) { string mwmName; - m2::PointD const mercatorPoint(MercatorBounds::FromLatLon(endSeg.lat2, endSeg.lon2)); + m2::PointD const & mercatorPoint = MercatorBounds::FromLatLon(endSeg.lat2, endSeg.lon2); m_countries.ForEachInRect(m2::RectD(mercatorPoint, mercatorPoint), [&](borders::CountryPolygons const & c) { if (c.m_name == countryName) @@ -123,11 +102,8 @@ void BuildRoutingIndex(string const & baseDir, string const & countryName, strin c.m_regions.ForEachInRect(m2::RectD(mercatorPoint, mercatorPoint), [&](m2::RegionD const & region) { // Sometimes Contains make errors for cases near the border. - if (region.Contains(mercatorPoint)) + if (region.Contains(mercatorPoint) || region.AtBorder(mercatorPoint, 0.01 /*Near border accuracy. In mercator.*/)) mwmName = c.m_name; - else - if (region.AtBorder(mercatorPoint, 0.01 /*Near border accuracy. In mercator.*/)) - mwmName = c.m_name; }); }); if (!mwmName.empty()) @@ -135,11 +111,139 @@ void BuildRoutingIndex(string const & baseDir, string const & countryName, strin else LOG(LINFO, ("Unknowing outgoing edge", endSeg.lat2, endSeg.lon2, startSeg.lat1, startSeg.lon1)); } - else if (!outStart && outEnd) - crossContext.addIngoingNode(nodeId, intersection); } } } + } +} + +void LoadRoutingFacade(string const & osrmFile, OsrmRawDataFacade & facade) +{ + ReaderSource edgeSource(new FileReader(osrmFile + "." + ROUTING_EDGEDATA_FILE_TAG)); + vector edgeBuffer(static_cast(edgeSource.Size())); + edgeSource.Read(&edgeBuffer[0], edgeSource.Size()); + + ReaderSource edgeIdsSource(new FileReader(osrmFile + "." + ROUTING_EDGEID_FILE_TAG)); + vector edgeIdsBuffer(static_cast(edgeIdsSource.Size())); + edgeIdsSource.Read(&edgeIdsBuffer[0], edgeIdsSource.Size()); + + ReaderSource shortcutsSource(new FileReader(osrmFile + "." + ROUTING_SHORTCUTS_FILE_TAG)); + vector shortcutsBuffer(static_cast(shortcutsSource.Size())); + shortcutsSource.Read(&shortcutsBuffer[0], shortcutsSource.Size()); + + ReaderSource matrixSource(new FileReader(osrmFile + "." + ROUTING_MATRIX_FILE_TAG)); + vector matrixBuffer(static_cast(matrixSource.Size())); + matrixSource.Read(&matrixBuffer[0], matrixSource.Size()); + + facade.LoadRawData(edgeBuffer.data(), edgeIdsBuffer.data(), shortcutsBuffer.data(), matrixBuffer.data()); +} + +void CalculateCrossAdjacency(OsrmRawDataFacade & facade, routing::CrossRoutingContextWriter & crossContext) +{ + LOG(LINFO, ("Calculating weight map between outgoing nodes")); + crossContext.reserveAdjacencyMatrix(); + auto const & in = crossContext.GetIngoingIterators(); + auto const & out = crossContext.GetOutgoingIterators(); + MultiroutingTaskPointT sources(distance(in.first, in.second)), targets(distance(out.first, out.second)); + size_t index = 0; + // Fill sources and targets with start node task for ingoing (true) and target node task + // (false) for outgoing nodes + for (auto i = in.first; i != in.second; ++i, ++index) + OsrmRouter::GenerateRoutingTaskFromNodeId(i->m_nodeId, true, sources[index]); + + index = 0; + for (auto i = out.first; i != out.second; ++i, ++index) + OsrmRouter::GenerateRoutingTaskFromNodeId(i->m_nodeId, false, targets[index]); + + vector costs; + OsrmRouter::FindWeightsMatrix(sources, targets, facade, costs); + auto res = costs.begin(); + for (auto i = in.first; i != in.second; ++i) + for (auto j = out.first; j != out.second; ++j) + { + EdgeWeight const & edgeWeigth = *(res++); + if (edgeWeigth != INVALID_EDGE_WEIGHT && edgeWeigth > 0) + crossContext.setAdjacencyCost(i, j, edgeWeigth); + } + LOG(LINFO, ("Calculation of weight map between outgoing nodes DONE")); +} + +void WriteCrossSection(routing::CrossRoutingContextWriter const & crossContext, string const & mwmFile) +{ + LOG(LINFO, ("Collect all data into one file...")); + string const fPath = mwmFile + ROUTING_FILE_EXTENSION; + + FilesContainerW routingCont(fPath, FileWriter::OP_WRITE_EXISTING); + + { + // Write version for routing file that is equal to correspondent mwm file. + FilesContainerR mwmCont(mwmFile); + + FileWriter w = routingCont.GetWriter(VERSION_FILE_TAG); + ReaderSource src(mwmCont.GetReader(VERSION_FILE_TAG)); + rw::ReadAndWrite(src, w); + w.WritePaddingByEnd(4); + } + + FileWriter w = routingCont.GetWriter(ROUTING_CROSS_CONTEXT_TAG); + size_t const start_size = w.Pos(); + crossContext.Save(w); + w.WritePaddingByEnd(4); + LOG(LINFO, ("Have written routing info, bytes written:", w.Pos() - start_size, "bytes")); + + routingCont.Finish(); +} + +void BuildCrossRoutingIndex(string const & baseDir, string const & countryName, string const & osrmFile) +{ + LOG(LINFO, ("Cross mwm routing section builder")); + string const mwmFile = baseDir + countryName + DATA_FILE_EXTENSION; + osrm::NodeDataVectorT nodeData; + gen::OsmID2FeatureID osm2ft; + if (!LoadIndexes(mwmFile, osrmFile, nodeData, osm2ft)) + return; + + routing::CrossRoutingContextWriter crossContext; + LOG(LINFO, ("Loading countries borders")); + borders::CountriesContainerT m_countries; + CHECK(borders::LoadCountriesList(baseDir, m_countries), + ("Error loading country polygons files")); + + FindCrossNodes(nodeData, osm2ft, m_countries, countryName, crossContext); + + // Load routing facade + OsrmRawDataFacade facade; + LoadRoutingFacade(osrmFile, facade); + CalculateCrossAdjacency(facade, crossContext); + + WriteCrossSection(crossContext, mwmFile); +} + +void BuildRoutingIndex(string const & baseDir, string const & countryName, string const & osrmFile) +{ + classificator::Load(); + + string const mwmFile = baseDir + countryName + DATA_FILE_EXTENSION; + Index index; + m2::RectD rect; + if (!index.Add(mwmFile, rect)) + { + LOG(LCRITICAL, ("MWM file not found")); + return; + } + + osrm::NodeDataVectorT nodeData; + gen::OsmID2FeatureID osm2ft; + if (!LoadIndexes(mwmFile, osrmFile, nodeData, osm2ft)) + return; + + OsrmFtSegMappingBuilder mapping; + + uint32_t found = 0, all = 0, multiple = 0, equal = 0, moreThan1Seg = 0, stored = 0; + + for (size_t nodeId = 0; nodeId < nodeData.size(); ++nodeId) + { + auto const & data = nodeData[nodeId]; OsrmFtSegMappingBuilder::FtSegVectorT vec; @@ -263,7 +367,7 @@ void BuildRoutingIndex(string const & baseDir, string const & countryName, strin LOG(LINFO, ("Collect all data into one file...")); string const fPath = mwmFile + ROUTING_FILE_EXTENSION; - FilesContainerW routingCont(fPath); + FilesContainerW routingCont(fPath /*, FileWriter::OP_APPEND*/); { // Write version for routing file that is equal to correspondent mwm file. @@ -289,65 +393,6 @@ void BuildRoutingIndex(string const & baseDir, string const & countryName, strin appendFile(ROUTING_MATRIX_FILE_TAG); appendFile(ROUTING_EDGEID_FILE_TAG); - { - // Load routing facade - OsrmRawDataFacade facade; - - ReaderSource edgeSource(new FileReader(osrmFile + "." + ROUTING_EDGEDATA_FILE_TAG)); - vector edgeBuffer(static_cast(edgeSource.Size())); - edgeSource.Read(&edgeBuffer[0], edgeSource.Size()); - - ReaderSource edgeIdsSource(new FileReader(osrmFile + "." + ROUTING_EDGEID_FILE_TAG)); - vector edgeIdsBuffer(static_cast(edgeIdsSource.Size())); - edgeIdsSource.Read(&edgeIdsBuffer[0], edgeIdsSource.Size()); - - ReaderSource shortcutsSource(new FileReader(osrmFile + "." + ROUTING_SHORTCUTS_FILE_TAG)); - vector shortcutsBuffer(static_cast(shortcutsSource.Size())); - shortcutsSource.Read(&shortcutsBuffer[0], shortcutsSource.Size()); - - ReaderSource matrixSource(new FileReader(osrmFile + "." + ROUTING_MATRIX_FILE_TAG)); - vector matrixBuffer(static_cast(matrixSource.Size())); - matrixSource.Read(&matrixBuffer[0], matrixSource.Size()); - - facade.LoadRawData(edgeBuffer.data(), edgeIdsBuffer.data(), shortcutsBuffer.data(), matrixBuffer.data()); - - LOG(LINFO, ("Calculating weight map between outgoing nodes")); - crossContext.reserveAdjacencyMatrix(); - auto in = crossContext.GetIngoingIterators(); - auto out = crossContext.GetOutgoingIterators(); - MultiroutingTaskPointT sources(distance(in.first, in.second)), targets(distance(out.first, out.second)); - size_t index = 0; - // Fill sources and targets with start node task for ingoing (true) and target node task - // (false) for outgoing nodes - for (auto i = in.first; i != in.second; ++i, ++index) - { - OsrmRouter::GenerateRoutingTaskFromNodeId(i->m_nodeId, true, sources[index]); - } - index = 0; - for (auto i = out.first; i != out.second; ++i, ++index) - { - OsrmRouter::GenerateRoutingTaskFromNodeId(i->m_nodeId, false, targets[index]); - } - - vector costs; - OsrmRouter::FindWeightsMatrix(sources, targets, facade, costs); - auto res = costs.begin(); - for (auto i = in.first; i < in.second; ++i) - for (auto j = out.first; j < out.second; ++j) - { - EdgeWeight const & edgeWeigth = *(res++); - if (edgeWeigth != INVALID_EDGE_WEIGHT && edgeWeigth > 0) - crossContext.setAdjacencyCost(i, j, edgeWeigth); - } - LOG(LINFO, ("Calculation of weight map between outgoing nodes DONE")); - } - - FileWriter w = routingCont.GetWriter(ROUTING_CROSS_CONTEXT_TAG); - size_t const start_size = w.Pos(); - crossContext.Save(w); - w.WritePaddingByEnd(4); - LOG(LINFO, ("Have written routing info, bytes written:", w.Pos() - start_size, "bytes")); - routingCont.Finish(); uint64_t sz; diff --git a/generator/routing_generator.hpp b/generator/routing_generator.hpp index 3e7fea3525..0fe0fa5e8d 100644 --- a/generator/routing_generator.hpp +++ b/generator/routing_generator.hpp @@ -11,6 +11,8 @@ namespace routing /// @param[in] osrmFile Full path to .osrm file (all prepared osrm files should be there). void BuildRoutingIndex(string const & baseDir, string const & countryName, string const & osrmFile); -/// @param[in] baseDir Full path to .mwm files directory. -void BuildCrossesRoutingIndex(string const & baseDir); +/// @param[in] baseDir Full path to .mwm files directory. +/// @param[in] countryName Country name same with .mwm and .border file name. +/// @param[in] osrmFile Full path to .osrm file (all prepared osrm files should be there). +void BuildCrossRoutingIndex(string const & baseDir, string const & countryName, string const & osrmFile); } diff --git a/routing/cross_routing_context.cpp b/routing/cross_routing_context.cpp index 887078ffa1..b0bcbf53d4 100644 --- a/routing/cross_routing_context.cpp +++ b/routing/cross_routing_context.cpp @@ -125,7 +125,7 @@ size_t CrossRoutingContextWriter::GetIndexInAdjMatrix(IngoingEdgeIteratorT ingoi return m_outgoingNodes.size() * ingoing_index + outgoing_index; } -void CrossRoutingContextWriter::Save(Writer & w) +void CrossRoutingContextWriter::Save(Writer & w) const { uint32_t size = static_cast(m_ingoingNodes.size()); w.Write(&size, sizeof(size)); diff --git a/routing/cross_routing_context.hpp b/routing/cross_routing_context.hpp index badffa7140..1990b7b83f 100644 --- a/routing/cross_routing_context.hpp +++ b/routing/cross_routing_context.hpp @@ -78,7 +78,7 @@ class CrossRoutingContextWriter size_t GetIndexInAdjMatrix(IngoingEdgeIteratorT ingoing, OutgoingEdgeIteratorT outgoing) const; public: - void Save(Writer & w); + void Save(Writer & w) const; void addIngoingNode(size_t const nodeId, m2::PointD const & point); diff --git a/tools/unix/make_routing.sh b/tools/unix/make_routing.sh index 9cb3ad5bea..c89e6e551c 100755 --- a/tools/unix/make_routing.sh +++ b/tools/unix/make_routing.sh @@ -15,4 +15,4 @@ then PV=pv fi -$PV ../../../omim-maps/$2.osm.bz2 | bzip2 -d | $GENERATOR_TOOL --osrm_file_name=../../../omim-maps/$2.osrm --output=$2 +$PV ../../../omim-maps/$2.osm.bz2 | bzip2 -d | $GENERATOR_TOOL --make_routing --make_cross_section --osrm_file_name=../../../omim-maps/$2.osrm --output=$2 diff --git a/tools/unix/routes_from_mwm_and_osrm.sh b/tools/unix/routes_from_mwm_and_osrm.sh index 43670dfcab..c3977d0e94 100755 --- a/tools/unix/routes_from_mwm_and_osrm.sh +++ b/tools/unix/routes_from_mwm_and_osrm.sh @@ -12,4 +12,9 @@ else GENERATOR_TOOL=~/omim-build-release/out/release/generator_tool fi -ls "$OSRM_DIR"/*.osrm | parallel -t -v "V=`basename -s .osrm {}`; $GENERATOR_TOOL --osrm_file_name={} --data_path=$DATA_DIR --output={/.}" +NUM_INSTANCES=8 + +OSRM_LIST=${OSRM_LIST-$(ls -1 $OSRM_DIR/*.osrm | xargs -d "\n" basename -s .osrm)} + +echo "$OSRM_LIST" | xargs -d "\n" -P $NUM_INSTANCES -I % $GENERATOR_TOOL --make_routing --make_cross_section --osrm_file_name=$OSRM_DIR/%.osrm --data_path=$DATA_DIR --output=% +