From 074165b581609f1a8120243a93e27cde0d72f429 Mon Sep 17 00:00:00 2001 From: Ilya Zverev Date: Wed, 7 Dec 2016 12:36:58 +0300 Subject: [PATCH 1/2] [generator] User only --make_new_routing for the new routing --- defines.hpp | 1 + generator/generate_info.hpp | 3 --- generator/generator_tool/generator_tool.cpp | 19 ++++++------------- generator/osm_source.cpp | 2 +- tools/unix/generate_planet.sh | 2 +- tools/unix/generate_planet_routing.sh | 5 +++-- 6 files changed, 12 insertions(+), 20 deletions(-) diff --git a/defines.hpp b/defines.hpp index ff4d6e5338..e79978cb1b 100644 --- a/defines.hpp +++ b/defines.hpp @@ -74,6 +74,7 @@ #define EXTERNAL_RESOURCES_FILE "external_resources.txt" #define GPS_TRACK_FILENAME "gps_track.dat" +#define RESTRICTIONS_FILENAME "restrictions.csv" #define TRAFFIC_FILE_EXTENSION ".traffic" diff --git a/generator/generate_info.hpp b/generator/generate_info.hpp index e5b52becc1..275ac2e9d8 100644 --- a/generator/generate_info.hpp +++ b/generator/generate_info.hpp @@ -36,9 +36,6 @@ struct GenerateInfo // Current generated file name if --output option is defined. string m_fileName; - // File name for a file with restrictions in osm id terms. - string m_restrictions; - NodeStorageType m_nodeStorageType; OsmSourceType m_osmFileType; string m_osmFileName; diff --git a/generator/generator_tool/generator_tool.cpp b/generator/generator_tool/generator_tool.cpp index a44271cf01..ccf1ec38b9 100644 --- a/generator/generator_tool/generator_tool.cpp +++ b/generator/generator_tool/generator_tool.cpp @@ -84,11 +84,7 @@ DEFINE_uint64(planet_version, my::SecondsSinceEpoch(), DEFINE_string(srtm_path, "", "Path to srtm directory. If it is set, generates section with altitude information " "about roads."); -DEFINE_string(restriction_name, "", "Name of file with relation restriction in osm id terms."); -DEFINE_string(feature_ids_to_osm_ids_name, "", - "Name of file with mapping from feature ids to osm ids."); -DEFINE_bool(generate_routing, false, "Generate section with routing information."); -DEFINE_bool(generate_restrictions, false, "Generate section with road restrictions."); +DEFINE_bool(make_routing_index, false, "Make sections with the routing information."); int main(int argc, char ** argv) { @@ -119,7 +115,6 @@ int main(int argc, char ** argv) } genInfo.m_osmFileName = FLAGS_osm_file_name; - genInfo.m_restrictions = FLAGS_restriction_name; genInfo.m_failOnCoasts = FLAGS_fail_on_coasts; genInfo.m_preloadCache = FLAGS_preload_cache; genInfo.m_bookingDatafileName = FLAGS_booking_data; @@ -152,7 +147,7 @@ int main(int argc, char ** argv) FLAGS_generate_index || FLAGS_generate_search_index || FLAGS_calc_statistics || FLAGS_type_statistics || FLAGS_dump_types || FLAGS_dump_prefixes || FLAGS_dump_feature_names != "" || FLAGS_check_mwm || FLAGS_srtm_path != "" || - FLAGS_generate_routing || FLAGS_generate_restrictions ) + FLAGS_make_routing_index) { classificator::Load(); classif().SortClassificator(); @@ -163,7 +158,6 @@ int main(int argc, char ** argv) { LOG(LINFO, ("Generating final data ...")); - genInfo.m_restrictions = FLAGS_restriction_name; genInfo.m_splitByPolygons = FLAGS_split_by_polygons; genInfo.m_createWorld = FLAGS_generate_world; genInfo.m_makeCoasts = FLAGS_make_coasts; @@ -239,14 +233,13 @@ int main(int argc, char ** argv) if (!FLAGS_srtm_path.empty()) routing::BuildRoadAltitudes(datFile, FLAGS_srtm_path); - if (FLAGS_generate_routing) - routing::BuildRoutingIndex(datFile, country); - - if (FLAGS_generate_restrictions) + if (FLAGS_make_routing_index) { routing::BuildRoadRestrictions( - datFile, genInfo.GetIntermediateFileName(genInfo.m_restrictions, "" /* extention */), + datFile, genInfo.GetIntermediateFileName(RESTRICTIONS_FILENAME, ""), genInfo.GetTargetFileName(country) + OSM2FEATURE_FILE_EXTENSION); + + routing::BuildRoutingIndex(datFile, country); } } diff --git a/generator/osm_source.cpp b/generator/osm_source.cpp index c39fc229e8..96e5880846 100644 --- a/generator/osm_source.cpp +++ b/generator/osm_source.cpp @@ -688,7 +688,7 @@ bool GenerateFeaturesImpl(feature::GenerateInfo & info, EmitterBase & emitter) // TODO(mgsergio): Get rid of EmitterBase template parameter. OsmToFeatureTranslator parser( emitter, cache, info.m_makeCoasts ? classif().GetCoastType() : 0, - info.GetAddressesFileName(), info.GetIntermediateFileName(info.m_restrictions, "")); + info.GetAddressesFileName(), info.GetIntermediateFileName(RESTRICTIONS_FILENAME, "")); TagAdmixer tagAdmixer(info.GetIntermediateFileName("ways", ".csv"), info.GetIntermediateFileName("towns", ".csv")); diff --git a/tools/unix/generate_planet.sh b/tools/unix/generate_planet.sh index 1ae31088bc..373c852bef 100755 --- a/tools/unix/generate_planet.sh +++ b/tools/unix/generate_planet.sh @@ -427,7 +427,7 @@ if [ "$MODE" == "features" ]; then # Checking for coastlines, can't build proper mwms without them if [ ! -s "$INTDIR/WorldCoasts.geom" ]; then COASTS="${COASTS-WorldCoasts.geom}" - if [ ! -s "$COASTS" -a -n "${OLD_INTDIR-}" -a -s "$OLD_INTDIR/WorldCoasts.geom" ]; then + if [ ! -s "$COASTS" -a -n "${OLD_INTDIR-}" -a -s "${OLD_INTDIR-}/WorldCoasts.geom" ]; then log "Using older coastlines." COASTS="$OLD_INTDIR/WorldCoasts.geom" fi diff --git a/tools/unix/generate_planet_routing.sh b/tools/unix/generate_planet_routing.sh index a85a442ce1..f63088cb1d 100755 --- a/tools/unix/generate_planet_routing.sh +++ b/tools/unix/generate_planet_routing.sh @@ -113,13 +113,14 @@ elif [ "$1" == "mwm" ]; then # Xargs has 255 chars limit for exec string, so we use short variable names. export G="$GENERATOR_TOOL" - export K="--make_routing --make_cross_section" + export K="--make_routing --make_cross_section --make_routing_index" + export I="--intermediate_data_path=$INTDIR" export TARGET export LOG_PATH export DATA_PATH set +e find "$INTDIR" -maxdepth 1 -name '*.osrm' -print0 | xargs -0 -P $NUM_PROCESSES -I % \ - sh -c 'O="%"; B="$(basename "$O" .osrm)"; "$G" $K --osrm_file_name="$O" --data_path="$TARGET" --user_resource_path="$DATA_PATH" --output="$B" 2>> "$LOG_PATH/$B.log"' + sh -c 'O="%"; B="$(basename "$O" .osrm)"; "$G" $K "$I" --osrm_file_name="$O" --data_path="$TARGET" --user_resource_path="$DATA_PATH" --output="$B" 2>> "$LOG_PATH/$B.log"' set -e if [ -n "${POLY_DIR-}" ]; then From e21ec340aa83c11283573cfe7d7bf23ed53f57b1 Mon Sep 17 00:00:00 2001 From: Ilya Zverev Date: Thu, 8 Dec 2016 13:28:52 +0300 Subject: [PATCH 2/2] [generator] Reorder command-line arguments --- generator/generator_tool/generator_tool.cpp | 73 ++++++++++++--------- 1 file changed, 42 insertions(+), 31 deletions(-) diff --git a/generator/generator_tool/generator_tool.cpp b/generator/generator_tool/generator_tool.cpp index ccf1ec38b9..aef0ff54f1 100644 --- a/generator/generator_tool/generator_tool.cpp +++ b/generator/generator_tool/generator_tool.cpp @@ -35,56 +35,67 @@ #include "3party/gflags/src/gflags/gflags.h" -DEFINE_bool(generate_classif, false, "Generate classificator."); - -DEFINE_bool(preprocess, false, "1st pass - create nodes/ways/relations data."); +// Coastlines. DEFINE_bool(make_coasts, false, "Create intermediate file with coasts data."); +DEFINE_bool(fail_on_coasts, false, "Stop and exit with '255' code if some coastlines are not merged."); DEFINE_bool(emit_coasts, false, "Push coasts features from intermediate file to out files/countries."); +// Generator settings and paths. +DEFINE_string(osm_file_name, "", "Input osm area file."); +DEFINE_string(osm_file_type, "xml", "Input osm area file type [xml, o5m]."); +DEFINE_string(data_path, "", "Working directory, 'path_to_exe/../../data' if empty."); +DEFINE_string(user_resource_path, "", "User defined resource path for classificator.txt and etc."); +DEFINE_string(intermediate_data_path, "", "Path to stored nodes, ways, relations."); +DEFINE_string(output, "", "File name for process (without 'mwm' ext)."); +DEFINE_bool(preload_cache, false, "Preload all ways and relations cache."); +DEFINE_string(node_storage, "map", + "Type of storage for intermediate points representation. Available: raw, map, mem."); +DEFINE_uint64(planet_version, my::SecondsSinceEpoch(), + "Version as seconds since epoch, by default - now."); + +// Preprocessing and feature generator. +DEFINE_bool(preprocess, false, "1st pass - create nodes/ways/relations data."); DEFINE_bool(generate_features, false, "2nd pass - generate intermediate features."); DEFINE_bool(generate_geometry, false, "3rd pass - split and simplify geometry and triangles for features."); DEFINE_bool(generate_index, false, "4rd pass - generate index."); DEFINE_bool(generate_search_index, false, "5th pass - generate search index."); -DEFINE_bool(calc_statistics, false, "Calculate feature statistics for specified mwm bucket files."); -DEFINE_bool(type_statistics, false, "Calculate statistics by type for specified mwm bucket files."); -DEFINE_bool(preload_cache, false, "Preload all ways and relations cache."); -DEFINE_string(node_storage, "map", - "Type of storage for intermediate points representation. Available: raw, map, mem."); -DEFINE_string(data_path, "", "Working directory, 'path_to_exe/../../data' if empty."); -DEFINE_string(output, "", "File name for process (without 'mwm' ext)."); -DEFINE_string(intermediate_data_path, "", "Path to stored nodes, ways, relations."); DEFINE_bool(generate_world, false, "Generate separate world file."); DEFINE_bool(split_by_polygons, false, "Use countries borders to split planet by regions and countries."); -DEFINE_bool(dump_types, false, "Prints all types combinations and their total count."); -DEFINE_bool(dump_prefixes, false, "Prints statistics on feature's' name prefixes."); -DEFINE_bool(dump_search_tokens, false, "Print statistics on search tokens."); -DEFINE_string(dump_feature_names, "", "Print all feature names by 2-letter locale."); -DEFINE_bool(unpack_mwm, false, "Unpack each section of mwm into a separate file with name filePath.sectionName."); -DEFINE_bool(generate_packed_borders, false, "Generate packed file with country polygons."); -DEFINE_string(unpack_borders, "", "Convert packed_polygons to a directory of polygon files (specify folder)."); -DEFINE_bool(check_mwm, false, "Check map file to be correct."); -DEFINE_string(delete_section, "", "Delete specified section (defines.hpp) from container."); -DEFINE_bool(fail_on_coasts, false, "Stop and exit with '255' code if some coastlines are not merged."); -DEFINE_bool(generate_addresses_file, false, "Generate .addr file (for '--output' option) with full addresses list."); + +// Routing. DEFINE_string(osrm_file_name, "", "Input osrm file to generate routing info."); DEFINE_bool(make_routing, false, "Make routing info based on osrm file."); DEFINE_bool(make_cross_section, false, "Make cross section in routing file for cross mwm routing."); -DEFINE_string(osm_file_name, "", "Input osm area file."); -DEFINE_string(osm_file_type, "xml", "Input osm area file type [xml, o5m]."); -DEFINE_string(user_resource_path, "", "User defined resource path for classificator.txt and etc."); +DEFINE_bool(make_routing_index, false, "Make sections with the routing information."); +DEFINE_string(srtm_path, "", + "Path to srtm directory. If set, generates a section with altitude information " + "about roads."); + +// Sponsored-related. DEFINE_string(booking_data, "", "Path to booking data in .tsv format."); DEFINE_string(booking_reference_path, "", "Path to mwm dataset for booking addresses matching."); DEFINE_string(opentable_data, "", "Path to opentable data in .tsv format."); DEFINE_string(opentable_reference_path, "", "Path to mwm dataset for opentable addresses matching."); -DEFINE_uint64(planet_version, my::SecondsSinceEpoch(), - "Version as seconds since epoch, by default - now."); -DEFINE_string(srtm_path, "", - "Path to srtm directory. If it is set, generates section with altitude information " - "about roads."); -DEFINE_bool(make_routing_index, false, "Make sections with the routing information."); + +// Printing stuff. +DEFINE_bool(calc_statistics, false, "Calculate feature statistics for specified mwm bucket files."); +DEFINE_bool(type_statistics, false, "Calculate statistics by type for specified mwm bucket files."); +DEFINE_bool(dump_types, false, "Prints all types combinations and their total count."); +DEFINE_bool(dump_prefixes, false, "Prints statistics on feature's' name prefixes."); +DEFINE_bool(dump_search_tokens, false, "Print statistics on search tokens."); +DEFINE_string(dump_feature_names, "", "Print all feature names by 2-letter locale."); + +// Service functions. +DEFINE_bool(generate_classif, false, "Generate classificator."); +DEFINE_bool(generate_packed_borders, false, "Generate packed file with country polygons."); +DEFINE_string(unpack_borders, "", "Convert packed_polygons to a directory of polygon files (specify folder)."); +DEFINE_bool(unpack_mwm, false, "Unpack each section of mwm into a separate file with name filePath.sectionName."); +DEFINE_bool(check_mwm, false, "Check map file to be correct."); +DEFINE_string(delete_section, "", "Delete specified section (defines.hpp) from container."); +DEFINE_bool(generate_addresses_file, false, "Generate .addr file (for '--output' option) with full addresses list."); int main(int argc, char ** argv) {