diff --git a/tools/unix/generate_planet.sh b/tools/unix/generate_planet.sh index ed6ab25f8c..e7ba167b25 100755 --- a/tools/unix/generate_planet.sh +++ b/tools/unix/generate_planet.sh @@ -86,7 +86,7 @@ while getopts ":cuwrh" opt; do done EXIT_ON_ERROR=${EXIT_ON_ERROR-1} -[ -n "${EXIT_ON_ERROR-}" ] && set -e # Exit when any of commands fail +[ -n "$EXIT_ON_ERROR" ] && set -e # Exit when any of commands fail set -o pipefail # Capture all errors in command chains set -u # Fail on undefined variables #set -x # Echo every script line @@ -105,6 +105,7 @@ OSMCTOOLS="${OSMCTOOLS:-$HOME/osmctools}" MERGE_COASTS_DELAY_SEC=2400 # set to "mem" if there is more than 64 GB of memory NODE_STORAGE=${NODE_STORAGE:-${NS:-mem}} +ASYNC_PBF=${ASYNC_PBF-} NUM_PROCESSES=${NUM_PROCESSES:-$(expr $(nproc || echo 8) - 1)} STATUS_FILE="$INTDIR/status" @@ -119,7 +120,7 @@ log "STATUS" "Start" source "$SCRIPTS_PATH/find_generator_tool.sh" # Prepare borders -[ -n "${EXIT_ON_ERROR-}" ] && set +e # Grep returns non-zero status +[ -n "$EXIT_ON_ERROR" ] && set +e # Grep returns non-zero status mkdir -p "$TARGET/borders" PREV_BORDERS="$(ls "$TARGET/borders" | grep \.poly)" if [ -n "${REGIONS:-}" ]; then @@ -137,7 +138,7 @@ elif [ -z "$PREV_BORDERS" ]; then cp "$BORDERS_PATH"/*.poly "$TARGET/borders/" fi [ -z "$(ls "$TARGET/borders" | grep \.poly)" ] && fail "No border polygons found, please use REGIONS or BORDER_PATH variables" -[ -n "${EXIT_ON_ERROR-}" ] && set -e +[ -n "$EXIT_ON_ERROR" ] && set -e ULIMIT_REQ=$(expr 3 \* $(ls "$TARGET/borders" | grep \.poly | wc -l)) [ $(ulimit -n) -lt $ULIMIT_REQ ] && fail "Ulimit is too small, you need at least $ULIMIT_REQ" @@ -172,7 +173,8 @@ fi if [ "$MODE" == "coast" ]; then putmode - [ ! -x "$OSMCTOOLS/osmconvert" ] && wget -q -O - http://m.m.i24.cc/osmconvert.c | cc -x c - -lz -O3 -o "$OSMCTOOLS/osmconvert" + # The patch increases number of nodes for osmconvert to avoid overflow crash + [ ! -x "$OSMCTOOLS/osmconvert" ] && wget -q -O - http://m.m.i24.cc/osmconvert.c | sed 's/60004/600004/' | cc -x c - -lz -O3 -o "$OSMCTOOLS/osmconvert" [ ! -x "$OSMCTOOLS/osmupdate" ] && wget -q -O - http://m.m.i24.cc/osmupdate.c | cc -x c - -o "$OSMCTOOLS/osmupdate" [ ! -x "$OSMCTOOLS/osmfilter" ] && wget -q -O - http://m.m.i24.cc/osmfilter.c | cc -x c - -O3 -o "$OSMCTOOLS/osmfilter" if [ -n "$OPT_DOWNLOAD" ]; then @@ -205,7 +207,7 @@ if [ "$MODE" == "coast" ]; then "$OSMCTOOLS/osmfilter" "$PLANET" --keep= --keep-ways="natural=coastline" -o=$COASTS # Preprocess coastlines to separate intermediate directory log "TIMEMARK" "Generate coastlines intermediate" - [ -n "${EXIT_ON_ERROR-}" ] && set +e # Temporary disable to read error code + [ -n "$EXIT_ON_ERROR" ] && set +e # Temporary disable to read error code "$GENERATOR_TOOL" --intermediate_data_path="$INTCOASTSDIR/" --node_storage=map --osm_file_type=o5m --osm_file_name="$COASTS" \ -preprocess 2>> "$GENERATOR_LOG" # Generate temporary coastlines file in the coasts intermediate dir @@ -213,7 +215,7 @@ if [ "$MODE" == "coast" ]; then "$GENERATOR_TOOL" --intermediate_data_path="$INTCOASTSDIR/" --node_storage=map --osm_file_type=o5m --osm_file_name="$COASTS" \ --user_resource_path="$DATA_PATH/" -make_coasts -fail_on_coasts 2>&1 | tee "$GENERATOR_LOG" EXIT_CODE=$? - [ -n "${EXIT_ON_ERROR-}" ] && set -e + [ -n "$EXIT_ON_ERROR" ] && set -e if [ $EXIT_CODE != 0 ]; then log "TIMEMARK" "Coastline merge failed" @@ -239,8 +241,17 @@ if [ -n "$OPT_ROUTING" ]; then if [ -e "$OSRM_FLAG" ]; then log "start_routing(): OSRM files have been already created, no need to repeat" else - putmode "Starting OSRM files generation" - ( bash "$ROUTING_SCRIPT" prepare &> "$ROUTING_LOG" ) + putmode "Step R: Starting OSRM files generation" + if [ -n "$ASYNC_PBF" ]; then + ( + bash "$ROUTING_SCRIPT" pbf &> "$ROUTING_LOG"; + bash "$ROUTING_SCRIPT" prepare &> "$ROUTING_LOG" + ) & + else + # Osmconvert takes too much memory: it makes sense to not extract pbfs asyncronously + bash "$ROUTING_SCRIPT" pbf &> "$ROUTING_LOG" + ( bash "$ROUTING_SCRIPT" prepare &> "$ROUTING_LOG" ) & + fi fi fi diff --git a/tools/unix/generate_planet_routing.sh b/tools/unix/generate_planet_routing.sh index 6b15e4768a..fe53516d70 100755 --- a/tools/unix/generate_planet_routing.sh +++ b/tools/unix/generate_planet_routing.sh @@ -8,7 +8,10 @@ set -u # Fail on undefined variables if [ $# -lt 1 ]; then echo '' - echo "Usage: $0 {prepare|mwm}" + echo "Usage:" + echo " $0 pbf" + echo " $0 prepare" + echo " $0 mwm" echo '' exit 1 fi @@ -27,15 +30,33 @@ mkdir -p "$INTDIR" NUM_PROCESSES=${NUM_PROCESSES:-8} OSRM_FLAG="${OSRM_FLAG:-$INTDIR/osrm_done}" -if [ "$1" == "prepare" ]; then +if [ "$1" == "pbf" ]; then rm -f "$OSRM_FLAG" PLANET="${PLANET:-$HOME/planet/planet-latest.o5m}" + OSMCTOOLS="${OSMCTOOLS:-$HOME/osmctools}" + [ ! -d "$OSMCTOOLS" ] && OSMCTOOLS="$INTDIR" + # The patch increases number of nodes for osmconvert to avoid overflow crash + [ ! -x "$OSMCTOOLS/osmconvert" ] && wget -q -O - http://m.m.i24.cc/osmconvert.c | sed 's/60004/600004/' | cc -x c - -lz -O3 -o "$OSMCTOOLS/osmconvert" + + TMPBORDERS="$INTDIR/tmpborders" + mkdir "$TMPBORDERS" + if [ -z "${REGIONS-}" ]; then + cp "$BORDERS_PATH"/*.poly "$TMPBORDERS" + else + echo "$REGIONS" | xargs -I % cp "$BORDERS_PATH/%.poly" "$TMPBORDERS" + fi + [ -z "$(ls "$TMPBORDERS"/*.poly)" ] && fail "No regions to create routing files for" + find "$TMPBORDERS" -name '*.poly' -print0 | xargs -0 -P $NUM_PROCESSES -I % \ + sh -c 'POLY="%"; "$OSMCTOOLS/osmconvert" $PLANET --hash-memory=2000 -B="$POLY" --complex-ways --out-pbf -o="$INTDIR/$(basename "$POLY" .poly).pbf"' + [ $? != 0 ] && fail "Failed to process all the regions" + rm -r "$TMPBORDERS" + +elif [ "$1" == "prepare" ]; then + rm -f "$OSRM_FLAG" + [ -z "$(ls "$INTDIR"/*.pbf)" ] && fail "Please build PBF files first" OSRM_PATH="${OSRM_PATH:-$OMIM_PATH/3party/osrm/osrm-backend}" OSRM_BUILD_PATH="${OSRM_BUILD_PATH:-$OSRM_PATH/build}" [ ! -x "$OSRM_BUILD_PATH/osrm-extract" ] && fail "Please compile OSRM binaries to $OSRM_BUILD_PATH" - OSMCTOOLS="${OSMCTOOLS:-$HOME/osmctools}" - [ ! -d "$OSMCTOOLS" ] && OSMCTOOLS="$INTDIR" - [ ! -x "$OSMCTOOLS/osmconvert" ] && wget -q -O - http://m.m.i24.cc/osmconvert.c | cc -x c - -lz -O3 -o "$OSMCTOOLS/osmconvert" OSRM_THREADS=${OSRM_THREADS:-15} OSRM_MEMORY=${OSRM_MEMORY:-50} @@ -48,17 +69,12 @@ if [ "$1" == "prepare" ]; then [ $# -gt 1 ] && PROFILE="$2" [ ! -r "$PROFILE" ] && fail "Lua profile $PROFILE is not found" - REGIONS=${REGIONS:-$(ls $BORDERS_PATH/*.poly | xargs -I % basename % .poly)} - [ -z "$REGIONS" ] && fail "No regions to create routing files for" - echo "$REGIONS" | xargs -P $NUM_PROCESSES -I % "$OSMCTOOLS/osmconvert" $PLANET --hash-memory=2000 -B=$BORDERS_PATH/%.poly --complex-ways --out-pbf -o=$INTDIR/%.pbf - [ $? != 0 ] && fail "Failed to process all the regions" - export STXXLCFG="$HOME/.stxxl" - echo "$REGIONS" | while read REGION ; do - OSRM_FILE="$INTDIR/$REGION.osrm" + for PBF in "$INTDIR"/*.pbf; do + OSRM_FILE="${PBF%.*}.osrm" rm -f "$OSRM_FILE" - "$OSRM_BUILD_PATH/osrm-extract" --config "$EXTRACT_CFG" --profile "$PROFILE" "$INTDIR/$REGION.pbf" - rm -f "$INTDIR/$REGION.pbf" + "$OSRM_BUILD_PATH/osrm-extract" --config "$EXTRACT_CFG" --profile "$PROFILE" "$PBF" + rm -f "$PBF" "$OSRM_BUILD_PATH/osrm-prepare" --config "$PREPARE_CFG" --profile "$PROFILE" "$OSRM_FILE" "$OSRM_BUILD_PATH/osrm-mapsme" -i "$OSRM_FILE" [ ! -f "$OSRM_FILE" ] && echo "Failed to create $OSRM_FILE" @@ -77,8 +93,8 @@ elif [ "$1" == "mwm" ]; then fi DATA_PATH="$OMIM_PATH/data/" - echo "${REGIONS:-$(ls $INTDIR/*.osrm | xargs -I % basename % .osrm)}" | xargs -P $NUM_PROCESSES -I % \ - $GENERATOR_TOOL --make_routing --make_cross_section --osrm_file_name="$INTDIR/%.osrm" --data_path="$TARGET" --user_resource_path="$DATA_PATH" --output="%" + find "$INTDIR" -name '*.osrm' -print0 | xargs -0 -P $NUM_PROCESSES -I % \ + sh -c 'OSRM="%"; $GENERATOR_TOOL --make_routing --make_cross_section --osrm_file_name="$OSRM" --data_path="$TARGET" --user_resource_path="$DATA_PATH" --output="$(basename "$OSRM" .osrm)"' if [ -n "${POLY_DIR-}" ]; then # delete temporary polygons