From 3c9f5d71906b8487f05358e561a665076347efff Mon Sep 17 00:00:00 2001 From: Ilya Zverev Date: Mon, 12 Oct 2015 20:39:23 +0300 Subject: [PATCH 1/2] [generator] Revert regions fix, warning on low disk space --- tools/unix/generate_planet.sh | 8 ++++---- tools/unix/generate_planet_routing.sh | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tools/unix/generate_planet.sh b/tools/unix/generate_planet.sh index 4c08193ae8..fa312b732d 100755 --- a/tools/unix/generate_planet.sh +++ b/tools/unix/generate_planet.sh @@ -129,6 +129,7 @@ DATA_PATH="$OMIM_PATH/data" TARGET="${TARGET:-$DATA_PATH}" mkdir -p "$TARGET" INTDIR="${INTDIR:-$TARGET/intermediate_data}" +[ "$(df -m "$INTDIR" | tail -n 1 | awk '{ printf "%d\n", $4 / 1024 }')" -lt "250" ] && echo "WARNING: You have less than 250 MB for intermediate data, that's not enough for the whole planet." OSMCTOOLS="${OSMCTOOLS:-$HOME/osmctools}" [ ! -d "$OSMCTOOLS" ] && OSMCTOOLS="$INTDIR" MERGE_INTERVAL=${MERGE_INTERVAL:-40} @@ -147,7 +148,7 @@ UPDATE_DATE="$(date +%y%m%d)" LOG_PATH="${LOG_PATH:-$TARGET/logs}" mkdir -p "$LOG_PATH" PLANET_LOG="$LOG_PATH/generate_planet.log" -[ -n "${MAIL-}" ] && trap "grep STATUS \"$PLANET_LOG\" | mailx -s \"Generate_planet: build failed\" \"$MAIL\"; exit 1" SIGINT SIGTERM +[ -n "${MAIL-}" ] && trap "grep STATUS \"$PLANET_LOG\" | mailx -s \"Generate_planet: build failed\" \"$MAIL\"; exit 1" SIGTERM ERR echo -e "\n\n----------------------------------------\n\n" >> "$PLANET_LOG" log "STATUS" "Start" @@ -165,9 +166,7 @@ if [ -n "${REGIONS:-}" ]; then log "BORDERS" "Note: old borders from $TARGET/borders were moved to $BORDERS_BACKUP_PATH" mv "$TARGET/borders"/*.poly "$BORDERS_BACKUP_PATH" fi - for i in $REGIONS; do - cp "$i" "$TARGET/borders/" - done + echo "$REGIONS" | xargs -I % cp "%" "$TARGET/borders/" elif [ -z "${REGIONS-1}" ]; then # A user asked specifically for no regions NO_REGIONS=1 @@ -443,4 +442,5 @@ rm "$STATUS_FILE" [ -f "$OSRM_FLAG" ] && rm "$OSRM_FLAG" [ -n "$(ls "$TARGET" | grep '\.mwm\.osm2ft')" ] && mv "$TARGET"/*.mwm.osm2ft "$INTDIR" [ -z "$KEEP_INTDIR" ] && rm -r "$INTDIR" +trap - SIGTERM ERR log "STATUS" "Done" diff --git a/tools/unix/generate_planet_routing.sh b/tools/unix/generate_planet_routing.sh index a435fa52e3..1eaadb6702 100755 --- a/tools/unix/generate_planet_routing.sh +++ b/tools/unix/generate_planet_routing.sh @@ -51,7 +51,7 @@ if [ "$1" == "pbf" ]; then export OSMCTOOLS export PLANET export INTDIR - find "$TMPBORDERS" -name '*.poly' -print0 | xargs -0 -P $NUM_PROCESSES -I % \ + find "$TMPBORDERS" -maxdepth 1 -name '*.poly' -print0 | xargs -0 -P $NUM_PROCESSES -I % \ sh -c '"$OSMCTOOLS/osmconvert" "$PLANET" --hash-memory=2000 -B="%" --complex-ways --out-pbf -o="$INTDIR/$(basename "%" .poly).pbf"' [ $? != 0 ] && fail "Failed to process all the regions" rm -r "$TMPBORDERS" @@ -111,7 +111,7 @@ elif [ "$1" == "mwm" ]; then export TARGET export LOG_PATH export DATA_PATH="$OMIM_PATH/data/" - find "$INTDIR" -name '*.osrm' -print0 | xargs -0 -P $NUM_PROCESSES -I % \ + find "$INTDIR" -maxdepth 1 -name '*.osrm' -print0 | xargs -0 -P $NUM_PROCESSES -I % \ sh -c 'O="%"; B="$(basename "$O" .osrm)"; "$G" $K --osrm_file_name="$O" --data_path="$TARGET" --user_resource_path="$DATA_PATH" --output="$B" 2>> "$LOG_PATH/$B.log"' if [ -n "${POLY_DIR-}" ]; then From 14107fd6e06e52829a4f475bf107307a8c1d575a Mon Sep 17 00:00:00 2001 From: Ilya Zverev Date: Tue, 13 Oct 2015 18:21:29 +0300 Subject: [PATCH 2/2] [docs] Clarify usage of REGIONS variable --- docs/MAPS.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/MAPS.md b/docs/MAPS.md index 8dceca293b..8932d1e4a9 100644 --- a/docs/MAPS.md +++ b/docs/MAPS.md @@ -135,8 +135,8 @@ but if this flag is set (e.g. to `1`), they are built asynchronously. But it can fail due to low memory. * `MERGE_INTERVAL`: delay in minutes between attempts to merge a coast line. Default is 40. -* `REGIONS`: a list of `.poly` files for regions to be built. Can be empty. -Example: `../../data/borders/UK* ../../data/borders/*Ireland*`. +* `REGIONS`: a list of `.poly` files for regions to be built. One for each line. +Can be empty. Example: `$(ls ../../data/borders/{UK*,Ireland}.poly)`. ### Testing