forked from organicmaps/organicmaps
[generator] Replace OSRM routing with a new A-star one
This commit is contained in:
Ilya Zverev
parent
32bce6ea7f
commit
126afc9f40
1 changed files with 53 additions and 89 deletions
|
|
@ -167,7 +167,6 @@ fi
|
|||
NUM_PROCESSES=${NUM_PROCESSES:-$(($CPUS - 1))}
|
||||
|
||||
STATUS_FILE="$INTDIR/status"
|
||||
OSRM_FLAG="$INTDIR/osrm_done"
|
||||
SCRIPTS_PATH="$(dirname "$0")"
|
||||
if [ -e "$SCRIPTS_PATH/hierarchy_to_countries.py" ]; then
|
||||
# In a compact packaging python scripts may be placed along with shell scripts.
|
||||
|
|
@ -175,7 +174,6 @@ if [ -e "$SCRIPTS_PATH/hierarchy_to_countries.py" ]; then
|
|||
else
|
||||
PYTHON_SCRIPTS_PATH="$OMIM_PATH/tools/python"
|
||||
fi
|
||||
ROUTING_SCRIPT="$SCRIPTS_PATH/generate_planet_routing.sh"
|
||||
ROADS_SCRIPT="$PYTHON_SCRIPTS_PATH/road_runner.py"
|
||||
HIERARCHY_SCRIPT="$PYTHON_SCRIPTS_PATH/hierarchy_to_countries.py"
|
||||
BOOKING_SCRIPT="$PYTHON_SCRIPTS_PATH/booking_hotels.py"
|
||||
|
|
@ -257,47 +255,47 @@ fi
|
|||
|
||||
# The building process starts here
|
||||
|
||||
# Download booking.com hotels. This takes around 3 hours, just like coastline processing.
|
||||
if [ ! -f "$BOOKING_FILE" -a -n "${BOOKING_USER-}" -a -n "${BOOKING_PASS-}" ]; then
|
||||
log "STATUS" "Step S1: Starting background hotels downloading"
|
||||
(
|
||||
$PYTHON $BOOKING_SCRIPT --user $BOOKING_USER --password $BOOKING_PASS --path "$INTDIR" --download --translate --output "$BOOKING_FILE" 2>"$LOG_PATH"/booking.log || true
|
||||
if [ -f "$BOOKING_FILE" -a "$(wc -l < "$BOOKING_FILE" || echo 0)" -gt 100 ]; then
|
||||
echo "Hotels have been downloaded. Please ensure this line is before Step 4." >> "$PLANET_LOG"
|
||||
else
|
||||
if [ -n "${OLD_INTDIR-}" -a -f "${OLD_INTDIR-}/$(basename "$BOOKING_FILE")" ]; then
|
||||
cp "$OLD_INTDIR/$(basename "$BOOKING_FILE")" "$INTDIR"
|
||||
warn "Failed to download hotels! Using older hotels list."
|
||||
else
|
||||
warn "Failed to download hotels!"
|
||||
fi
|
||||
[ -n "${MAIL-}" ] && tail "$LOG_PATH/booking.log" | mailx -s "Failed to download hotels at $(hostname), please hurry to fix" "$MAIL"
|
||||
fi
|
||||
) &
|
||||
fi
|
||||
|
||||
# Download opentable.com restaurants. This takes around 30 minutes.
|
||||
if [ ! -f "$OPENTABLE_FILE" -a -n "${OPENTABLE_USER-}" -a -n "${OPENTABLE_PASS-}" ]; then
|
||||
log "STATUS" "Step S2: Starting background restaurants downloading"
|
||||
(
|
||||
$PYTHON $OPENTABLE_SCRIPT --client $OPENTABLE_USER --secret $OPENTABLE_PASS --opentable_data "$INTDIR"/opentable.json --download --tsv "$OPENTABLE_FILE" 2>"$LOG_PATH"/opentable.log || true
|
||||
if [ -f "$OPENTABLE_FILE" -a "$(wc -l < "$OPENTABLE_FILE" || echo 0)" -gt 100 ]; then
|
||||
echo "Restaurants have been downloaded. Please ensure this line is before Step 4." >> "$PLANET_LOG"
|
||||
else
|
||||
if [ -n "${OLD_INTDIR-}" -a -f "${OLD_INTDIR-}/$(basename "$OPENTABLE_FILE")" ]; then
|
||||
cp "$OLD_INTDIR/$(basename "$OPENTABLE_FILE")" "$INTDIR"
|
||||
warn "Failed to download restaurants! Using older restaurants list."
|
||||
else
|
||||
warn "Failed to download restaurants!"
|
||||
fi
|
||||
[ -n "${MAIL-}" ] && tail "$LOG_PATH/opentable.log" | mailx -s "Failed to download restaurants at $(hostname), please hurry to fix" "$MAIL"
|
||||
fi
|
||||
) &
|
||||
fi
|
||||
|
||||
if [ "$MODE" == "coast" ]; then
|
||||
putmode
|
||||
|
||||
# Download booking.com hotels. This takes around 3 hours, just like coastline processing.
|
||||
if [ ! -f "$BOOKING_FILE" -a -n "${BOOKING_USER-}" -a -n "${BOOKING_PASS-}" ]; then
|
||||
log "STATUS" "Step S1: Starting background hotels downloading"
|
||||
(
|
||||
$PYTHON $BOOKING_SCRIPT --user $BOOKING_USER --password $BOOKING_PASS --path "$INTDIR" --download --translate --output "$BOOKING_FILE" 2>"$LOG_PATH"/booking.log || true
|
||||
if [ -f "$BOOKING_FILE" -a "$(wc -l < "$BOOKING_FILE" || echo 0)" -gt 100 ]; then
|
||||
echo "Hotels have been downloaded. Please ensure this line is before Step 4." >> "$PLANET_LOG"
|
||||
else
|
||||
if [ -n "${OLD_INTDIR-}" -a -f "${OLD_INTDIR-}/$(basename "$BOOKING_FILE")" ]; then
|
||||
cp "$OLD_INTDIR/$(basename "$BOOKING_FILE")" "$INTDIR"
|
||||
warn "Failed to download hotels! Using older hotels list."
|
||||
else
|
||||
warn "Failed to download hotels!"
|
||||
fi
|
||||
[ -n "${MAIL-}" ] && tail "$LOG_PATH/booking.log" | mailx -s "Failed to download hotels at $(hostname), please hurry to fix" "$MAIL"
|
||||
fi
|
||||
) &
|
||||
fi
|
||||
|
||||
# Download opentable.com restaurants. This takes around 30 minutes.
|
||||
if [ ! -f "$OPENTABLE_FILE" -a -n "${OPENTABLE_USER-}" -a -n "${OPENTABLE_PASS-}" ]; then
|
||||
log "STATUS" "Step S2: Starting background restaurants downloading"
|
||||
(
|
||||
$PYTHON $OPENTABLE_SCRIPT --client $OPENTABLE_USER --secret $OPENTABLE_PASS --opentable_data "$INTDIR"/opentable.json --download --tsv "$OPENTABLE_FILE" 2>"$LOG_PATH"/opentable.log || true
|
||||
if [ -f "$OPENTABLE_FILE" -a "$(wc -l < "$OPENTABLE_FILE" || echo 0)" -gt 100 ]; then
|
||||
echo "Restaurants have been downloaded. Please ensure this line is before Step 4." >> "$PLANET_LOG"
|
||||
else
|
||||
if [ -n "${OLD_INTDIR-}" -a -f "${OLD_INTDIR-}/$(basename "$OPENTABLE_FILE")" ]; then
|
||||
cp "$OLD_INTDIR/$(basename "$OPENTABLE_FILE")" "$INTDIR"
|
||||
warn "Failed to download restaurants! Using older restaurants list."
|
||||
else
|
||||
warn "Failed to download restaurants!"
|
||||
fi
|
||||
[ -n "${MAIL-}" ] && tail "$LOG_PATH/opentable.log" | mailx -s "Failed to download restaurants at $(hostname), please hurry to fix" "$MAIL"
|
||||
fi
|
||||
) &
|
||||
fi
|
||||
|
||||
[ ! -x "$OSMCTOOLS/osmupdate" ] && cc -x c "$OMIM_PATH/tools/osmctools/osmupdate.c" -o "$OSMCTOOLS/osmupdate"
|
||||
[ ! -x "$OSMCTOOLS/osmfilter" ] && cc -x c -O3 "$OMIM_PATH/tools/osmctools/osmfilter.c" -o "$OSMCTOOLS/osmfilter"
|
||||
if [ -n "$OPT_DOWNLOAD" ]; then
|
||||
|
|
@ -383,43 +381,6 @@ if [ "$MODE" == "roads" ]; then
|
|||
MODE=inter
|
||||
fi
|
||||
|
||||
# Starting routing generation as early as we can, since it's done in parallel
|
||||
if [ -n "$OPT_ROUTING" -a -z "$NO_REGIONS" ]; then
|
||||
if [ -e "$OSRM_FLAG" ]; then
|
||||
log "start_routing(): OSRM files have been already created, no need to repeat"
|
||||
else
|
||||
putmode "Step R: Starting OSRM files generation"
|
||||
PBF_FLAG="${OSRM_FLAG}_pbf"
|
||||
if [ -n "$ASYNC_PBF" -a ! -e "$PBF_FLAG" ]; then
|
||||
(
|
||||
bash "$ROUTING_SCRIPT" pbf >> "$PLANET_LOG" 2>&1
|
||||
touch "$PBF_FLAG"
|
||||
bash "$ROUTING_SCRIPT" prepare >> "$PLANET_LOG" 2>&1
|
||||
touch "$OSRM_FLAG"
|
||||
rm "$PBF_FLAG"
|
||||
) &
|
||||
else
|
||||
# Osmconvert takes too much memory: it makes sense to not extract pbfs asyncronously
|
||||
if [ -e "$PBF_FLAG" ]; then
|
||||
log "start_routing(): PBF files have been already created, skipping that step"
|
||||
else
|
||||
bash "$ROUTING_SCRIPT" pbf >> "$PLANET_LOG" 2>&1
|
||||
touch "$PBF_FLAG"
|
||||
fi
|
||||
(
|
||||
bash "$ROUTING_SCRIPT" prepare >> "$PLANET_LOG" 2>&1
|
||||
touch "$OSRM_FLAG"
|
||||
rm "$PBF_FLAG"
|
||||
) &
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ -n "$OPT_ONLINE_ROUTING" -a -z "$NO_REGIONS" ]; then
|
||||
putmode "Step RO: Generating OSRM files for osrm-routed server."
|
||||
bash "$ROUTING_SCRIPT" online >> "$PLANET_LOG" 2>&1
|
||||
fi
|
||||
|
||||
if [ "$MODE" == "inter" ]; then
|
||||
putmode "Step 3: Generating intermediate data for all MWMs"
|
||||
# 1st pass, run in parallel - preprocess whole planet to speed up generation if all coastlines are correct
|
||||
|
|
@ -428,6 +389,9 @@ if [ "$MODE" == "inter" ]; then
|
|||
MODE=features
|
||||
fi
|
||||
|
||||
# Wait until booking and restaurants lists are downloaded.
|
||||
wait
|
||||
|
||||
if [ "$MODE" == "features" ]; then
|
||||
putmode "Step 4: Generating features of everything into $TARGET"
|
||||
# Checking for coastlines, can't build proper mwms without them
|
||||
|
|
@ -495,6 +459,7 @@ if [ "$MODE" == "mwm" ]; then
|
|||
forky
|
||||
fi
|
||||
done
|
||||
wait
|
||||
fi
|
||||
|
||||
if [ -n "$OPT_ROUTING" -a -z "$NO_REGIONS" ]; then
|
||||
|
|
@ -504,25 +469,24 @@ if [ "$MODE" == "mwm" ]; then
|
|||
fi
|
||||
fi
|
||||
|
||||
# The following steps require *.mwm and *.osrm files complete
|
||||
wait
|
||||
|
||||
if [ "$MODE" == "routing" ]; then
|
||||
putmode "Step 6: Using freshly generated *.mwm and *.osrm to create routing files"
|
||||
if [ ! -e "$OSRM_FLAG" ]; then
|
||||
warn "OSRM files are missing, skipping routing step."
|
||||
else
|
||||
# If *.mwm.osm2ft were moved to INTDIR, let's put them back
|
||||
[ -z "$(ls "$TARGET" | grep '\.mwm\.osm2ft')" -a -n "$(ls "$INTDIR" | grep '\.mwm\.osm2ft')" ] && mv "$INTDIR"/*.mwm.osm2ft "$TARGET"
|
||||
bash "$ROUTING_SCRIPT" mwm >> "$PLANET_LOG" 2>&1
|
||||
fi
|
||||
# If *.mwm.osm2ft were moved to INTDIR, let's put them back
|
||||
[ -z "$(ls "$TARGET" | grep '\.mwm\.osm2ft')" -a -n "$(ls "$INTDIR" | grep '\.mwm\.osm2ft')" ] && mv "$INTDIR"/*.mwm.osm2ft "$TARGET"
|
||||
|
||||
for file in "$TARGET"/*.mwm; do
|
||||
if [[ "$file" != *minsk-pass* && "$file" != *World* ]]; then
|
||||
BASENAME="$(basename "$file" .mwm)"
|
||||
"$GENERATOR_TOOL" --data_path="$TARGET" --intermediate_data_path="$INTDIR/" --user_resource_path="$DATA_PATH/" --make_cross_mwm --disable_cross_mwm_progress --make_routing_index --generate_traffic_keys --output="$BASENAME" 2>> "$LOG_PATH/$BASENAME.log" &
|
||||
forky
|
||||
fi
|
||||
done
|
||||
wait
|
||||
MODE=resources
|
||||
fi
|
||||
|
||||
# Clean up temporary routing files
|
||||
[ -f "$OSRM_FLAG" ] && rm "$OSRM_FLAG"
|
||||
[ -n "$(ls "$TARGET" | grep '\.mwm\.osm2ft')" ] && mv "$TARGET"/*.mwm.osm2ft "$INTDIR"
|
||||
[ -n "$(ls "$TARGET" | grep '\.mwm\.norouting')" ] && mkdir -p "$INTDIR/norouting" && mv "$TARGET"/*.mwm.norouting "$INTDIR/norouting"
|
||||
|
||||
if [ "$MODE" == "resources" ]; then
|
||||
putmode "Step 7: Updating resource lists"
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue