From 59ae727bef8a55c543d8891d55d016fd956707cc Mon Sep 17 00:00:00 2001 From: Yury Melnichek Date: Mon, 10 Sep 2012 23:59:48 +0200 Subject: [PATCH] [crawler] Crawl more Wikitravel pages. --- crawler/wikitravel-download-lists.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/crawler/wikitravel-download-lists.sh b/crawler/wikitravel-download-lists.sh index 16f6c8f764..0b3a0136ec 100755 --- a/crawler/wikitravel-download-lists.sh +++ b/crawler/wikitravel-download-lists.sh @@ -9,6 +9,7 @@ wget $LONGPAGES_URL"&limit=5000&offset=0" -O wikitravel-pages-0.html && slee wget $LONGPAGES_URL"&limit=5000&offset=5000" -O wikitravel-pages-1.html && sleep 10s wget $LONGPAGES_URL"&limit=5000&offset=10000" -O wikitravel-pages-2.html && sleep 10s wget $LONGPAGES_URL"&limit=5000&offset=15000" -O wikitravel-pages-3.html && sleep 10s +wget $LONGPAGES_URL"&limit=5000&offset=20000" -O wikitravel-pages-4.html && sleep 10s # Get all redirects. wget $REDIRECTS_URL"&limit=5000&offset=0" -O wikitravel-redirects-0.html && sleep 10s