From 1ea67cca63496c571eac497772bf06ec41acbeb7 Mon Sep 17 00:00:00 2001
From: Yury Melnichek <melnichek@gmail.com>
Date: Sun, 3 Apr 2011 03:12:37 +0200
Subject: [PATCH] Better error handling in crawler.

---
 crawler/wikitravel-download-pages.py | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/crawler/wikitravel-download-pages.py b/crawler/wikitravel-download-pages.py
index 3d2f5db22a..7e5502739b 100755
--- a/crawler/wikitravel-download-pages.py
+++ b/crawler/wikitravel-download-pages.py
@@ -12,9 +12,22 @@ for i, line in enumerate(sys.stdin):
   else:
     sys.stderr.write('Downloading {0} {1}\n'.format(i, fileName))
 
-    remoteFile = urllib2.urlopen(url)
-    data = remoteFile.read();
-    remoteFile.close()
+    tryCount = 0
+    while True:
+      try:
+        tryCount = tryCount + 1
+        remoteFile = urllib2.urlopen(url)
+        try:
+          data = remoteFile.read();
+        finally:
+          remoteFile.close()
+        break
+      except IOError as error:
+        sys.stderr.write('Try {0}, error: {1}\n'.format(tryCount, error))
+        if tryCount >= 5:
+          raise
+        else:
+          time.sleep(120)
 
     localFile = open(fileName, 'w')
     localFile.write(data)