diff --git a/crawler/wikitravel-download-pages.py b/crawler/wikitravel-download-pages.py index 3d2f5db22a..7e5502739b 100755 --- a/crawler/wikitravel-download-pages.py +++ b/crawler/wikitravel-download-pages.py @@ -12,9 +12,22 @@ for i, line in enumerate(sys.stdin): else: sys.stderr.write('Downloading {0} {1}\n'.format(i, fileName)) - remoteFile = urllib2.urlopen(url) - data = remoteFile.read(); - remoteFile.close() + tryCount = 0 + while True: + try: + tryCount = tryCount + 1 + remoteFile = urllib2.urlopen(url) + try: + data = remoteFile.read(); + finally: + remoteFile.close() + break + except IOError as error: + sys.stderr.write('Try {0}, error: {1}\n'.format(tryCount, error)) + if tryCount >= 5: + raise + else: + time.sleep(120) localFile = open(fileName, 'w') localFile.write(data)