tfix multiprocessing issue with download() - amprolla - devuan's apt repo merger
HTML git clone git://parazyd.org/amprolla.git
DIR Log
DIR Files
DIR Refs
DIR README
DIR LICENSE
---
DIR commit e57b27612f2df0ade2d8f4dba62943712d6b74b4
DIR parent f82aba29785dd49fa0ec1425798de134366e2abd
HTML Author: parazyd <parazyd@dyne.org>
Date: Sun, 23 Jul 2017 17:58:47 +0200
fix multiprocessing issue with download()
tthe process should not forcefully exit with die() since it causes
a deadlock for multiprocessing.Pool
tthis commit also reimplements handling of
requests.exceptions.ReadTimeout
Diffstat:
M lib/net.py | 10 ++++------
1 file changed, 4 insertions(+), 6 deletions(-)
---
DIR diff --git a/lib/net.py b/lib/net.py
t@@ -21,16 +21,14 @@ def download(uris):
try:
r = requests.get(url, stream=True, timeout=20)
- # TODO: investigate also requests.exceptions.ReadTimeout
- except requests.exceptions.ConnectionError as e:
+ except (requests.exceptions.ConnectionError,
+ requests.exceptions.ReadTimeout) as e:
warn('Caught exception: "%s". Retrying...' % e)
return download(uris)
- if r.status_code == 404:
- warn('failed: 404 not found!')
+ if r.status_code != 200:
+ warn('%s failed: %d' % (url, r.status_code))
return
- elif r.status_code != 200:
- die('failed: %d' % r.status_code)
makedirs(dirname(path), exist_ok=True)
f = open(path, 'wb')