From 4229680832ebdd7dc42475db9127741c488a2e21 Mon Sep 17 00:00:00 2001 From: Frank Ch. Eigler Date: Jan 13 2020 16:12:43 +0000 Subject: [PATCH 1/2] issues/1880: send Range: header to accelerate duplicate/partial downloads --- diff --git a/cli/koji_cli/lib.py b/cli/koji_cli/lib.py index 2243960..c3a8cde 100644 --- a/cli/koji_cli/lib.py +++ b/cli/koji_cli/lib.py @@ -492,20 +492,30 @@ def download_file(url, relpath, quiet=False, noprogress=False, size=None, num=No else: print(_("Downloading: %s") % relpath) + f = open(relpath, 'ab') + headers = {} + pos = f.tell() + if pos: + headers['Range'] = f'bytes={pos}-' + # closing needs to be used for requests < 2.18.0 - with closing(requests.get(url, stream=True)) as response: - # raise error if occured - response.raise_for_status() + with closing(requests.get(url, headers=headers, stream=True)) as response: + if (response.status_code == 200): # full content provided? + f.close() + f = open(relpath, 'wb') + elif not (response.status_code == 416 and pos): # error? + response.raise_for_status() length = int(response.headers.get('content-length') or 0) - with open(relpath, 'wb') as f: - l = 0 - for chunk in response.iter_content(chunk_size=65536): - l += len(chunk) - f.write(chunk) - if not (quiet or noprogress): - _download_progress(length, l) - if not length and not (quiet or noprogress): - _download_progress(l, l) + + l = 0 + for chunk in response.iter_content(chunk_size=65536): + l += len(chunk) + f.write(chunk) + if not (quiet or noprogress): + _download_progress(length, l) + if not length and not (quiet or noprogress): + _download_progress(l, l) + f.close() if not (quiet or noprogress): print('') From 5e306ef0d121316d7e6f31b8c5e87452147ab817 Mon Sep 17 00:00:00 2001 From: Frank Ch. Eigler Date: Jan 20 2020 14:43:00 +0000 Subject: [PATCH 2/2] Range: header: use old school fmt % operator This should be more compatible with old pythons. --- diff --git a/cli/koji_cli/lib.py b/cli/koji_cli/lib.py index c3a8cde..745c979 100644 --- a/cli/koji_cli/lib.py +++ b/cli/koji_cli/lib.py @@ -496,7 +496,7 @@ def download_file(url, relpath, quiet=False, noprogress=False, size=None, num=No headers = {} pos = f.tell() if pos: - headers['Range'] = f'bytes={pos}-' + headers['Range'] = ('bytes=%d-' % pos) # closing needs to be used for requests < 2.18.0 with closing(requests.get(url, headers=headers, stream=True)) as response: