From 0df536124b5f25417569a6251a41f54846e18310 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: May 03 2017 14:06:31 +0000 Subject: [PATCH 1/3] Import GET for compatrequests Signed-off-by: Patrick Uiterwijk --- diff --git a/koji/compatrequests.py b/koji/compatrequests.py index d56f0f2..cba353e 100644 --- a/koji/compatrequests.py +++ b/koji/compatrequests.py @@ -23,8 +23,19 @@ class Session(object): def __init__(self): self.connection = None + def get(self, url, headers=None, stream=None, verify=None, cert=None, + timeout=None): + return self._perform('GET', url=url, headers=headers, stream=stream, + verify=verify, cert=cert) + def post(self, url, data=None, headers=None, stream=None, verify=None, cert=None, timeout=None): + return self._perform('POST', url=url, data=data, headers=headers, + stream=stream, verify=verify, cert=cert, + timeout=timeout) + + def _perform(self, method, url, data=None, headers=None, stream=None, + verify=None, cert=None, timeout=None): uri = urlparse.urlsplit(url) if uri[3]: handler = "%s?%s" % (uri[2], uri[3]) @@ -32,7 +43,7 @@ class Session(object): handler = uri[2] cnx = self.get_connection(uri, cert, verify, timeout) #cnx.set_debuglevel(1) - cnx.putrequest('POST', handler) + cnx.putrequest(method, handler) if headers: for k in headers: cnx.putheader(k, headers[k]) From eded9fce83de48b4482302285a90d7281a9e24c7 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: May 03 2017 14:07:04 +0000 Subject: [PATCH 2/3] Make iter_content handle chunk_size Signed-off-by: Patrick Uiterwijk --- diff --git a/koji/compatrequests.py b/koji/compatrequests.py index cba353e..cbd1e30 100644 --- a/koji/compatrequests.py +++ b/koji/compatrequests.py @@ -138,7 +138,12 @@ class Response(object): self.response.reason)) - def iter_content(self, blocksize=1): + def iter_content(self, chunk_size=8192, blocksize=None): + if blocksize is not None: + # This was called blocksize in a previous release, while requests + # uses chunk_size. So let's stay backwards compatible in case any + # places that use it decided to use blocksize as a keyword argument + chunk_size = blocksize # should we check this in Session.post()? # should we even check this here? if self.response.status != 200: @@ -147,7 +152,7 @@ class Response(object): # XXX wrong exception raise Exception("Server status: %s" % self.response.status) while True: - chunk = self.response.read(blocksize) + chunk = self.response.read(chunk_size) if not chunk: break yield chunk From 6b4770f2d75baeb03593ad1f0b63a883d58b7a0d Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: May 03 2017 14:07:04 +0000 Subject: [PATCH 3/3] Implement openRemoteFile with requests or compatrequests Signed-off-by: Patrick Uiterwijk --- diff --git a/koji/__init__.py b/koji/__init__.py index 7bafee7..a099577 100644 --- a/koji/__init__.py +++ b/koji/__init__.py @@ -70,7 +70,6 @@ import tempfile import time import traceback import urllib -import urllib2 import urlparse import util import warnings @@ -1560,11 +1559,19 @@ def openRemoteFile(relpath, topurl=None, topdir=None, tempdir=None): This is done either via a mounted filesystem (nfs) or http, depending on options""" if topurl: + if requests is None: + import koji.compatrequests + rsession = koji.compatrequests.Session() + else: + rsession = requests.Session() + url = "%s/%s" % (topurl, relpath) - src = urllib2.urlopen(url) + resp = rsession.get(url) fo = tempfile.TemporaryFile(dir=tempdir) - shutil.copyfileobj(src, fo) - src.close() + for chunk in resp.iter_content(chunk_size=8192): + if chunk: + fo.write(chunk) + resp.close() fo.seek(0) elif topdir: fn = "%s/%s" % (topdir, relpath)