From a3b20e216f3e4ec1793dbfef3471627dff49b324 Mon Sep 17 00:00:00 2001 From: Alex Iribarren Date: May 21 2020 12:14:28 +0000 Subject: [PATCH 1/4] Resolve a single Hub's IP so GSSAPI authentication and all requests for a session end up on the same host --- diff --git a/cli/koji_cli/commands.py b/cli/koji_cli/commands.py index ed96121..d660d6b 100644 --- a/cli/koji_cli/commands.py +++ b/cli/koji_cli/commands.py @@ -7367,7 +7367,7 @@ def handle_moshimoshi(options, session, args): u = {'name': 'anonymous user'} print("%s, %s!" % (_printable_unicode(random.choice(greetings)), u["name"])) print("") - print("You are using the hub at %s" % session.baseurl) + print("You are using the hub at %s" % session.original_baseurl) authtype = u.get('authtype', getattr(session, 'authtype', None)) if authtype == koji.AUTHTYPE_NORMAL: print("Authenticated via password") diff --git a/koji.spec b/koji.spec index 96d7412..9b8bbf3 100644 --- a/koji.spec +++ b/koji.spec @@ -130,6 +130,7 @@ Requires: python-requests-kerberos %else Requires: python-krbV >= 1.0.13 %endif +Requires: python-requests-toolbelt Requires: python-dateutil Requires: python-six @@ -150,6 +151,7 @@ Requires: rpm-python%{python3_pkgversion} Requires: python%{python3_pkgversion}-pyOpenSSL Requires: python%{python3_pkgversion}-requests Requires: python%{python3_pkgversion}-requests-kerberos +Requires: python%{python3_pkgversion}-requests-toolbelt Requires: python%{python3_pkgversion}-dateutil Requires: python%{python3_pkgversion}-six diff --git a/koji/__init__.py b/koji/__init__.py index 27536b2..c48fd23 100644 --- a/koji/__init__.py +++ b/koji/__init__.py @@ -51,6 +51,7 @@ import xml.sax.handler from fnmatch import fnmatch import requests +from requests_toolbelt.adapters import host_header_ssl import six import six.moves.configparser import six.moves.http_client @@ -2341,6 +2342,7 @@ class ClientSession(object): opts = opts.copy() self._apidoc = None self.baseurl = baseurl + self.original_baseurl = baseurl self.opts = opts self.authtype = None self.setSession(sinfo) @@ -2373,7 +2375,19 @@ class ClientSession(object): self.logger.debug("Opening new requests session") if self.rsession: self.rsession.close() + + # Resolve the hub we're going to talk to + uri = six.moves.urllib.parse.urlsplit(self.original_baseurl) + for addr in socket.getaddrinfo(uri[1], uri[0], 0, socket.SOCK_STREAM, socket.SOL_TCP): + s = socket.socket(addr[0], addr[1], addr[2]) + closed = s.connect_ex(addr[4]) + s.close() + if not closed: + self.baseurl = six.moves.urllib.parse.urlunsplit(uri._replace(netloc=socket.gethostbyaddr(addr[4][0])[0])) + break + self.rsession = requests.Session() + self.rsession.mount('https://', host_header_ssl.HostHeaderSSLAdapter()) def setSession(self, sinfo): """Set the session info @@ -2400,7 +2414,7 @@ class ClientSession(object): def subsession(self): "Create a subsession" sinfo = self.callMethod('subsession') - return type(self)(self.baseurl, self.opts, sinfo) + return type(self)(self.original_baseurl, self.opts, sinfo) def krb_login(self, principal=None, keytab=None, ccache=None, proxyuser=None, ctx=None): """Log in using Kerberos. If principal is not None and keytab is @@ -2693,7 +2707,7 @@ class ClientSession(object): # xml declaration for encoding as UTF-8". request = request.encode('utf-8') headers = [ - # connection class handles Host + ('Host', six.moves.urllib.parse.urlsplit(self.original_baseurl)[1]), ('User-Agent', 'koji/1'), ('Content-Type', 'text/xml'), ('Content-Length', str(len(request))), From 7b4cbe5db1b6a8fc97672dedda044876981aea18 Mon Sep 17 00:00:00 2001 From: Alex Iribarren Date: May 26 2020 08:39:36 +0000 Subject: [PATCH 2/4] Drop dependency of python-requests-toolbelt --- diff --git a/koji.spec b/koji.spec index 9b8bbf3..96d7412 100644 --- a/koji.spec +++ b/koji.spec @@ -130,7 +130,6 @@ Requires: python-requests-kerberos %else Requires: python-krbV >= 1.0.13 %endif -Requires: python-requests-toolbelt Requires: python-dateutil Requires: python-six @@ -151,7 +150,6 @@ Requires: rpm-python%{python3_pkgversion} Requires: python%{python3_pkgversion}-pyOpenSSL Requires: python%{python3_pkgversion}-requests Requires: python%{python3_pkgversion}-requests-kerberos -Requires: python%{python3_pkgversion}-requests-toolbelt Requires: python%{python3_pkgversion}-dateutil Requires: python%{python3_pkgversion}-six diff --git a/koji/__init__.py b/koji/__init__.py index c48fd23..500de68 100644 --- a/koji/__init__.py +++ b/koji/__init__.py @@ -51,7 +51,6 @@ import xml.sax.handler from fnmatch import fnmatch import requests -from requests_toolbelt.adapters import host_header_ssl import six import six.moves.configparser import six.moves.http_client @@ -2387,7 +2386,7 @@ class ClientSession(object): break self.rsession = requests.Session() - self.rsession.mount('https://', host_header_ssl.HostHeaderSSLAdapter()) + self.rsession.mount('https://', util.HostHeaderSSLAdapter()) def setSession(self, sinfo): """Set the session info diff --git a/koji/util.py b/koji/util.py index f2f4c34..b5e9d8b 100644 --- a/koji/util.py +++ b/koji/util.py @@ -37,6 +37,7 @@ import time import warnings from fnmatch import fnmatch from zlib import adler32 +from requests.adapters import HTTPAdapter import six from six.moves import range, zip @@ -851,3 +852,33 @@ def to_list(lst): return lst else: return list(lst) + +# Shamelessly copied from requests_toolbelt.adapters.host_header_ssl +class HostHeaderSSLAdapter(HTTPAdapter): + """ + A HTTPS Adapter for Python Requests that sets the hostname for certificate + verification based on the Host header. + This allows requesting the IP address directly via HTTPS without getting + a "hostname doesn't match" exception. + Example usage: + >>> s.mount('https://', HostHeaderSSLAdapter()) + >>> s.get("https://93.184.216.34", headers={"Host": "example.org"}) + """ + + def send(self, request, **kwargs): + # HTTP headers are case-insensitive (RFC 7230) + host_header = None + for header in request.headers: + if header.lower() == "host": + host_header = request.headers[header] + break + + connection_pool_kwargs = self.poolmanager.connection_pool_kw + + if host_header: + connection_pool_kwargs["assert_hostname"] = host_header + elif "assert_hostname" in connection_pool_kwargs: + # an assert_hostname from a previous request may have been left + connection_pool_kwargs.pop("assert_hostname", None) + + return super(HostHeaderSSLAdapter, self).send(request, **kwargs) From 6113cd6fef1eb190317a0098ee4634070e4eab98 Mon Sep 17 00:00:00 2001 From: Alex Iribarren Date: May 26 2020 10:05:13 +0000 Subject: [PATCH 3/4] Add new session_balance option --- diff --git a/docs/source/kojid_conf.rst b/docs/source/kojid_conf.rst index 2f61507..f34a1cd 100644 --- a/docs/source/kojid_conf.rst +++ b/docs/source/kojid_conf.rst @@ -37,6 +37,10 @@ General Turn off SSL verification for https calls. It is strongly advised to not turn off this verification. + session_balance=False + Load-balance requests to hubs by session instead of per request. This + allows GSSAPI authentication to work with multiple load-balanced hubs. + offline_retry=True The hub returns a special error code when it is placed in offline mode or when the database is unavailable. This setting controls diff --git a/koji/__init__.py b/koji/__init__.py index 500de68..1fe5258 100644 --- a/koji/__init__.py +++ b/koji/__init__.py @@ -1838,6 +1838,7 @@ def read_config(profile_name, user_config=None): 'ca': '', # FIXME: remove in next major release 'serverca': None, 'no_ssl_verify': False, + 'session_balance': False, 'authtype': None, 'debug': False, 'debug_xmlrpc': False, @@ -1880,7 +1881,7 @@ def read_config(profile_name, user_config=None): if name in result: if name in ('anon_retry', 'offline_retry', 'use_fast_upload', 'krb_rdns', 'debug', - 'debug_xmlrpc', 'krb_canon_host'): + 'debug_xmlrpc', 'krb_canon_host', 'session_balance'): result[name] = config.getboolean(profile_name, name) elif name in ('max_retries', 'retry_interval', 'offline_retry_interval', 'poll_interval', @@ -2316,6 +2317,7 @@ def grab_session_options(options): 'krb_canon_host', 'krb_server_realm', 'no_ssl_verify', + 'session_balance', 'serverca', ) # cert is omitted for now @@ -2375,15 +2377,16 @@ class ClientSession(object): if self.rsession: self.rsession.close() - # Resolve the hub we're going to talk to - uri = six.moves.urllib.parse.urlsplit(self.original_baseurl) - for addr in socket.getaddrinfo(uri[1], uri[0], 0, socket.SOCK_STREAM, socket.SOL_TCP): - s = socket.socket(addr[0], addr[1], addr[2]) - closed = s.connect_ex(addr[4]) - s.close() - if not closed: - self.baseurl = six.moves.urllib.parse.urlunsplit(uri._replace(netloc=socket.gethostbyaddr(addr[4][0])[0])) - break + if self.opts['session_balance']: + # Resolve the hub we're going to talk to + uri = six.moves.urllib.parse.urlsplit(self.original_baseurl) + for addr in socket.getaddrinfo(uri[1], uri[0], 0, socket.SOCK_STREAM, socket.SOL_TCP): + s = socket.socket(addr[0], addr[1], addr[2]) + closed = s.connect_ex(addr[4]) + s.close() + if not closed: + self.baseurl = six.moves.urllib.parse.urlunsplit(uri._replace(netloc=socket.gethostbyaddr(addr[4][0])[0])) + break self.rsession = requests.Session() self.rsession.mount('https://', util.HostHeaderSSLAdapter()) From 3e2b5d88c3faa9c12a8d51e30b1f4b77d4f4da62 Mon Sep 17 00:00:00 2001 From: Alex Iribarren Date: May 26 2020 16:52:45 +0000 Subject: [PATCH 4/4] Add new option everywhere --- diff --git a/builder/kojid b/builder/kojid index 63467ac..0a4c230 100755 --- a/builder/kojid +++ b/builder/kojid @@ -6390,6 +6390,7 @@ def get_options(): 'log_timestamps': False, 'timeout': None, 'no_ssl_verify': False, + 'session_balance': False, 'use_fast_upload': True, 'use_createrepo_c': True, 'createrepo_skip_stat': True, @@ -6423,7 +6424,7 @@ def get_options(): elif name in ['offline_retry', 'use_createrepo_c', 'createrepo_skip_stat', 'createrepo_update', 'use_fast_upload', 'support_rpm_source_layout', 'krb_rdns', 'krb_canon_host', 'build_arch_can_fail', 'no_ssl_verify', - 'log_timestamps']: + 'log_timestamps', 'session_balance']: defaults[name] = config.getboolean('kojid', name) elif name in ['plugin', 'plugins']: defaults['plugin'] = value.split() diff --git a/util/koji-gc b/util/koji-gc index e107a94..de800d1 100755 --- a/util/koji-gc +++ b/util/koji-gc @@ -149,6 +149,7 @@ def get_options(): ['grace_period', None, 'string'], ['trashcan_tag', None, 'string'], ['no_ssl_verify', None, 'boolean'], + ['session_balance', None, 'boolean'], ['timeout', None, 'integer'], ['lock_file', None, 'string'], ['exit_on_lock', None, 'boolean'], diff --git a/util/koji-sidetag-cleanup b/util/koji-sidetag-cleanup index a77d73d..57df33a 100644 --- a/util/koji-sidetag-cleanup +++ b/util/koji-sidetag-cleanup @@ -99,6 +99,7 @@ def get_options(): ['serverca', None, 'string'], ['server', None, 'string'], ['no_ssl_verify', None, 'boolean'], + ['session_balance', None, 'boolean'], ] for name, alias, type in cfgmap: if alias is None: diff --git a/util/kojira b/util/kojira index d1ba71f..cccdbc0 100755 --- a/util/kojira +++ b/util/kojira @@ -1033,6 +1033,7 @@ def get_options(): 'offline_retry': True, 'offline_retry_interval': 120, 'no_ssl_verify': False, + 'session_balance': False, 'max_delete_processes': 4, 'max_repo_tasks': 4, 'max_repo_tasks_maven': 2, @@ -1058,7 +1059,7 @@ def get_options(): str_opts = ('topdir', 'server', 'user', 'password', 'logfile', 'principal', 'keytab', 'krbservice', 'cert', 'ca', 'serverca', 'debuginfo_tags', 'queue_file', 'source_tags', 'separate_source_tags', 'ignore_tags') # FIXME: remove ca here - bool_opts = ('verbose', 'debug', 'ignore_stray_repos', 'offline_retry', + bool_opts = ('verbose', 'debug', 'ignore_stray_repos', 'offline_retry', 'session_balance', 'krb_rdns', 'krb_canon_host', 'no_ssl_verify', 'check_external_repos') legacy_opts = ('with_src') for name in config.options(section): diff --git a/vm/kojivmd b/vm/kojivmd index de06624..bed17bc 100755 --- a/vm/kojivmd +++ b/vm/kojivmd @@ -143,6 +143,7 @@ def get_options(): 'user': None, 'password': None, 'no_ssl_verify': False, + 'session_balance', False, 'retry_interval': 60, 'max_retries': 120, 'offline_retry': True, @@ -161,7 +162,7 @@ def get_options(): except ValueError: quit("value for %s option must be a valid integer" % name) elif name in ['offline_retry', 'krb_rdns', 'krb_canon_host', - 'no_ssl_verify']: + 'no_ssl_verify', 'session_balance']: defaults[name] = config.getboolean('kojivmd', name) elif name in ['plugin', 'plugins']: defaults['plugin'] = value.split() diff --git a/www/kojiweb/index.py b/www/kojiweb/index.py index d27e4f0..4137dfe 100644 --- a/www/kojiweb/index.py +++ b/www/kojiweb/index.py @@ -184,7 +184,8 @@ def _getServer(environ): s_opts = {'krbservice': opts['KrbService'], 'krb_rdns': opts['KrbRDNS'], 'krb_canon_host': opts['KrbCanonHost'], - 'krb_server_realm': opts['KrbServerRealm'] + 'krb_server_realm': opts['KrbServerRealm'], + 'session_balance': opts['session_balance'], } session = koji.ClientSession(opts['KojiHubURL'], opts=s_opts) diff --git a/www/kojiweb/wsgi_publisher.py b/www/kojiweb/wsgi_publisher.py index 44167bd..d8a9078 100644 --- a/www/kojiweb/wsgi_publisher.py +++ b/www/kojiweb/wsgi_publisher.py @@ -84,6 +84,7 @@ class Dispatcher(object): ['WebCert', 'string', None], ['KojiHubCA', 'string', '/etc/kojiweb/kojihubca.crt'], + ['session_balance', 'boolean', False], ['PythonDebug', 'boolean', False],