From e60d07c3bfd472cf61c6469087ca3c0f7f1b8a76 Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Jun 22 2017 08:49:21 +0000 Subject: [PATCH 1/3] Patch xmlrpclib for 64-bit integers Related: https://pagure.io/koji/issue/464 --- diff --git a/hub/kojixmlrpc.py b/hub/kojixmlrpc.py index 6f848d8..3be9ce7 100644 --- a/hub/kojixmlrpc.py +++ b/hub/kojixmlrpc.py @@ -61,6 +61,24 @@ class Marshaller(xmlrpclib.Marshaller): self.dump_string(value, write) dispatch[datetime.datetime] = dump_datetime + MAXI8 = 2 ** 64 - 1 + MINI8 = -2 ** 64 + def dump_i8(self, value, write): + # python2's xmlrpclib doesn't support i8 extension for marshalling, + # but can unmarshall it correctly. + if value > Marshaller.MAXI8 or value < Marshaller.MINI8: + raise OverflowError, "long int exceeds XML-RPC limits" + elif value > xmlrpclib.MAXINT or value < xmlrpclib.MININT: + write("") + write(str(int(value))) + write("\n") + else: + write("") + write(str(int(value))) + write("\n") + dispatch[types.LongType] = dump_i8 + dispatch[types.IntType] = dump_i8 + xmlrpclib.Marshaller = Marshaller From e345ba199bdc8e44c3efca564c6038e09fd74b45 Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Jun 22 2017 08:49:21 +0000 Subject: [PATCH 2/3] remove encode_int in favour of i8 xmlrpc extension --- diff --git a/devtools/fakehub b/devtools/fakehub index fed749d..4e4ba07 100755 --- a/devtools/fakehub +++ b/devtools/fakehub @@ -7,7 +7,6 @@ import os import os.path import pprint import sys -import six.moves.xmlrpc_client from six.moves import cStringIO from six.moves.urllib.parse import quote @@ -58,7 +57,7 @@ def get_request(): else: args.append(nice_literal(s)) args = koji.encode_args(*args, **kwargs) - request = six.moves.xmlrpc_client.dumps(args, method, allow_none=1) + request = koji.dumps(args, method, allow_none=1) return request @@ -68,7 +67,7 @@ def start_response(status, headers): def parse_response(data): - p, u = six.moves.xmlrpc_client.getparser() + p, u = koji.getparser() for chunk in data: p.feed(chunk) p.close() diff --git a/hub/kojihub.py b/hub/kojihub.py index 4fce02b..4a1f041 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -559,8 +559,7 @@ def make_task(method, arglist, **opts): raise koji.GenericError("invalid channel policy") # encode xmlrpc request - opts['request'] = xmlrpclib.dumps(tuple(arglist), methodname=method, - allow_none=1) + opts['request'] = koji.dumps(tuple(arglist), methodname=method, allow_none=1) opts['state'] = koji.TASK_STATES['FREE'] opts['method'] = method koji.plugin.run_callbacks('preTaskStateChange', attribute='state', old=None, new='FREE', info=opts) @@ -3599,7 +3598,7 @@ def get_next_release(build_info): def _fix_rpm_row(row): if 'size' in row: - row['size'] = koji.encode_int(row['size']) + row['size'] = row['size'] if 'extra' in row: row['extra'] = parse_json(row['extra'], desc='rpm extra') return row @@ -8809,8 +8808,6 @@ class RootExports(object): contents = base64.decodestring(data) del data # we will accept offset and size as strings to work around xmlrpc limits - offset = koji.decode_int(offset) - size = koji.decode_int(size) if isinstance(md5sum, basestring): # this case is for backwards compatibility verify = "md5" @@ -8911,7 +8908,7 @@ class RootExports(object): st = os.fstat(fd) if not stat.S_ISREG(st.st_mode): raise koji.GenericError("Not a regular file: %s" % fn) - data['size'] = koji.encode_int(st.st_size) + data['size'] = st.st_size data['mtime'] = st.st_mtime if verify: sum_cls = get_verify_class(verify) @@ -8927,7 +8924,7 @@ class RootExports(object): length += len(chunk) chksum.update(chunk) chunk = os.read(fd, 8192) - data['sumlength'] = koji.encode_int(length) + data['sumlength'] = length data['hexdigest'] = chksum.hexdigest() return data finally: @@ -12577,9 +12574,9 @@ def handle_upload(environ): # this will also remove our lock os.close(fd) ret = { - 'size': koji.encode_int(size), + 'size': size, 'fileverify': verify, - 'offset': koji.encode_int(offset), + 'offset': offset, } if verify: # unsigned 32bit - could be too big for xmlrpc diff --git a/hub/kojixmlrpc.py b/hub/kojixmlrpc.py index 3be9ce7..d826d5c 100644 --- a/hub/kojixmlrpc.py +++ b/hub/kojixmlrpc.py @@ -19,18 +19,14 @@ # Mike McLean from ConfigParser import RawConfigParser -import datetime import inspect import logging import os import sys import time import traceback -import types import pprint import resource -import xmlrpclib -from xmlrpclib import getparser, dumps, Fault from koji.server import WSGIWrapper import koji @@ -39,49 +35,11 @@ import koji.db import koji.plugin import koji.policy import koji.util +# import xmlrpclib functions from koji to use tweaked Marshaller +from koji import getparser, dumps, Fault from koji.context import context -# Workaround to allow xmlrpclib deal with iterators -class Marshaller(xmlrpclib.Marshaller): - - dispatch = xmlrpclib.Marshaller.dispatch.copy() - - def dump_generator(self, value, write): - dump = self.__dump - write("\n") - for v in value: - dump(v, write) - write("\n") - dispatch[types.GeneratorType] = dump_generator - - def dump_datetime(self, value, write): - # For backwards compatibility, we return datetime objects as strings - value = value.isoformat(' ') - self.dump_string(value, write) - dispatch[datetime.datetime] = dump_datetime - - MAXI8 = 2 ** 64 - 1 - MINI8 = -2 ** 64 - def dump_i8(self, value, write): - # python2's xmlrpclib doesn't support i8 extension for marshalling, - # but can unmarshall it correctly. - if value > Marshaller.MAXI8 or value < Marshaller.MINI8: - raise OverflowError, "long int exceeds XML-RPC limits" - elif value > xmlrpclib.MAXINT or value < xmlrpclib.MININT: - write("") - write(str(int(value))) - write("\n") - else: - write("") - write(str(int(value))) - write("\n") - dispatch[types.LongType] = dump_i8 - dispatch[types.IntType] = dump_i8 - -xmlrpclib.Marshaller = Marshaller - - class HandlerRegistry(object): """Track handlers for RPC calls""" diff --git a/koji/__init__.py b/koji/__init__.py index 2b9fe24..51ee1c5 100644 --- a/koji/__init__.py +++ b/koji/__init__.py @@ -73,13 +73,55 @@ import struct import tempfile import time import traceback +import types from . import util import warnings -import six.moves.xmlrpc_client import xml.sax import xml.sax.handler -from six.moves.xmlrpc_client import loads, dumps, Fault import six.moves.urllib +from six.moves.xmlrpc_client import getparser, loads, dumps, Fault + +# Workaround to allow xmlrpclib deal with iterators and 64-bit ints +class Marshaller(six.moves.xmlrpc_client.Marshaller): + + dispatch = six.moves.xmlrpc_client.Marshaller.dispatch.copy() + + def dump_generator(self, value, write): + dump = self.__dump + write("\n") + for v in value: + dump(v, write) + write("\n") + dispatch[types.GeneratorType] = dump_generator + + def dump_datetime(self, value, write): + # For backwards compatibility, we return datetime objects as strings + value = value.isoformat(' ') + self.dump_string(value, write) + dispatch[datetime.datetime] = dump_datetime + + MAXI8 = 2 ** 64 - 1 + MINI8 = -2 ** 64 + def dump_i8(self, value, write): + # python2's xmlrpclib doesn't support i8 extension for marshalling, + # but can unmarshall it correctly. + if value > Marshaller.MAXI8 or value < Marshaller.MINI8: + raise OverflowError, "long int exceeds XML-RPC limits" + elif value > six.moves.xmlrpc_client.MAXINT or \ + value < six.moves.xmlrpc_client.MININT: + write("") + write(str(int(value))) + write("\n") + else: + write("") + write(str(int(value))) + write("\n") + dispatch[types.LongType] = dump_i8 + dispatch[types.IntType] = dump_i8 + +six.moves.xmlrpc_client.Marshaller = Marshaller +six.moves.xmlrpc_client.FastMarshaller = None + PROFILE_MODULES = {} # {module_name: module_instance} @@ -452,23 +494,6 @@ def decode_args2(args, names, strict=True): ret.update(opts) return ret -## BEGIN kojikamid dup - -def encode_int(n): - """If n is too large for a 32bit signed, convert it to a string""" - if n <= 2147483647: - return n - #else - return str(n) -## END kojikamid dup - -def decode_int(n): - """If n is not an integer, attempt to convert it""" - if isinstance(n, six.integer_types): - return n - #else - return int(n) - #commonly used functions def safe_xmlrpc_loads(s): @@ -2382,7 +2407,7 @@ class ClientSession(object): return ret def _read_xmlrpc_response(self, response): - p, u = six.moves.xmlrpc_client.getparser() + p, u = getparser() for chunk in response.iter_content(8192): if self.opts.get('debug_xmlrpc', False): print("body: %r" % chunk) @@ -2643,7 +2668,7 @@ class ClientSession(object): while True: if debug: self.logger.debug("uploadFile(%r,%r,%r,%r,%r,...)" %(path, name, sz, digest, offset)) - if self.callMethod('uploadFile', path, name, encode_int(sz), digest, encode_int(offset), data, **volopts): + if self.callMethod('uploadFile', path, name, sz, digest, offset, data, **volopts): break if tries <= retries: tries += 1 diff --git a/koji/daemon.py b/koji/daemon.py index 9706437..71f8bf0 100644 --- a/koji/daemon.py +++ b/koji/daemon.py @@ -34,7 +34,6 @@ import time import sys import traceback import errno -import xmlrpclib def incremental_upload(session, fname, fd, path, retries=5, logger=None): @@ -61,7 +60,7 @@ def incremental_upload(session, fname, fd, path, retries=5, logger=None): tries = 0 while True: - if session.uploadFile(path, fname, koji.encode_int(size), digest, koji.encode_int(offset), data): + if session.uploadFile(path, fname, size, digest, offset, data): break if tries <= retries: @@ -1209,12 +1208,12 @@ class TaskManager(object): try: response = (handler.run(),) # note that we wrap response in a singleton tuple - response = xmlrpclib.dumps(response, methodresponse=1, allow_none=1) + response = koji.dumps(response, methodresponse=1, allow_none=1) self.logger.info("RESPONSE: %r" % response) self.session.host.closeTask(handler.id, response) return - except xmlrpclib.Fault, fault: - response = xmlrpclib.dumps(fault) + except koji.Fault, fault: + response = koji.dumps(fault) tb = ''.join(traceback.format_exception(*sys.exc_info())).replace(r"\n", "\n") self.logger.warn("FAULT:\n%s" % tb) except (SystemExit, koji.tasks.ServerExit, KeyboardInterrupt): @@ -1233,7 +1232,7 @@ class TaskManager(object): if issubclass(e_class, koji.GenericError): #just pass it through tb = str(e) - response = xmlrpclib.dumps(xmlrpclib.Fault(faultCode, tb)) + response = koji.dumps(koji.Fault(faultCode, tb)) # if we get here, then we're handling an exception, so fail the task self.session.host.failTask(handler.id, response) diff --git a/vm/kojikamid.py b/vm/kojikamid.py index 0499c74..386f86e 100755 --- a/vm/kojikamid.py +++ b/vm/kojikamid.py @@ -305,7 +305,7 @@ class WindowsBuild(object): offset = 0 checksum = hashlib.md5() while True: - encoded = self.server.getFile(buildinfo, fileinfo, encode_int(offset), 1048576, type) + encoded = self.server.getFile(buildinfo, fileinfo, offset, 1048576, type) if not encoded: break data = base64.b64decode(encoded) @@ -568,7 +568,7 @@ def upload_file(server, prefix, path): if not data: break encoded = base64.b64encode(data) - server.upload(path, encode_int(offset), encoded) + server.upload(path, offset, encoded) offset += len(data) sum.update(data) fobj.close() diff --git a/vm/kojivmd b/vm/kojivmd index 3ca7342..b4774bf 100755 --- a/vm/kojivmd +++ b/vm/kojivmd @@ -720,8 +720,8 @@ class VMExecTask(BaseTaskHandler): """ remotepath = os.path.dirname(os.path.join(self.getUploadDir(), filepath)) filename = os.path.basename(filepath) - self.session.uploadFile(remotepath, filename, koji.encode_int(size), - md5sum, koji.encode_int(offset), data) + self.session.uploadFile(remotepath, filename, size, + md5sum, offset, data) def verifyChecksum(self, path, checksum, algo='sha1'): local_path = os.path.abspath(os.path.join(self.output_dir, path)) diff --git a/www/kojiweb/index.py b/www/kojiweb/index.py index caa65a7..237e7d0 100644 --- a/www/kojiweb/index.py +++ b/www/kojiweb/index.py @@ -774,14 +774,13 @@ def getfile(environ, taskID, name, volume='DEFAULT', offset=None, size=None): def _chunk_file(server, environ, taskID, name, offset, size, volume): remaining = size - encode_int = koji.encode_int while True: if remaining <= 0: break chunk_size = 1048576 if remaining < chunk_size: chunk_size = remaining - content = server.downloadTaskOutput(taskID, name, offset=encode_int(offset), size=chunk_size, volume=volume) + content = server.downloadTaskOutput(taskID, name, offset=offset, size=chunk_size, volume=volume) if not content: break yield content From 6eea21cd36512abc09b9a3f9183fa26f10885e5c Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Jun 22 2017 08:50:15 +0000 Subject: [PATCH 3/3] docs update for xmlrpc i8 extension --- diff --git a/docs/source/using_the_koji_build_system.rst b/docs/source/using_the_koji_build_system.rst index d913219..b2f1d57 100644 --- a/docs/source/using_the_koji_build_system.rst +++ b/docs/source/using_the_koji_build_system.rst @@ -566,3 +566,6 @@ All features supported by command-line client are also accessible by XMLRPC API. You can get listing of all available calls, arguments and basic help via calling `koji list-api` command. This call will also provide you API extensions provided by plugins in that particular koji instance. + +In addition to xmlrpc standard we are using `i8` extension for large +integers. Datetimes are exchanged as text strings in iso format.