From 802f97a7499c18a4d6b5f52e5e206fc017c7bbe6 Mon Sep 17 00:00:00 2001 From: Xibo Ning Date: Jun 07 2017 09:19:31 +0000 Subject: [PATCH 1/11] patch2 for issue 288 --- diff --git a/builder/kojid b/builder/kojid index 235cbeb..0820f05 100755 --- a/builder/kojid +++ b/builder/kojid @@ -1364,6 +1364,7 @@ class BuildMavenTask(BaseBuildTask): scm = SCM(url) scm.assert_allowed(self.options.allowed_scms) + self.run_plugin('preSCMCheckout', scminfo=scm.contextData()) repo_id = opts.get('repo_id') if not repo_id: @@ -1417,6 +1418,9 @@ class BuildMavenTask(BaseBuildTask): # Check out sources from the SCM sourcedir = scm.checkout(scmdir, self.session, uploadpath, logfile) + self.run_plugin("postSCMCheckout", + scminfo=scm.contextData(), + extra_keys={"sourcedir": sourcedir}) # zip up pristine sources for auditing purposes self._zip_dir(sourcedir, os.path.join(outputdir, 'scm-sources.zip')) @@ -1426,9 +1430,13 @@ class BuildMavenTask(BaseBuildTask): patchlog = self.workdir + '/patches.log' patch_scm = SCM(self.opts.get('patches')) patch_scm.assert_allowed(self.options.allowed_scms) + self.run_plugin('preSCMCheckout', scminfo=patch_scm.contextData()) # never try to check out a common/ dir when checking out patches patch_scm.use_common = False patchcheckoutdir = patch_scm.checkout(patchdir, self.session, uploadpath, patchlog) + self.run_plugin("postSCMCheckout", + scminfo=patch_scm.contextData(), + extra_keys={"sourcedir": patchcheckoutdir}) self._zip_dir(patchcheckoutdir, os.path.join(outputdir, 'patches.zip')) # Apply patches, if present @@ -1691,6 +1699,7 @@ class WrapperRPMTask(BaseBuildTask): scm = SCM(spec_url) scm.assert_allowed(self.options.allowed_scms) + self.run_plugin('preSCMCheckout', scminfo=scm.contextData()) repo_id = opts.get('repo_id') if not repo_id: @@ -1710,6 +1719,9 @@ class WrapperRPMTask(BaseBuildTask): scmdir = buildroot.rootdir() + '/tmp/scmroot' koji.ensuredir(scmdir) specdir = scm.checkout(scmdir, self.session, self.getUploadDir(), logfile) + self.run_plugin("postSCMCheckout", + scminfo=scm.contextData(), + extra_keys={"sourcedir": specdir}) spec_template = None for path, dir, files in os.walk(specdir): @@ -2641,8 +2653,12 @@ class ImageTask(BaseTaskHandler): if self.opts.get('ksurl'): scm = SCM(self.opts['ksurl']) scm.assert_allowed(self.options.allowed_scms) + self.run_plugin('preSCMCheckout', scminfo=scm.contextData()) logfile = os.path.join(self.workdir, 'checkout.log') scmsrcdir = scm.checkout(scmdir, self.session, self.getUploadDir(), logfile) + self.run_plugin("postSCMCheckout", + scminfo=scm.contextData(), + extra_keys={"sourcedir": scmsrcdir}) kspath = os.path.join(scmsrcdir, ksfile) else: kspath = self.localPath("work/%s" % ksfile) @@ -3253,9 +3269,13 @@ class OzImageTask(BaseTaskHandler): if self.opts.get('ksurl'): scm = SCM(self.opts['ksurl']) scm.assert_allowed(self.options.allowed_scms) + self.run_plugin('preSCMCheckout', scminfo=scm.contextData()) logfile = os.path.join(self.workdir, 'checkout-%s.log' % self.arch) scmsrcdir = scm.checkout(self.workdir, self.session, self.getUploadDir(), logfile) + self.run_plugin("postSCMCheckout", + scminfo=scm.contextData(), + extra_keys={"sourcedir": scmsrcdir}) kspath = os.path.join(scmsrcdir, os.path.basename(ksfile)) else: tops = dict([(k, getattr(self.options, k)) for k in 'topurl','topdir']) @@ -4060,9 +4080,13 @@ class BuildIndirectionImageTask(OzImageTask): if fileurl: scm = SCM(fileurl) scm.assert_allowed(self.options.allowed_scms) + self.run_plugin('preSCMCheckout', scminfo=scm.contextData()) logfile = os.path.join(self.workdir, 'checkout.log') scmsrcdir = scm.checkout(self.workdir, self.session, self.getUploadDir(), logfile) + self.run_plugin("postSCMCheckout", + scminfo=scm.contextData(), + extra_keys={"sourcedir": scmsrcdir}) final_path = os.path.join(scmsrcdir, os.path.basename(filepath)) else: tops = dict([(k, getattr(self.options, k)) for k in 'topurl','topdir']) @@ -4372,6 +4396,7 @@ class BuildSRPMFromSCMTask(BaseBuildTask): # will throw a BuildError if the url is invalid scm = SCM(url) scm.assert_allowed(self.options.allowed_scms) + self.run_plugin('preSCMCheckout', scminfo=scm.contextData()) if opts is None: opts = {} @@ -4405,6 +4430,9 @@ class BuildSRPMFromSCMTask(BaseBuildTask): # Check out spec file, etc. from SCM sourcedir = scm.checkout(scmdir, self.session, uploadpath, logfile) + self.run_plugin("postSCMCheckout", + scminfo=scm.contextData(), + extra_keys={"sourcedir": sourcedir}) # chown the sourcedir and everything under it to the mockuser # so we can build the srpm as non-root uid = pwd.getpwnam(self.options.mockuser)[2] diff --git a/koji/daemon.py b/koji/daemon.py index 637566f..9e12b3e 100644 --- a/koji/daemon.py +++ b/koji/daemon.py @@ -229,6 +229,11 @@ class SCM(object): # should never happen raise koji.GenericError('Invalid SCM URL: %s' % url) + def contextData(self, keys=None): + if keys is None: + keys = ["url", "scheme", "user", "host", "path", "query", "revision", "scmtype"] + return dict([(k, vars(self)[k]) for k in keys if vars(self).has_key(k)]) + def _parse_url(self): """ Parse the SCM url into usable components. diff --git a/koji/plugin.py b/koji/plugin.py index e4fe19d..b17632e 100644 --- a/koji/plugin.py +++ b/koji/plugin.py @@ -48,6 +48,8 @@ callbacks = { 'postRepoDone': [], 'preCommit': [], 'postCommit': [], + 'preSCMCheckout': [], + 'postSCMCheckout': [], } class PluginTracker(object): diff --git a/koji/tasks.py b/koji/tasks.py index 36c50cf..45d2fdc 100644 --- a/koji/tasks.py +++ b/koji/tasks.py @@ -31,6 +31,7 @@ import random import time import pprint import six.moves.urllib.request +import inspect def scan_mounts(topdir): """Search path for mountpoints""" @@ -382,6 +383,45 @@ class BaseTaskHandler(object): repo_info = self.wait(task_id)[task_id] return repo_info + @classmethod + def request_keys(cls, method_name='handler', filter=None): + try: + keys = cls.requestKeys + except AttributeError: + pass + else: + return keys + + if filter is None: + filter = (lambda name, obj: + inspect.ismethod(obj) and name == method_name) + methods = [(name, obj) for name, obj in inspect.getmembers(cls) + if filter(name, obj)] + _, method = methods[0] + keys, args, kwargs, default_vals = tuple(inspect.getargspec(method)) + keys = keys + [item for item in args, kwargs if item is not None] + return keys[1:] + + def parsed_request(self, request=None, keys=None): + if request is None: + request = self.session.getTaskRequest(self.id) + if keys is None: + keys = self.request_keys() + + return zip(keys, request) + + def contextData(self): + taskinfo = self.session.getTaskInfo(self.id, request=True) + request = self.parsed_request(taskinfo['request']) + return taskinfo.update(request) + + def run_plugin(self, plugin, scminfo=None, extra_keys=None): + taskinfo = self.contextData() + if extra_keys: + scminfo.update(extra_keys) + + koji.plugin.run_callbacks(plugin, taskinfo=taskinfo, scminfo=scminfo) + class FakeTask(BaseTaskHandler): Methods = ['someMethod'] diff --git a/vm/kojivmd b/vm/kojivmd index 3ca7342..cbabac9 100755 --- a/vm/kojivmd +++ b/vm/kojivmd @@ -305,6 +305,7 @@ class WinBuildTask(MultiPlatformTask): for url in [source_url] + subopts.values(): scm = SCM(url) scm.assert_allowed(self.options.allowed_scms) + self.run_plugin('preSCMCheckout', scminfo=scm.contextData()) task_info = self.session.getTaskInfo(self.id) target_info = self.session.getBuildTarget(target) From c1ca9c7360d0a5a5328ac5513edb6950f25c28f1 Mon Sep 17 00:00:00 2001 From: Yuming Zhu Date: Jun 07 2017 09:19:31 +0000 Subject: [PATCH 2/11] only path taskinfo, scminfo, [srcdir] to plugin --- diff --git a/builder/kojid b/builder/kojid index 0820f05..43fd1d9 100755 --- a/builder/kojid +++ b/builder/kojid @@ -1364,8 +1364,6 @@ class BuildMavenTask(BaseBuildTask): scm = SCM(url) scm.assert_allowed(self.options.allowed_scms) - self.run_plugin('preSCMCheckout', scminfo=scm.contextData()) - repo_id = opts.get('repo_id') if not repo_id: raise koji.BuildError('A repo_id must be provided') @@ -1416,11 +1414,13 @@ class BuildMavenTask(BaseBuildTask): logfile = self.workdir + '/checkout.log' uploadpath = self.getUploadDir() + + self.run_plugin('preSCMCheckout', scminfo=scm.get_info()) # Check out sources from the SCM sourcedir = scm.checkout(scmdir, self.session, uploadpath, logfile) self.run_plugin("postSCMCheckout", - scminfo=scm.contextData(), - extra_keys={"sourcedir": sourcedir}) + scminfo=scm.get_info(), + srcdir=sourcedir) # zip up pristine sources for auditing purposes self._zip_dir(sourcedir, os.path.join(outputdir, 'scm-sources.zip')) @@ -1430,13 +1430,13 @@ class BuildMavenTask(BaseBuildTask): patchlog = self.workdir + '/patches.log' patch_scm = SCM(self.opts.get('patches')) patch_scm.assert_allowed(self.options.allowed_scms) - self.run_plugin('preSCMCheckout', scminfo=patch_scm.contextData()) + self.run_plugin('preSCMCheckout', scminfo=patch_scm.get_info()) # never try to check out a common/ dir when checking out patches patch_scm.use_common = False patchcheckoutdir = patch_scm.checkout(patchdir, self.session, uploadpath, patchlog) self.run_plugin("postSCMCheckout", - scminfo=patch_scm.contextData(), - extra_keys={"sourcedir": patchcheckoutdir}) + scminfo=patch_scm.get_info(), + srcdir=patchcheckoutdir) self._zip_dir(patchcheckoutdir, os.path.join(outputdir, 'patches.zip')) # Apply patches, if present @@ -1699,7 +1699,6 @@ class WrapperRPMTask(BaseBuildTask): scm = SCM(spec_url) scm.assert_allowed(self.options.allowed_scms) - self.run_plugin('preSCMCheckout', scminfo=scm.contextData()) repo_id = opts.get('repo_id') if not repo_id: @@ -1718,10 +1717,11 @@ class WrapperRPMTask(BaseBuildTask): logfile = os.path.join(self.workdir, 'checkout.log') scmdir = buildroot.rootdir() + '/tmp/scmroot' koji.ensuredir(scmdir) + self.run_plugin('preSCMCheckout', scminfo=scm.get_info()) specdir = scm.checkout(scmdir, self.session, self.getUploadDir(), logfile) self.run_plugin("postSCMCheckout", - scminfo=scm.contextData(), - extra_keys={"sourcedir": specdir}) + scminfo=scm.get_info(), + srcdir=specdir) spec_template = None for path, dir, files in os.walk(specdir): @@ -2653,12 +2653,12 @@ class ImageTask(BaseTaskHandler): if self.opts.get('ksurl'): scm = SCM(self.opts['ksurl']) scm.assert_allowed(self.options.allowed_scms) - self.run_plugin('preSCMCheckout', scminfo=scm.contextData()) logfile = os.path.join(self.workdir, 'checkout.log') + self.run_plugin('preSCMCheckout', scminfo=scm.get_info()) scmsrcdir = scm.checkout(scmdir, self.session, self.getUploadDir(), logfile) self.run_plugin("postSCMCheckout", - scminfo=scm.contextData(), - extra_keys={"sourcedir": scmsrcdir}) + scminfo=scm.get_info(), + srcdir=scmsrcdir) kspath = os.path.join(scmsrcdir, ksfile) else: kspath = self.localPath("work/%s" % ksfile) @@ -3269,13 +3269,13 @@ class OzImageTask(BaseTaskHandler): if self.opts.get('ksurl'): scm = SCM(self.opts['ksurl']) scm.assert_allowed(self.options.allowed_scms) - self.run_plugin('preSCMCheckout', scminfo=scm.contextData()) logfile = os.path.join(self.workdir, 'checkout-%s.log' % self.arch) + self.run_plugin('preSCMCheckout', scminfo=scm.get_info()) scmsrcdir = scm.checkout(self.workdir, self.session, self.getUploadDir(), logfile) self.run_plugin("postSCMCheckout", - scminfo=scm.contextData(), - extra_keys={"sourcedir": scmsrcdir}) + scminfo=scm.get_info(), + srcdir=scmsrcdir) kspath = os.path.join(scmsrcdir, os.path.basename(ksfile)) else: tops = dict([(k, getattr(self.options, k)) for k in 'topurl','topdir']) @@ -4080,13 +4080,13 @@ class BuildIndirectionImageTask(OzImageTask): if fileurl: scm = SCM(fileurl) scm.assert_allowed(self.options.allowed_scms) - self.run_plugin('preSCMCheckout', scminfo=scm.contextData()) + self.run_plugin('preSCMCheckout', scminfo=scm.get_info()) logfile = os.path.join(self.workdir, 'checkout.log') scmsrcdir = scm.checkout(self.workdir, self.session, self.getUploadDir(), logfile) self.run_plugin("postSCMCheckout", - scminfo=scm.contextData(), - extra_keys={"sourcedir": scmsrcdir}) + scminfo=scm.get_info(), + srcdir=scmsrcdir) final_path = os.path.join(scmsrcdir, os.path.basename(filepath)) else: tops = dict([(k, getattr(self.options, k)) for k in 'topurl','topdir']) @@ -4396,7 +4396,6 @@ class BuildSRPMFromSCMTask(BaseBuildTask): # will throw a BuildError if the url is invalid scm = SCM(url) scm.assert_allowed(self.options.allowed_scms) - self.run_plugin('preSCMCheckout', scminfo=scm.contextData()) if opts is None: opts = {} @@ -4428,11 +4427,12 @@ class BuildSRPMFromSCMTask(BaseBuildTask): logfile = self.workdir + '/checkout.log' uploadpath = self.getUploadDir() + self.run_plugin('preSCMCheckout', scminfo=scm.get_info()) # Check out spec file, etc. from SCM sourcedir = scm.checkout(scmdir, self.session, uploadpath, logfile) self.run_plugin("postSCMCheckout", - scminfo=scm.contextData(), - extra_keys={"sourcedir": sourcedir}) + scminfo=scm.get_info(), + srcdir=sourcedir) # chown the sourcedir and everything under it to the mockuser # so we can build the srpm as non-root uid = pwd.getpwnam(self.options.mockuser)[2] diff --git a/koji/daemon.py b/koji/daemon.py index 9e12b3e..8b72e5b 100644 --- a/koji/daemon.py +++ b/koji/daemon.py @@ -229,7 +229,7 @@ class SCM(object): # should never happen raise koji.GenericError('Invalid SCM URL: %s' % url) - def contextData(self, keys=None): + def get_info(self, keys=None): if keys is None: keys = ["url", "scheme", "user", "host", "path", "query", "revision", "scmtype"] return dict([(k, vars(self)[k]) for k in keys if vars(self).has_key(k)]) diff --git a/koji/tasks.py b/koji/tasks.py index 45d2fdc..2c5fed2 100644 --- a/koji/tasks.py +++ b/koji/tasks.py @@ -383,44 +383,16 @@ class BaseTaskHandler(object): repo_info = self.wait(task_id)[task_id] return repo_info - @classmethod - def request_keys(cls, method_name='handler', filter=None): - try: - keys = cls.requestKeys - except AttributeError: - pass - else: - return keys - - if filter is None: - filter = (lambda name, obj: - inspect.ismethod(obj) and name == method_name) - methods = [(name, obj) for name, obj in inspect.getmembers(cls) - if filter(name, obj)] - _, method = methods[0] - keys, args, kwargs, default_vals = tuple(inspect.getargspec(method)) - keys = keys + [item for item in args, kwargs if item is not None] - return keys[1:] - - def parsed_request(self, request=None, keys=None): - if request is None: - request = self.session.getTaskRequest(self.id) - if keys is None: - keys = self.request_keys() - - return zip(keys, request) - - def contextData(self): - taskinfo = self.session.getTaskInfo(self.id, request=True) - request = self.parsed_request(taskinfo['request']) - return taskinfo.update(request) - - def run_plugin(self, plugin, scminfo=None, extra_keys=None): - taskinfo = self.contextData() - if extra_keys: - scminfo.update(extra_keys) - - koji.plugin.run_callbacks(plugin, taskinfo=taskinfo, scminfo=scminfo) + + def run_plugin(self, plugin, *args, **kwargs): + if 'taskinfo' not in kwargs: + try: + taskinfo = self.taskinfo + except AttributeError: + self.taskinfo = self.session.getTaskInfo(self.id, request=True) + taskinfo = self.taskinfo + kwargs['taskinfo'] = taskinfo + koji.plugin.run_callbacks(plugin, *args, **kwargs) class FakeTask(BaseTaskHandler): diff --git a/vm/kojikamid.py b/vm/kojikamid.py index 0499c74..9249e1e 100755 --- a/vm/kojikamid.py +++ b/vm/kojikamid.py @@ -67,7 +67,7 @@ class WindowsBuild(object): """Get task info and setup build directory""" self.logger = logging.getLogger('koji.vm') self.server = server - info = server.getTaskInfo() + self.taskinfo = server.getTaskInfo() self.source_url = info[0] self.build_tag = info[1] if len(info) > 2: @@ -166,17 +166,29 @@ class WindowsBuild(object): def checkout(self): """Checkout sources, winspec, and patches, and apply patches""" src_scm = SCM(self.source_url) + koji.plugin.run_callbacks('preSCMCheckout', taskinfo=self.taskinfo, scminfo=src_scm.get_info()) self.source_dir = src_scm.checkout(ensuredir(os.path.join(self.workdir, 'source'))) + koji.plugin.run_callbacks('postSCMCheckout', taskinfo=self.taskinfo, + scminfo=src_scm.get_info(), + srcdir=self.source_dir) self.zipDir(self.source_dir, os.path.join(self.workdir, 'sources.zip')) if 'winspec' in self.task_opts: spec_scm = SCM(self.task_opts['winspec']) + koji.plugin.run_callbacks('preSCMCheckout', taskinfo=self.taskinfo, scminfo=spec_scm.get_info()) self.spec_dir = spec_scm.checkout(ensuredir(os.path.join(self.workdir, 'spec'))) + koji.plugin.run_callbacks('postSCMCheckout', taskinfo=self.taskinfo, + scminfo=spec_scm.get_info(), + srcdir=self.spec_dir) self.zipDir(self.spec_dir, os.path.join(self.workdir, 'spec.zip')) else: self.spec_dir = self.source_dir if 'patches' in self.task_opts: patch_scm = SCM(self.task_opts['patches']) + koji.plugin.run_callbacks('preSCMCheckout', taskinfo=self.taskinfo, scminfo=patch_scm.get_info()) self.patches_dir = patch_scm.checkout(ensuredir(os.path.join(self.workdir, 'patches'))) + koji.plugin.run_callbacks('postSCMCheckout', taskinfo=self.taskinfo, + scminfo=patch_scm.get_info(), + srcdir=self.patch_dir) self.zipDir(self.patches_dir, os.path.join(self.workdir, 'patches.zip')) self.applyPatches(self.source_dir, self.patches_dir) self.virusCheck(self.workdir) diff --git a/vm/kojivmd b/vm/kojivmd index cbabac9..75e3eb9 100755 --- a/vm/kojivmd +++ b/vm/kojivmd @@ -305,8 +305,6 @@ class WinBuildTask(MultiPlatformTask): for url in [source_url] + subopts.values(): scm = SCM(url) scm.assert_allowed(self.options.allowed_scms) - self.run_plugin('preSCMCheckout', scminfo=scm.contextData()) - task_info = self.session.getTaskInfo(self.id) target_info = self.session.getBuildTarget(target) if not target_info: From 6c95b2eab53fee1206c292f14ac03f6ba63f2e2e Mon Sep 17 00:00:00 2001 From: Yuming Zhu Date: Jun 07 2017 09:20:03 +0000 Subject: [PATCH 3/11] remove useless import --- diff --git a/koji/tasks.py b/koji/tasks.py index 2c5fed2..3960901 100644 --- a/koji/tasks.py +++ b/koji/tasks.py @@ -31,7 +31,6 @@ import random import time import pprint import six.moves.urllib.request -import inspect def scan_mounts(topdir): """Search path for mountpoints""" From ba7c13f21afa6eb488b0416cc5c0c3821131ccf4 Mon Sep 17 00:00:00 2001 From: Yuming Zhu Date: Jun 07 2017 09:20:03 +0000 Subject: [PATCH 4/11] fix reference problem --- diff --git a/vm/kojikamid.py b/vm/kojikamid.py index 9249e1e..ff38b41 100755 --- a/vm/kojikamid.py +++ b/vm/kojikamid.py @@ -67,9 +67,9 @@ class WindowsBuild(object): """Get task info and setup build directory""" self.logger = logging.getLogger('koji.vm') self.server = server - self.taskinfo = server.getTaskInfo() - self.source_url = info[0] - self.build_tag = info[1] + self.task_info = server.getTaskInfo() + self.source_url = self.task_info[0] + self.build_tag = self.task_info[1] if len(info) > 2: self.task_opts = info[2] else: @@ -166,17 +166,17 @@ class WindowsBuild(object): def checkout(self): """Checkout sources, winspec, and patches, and apply patches""" src_scm = SCM(self.source_url) - koji.plugin.run_callbacks('preSCMCheckout', taskinfo=self.taskinfo, scminfo=src_scm.get_info()) + koji.plugin.run_callbacks('preSCMCheckout', taskinfo=self.task_info, scminfo=src_scm.get_info()) self.source_dir = src_scm.checkout(ensuredir(os.path.join(self.workdir, 'source'))) - koji.plugin.run_callbacks('postSCMCheckout', taskinfo=self.taskinfo, + koji.plugin.run_callbacks('postSCMCheckout', taskinfo=self.task_info, scminfo=src_scm.get_info(), srcdir=self.source_dir) self.zipDir(self.source_dir, os.path.join(self.workdir, 'sources.zip')) if 'winspec' in self.task_opts: spec_scm = SCM(self.task_opts['winspec']) - koji.plugin.run_callbacks('preSCMCheckout', taskinfo=self.taskinfo, scminfo=spec_scm.get_info()) + koji.plugin.run_callbacks('preSCMCheckout', taskinfo=self.task_info, scminfo=spec_scm.get_info()) self.spec_dir = spec_scm.checkout(ensuredir(os.path.join(self.workdir, 'spec'))) - koji.plugin.run_callbacks('postSCMCheckout', taskinfo=self.taskinfo, + koji.plugin.run_callbacks('postSCMCheckout', taskinfo=self.task_info, scminfo=spec_scm.get_info(), srcdir=self.spec_dir) self.zipDir(self.spec_dir, os.path.join(self.workdir, 'spec.zip')) @@ -184,9 +184,9 @@ class WindowsBuild(object): self.spec_dir = self.source_dir if 'patches' in self.task_opts: patch_scm = SCM(self.task_opts['patches']) - koji.plugin.run_callbacks('preSCMCheckout', taskinfo=self.taskinfo, scminfo=patch_scm.get_info()) + koji.plugin.run_callbacks('preSCMCheckout', taskinfo=self.task_info, scminfo=patch_scm.get_info()) self.patches_dir = patch_scm.checkout(ensuredir(os.path.join(self.workdir, 'patches'))) - koji.plugin.run_callbacks('postSCMCheckout', taskinfo=self.taskinfo, + koji.plugin.run_callbacks('postSCMCheckout', taskinfo=self.task_info, scminfo=patch_scm.get_info(), srcdir=self.patch_dir) self.zipDir(self.patches_dir, os.path.join(self.workdir, 'patches.zip')) From e9205ac83f9f01e746b336078f21a4fd8ce94993 Mon Sep 17 00:00:00 2001 From: Yuming Zhu Date: Jun 07 2017 09:20:03 +0000 Subject: [PATCH 5/11] use util.dslice instead --- diff --git a/koji/daemon.py b/koji/daemon.py index 8b72e5b..5c35486 100644 --- a/koji/daemon.py +++ b/koji/daemon.py @@ -23,7 +23,7 @@ import koji import koji.tasks from koji.tasks import safe_rmtree -from koji.util import md5_constructor, adler32_constructor, parseStatus +from koji.util import md5_constructor, adler32_constructor, parseStatus, dslice import os import signal import logging @@ -232,7 +232,7 @@ class SCM(object): def get_info(self, keys=None): if keys is None: keys = ["url", "scheme", "user", "host", "path", "query", "revision", "scmtype"] - return dict([(k, vars(self)[k]) for k in keys if vars(self).has_key(k)]) + return dslice(vars(self), keys) def _parse_url(self): """ From 11e1c5c670d7ebf34aa3e888ce988ddc4a44ba37 Mon Sep 17 00:00:00 2001 From: Yuming Zhu Date: Jun 07 2017 09:20:03 +0000 Subject: [PATCH 6/11] fix fields in SCM --- diff --git a/koji/daemon.py b/koji/daemon.py index 5c35486..421a2b1 100644 --- a/koji/daemon.py +++ b/koji/daemon.py @@ -231,7 +231,7 @@ class SCM(object): def get_info(self, keys=None): if keys is None: - keys = ["url", "scheme", "user", "host", "path", "query", "revision", "scmtype"] + keys = ["url", "scheme", "user", "host", "repository", "module", "revision", "scmtype"] return dslice(vars(self), keys) def _parse_url(self): From bfc1689eefb7353a4571a76a5930ba7790b74310 Mon Sep 17 00:00:00 2001 From: yzhu Date: Jun 07 2017 09:20:03 +0000 Subject: [PATCH 7/11] enable callback plugin for builder --- diff --git a/koji/daemon.py b/koji/daemon.py index 421a2b1..9706437 100644 --- a/koji/daemon.py +++ b/koji/daemon.py @@ -518,13 +518,29 @@ class TaskManager(object): def findHandlers(self, vars): """Find and index task handlers""" for v in vars.values(): - if isinstance(v, type(koji.tasks.BaseTaskHandler)) and issubclass(v, koji.tasks.BaseTaskHandler): - for method in v.Methods: - self.handlers[method] = v + self.registerHandler(v) + + def registerHandler(self, entry): + """register and index task handler""" + if isinstance(entry, type(koji.tasks.BaseTaskHandler)) and issubclass(entry, koji.tasks.BaseTaskHandler): + for method in entry.Methods: + self.handlers[method] = entry + + def registerCallback(self, entry): + """register and index callback plugins""" + if callable(entry) and getattr(entry, 'callbacks', None): + for cbtype in entry.callbacks: + koji.plugin.register_callback(cbtype, entry) + + def registerEntries(self, vars): + """Register task handlers and other plugins""" + for v in vars.values(): + self.registerHandler(v) + self.registerCallback(v) def scanPlugin(self, plugin): """Find task handlers in a plugin""" - self.findHandlers(vars(plugin)) + self.registerEntries(vars(plugin)) def shutdown(self): """Attempt to shut down cleanly""" From a174e5236a6cc93bf1a7080adea40085778eed10 Mon Sep 17 00:00:00 2001 From: Yuming Zhu Date: Jun 07 2017 09:20:03 +0000 Subject: [PATCH 8/11] push build_tag, session, scratch into callbacks --- diff --git a/builder/kojid b/builder/kojid index 43fd1d9..047cdaf 100755 --- a/builder/kojid +++ b/builder/kojid @@ -899,7 +899,7 @@ class BuildTask(BaseTaskHandler): def getSRPMFromSCM(self, url, build_tag, repo_id): #TODO - allow different ways to get the srpm task_id = self.session.host.subtask(method='buildSRPMFromSCM', - arglist=[url, build_tag, {'repo_id': repo_id}], + arglist=[url, build_tag, {'repo_id': repo_id, 'scratch': self.opts.get('scratch')}], label='srpm', parent=self.id) # wait for subtask to finish @@ -1264,7 +1264,7 @@ class MavenTask(MultiPlatformTask): raise koji.BuildError('no repo for tag %s' % build_tag['name']) build_opts = dslice(opts, ['goals', 'profiles', 'properties', 'envs', 'patches', - 'packages', 'jvm_options', 'maven_options', 'deps'], + 'packages', 'jvm_options', 'maven_options', 'deps', 'scratch'], strict=False) build_opts['repo_id'] = repo_id @@ -1415,11 +1415,12 @@ class BuildMavenTask(BaseBuildTask): uploadpath = self.getUploadDir() - self.run_plugin('preSCMCheckout', scminfo=scm.get_info()) + self.run_plugin('preSCMCheckout', scminfo=scm.get_info(), build_tag=build_tag) # Check out sources from the SCM sourcedir = scm.checkout(scmdir, self.session, uploadpath, logfile) self.run_plugin("postSCMCheckout", scminfo=scm.get_info(), + scratch=opts.get('scratch'), srcdir=sourcedir) # zip up pristine sources for auditing purposes @@ -1430,12 +1431,13 @@ class BuildMavenTask(BaseBuildTask): patchlog = self.workdir + '/patches.log' patch_scm = SCM(self.opts.get('patches')) patch_scm.assert_allowed(self.options.allowed_scms) - self.run_plugin('preSCMCheckout', scminfo=patch_scm.get_info()) + self.run_plugin('preSCMCheckout', scminfo=patch_scm.get_info(), build_tag=build_tag) # never try to check out a common/ dir when checking out patches patch_scm.use_common = False patchcheckoutdir = patch_scm.checkout(patchdir, self.session, uploadpath, patchlog) self.run_plugin("postSCMCheckout", scminfo=patch_scm.get_info(), + scratch=opts.get('scratch'), srcdir=patchcheckoutdir) self._zip_dir(patchcheckoutdir, os.path.join(outputdir, 'patches.zip')) @@ -1717,10 +1719,11 @@ class WrapperRPMTask(BaseBuildTask): logfile = os.path.join(self.workdir, 'checkout.log') scmdir = buildroot.rootdir() + '/tmp/scmroot' koji.ensuredir(scmdir) - self.run_plugin('preSCMCheckout', scminfo=scm.get_info()) + self.run_plugin('preSCMCheckout', scminfo=scm.get_info(), build_tag=build_tag) specdir = scm.checkout(scmdir, self.session, self.getUploadDir(), logfile) self.run_plugin("postSCMCheckout", scminfo=scm.get_info(), + scratch=opts.get('scratch'), srcdir=specdir) spec_template = None @@ -2631,7 +2634,7 @@ class ImageTask(BaseTaskHandler): self.logger.debug("Image buildroot ready: " + broot.rootdir()) return broot - def fetchKickstart(self, broot, ksfile): + def fetchKickstart(self, broot, ksfile, build_tag): """ Retrieve the kickstart file we were given (locally or remotely) and upload it. @@ -2645,6 +2648,7 @@ class ImageTask(BaseTaskHandler): @args: broot: a buildroot object ksfile: path to a kickstart file + build_tag: build tag name @returns: absolute path to the retrieved kickstart file """ scmdir = os.path.join(broot.rootdir(), 'tmp') @@ -2654,10 +2658,11 @@ class ImageTask(BaseTaskHandler): scm = SCM(self.opts['ksurl']) scm.assert_allowed(self.options.allowed_scms) logfile = os.path.join(self.workdir, 'checkout.log') - self.run_plugin('preSCMCheckout', scminfo=scm.get_info()) + self.run_plugin('preSCMCheckout', scminfo=scm.get_info(), build_tag=build_tag) scmsrcdir = scm.checkout(scmdir, self.session, self.getUploadDir(), logfile) self.run_plugin("postSCMCheckout", scminfo=scm.get_info(), + scratch=self.opts.get('scratch'), srcdir=scmsrcdir) kspath = os.path.join(scmsrcdir, ksfile) else: @@ -2832,7 +2837,7 @@ class ApplianceTask(ImageTask): self.opts = opts broot = self.makeImgBuildRoot(build_tag, repo_info, arch, 'appliance-build') - kspath = self.fetchKickstart(broot, ksfile) + kspath = self.fetchKickstart(broot, ksfile, target_info['build_tag_name']) self.readKickstart(kspath, opts) kskoji = self.prepareKickstart(repo_info, target_info, arch, broot, opts) # Figure out appliance-creator arguments, let it fail if something @@ -2981,7 +2986,7 @@ class LiveCDTask(ImageTask): broot = self.makeImgBuildRoot(build_tag, repo_info, arch, 'livecd-build') - kspath = self.fetchKickstart(broot, ksfile) + kspath = self.fetchKickstart(broot, ksfile, target_info['build_tag_name']) self.readKickstart(kspath, opts) kskoji = self.prepareKickstart(repo_info, target_info, arch, broot, opts) @@ -3131,7 +3136,7 @@ class LiveMediaTask(ImageTask): broot = self.makeImgBuildRoot(build_tag, repo_info, arch, 'livemedia-build') - kspath = self.fetchKickstart(broot, ksfile) + kspath = self.fetchKickstart(broot, ksfile, target_info['build_tag_name']) self.readKickstart(kspath, opts) kskoji = self.prepareKickstart(repo_info, target_info, arch, broot, opts) @@ -3249,7 +3254,7 @@ class LiveMediaTask(ImageTask): class OzImageTask(BaseTaskHandler): Methods = [] - def fetchKickstart(self): + def fetchKickstart(self, build_tag): """ Retrieve the kickstart file we were given (locally or remotely) and upload it to the hub. @@ -3260,7 +3265,8 @@ class OzImageTask(BaseTaskHandler): relative path in a remote scm. The user should have passed in an scm url with --ksurl. - @args: None, use self.opts for options + @args: build_tag: build tag name + use self.opts for options @returns: absolute path to the retrieved kickstart file """ @@ -3270,11 +3276,12 @@ class OzImageTask(BaseTaskHandler): scm = SCM(self.opts['ksurl']) scm.assert_allowed(self.options.allowed_scms) logfile = os.path.join(self.workdir, 'checkout-%s.log' % self.arch) - self.run_plugin('preSCMCheckout', scminfo=scm.get_info()) + self.run_plugin('preSCMCheckout', scminfo=scm.get_info(), build_tag=build_tag) scmsrcdir = scm.checkout(self.workdir, self.session, self.getUploadDir(), logfile) self.run_plugin("postSCMCheckout", scminfo=scm.get_info(), + scratch=self.opts.get('scratch'), srcdir=scmsrcdir) kspath = os.path.join(scmsrcdir, os.path.basename(ksfile)) else: @@ -3929,7 +3936,7 @@ class BaseImageTask(OzImageTask): self.formats = self._format_deps(opts.get('format')) # First, prepare the kickstart to use the repos we tell it - kspath = self.fetchKickstart() + kspath = self.fetchKickstart(build_tag=target_info['build_tag_name']) ks = self.prepareKickstart(kspath, inst_tree) kskoji = self.writeKickstart(ks, os.path.join(self.workdir, 'koji-%s-%i-base.ks' % @@ -4060,7 +4067,7 @@ class BuildIndirectionImageTask(OzImageTask): # END inefficient base image task method copies - def fetchHubOrSCM(self, filepath, fileurl): + def fetchHubOrSCM(self, filepath, fileurl, build_tag): """ Retrieve a file either from the hub or a remote scm @@ -4080,12 +4087,13 @@ class BuildIndirectionImageTask(OzImageTask): if fileurl: scm = SCM(fileurl) scm.assert_allowed(self.options.allowed_scms) - self.run_plugin('preSCMCheckout', scminfo=scm.get_info()) + self.run_plugin('preSCMCheckout', scminfo=scm.get_info(), build_tag=build_tag) logfile = os.path.join(self.workdir, 'checkout.log') scmsrcdir = scm.checkout(self.workdir, self.session, self.getUploadDir(), logfile) self.run_plugin("postSCMCheckout", scminfo=scm.get_info(), + scratch=self.opts.get('scratch'), srcdir=scmsrcdir) final_path = os.path.join(scmsrcdir, os.path.basename(filepath)) else: @@ -4236,7 +4244,8 @@ class BuildIndirectionImageTask(OzImageTask): raise koji.ApplianceError('The Release may not have a hyphen') indirection_template = self.fetchHubOrSCM(opts.get('indirection_template'), - opts.get('indirection_template_url')) + opts.get('indirection_template_url'), + target_info['build_tag_name']) self.logger.debug('Got indirection template %s' % (indirection_template)) @@ -4427,11 +4436,12 @@ class BuildSRPMFromSCMTask(BaseBuildTask): logfile = self.workdir + '/checkout.log' uploadpath = self.getUploadDir() - self.run_plugin('preSCMCheckout', scminfo=scm.get_info()) + self.run_plugin('preSCMCheckout', scminfo=scm.get_info(), build_tag=build_tag) # Check out spec file, etc. from SCM sourcedir = scm.checkout(scmdir, self.session, uploadpath, logfile) self.run_plugin("postSCMCheckout", scminfo=scm.get_info(), + scratch=self.opts.get('scratch'), srcdir=sourcedir) # chown the sourcedir and everything under it to the mockuser # so we can build the srpm as non-root diff --git a/koji/tasks.py b/koji/tasks.py index 3960901..5d7ac98 100644 --- a/koji/tasks.py +++ b/koji/tasks.py @@ -391,6 +391,7 @@ class BaseTaskHandler(object): self.taskinfo = self.session.getTaskInfo(self.id, request=True) taskinfo = self.taskinfo kwargs['taskinfo'] = taskinfo + kwargs['session'] = self.session koji.plugin.run_callbacks(plugin, *args, **kwargs) diff --git a/vm/kojikamid.py b/vm/kojikamid.py index ff38b41..5882c16 100755 --- a/vm/kojikamid.py +++ b/vm/kojikamid.py @@ -47,19 +47,20 @@ MANAGER_PORT = 7000 KOJIKAMID = True + ## INSERT kojikamid dup class fakemodule(object): pass -#make parts of the above insert accessible as koji.X + +# make parts of the above insert accessible as koji.X koji = fakemodule() koji.GenericError = GenericError koji.BuildError = BuildError class WindowsBuild(object): - LEADING_CHAR = re.compile('^[^A-Za-z_]') VAR_CHARS = re.compile('[^A-Za-z0-9_]') @@ -67,9 +68,9 @@ class WindowsBuild(object): """Get task info and setup build directory""" self.logger = logging.getLogger('koji.vm') self.server = server - self.task_info = server.getTaskInfo() - self.source_url = self.task_info[0] - self.build_tag = self.task_info[1] + self.info = server.getTaskInfo() + self.source_url = self.info[0] + self.build_tag = self.info[1] if len(info) > 2: self.task_opts = info[2] else: @@ -136,7 +137,7 @@ class WindowsBuild(object): self.logger.info('file %s exists', entry) if errors: raise BuildError('error validating build environment: %s' % \ - ', '.join(errors)) + ', '.join(errors)) def updateClam(self): """update ClamAV virus definitions""" @@ -166,17 +167,17 @@ class WindowsBuild(object): def checkout(self): """Checkout sources, winspec, and patches, and apply patches""" src_scm = SCM(self.source_url) - koji.plugin.run_callbacks('preSCMCheckout', taskinfo=self.task_info, scminfo=src_scm.get_info()) + self.server.runCallBacks('preSCMCheckout', scminfo=src_scm.get_info()) self.source_dir = src_scm.checkout(ensuredir(os.path.join(self.workdir, 'source'))) - koji.plugin.run_callbacks('postSCMCheckout', taskinfo=self.task_info, + self.server.runCallBacks('postSCMCheckout', scminfo=src_scm.get_info(), srcdir=self.source_dir) self.zipDir(self.source_dir, os.path.join(self.workdir, 'sources.zip')) if 'winspec' in self.task_opts: spec_scm = SCM(self.task_opts['winspec']) - koji.plugin.run_callbacks('preSCMCheckout', taskinfo=self.task_info, scminfo=spec_scm.get_info()) + self.server.runCallBacks('preSCMCheckout', scminfo=spec_scm.get_info()) self.spec_dir = spec_scm.checkout(ensuredir(os.path.join(self.workdir, 'spec'))) - koji.plugin.run_callbacks('postSCMCheckout', taskinfo=self.task_info, + self.server.runCallBacks('postSCMCheckout', scminfo=spec_scm.get_info(), srcdir=self.spec_dir) self.zipDir(self.spec_dir, os.path.join(self.workdir, 'spec.zip')) @@ -184,9 +185,9 @@ class WindowsBuild(object): self.spec_dir = self.source_dir if 'patches' in self.task_opts: patch_scm = SCM(self.task_opts['patches']) - koji.plugin.run_callbacks('preSCMCheckout', taskinfo=self.task_info, scminfo=patch_scm.get_info()) + self.server.runCallBacks('preSCMCheckout', scminfo=patch_scm.get_info()) self.patches_dir = patch_scm.checkout(ensuredir(os.path.join(self.workdir, 'patches'))) - koji.plugin.run_callbacks('postSCMCheckout', taskinfo=self.task_info, + self.server.runCallBacks('postSCMCheckout', scminfo=patch_scm.get_info(), srcdir=self.patch_dir) self.zipDir(self.patches_dir, os.path.join(self.workdir, 'patches.zip')) @@ -330,7 +331,7 @@ class WindowsBuild(object): # rpms don't have a md5sum in the fileinfo, but check it for everything else if ('md5sum' in fileinfo) and (digest != fileinfo['md5sum']): raise BuildError('md5 checksum validation failed for %s, %s (computed) != %s (provided)' % \ - (destpath, digest, fileinfo['md5sum'])) + (destpath, digest, fileinfo['md5sum'])) self.logger.info('Retrieved %s (%s bytes, md5: %s)', destpath, offset, digest) def fetchBuildReqs(self): @@ -480,7 +481,7 @@ class WindowsBuild(object): self.virusCheck(self.workdir) if errors: raise BuildError('error validating build output: %s' % \ - ', '.join(errors)) + ', '.join(errors)) def virusCheck(self, path): """ensure a path is virus free with ClamAV. path should be absolute""" @@ -511,6 +512,7 @@ class WindowsBuild(object): self.expireBuildroot() return self.gatherResults() + def run(cmd, chdir=None, fatal=False, log=True): global logfd output = '' @@ -540,6 +542,7 @@ def run(cmd, chdir=None, fatal=False, log=True): raise BuildError(msg) return ret, output + def find_net_info(): """ Find the network gateway configured for this VM. @@ -568,6 +571,7 @@ def find_net_info(): gateway = None return macaddr, gateway + def upload_file(server, prefix, path): """upload a single file to the vmd""" logger = logging.getLogger('koji.vm') @@ -588,6 +592,7 @@ def upload_file(server, prefix, path): server.verifyChecksum(path, digest, 'md5') logger.info('Uploaded %s (%s bytes, md5: %s)', destpath, offset, digest) + def get_mgmt_server(): """Get a ServerProxy object we can use to retrieve task info""" logger = logging.getLogger('koji.vm') @@ -606,6 +611,7 @@ def get_mgmt_server(): logger.debug('found task-specific port %s', task_port) return xmlrpclib.ServerProxy('http://%s:%s/' % (gateway, task_port), allow_none=True) + def get_options(): """handle usage and parse options""" usage = """%prog [options] @@ -615,10 +621,12 @@ def get_options(): parser = OptionParser(usage=usage) parser.add_option('-d', '--debug', action='store_true', help='Log debug statements') parser.add_option('-i', '--install', action='store_true', help='Install this daemon as a service', default=False) - parser.add_option('-u', '--uninstall', action='store_true', help='Uninstall this daemon if it was installed previously as a service', default=False) + parser.add_option('-u', '--uninstall', action='store_true', + help='Uninstall this daemon if it was installed previously as a service', default=False) (options, args) = parser.parse_args() return options + def setup_logging(opts): global logfile, logfd logger = logging.getLogger('koji.vm') @@ -633,11 +641,13 @@ def setup_logging(opts): logger.addHandler(handler) return handler + def log_local(msg): tb = ''.join(traceback.format_exception(*sys.exc_info())) sys.stderr.write('%s: %s\n' % (time.ctime(), msg)) sys.stderr.write(tb) + def stream_logs(server, handler, builds): """Stream logs incrementally to the server. The global logfile will always be streamed. @@ -675,6 +685,7 @@ def stream_logs(server, handler, builds): log_local('error uploading %s' % relpath) time.sleep(1) + def fail(server, handler): """do the right thing when a build fails""" global logfile, logfd @@ -701,6 +712,7 @@ def fail(server, handler): logfile = '/tmp/build.log' logfd = None + def main(): prog = os.path.basename(sys.argv[0]) opts = get_options() diff --git a/vm/kojivmd b/vm/kojivmd index 75e3eb9..7609253 100755 --- a/vm/kojivmd +++ b/vm/kojivmd @@ -298,7 +298,7 @@ class WinBuildTask(MultiPlatformTask): if not opts: opts = {} - subopts = koji.util.dslice(opts, ['winspec', 'patches'], + subopts = koji.util.dslice(opts, ['winspec', 'patches', 'scratch'], strict=False) # winspec and patches options are urls # verify the urls before passing them to the VM @@ -531,6 +531,20 @@ class VMExecTask(BaseTaskHandler): """ return self.task_info + def runCallbacks(self, method, **opts): + """ + Run callback plugins. + """ + taskinfo = self.session.getTaskInfo(self.id, request=True) + opts['taskinfo'] = taskinfo + opts['session'] = self.session + if method == 'preSCMCheckout': + opts['build_tag'] = self.task_info.get('build_tag') + elif method == 'postSCMCheckout': + opts['scratch'] = self.task_info.get('scratch') + koji.plugin.run_callbacks(method, **opts) + + def initBuildroot(self, repo_id, platform): """ Create the buildroot object on the hub. From f33e6343af4ed2729b8dad1c718b8c33438927dc Mon Sep 17 00:00:00 2001 From: Yuming Zhu Date: Jun 07 2017 09:20:03 +0000 Subject: [PATCH 9/11] remove callbacks in win builder --- diff --git a/vm/kojikamid.py b/vm/kojikamid.py index 5882c16..710671a 100755 --- a/vm/kojikamid.py +++ b/vm/kojikamid.py @@ -68,9 +68,9 @@ class WindowsBuild(object): """Get task info and setup build directory""" self.logger = logging.getLogger('koji.vm') self.server = server - self.info = server.getTaskInfo() - self.source_url = self.info[0] - self.build_tag = self.info[1] + info = server.getTaskInfo() + self.source_url = info[0] + self.build_tag = info[1] if len(info) > 2: self.task_opts = info[2] else: @@ -167,29 +167,17 @@ class WindowsBuild(object): def checkout(self): """Checkout sources, winspec, and patches, and apply patches""" src_scm = SCM(self.source_url) - self.server.runCallBacks('preSCMCheckout', scminfo=src_scm.get_info()) self.source_dir = src_scm.checkout(ensuredir(os.path.join(self.workdir, 'source'))) - self.server.runCallBacks('postSCMCheckout', - scminfo=src_scm.get_info(), - srcdir=self.source_dir) self.zipDir(self.source_dir, os.path.join(self.workdir, 'sources.zip')) if 'winspec' in self.task_opts: spec_scm = SCM(self.task_opts['winspec']) - self.server.runCallBacks('preSCMCheckout', scminfo=spec_scm.get_info()) self.spec_dir = spec_scm.checkout(ensuredir(os.path.join(self.workdir, 'spec'))) - self.server.runCallBacks('postSCMCheckout', - scminfo=spec_scm.get_info(), - srcdir=self.spec_dir) self.zipDir(self.spec_dir, os.path.join(self.workdir, 'spec.zip')) else: self.spec_dir = self.source_dir if 'patches' in self.task_opts: patch_scm = SCM(self.task_opts['patches']) - self.server.runCallBacks('preSCMCheckout', scminfo=patch_scm.get_info()) self.patches_dir = patch_scm.checkout(ensuredir(os.path.join(self.workdir, 'patches'))) - self.server.runCallBacks('postSCMCheckout', - scminfo=patch_scm.get_info(), - srcdir=self.patch_dir) self.zipDir(self.patches_dir, os.path.join(self.workdir, 'patches.zip')) self.applyPatches(self.source_dir, self.patches_dir) self.virusCheck(self.workdir) diff --git a/vm/kojivmd b/vm/kojivmd index 7609253..3ca7342 100755 --- a/vm/kojivmd +++ b/vm/kojivmd @@ -298,13 +298,14 @@ class WinBuildTask(MultiPlatformTask): if not opts: opts = {} - subopts = koji.util.dslice(opts, ['winspec', 'patches', 'scratch'], + subopts = koji.util.dslice(opts, ['winspec', 'patches'], strict=False) # winspec and patches options are urls # verify the urls before passing them to the VM for url in [source_url] + subopts.values(): scm = SCM(url) scm.assert_allowed(self.options.allowed_scms) + task_info = self.session.getTaskInfo(self.id) target_info = self.session.getBuildTarget(target) if not target_info: @@ -531,20 +532,6 @@ class VMExecTask(BaseTaskHandler): """ return self.task_info - def runCallbacks(self, method, **opts): - """ - Run callback plugins. - """ - taskinfo = self.session.getTaskInfo(self.id, request=True) - opts['taskinfo'] = taskinfo - opts['session'] = self.session - if method == 'preSCMCheckout': - opts['build_tag'] = self.task_info.get('build_tag') - elif method == 'postSCMCheckout': - opts['scratch'] = self.task_info.get('scratch') - koji.plugin.run_callbacks(method, **opts) - - def initBuildroot(self, repo_id, platform): """ Create the buildroot object on the hub. From fcc3e29081333f1b42ac538ec55efa8190185338 Mon Sep 17 00:00:00 2001 From: Yuming Zhu Date: Jun 07 2017 09:20:03 +0000 Subject: [PATCH 10/11] change `run_plugin` to `run_callbacks` and remove the modification for kojikamid.py --- diff --git a/builder/kojid b/builder/kojid index 047cdaf..33c2bd8 100755 --- a/builder/kojid +++ b/builder/kojid @@ -1415,13 +1415,13 @@ class BuildMavenTask(BaseBuildTask): uploadpath = self.getUploadDir() - self.run_plugin('preSCMCheckout', scminfo=scm.get_info(), build_tag=build_tag) + self.run_callbacks('preSCMCheckout', scminfo=scm.get_info(), build_tag=build_tag) # Check out sources from the SCM sourcedir = scm.checkout(scmdir, self.session, uploadpath, logfile) - self.run_plugin("postSCMCheckout", - scminfo=scm.get_info(), - scratch=opts.get('scratch'), - srcdir=sourcedir) + self.run_callbacks("postSCMCheckout", + scminfo=scm.get_info(), + scratch=opts.get('scratch'), + srcdir=sourcedir) # zip up pristine sources for auditing purposes self._zip_dir(sourcedir, os.path.join(outputdir, 'scm-sources.zip')) @@ -1431,14 +1431,14 @@ class BuildMavenTask(BaseBuildTask): patchlog = self.workdir + '/patches.log' patch_scm = SCM(self.opts.get('patches')) patch_scm.assert_allowed(self.options.allowed_scms) - self.run_plugin('preSCMCheckout', scminfo=patch_scm.get_info(), build_tag=build_tag) + self.run_callbacks('preSCMCheckout', scminfo=patch_scm.get_info(), build_tag=build_tag) # never try to check out a common/ dir when checking out patches patch_scm.use_common = False patchcheckoutdir = patch_scm.checkout(patchdir, self.session, uploadpath, patchlog) - self.run_plugin("postSCMCheckout", - scminfo=patch_scm.get_info(), - scratch=opts.get('scratch'), - srcdir=patchcheckoutdir) + self.run_callbacks("postSCMCheckout", + scminfo=patch_scm.get_info(), + scratch=opts.get('scratch'), + srcdir=patchcheckoutdir) self._zip_dir(patchcheckoutdir, os.path.join(outputdir, 'patches.zip')) # Apply patches, if present @@ -1719,12 +1719,12 @@ class WrapperRPMTask(BaseBuildTask): logfile = os.path.join(self.workdir, 'checkout.log') scmdir = buildroot.rootdir() + '/tmp/scmroot' koji.ensuredir(scmdir) - self.run_plugin('preSCMCheckout', scminfo=scm.get_info(), build_tag=build_tag) + self.run_callbacks('preSCMCheckout', scminfo=scm.get_info(), build_tag=build_tag) specdir = scm.checkout(scmdir, self.session, self.getUploadDir(), logfile) - self.run_plugin("postSCMCheckout", - scminfo=scm.get_info(), - scratch=opts.get('scratch'), - srcdir=specdir) + self.run_callbacks("postSCMCheckout", + scminfo=scm.get_info(), + scratch=opts.get('scratch'), + srcdir=specdir) spec_template = None for path, dir, files in os.walk(specdir): @@ -2658,12 +2658,12 @@ class ImageTask(BaseTaskHandler): scm = SCM(self.opts['ksurl']) scm.assert_allowed(self.options.allowed_scms) logfile = os.path.join(self.workdir, 'checkout.log') - self.run_plugin('preSCMCheckout', scminfo=scm.get_info(), build_tag=build_tag) + self.run_callbacks('preSCMCheckout', scminfo=scm.get_info(), build_tag=build_tag) scmsrcdir = scm.checkout(scmdir, self.session, self.getUploadDir(), logfile) - self.run_plugin("postSCMCheckout", - scminfo=scm.get_info(), - scratch=self.opts.get('scratch'), - srcdir=scmsrcdir) + self.run_callbacks("postSCMCheckout", + scminfo=scm.get_info(), + scratch=self.opts.get('scratch'), + srcdir=scmsrcdir) kspath = os.path.join(scmsrcdir, ksfile) else: kspath = self.localPath("work/%s" % ksfile) @@ -3276,13 +3276,13 @@ class OzImageTask(BaseTaskHandler): scm = SCM(self.opts['ksurl']) scm.assert_allowed(self.options.allowed_scms) logfile = os.path.join(self.workdir, 'checkout-%s.log' % self.arch) - self.run_plugin('preSCMCheckout', scminfo=scm.get_info(), build_tag=build_tag) + self.run_callbacks('preSCMCheckout', scminfo=scm.get_info(), build_tag=build_tag) scmsrcdir = scm.checkout(self.workdir, self.session, self.getUploadDir(), logfile) - self.run_plugin("postSCMCheckout", - scminfo=scm.get_info(), - scratch=self.opts.get('scratch'), - srcdir=scmsrcdir) + self.run_callbacks("postSCMCheckout", + scminfo=scm.get_info(), + scratch=self.opts.get('scratch'), + srcdir=scmsrcdir) kspath = os.path.join(scmsrcdir, os.path.basename(ksfile)) else: tops = dict([(k, getattr(self.options, k)) for k in 'topurl','topdir']) @@ -4087,14 +4087,14 @@ class BuildIndirectionImageTask(OzImageTask): if fileurl: scm = SCM(fileurl) scm.assert_allowed(self.options.allowed_scms) - self.run_plugin('preSCMCheckout', scminfo=scm.get_info(), build_tag=build_tag) + self.run_callbacks('preSCMCheckout', scminfo=scm.get_info(), build_tag=build_tag) logfile = os.path.join(self.workdir, 'checkout.log') scmsrcdir = scm.checkout(self.workdir, self.session, self.getUploadDir(), logfile) - self.run_plugin("postSCMCheckout", - scminfo=scm.get_info(), - scratch=self.opts.get('scratch'), - srcdir=scmsrcdir) + self.run_callbacks("postSCMCheckout", + scminfo=scm.get_info(), + scratch=self.opts.get('scratch'), + srcdir=scmsrcdir) final_path = os.path.join(scmsrcdir, os.path.basename(filepath)) else: tops = dict([(k, getattr(self.options, k)) for k in 'topurl','topdir']) @@ -4436,13 +4436,13 @@ class BuildSRPMFromSCMTask(BaseBuildTask): logfile = self.workdir + '/checkout.log' uploadpath = self.getUploadDir() - self.run_plugin('preSCMCheckout', scminfo=scm.get_info(), build_tag=build_tag) + self.run_callbacks('preSCMCheckout', scminfo=scm.get_info(), build_tag=build_tag) # Check out spec file, etc. from SCM sourcedir = scm.checkout(scmdir, self.session, uploadpath, logfile) - self.run_plugin("postSCMCheckout", - scminfo=scm.get_info(), - scratch=self.opts.get('scratch'), - srcdir=sourcedir) + self.run_callbacks("postSCMCheckout", + scminfo=scm.get_info(), + scratch=self.opts.get('scratch'), + srcdir=sourcedir) # chown the sourcedir and everything under it to the mockuser # so we can build the srpm as non-root uid = pwd.getpwnam(self.options.mockuser)[2] diff --git a/koji/tasks.py b/koji/tasks.py index 5d7ac98..a11bd3e 100644 --- a/koji/tasks.py +++ b/koji/tasks.py @@ -383,7 +383,7 @@ class BaseTaskHandler(object): return repo_info - def run_plugin(self, plugin, *args, **kwargs): + def run_callbacks(self, plugin, *args, **kwargs): if 'taskinfo' not in kwargs: try: taskinfo = self.taskinfo diff --git a/vm/kojikamid.py b/vm/kojikamid.py index 710671a..0499c74 100755 --- a/vm/kojikamid.py +++ b/vm/kojikamid.py @@ -47,20 +47,19 @@ MANAGER_PORT = 7000 KOJIKAMID = True - ## INSERT kojikamid dup class fakemodule(object): pass - -# make parts of the above insert accessible as koji.X +#make parts of the above insert accessible as koji.X koji = fakemodule() koji.GenericError = GenericError koji.BuildError = BuildError class WindowsBuild(object): + LEADING_CHAR = re.compile('^[^A-Za-z_]') VAR_CHARS = re.compile('[^A-Za-z0-9_]') @@ -137,7 +136,7 @@ class WindowsBuild(object): self.logger.info('file %s exists', entry) if errors: raise BuildError('error validating build environment: %s' % \ - ', '.join(errors)) + ', '.join(errors)) def updateClam(self): """update ClamAV virus definitions""" @@ -319,7 +318,7 @@ class WindowsBuild(object): # rpms don't have a md5sum in the fileinfo, but check it for everything else if ('md5sum' in fileinfo) and (digest != fileinfo['md5sum']): raise BuildError('md5 checksum validation failed for %s, %s (computed) != %s (provided)' % \ - (destpath, digest, fileinfo['md5sum'])) + (destpath, digest, fileinfo['md5sum'])) self.logger.info('Retrieved %s (%s bytes, md5: %s)', destpath, offset, digest) def fetchBuildReqs(self): @@ -469,7 +468,7 @@ class WindowsBuild(object): self.virusCheck(self.workdir) if errors: raise BuildError('error validating build output: %s' % \ - ', '.join(errors)) + ', '.join(errors)) def virusCheck(self, path): """ensure a path is virus free with ClamAV. path should be absolute""" @@ -500,7 +499,6 @@ class WindowsBuild(object): self.expireBuildroot() return self.gatherResults() - def run(cmd, chdir=None, fatal=False, log=True): global logfd output = '' @@ -530,7 +528,6 @@ def run(cmd, chdir=None, fatal=False, log=True): raise BuildError(msg) return ret, output - def find_net_info(): """ Find the network gateway configured for this VM. @@ -559,7 +556,6 @@ def find_net_info(): gateway = None return macaddr, gateway - def upload_file(server, prefix, path): """upload a single file to the vmd""" logger = logging.getLogger('koji.vm') @@ -580,7 +576,6 @@ def upload_file(server, prefix, path): server.verifyChecksum(path, digest, 'md5') logger.info('Uploaded %s (%s bytes, md5: %s)', destpath, offset, digest) - def get_mgmt_server(): """Get a ServerProxy object we can use to retrieve task info""" logger = logging.getLogger('koji.vm') @@ -599,7 +594,6 @@ def get_mgmt_server(): logger.debug('found task-specific port %s', task_port) return xmlrpclib.ServerProxy('http://%s:%s/' % (gateway, task_port), allow_none=True) - def get_options(): """handle usage and parse options""" usage = """%prog [options] @@ -609,12 +603,10 @@ def get_options(): parser = OptionParser(usage=usage) parser.add_option('-d', '--debug', action='store_true', help='Log debug statements') parser.add_option('-i', '--install', action='store_true', help='Install this daemon as a service', default=False) - parser.add_option('-u', '--uninstall', action='store_true', - help='Uninstall this daemon if it was installed previously as a service', default=False) + parser.add_option('-u', '--uninstall', action='store_true', help='Uninstall this daemon if it was installed previously as a service', default=False) (options, args) = parser.parse_args() return options - def setup_logging(opts): global logfile, logfd logger = logging.getLogger('koji.vm') @@ -629,13 +621,11 @@ def setup_logging(opts): logger.addHandler(handler) return handler - def log_local(msg): tb = ''.join(traceback.format_exception(*sys.exc_info())) sys.stderr.write('%s: %s\n' % (time.ctime(), msg)) sys.stderr.write(tb) - def stream_logs(server, handler, builds): """Stream logs incrementally to the server. The global logfile will always be streamed. @@ -673,7 +663,6 @@ def stream_logs(server, handler, builds): log_local('error uploading %s' % relpath) time.sleep(1) - def fail(server, handler): """do the right thing when a build fails""" global logfile, logfd @@ -700,7 +689,6 @@ def fail(server, handler): logfile = '/tmp/build.log' logfd = None - def main(): prog = os.path.basename(sys.argv[0]) opts = get_options() From 620b2ce1c8dbcecd571483c933e0c58d83f8dddb Mon Sep 17 00:00:00 2001 From: Yuming Zhu Date: Jun 07 2017 09:20:32 +0000 Subject: [PATCH 11/11] add scratch param in preSCMCheckout --- diff --git a/builder/kojid b/builder/kojid index 33c2bd8..5db734e 100755 --- a/builder/kojid +++ b/builder/kojid @@ -1415,7 +1415,7 @@ class BuildMavenTask(BaseBuildTask): uploadpath = self.getUploadDir() - self.run_callbacks('preSCMCheckout', scminfo=scm.get_info(), build_tag=build_tag) + self.run_callbacks('preSCMCheckout', scminfo=scm.get_info(), build_tag=build_tag, scratch=opts.get('scratch')) # Check out sources from the SCM sourcedir = scm.checkout(scmdir, self.session, uploadpath, logfile) self.run_callbacks("postSCMCheckout", @@ -1431,7 +1431,7 @@ class BuildMavenTask(BaseBuildTask): patchlog = self.workdir + '/patches.log' patch_scm = SCM(self.opts.get('patches')) patch_scm.assert_allowed(self.options.allowed_scms) - self.run_callbacks('preSCMCheckout', scminfo=patch_scm.get_info(), build_tag=build_tag) + self.run_callbacks('preSCMCheckout', scminfo=patch_scm.get_info(), build_tag=build_tag, scratch=opts.get('scratch')) # never try to check out a common/ dir when checking out patches patch_scm.use_common = False patchcheckoutdir = patch_scm.checkout(patchdir, self.session, uploadpath, patchlog) @@ -1719,7 +1719,7 @@ class WrapperRPMTask(BaseBuildTask): logfile = os.path.join(self.workdir, 'checkout.log') scmdir = buildroot.rootdir() + '/tmp/scmroot' koji.ensuredir(scmdir) - self.run_callbacks('preSCMCheckout', scminfo=scm.get_info(), build_tag=build_tag) + self.run_callbacks('preSCMCheckout', scminfo=scm.get_info(), build_tag=build_tag, scratch=opts.get('scratch')) specdir = scm.checkout(scmdir, self.session, self.getUploadDir(), logfile) self.run_callbacks("postSCMCheckout", scminfo=scm.get_info(), @@ -2658,7 +2658,7 @@ class ImageTask(BaseTaskHandler): scm = SCM(self.opts['ksurl']) scm.assert_allowed(self.options.allowed_scms) logfile = os.path.join(self.workdir, 'checkout.log') - self.run_callbacks('preSCMCheckout', scminfo=scm.get_info(), build_tag=build_tag) + self.run_callbacks('preSCMCheckout', scminfo=scm.get_info(), build_tag=build_tag, scratch=self.opts.get('scratch')) scmsrcdir = scm.checkout(scmdir, self.session, self.getUploadDir(), logfile) self.run_callbacks("postSCMCheckout", scminfo=scm.get_info(), @@ -3276,7 +3276,7 @@ class OzImageTask(BaseTaskHandler): scm = SCM(self.opts['ksurl']) scm.assert_allowed(self.options.allowed_scms) logfile = os.path.join(self.workdir, 'checkout-%s.log' % self.arch) - self.run_callbacks('preSCMCheckout', scminfo=scm.get_info(), build_tag=build_tag) + self.run_callbacks('preSCMCheckout', scminfo=scm.get_info(), build_tag=build_tag, scratch=self.opts.get('scratch')) scmsrcdir = scm.checkout(self.workdir, self.session, self.getUploadDir(), logfile) self.run_callbacks("postSCMCheckout", @@ -4087,7 +4087,7 @@ class BuildIndirectionImageTask(OzImageTask): if fileurl: scm = SCM(fileurl) scm.assert_allowed(self.options.allowed_scms) - self.run_callbacks('preSCMCheckout', scminfo=scm.get_info(), build_tag=build_tag) + self.run_callbacks('preSCMCheckout', scminfo=scm.get_info(), build_tag=build_tag, scratch=self.opts.get('scratch')) logfile = os.path.join(self.workdir, 'checkout.log') scmsrcdir = scm.checkout(self.workdir, self.session, self.getUploadDir(), logfile) @@ -4436,7 +4436,7 @@ class BuildSRPMFromSCMTask(BaseBuildTask): logfile = self.workdir + '/checkout.log' uploadpath = self.getUploadDir() - self.run_callbacks('preSCMCheckout', scminfo=scm.get_info(), build_tag=build_tag) + self.run_callbacks('preSCMCheckout', scminfo=scm.get_info(), build_tag=build_tag, scratch=self.opts.get('scratch')) # Check out spec file, etc. from SCM sourcedir = scm.checkout(scmdir, self.session, uploadpath, logfile) self.run_callbacks("postSCMCheckout",