From bf690318a75299c28047f4212131a174f648eb15 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Sep 07 2016 10:15:01 +0000 Subject: [PATCH 1/10] Small style fix in pagure_ci_server --- diff --git a/pagure-ci/pagure_ci_server.py b/pagure-ci/pagure_ci_server.py index a3470d9..aedeec3 100644 --- a/pagure-ci/pagure_ci_server.py +++ b/pagure-ci/pagure_ci_server.py @@ -93,8 +93,9 @@ def handle_messages(): base[:-1] base += '/%s' % project.path - log.info("Trigger on %s PR #%s from %s: %s", - project.fullname, pr_id, repo, branch) + log.info( + "Trigger on %s PR #%s from %s: %s", + project.fullname, pr_id, repo, branch) url = project.ci_hook.ci_url From 9ef674317b25ea6628a583a939c6f2e200855ce1 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Sep 07 2016 10:15:31 +0000 Subject: [PATCH 2/10] Fix building the jenkins URL to call to trigger the build in pagure-ci --- diff --git a/pagure-ci/pagure_ci_server.py b/pagure-ci/pagure_ci_server.py index aedeec3..99ff2fb 100644 --- a/pagure-ci/pagure_ci_server.py +++ b/pagure-ci/pagure_ci_server.py @@ -97,10 +97,10 @@ def handle_messages(): "Trigger on %s PR #%s from %s: %s", project.fullname, pr_id, repo, branch) - url = project.ci_hook.ci_url + url = project.ci_hook.ci_url.rstrip('/') if data['ci_type'] == 'jenkins': - url = urlparse.urljoin(url, '/buildWithParameters') + url = url + '/buildWithParameters' log.info('Triggering the build at: %s', url) requests.post( url, From 1ed4176981d871ae17f6f757663731956679427f Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Sep 07 2016 10:15:31 +0000 Subject: [PATCH 3/10] Drop un-used import --- diff --git a/pagure-ci/pagure_ci_server.py b/pagure-ci/pagure_ci_server.py index 99ff2fb..72ba82a 100644 --- a/pagure-ci/pagure_ci_server.py +++ b/pagure-ci/pagure_ci_server.py @@ -16,22 +16,14 @@ receiving end is offline or so. """ -import datetime -import hashlib -import hmac import json import logging import os import requests -import time -import urlparse -import uuid -import six import trollius import trollius_redis -from kitchen.text.converters import to_bytes log = logging.getLogger(__name__) From d31d7e2fc24333f5d91c710205f3a61d2630245f Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Sep 07 2016 10:15:31 +0000 Subject: [PATCH 4/10] pylint fixes to pagure-ci --- diff --git a/pagure-ci/pagure_ci_server.py b/pagure-ci/pagure_ci_server.py index 72ba82a..9e4f13e 100644 --- a/pagure-ci/pagure_ci_server.py +++ b/pagure-ci/pagure_ci_server.py @@ -25,8 +25,7 @@ import trollius import trollius_redis - -log = logging.getLogger(__name__) +LOG = logging.getLogger(__name__) if 'PAGURE_CONFIG' not in os.environ \ and os.path.exists('/etc/pagure/pagure.cfg'): @@ -36,16 +35,20 @@ if 'PAGURE_CONFIG' not in os.environ \ import pagure import pagure.lib -from pagure.exceptions import PagureEvException @trollius.coroutine def handle_messages(): + ''' Handles connecting to redis and acting upon messages received. + In this case, it means triggering a build on jenkins based on the + information provided. + ''' + host = pagure.APP.config.get('REDIS_HOST', '0.0.0.0') port = pagure.APP.config.get('REDIS_PORT', 6379) - db = pagure.APP.config.get('REDIS_DB', 0) + dbname = pagure.APP.config.get('REDIS_DB', 0) connection = yield trollius.From(trollius_redis.Connection.create( - host=host, port=port, db=db)) + host=host, port=port, db=dbname)) # Create subscriber. subscriber = yield trollius.From(connection.start_subscribe()) @@ -56,7 +59,7 @@ def handle_messages(): # Inside a while loop, wait for incoming events. while True: reply = yield trollius.From(subscriber.next_published()) - log.info( + LOG.info( 'Received: %s on channel: %s', repr(reply.value), reply.channel) data = json.loads(reply.value) @@ -74,26 +77,19 @@ def handle_messages(): session=pagure.SESSION, name=projectname, user=username) if not project: - log.warning( - 'No project could be found from the message %s' % data) + LOG.warning( + 'No project could be found from the message %s', data) continue - repo = data['pr'].get('remote_git') - if not repo: - base = pagure.APP.config['APP_URL'] - if base.endswith('/'): - base[:-1] - base += '/%s' % project.path - - log.info( + LOG.info( "Trigger on %s PR #%s from %s: %s", - project.fullname, pr_id, repo, branch) + project.fullname, pr_id, project.fullname, branch) url = project.ci_hook.ci_url.rstrip('/') if data['ci_type'] == 'jenkins': url = url + '/buildWithParameters' - log.info('Triggering the build at: %s', url) + LOG.info('Triggering the build at: %s', url) requests.post( url, data={ @@ -104,13 +100,14 @@ def handle_messages(): } ) else: - log.warning('Un-supported CI type') + LOG.warning('Un-supported CI type') - log.info('Ready for another') + LOG.info('Ready for another') def main(): - server = None + ''' Start the main async loop. ''' + try: loop = trollius.get_event_loop() tasks = [ @@ -123,24 +120,23 @@ def main(): except trollius.ConnectionResetError: pass - log.info("End Connection") + LOG.info("End Connection") loop.close() - log.info("End") + LOG.info("End") if __name__ == '__main__': - log = logging.getLogger("") formatter = logging.Formatter( "%(asctime)s %(levelname)s [%(module)s:%(lineno)d] %(message)s") # setup console logging - log.setLevel(logging.DEBUG) - ch = logging.StreamHandler() - ch.setLevel(logging.DEBUG) + LOG.setLevel(logging.DEBUG) + shellhandler = logging.StreamHandler() + shellhandler.setLevel(logging.DEBUG) aslog = logging.getLogger("asyncio") aslog.setLevel(logging.DEBUG) - ch.setFormatter(formatter) - log.addHandler(ch) + shellhandler.setFormatter(formatter) + LOG.addHandler(shellhandler) main() From b1b1f1db7dcbeb7a0bdad3f873cc9bc026e5783e Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Sep 07 2016 10:15:31 +0000 Subject: [PATCH 5/10] Make the jenkins_ci_notifications API endpoint marked as such This will allow catching all the APIError exception raised and turn them into valid JSON messages for the client --- diff --git a/pagure/api/ci/jenkins.py b/pagure/api/ci/jenkins.py index e0d45ac..22799fc 100644 --- a/pagure/api/ci/jenkins.py +++ b/pagure/api/ci/jenkins.py @@ -18,13 +18,14 @@ import pagure.exceptions import pagure.lib import pagure.lib.lib_ci as lib_ci from pagure import APP, SESSION -from pagure.api import API, APIERROR +from pagure.api import API, APIERROR, api_method @API.route('/ci/jenkins///build-finished', methods=['POST']) @API.route('/ci/jenkins/forks///' '/build-finished', methods=['POST']) +@api_method def jenkins_ci_notification(repo, pagure_ci_token, username=None): """ Jenkins Build Notification From abe18c084524e22e00f57f64861e6d053113b38b Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Sep 07 2016 10:15:32 +0000 Subject: [PATCH 6/10] Let process_jenkins_build be a little more flexible about the data retrieve Apparently, not all the builds have a 'result' key, so let's not assume there is always one. --- diff --git a/pagure/lib/lib_ci.py b/pagure/lib/lib_ci.py index ecb3913..db719e9 100644 --- a/pagure/lib/lib_ci.py +++ b/pagure/lib/lib_ci.py @@ -47,7 +47,7 @@ def process_jenkins_build(session, project, build_id, requestfolder): jenkins_name = project.ci_hook.ci_url.split( '/job/', 1)[1].split('/', 1)[0] build_info = jenk.get_build_info(jenkins_name, build_id) - result = build_info['result'] + result = build_info.get('result') url = build_info['url'] pr_id = None From 3e1101ec0e114c16b0f74eaf027a0a5d91baf6d7 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Sep 07 2016 10:15:32 +0000 Subject: [PATCH 7/10] Make the REPO variable be the public url where it can be cloned --- diff --git a/pagure-ci/pagure_ci_server.py b/pagure-ci/pagure_ci_server.py index 9e4f13e..69cc2a5 100644 --- a/pagure-ci/pagure_ci_server.py +++ b/pagure-ci/pagure_ci_server.py @@ -89,13 +89,16 @@ def handle_messages(): if data['ci_type'] == 'jenkins': url = url + '/buildWithParameters' - LOG.info('Triggering the build at: %s', url) + repo = '%s/%s' % ( + pagure.APP.config['GIT_URL_GIT'].rstrip('/'), project.path) + LOG.info( + 'Triggering the build at: %s, for repo: %s', url, repo) requests.post( url, data={ 'token': project.ci_hook.pagure_ci_token, 'cause': pr_id, - 'REPO': project.fullname, + 'REPO': repo, 'BRANCH': branch } ) From feb4b60541e4b8cbfc86e397a7a75fc57b5b3f57 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Sep 07 2016 10:15:32 +0000 Subject: [PATCH 8/10] Fix retrieving the username from the data when the repo is a fork Thanks @vivekanand1101 --- diff --git a/pagure-ci/pagure_ci_server.py b/pagure-ci/pagure_ci_server.py index 69cc2a5..96d8c74 100644 --- a/pagure-ci/pagure_ci_server.py +++ b/pagure-ci/pagure_ci_server.py @@ -71,7 +71,7 @@ def handle_messages(): username = None projectname = data['pr']['project']['name'] if data['pr'].get('parent'): - username, data['pr']['project']['user']['user'] + username = data['pr']['project']['user']['user'] project = pagure.lib.get_project( session=pagure.SESSION, name=projectname, user=username) From 2ae1ff01dbb2af55fbc4fd2ae38599266bcc310f Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Sep 07 2016 10:15:32 +0000 Subject: [PATCH 9/10] Drop un-used variable --- diff --git a/pagure-ci/pagure_ci_server.py b/pagure-ci/pagure_ci_server.py index 96d8c74..b9d87e8 100644 --- a/pagure-ci/pagure_ci_server.py +++ b/pagure-ci/pagure_ci_server.py @@ -65,11 +65,10 @@ def handle_messages(): data = json.loads(reply.value) pr_id = data['pr']['id'] - project = data['pr']['project']['name'] branch = data['pr']['branch_from'] + projectname = data['pr']['project']['name'] username = None - projectname = data['pr']['project']['name'] if data['pr'].get('parent'): username = data['pr']['project']['user']['user'] From 52ed110a52259f30b57370c39bb4ce27b4bcec65 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Sep 07 2016 10:22:24 +0000 Subject: [PATCH 10/10] Drop get_project_by_ci_token from pagure.lib_ci since it not used anywhere --- diff --git a/pagure/lib/lib_ci.py b/pagure/lib/lib_ci.py index db719e9..33ddd98 100644 --- a/pagure/lib/lib_ci.py +++ b/pagure/lib/lib_ci.py @@ -24,19 +24,6 @@ BUILD_STATS = { } -def get_project_by_ci_token(session, ci_token): - """ Return the project corresponding to the provided ci_token. """ - query = session.query( - model.Project - ).filter( - model.Project.id == pagure_ci.PagureCITable.project_id - ).filter( - pagure_ci.PagureCITable.pagure_ci_token == ci_token - ) - - return query.first() - - def process_jenkins_build(session, project, build_id, requestfolder): """ Gets the build info from jenkins and flags that particular pull-request.