From 2bf8f5d1b5347a8c5c22afeab71e1e03d769598e Mon Sep 17 00:00:00 2001 From: farhaanbukhsh Date: Jul 26 2016 07:41:45 +0000 Subject: [PATCH 1/25] Add jenkins hook form --- diff --git a/pagure/hooks/jenkins_hook.py b/pagure/hooks/jenkins_hook.py new file mode 100644 index 0000000..379fcd4 --- /dev/null +++ b/pagure/hooks/jenkins_hook.py @@ -0,0 +1,174 @@ +# -*- coding: utf-8 -*- + +""" + (c) 2014 - Copyright Red Hat Inc + + Authors: + Pierre-Yves Chibon + +""" + +import os + +import sqlalchemy as sa +import pygit2 +from wtforms import validators, TextField +from flask.ext import wtf +from sqlalchemy.orm import relation +from sqlalchemy.orm import backref + +from pagure.hooks import BaseHook, RequiredIf +from pagure.lib.model import BASE, Project +from pagure import get_repo_path + + + +class PagureCI(BASE): + __tablename__ = 'hook_pagure_ci' + __table_args__ = {'extend_existing': True} + + name = sa.Column(sa.String(64), primary_key=True, unique=True) + display_name = sa.Column(sa.String(64), nullable=False, default='Jenkins') + owner = sa.Column(sa.String(64)) + + pagure_name = sa.Column(sa.String(255)) + pagure_url = sa.Column(sa.String(255)) + pagure_token = sa.Column(sa.String(64)) + + jenkins_name = sa.Column(sa.String(255)) + jenkins_url = sa.Column(sa.String(255)) + jenkins_token = sa.Column(sa.String(64)) + + hook_token = sa.Column(sa.String(64)) + + '''def __init__(self, name, display_name, owner, + pagure_name, pagure_url, pagure_token, + jenkins_name, jenkins_url, jenkins_token, + hook_token): + self.name = name + self.display_name = display_name + self.owner = owner + self.pagure_name = pagure_name + self.pagure_url = pagure_url + self.pagure_token = pagure_token + + self.jenkins_name = jenkins_name + self.jenkins_url = jenkins_url + self.jenkins_token = jenkins_token + + self.hook_token = hook_token''' + + def __repr__(self): + return ''.format(self) + + +def init_db(db): + from sqlalchemy import create_engine + engine = create_engine(db, convert_unicode=True) + Base.metadata.create_all(bind=engine) + + +class ConfigNotFound(Exception): + pass + + +class Service(object): + PAGURE = PagureCI.pagure_name + JENKINS = PagureCI.jenkins_name + + +def get_configs(project_name, service): + """Returns all configurations with given name on a service. + + :raises ConfigNotFound: when no configuration matches + """ + cfg = PagureCI.query.filter(service == project_name).all() + if len(cfg) == 0: + raise ConfigNotFound(project_name) + return cfg + + + +class JenkinsForm(wtf.Form): + + '''Form to configure Jenkins hook''' + name = TextField('Name', + [validators.Required(), + validators.Length(max=64)]) + display_name = TextField('Display name', + [validators.Required(), + validators.Length(max=64)], + default='Jenkins') + pagure_name = TextField('Name of project in Pagure', + [validators.Required(), + validators.Length(max=255)]) + pagure_url = TextField('Pagure URL', + [validators.Required(), + validators.Length(max=255)], + default='https://pagure.io/') + pagure_token = TextField('Pagure token', + [validators.Required()]) + + jenkins_name = TextField('Name of project in Jenkins', + [validators.Required(), + validators.Length(max=255)]) + jenkins_url = TextField('Jenkins URL', + [validators.Required(), + validators.Length(max=255)], + default='http://jenkins.fedorainfracloud.org/') + jenkins_token = TextField('Jenkins token', + [validators.Required()]) + + +class Hook(BaseHook): + ''' Jenkins hooks. ''' + + name = 'Jenkins Hook' + description = 'This hook help to set up CI for the project'\ + ' the changes made by the pushes to the git repository.' + form = JenkinsForm + db_object = PagureCI + backref = 'pagure_ci' + form_fields = [ + 'name', 'pagure_name', 'pagure_url', 'pagure_token', 'jenkins_name', + 'jenkins_url', 'jenkins_token' + ] + + @classmethod + def install(cls, project, dbobj): + ''' Method called to install the hook for a project. + + :arg project: a ``pagure.model.Project`` object to which the hook + should be installed + + ''' + repopath = get_repo_path(project) + + hook_files = os.path.join( + os.path.dirname(os.path.realpath(__file__)), 'files') + repo_obj = pygit2.Repository(repopath) + + # Configure the hook + # repo_obj.config.set_multivar() + + # Install the hook itself + #hook_file = os.path.join(hook_files, 'git_irc.py') + #if not os.path.exists(hook_file): + #os.symlink( + #hook_file, + #os.path.join(repopath, 'hooks', 'post-receive.irc') + #) + + @classmethod + def remove(cls, project): + ''' Method called to remove the hook of a project. + + :arg project: a ``pagure.model.Project`` object to which the hook + should be installed + + ''' + repopath = get_repo_path(project) + + #hook_path = os.path.join(repopath, 'hooks', 'post-receive.irc') + #if os.path.exists(hook_path): + #os.unlink(hook_path) From 2c9bb9275119c9c60d0f0373ee04f2f01562fc08 Mon Sep 17 00:00:00 2001 From: farhaanbukhsh Date: Jul 26 2016 07:41:45 +0000 Subject: [PATCH 2/25] Fix database changes --- diff --git a/pagure/hooks/jenkins_hook.py b/pagure/hooks/jenkins_hook.py index 379fcd4..1305078 100644 --- a/pagure/hooks/jenkins_hook.py +++ b/pagure/hooks/jenkins_hook.py @@ -27,6 +27,15 @@ class PagureCI(BASE): __tablename__ = 'hook_pagure_ci' __table_args__ = {'extend_existing': True} + id = sa.Column(sa.Integer, primary_key=True) + project_id = sa.Column( + sa.Integer, + sa.ForeignKey('projects.id', onupdate='CASCADE'), + nullable=False, + unique=True, + index=True) + + name = sa.Column(sa.String(64), primary_key=True, unique=True) display_name = sa.Column(sa.String(64), nullable=False, default='Jenkins') owner = sa.Column(sa.String(64)) @@ -41,52 +50,12 @@ class PagureCI(BASE): hook_token = sa.Column(sa.String(64)) - '''def __init__(self, name, display_name, owner, - pagure_name, pagure_url, pagure_token, - jenkins_name, jenkins_url, jenkins_token, - hook_token): - self.name = name - self.display_name = display_name - self.owner = owner - self.pagure_name = pagure_name - self.pagure_url = pagure_url - self.pagure_token = pagure_token - - self.jenkins_name = jenkins_name - self.jenkins_url = jenkins_url - self.jenkins_token = jenkins_token - - self.hook_token = hook_token''' - - def __repr__(self): - return ''.format(self) - - -def init_db(db): - from sqlalchemy import create_engine - engine = create_engine(db, convert_unicode=True) - Base.metadata.create_all(bind=engine) - - -class ConfigNotFound(Exception): - pass - - -class Service(object): - PAGURE = PagureCI.pagure_name - JENKINS = PagureCI.jenkins_name - - -def get_configs(project_name, service): - """Returns all configurations with given name on a service. - - :raises ConfigNotFound: when no configuration matches - """ - cfg = PagureCI.query.filter(service == project_name).all() - if len(cfg) == 0: - raise ConfigNotFound(project_name) - return cfg - + project = relation( + 'Project', remote_side=[Project.id], + backref=backref( + 'jenkins_hook', cascade="delete, delete-orphan", + single_parent=True) + ) class JenkinsForm(wtf.Form): @@ -128,7 +97,7 @@ class Hook(BaseHook): ' the changes made by the pushes to the git repository.' form = JenkinsForm db_object = PagureCI - backref = 'pagure_ci' + backref = 'jenkins_hook' form_fields = [ 'name', 'pagure_name', 'pagure_url', 'pagure_token', 'jenkins_name', 'jenkins_url', 'jenkins_token' @@ -149,15 +118,15 @@ class Hook(BaseHook): repo_obj = pygit2.Repository(repopath) # Configure the hook - # repo_obj.config.set_multivar() + #repo_obj.config.set_multivar(dbobj) # Install the hook itself - #hook_file = os.path.join(hook_files, 'git_irc.py') - #if not os.path.exists(hook_file): - #os.symlink( - #hook_file, - #os.path.join(repopath, 'hooks', 'post-receive.irc') - #) + hook_file = os.path.join(hook_files, 'jenkins_hook.py') + if not os.path.exists(hook_file): + os.symlink( + hook_file, + os.path.join(repopath, 'hooks', 'post-receive.irc') + ) @classmethod def remove(cls, project): From fa5ca970b2566a0ced96c7dd89351e845b5ef1d0 Mon Sep 17 00:00:00 2001 From: farhaanbukhsh Date: Jul 26 2016 07:41:45 +0000 Subject: [PATCH 3/25] Integration PMCI --- diff --git a/pagure/consumer.py b/pagure/consumer.py new file mode 100644 index 0000000..b01496b --- /dev/null +++ b/pagure/consumer.py @@ -0,0 +1,74 @@ +# -*- coding: utf-8 -*- +import fedmsg.consumers +from pagure.hooks import jenkins_hook +from pagure.lib import pagure_ci +from pagure.lib.model import BASE, Project, User +PAGURE_MAIN_REPO = '{base}{name}.git' +PAGURE_FORK_REPO = '{base}forks/{user}/{name}.git' + + +class Integrator(fedmsg.consumers.FedmsgConsumer): + topic = [ + 'io.pagure.prod.pagure.pull-request.comment.added', + 'org.fedoraproject.dev.pagure.pull-request.new', + 'org.fedoraproject.dev.pagure.pull-request.comment.added', + 'io.pagure.prod.pagure.pull-request.new', + 'org.fedoraproject.prod.jenkins.build', + ] + + config_key = 'integrator.enabled' + + def __init__(self, hub): + super(Integrator, self).__init__(hub) + pagure_ci.connect_db() + + def consume(self, msg): + topic, msg = msg['topic'], msg['body'] + self.log.info("Received %r, %r", topic, msg.get('msg_id', None)) + msg = msg['msg'] + try: + if topic.endswith('.pull-request.comment.added'): + if is_rebase(msg): + self.trigger_build(msg) + elif topic.endswith('.pull-request.new'): + self.trigger_build(msg) + else: + self.process_build(msg) + except jenkins_hook.ConfigNotFound as exc: + self.log.info('Unconfigured project %r', str(exc)) + + def trigger_build(self, msg): + pr_id = msg['pullrequest']['id'] + project = msg['pullrequest']['project']['name'] + branch = msg['pullrequest']['branch_from'] + print project, jenkins_hook.Service.PAGURE + for cfg in jenkins_hook.get_configs(project, jenkins_hook.Service.PAGURE): + repo = msg['pullrequest'].get('remote_git') or get_repo(cfg, msg) + + self.log.info("Trigger on %s PR #%s from %s: %s", + project, pr_id, repo, branch) + + pagure_ci.process_pr(self.log, cfg, pr_id, repo, branch) + + def process_build(self, msg): + for cfg in jenkins_hook.get_configs(msg['project'], jenkins_hook.Service.JENKINS): + pagure_ci.process_build(self.log, cfg, msg['build']) + + +def get_repo(cfg, msg): + url = PAGURE_MAIN_REPO + if msg['pullrequest']['repo_from']['parent']: + url = PAGURE_FORK_REPO + return url.format( + base=cfg.pagure_url, + user=msg['pullrequest']['repo_from']['user']['name'], + name=msg['pullrequest']['repo_from']['name']) + + +def is_rebase(msg): + if msg['pullrequest']['status'] != 'Open': + return False + try: + return msg['pullrequest']['comments'][-1]['notification'] + except (IndexError, KeyError): + return False diff --git a/pagure/hooks/jenkins_hook.py b/pagure/hooks/jenkins_hook.py index 1305078..902c87d 100644 --- a/pagure/hooks/jenkins_hook.py +++ b/pagure/hooks/jenkins_hook.py @@ -12,34 +12,32 @@ import os import sqlalchemy as sa import pygit2 -from wtforms import validators, TextField +from wtforms import validators, TextField, BooleanField from flask.ext import wtf from sqlalchemy.orm import relation from sqlalchemy.orm import backref +from sqlalchemy.ext.declarative import declarative_base from pagure.hooks import BaseHook, RequiredIf -from pagure.lib.model import BASE, Project +from pagure.lib.model import BASE, Project, User from pagure import get_repo_path - class PagureCI(BASE): + __tablename__ = 'hook_pagure_ci' - __table_args__ = {'extend_existing': True} id = sa.Column(sa.Integer, primary_key=True) project_id = sa.Column( - sa.Integer, + sa.Integer, sa.ForeignKey('projects.id', onupdate='CASCADE'), nullable=False, - unique=True, + unique=False, index=True) + active = sa.Column(sa.Boolean, nullable=False, default=False) - name = sa.Column(sa.String(64), primary_key=True, unique=True) - display_name = sa.Column(sa.String(64), nullable=False, default='Jenkins') - owner = sa.Column(sa.String(64)) - + name = sa.Column(sa.String(64)) pagure_name = sa.Column(sa.String(255)) pagure_url = sa.Column(sa.String(255)) pagure_token = sa.Column(sa.String(64)) @@ -47,19 +45,67 @@ class PagureCI(BASE): jenkins_name = sa.Column(sa.String(255)) jenkins_url = sa.Column(sa.String(255)) jenkins_token = sa.Column(sa.String(64)) - - hook_token = sa.Column(sa.String(64)) + hook_token = sa.Column(sa.String(64), + nullable=True, + unique=True, + index=True) project = relation( - 'Project', remote_side=[Project.id], + 'Project', + foreign_keys=[project_id], + remote_side=[Project.id], backref=backref( - 'jenkins_hook', cascade="delete, delete-orphan", + 'hook_pagure_ci', cascade="delete, delete-orphan", single_parent=True) ) + def __init__(self, name = None, display_name = None, owner = None, + pagure_name = None, pagure_url = None, pagure_token = None, + jenkins_name = None, jenkins_url = None, jenkins_token = None, + hook_token = None, active = False): + self.name = name + self.display_name = display_name + self.owner = owner + self.pagure_name = pagure_name + self.pagure_url = pagure_url + self.pagure_token = pagure_token + + self.jenkins_name = jenkins_name + self.jenkins_url = jenkins_url + self.jenkins_token = jenkins_token + + self.hook_token = hook_token + self.active = active + + def __repr__(self): + return ''.format(self) + +def init_db(db): + from sqlalchemy import create_engine + engine = create_engine(db, convert_unicode=True) + BASE.metadata.create_all(bind=engine) + +class ConfigNotFound(Exception): + pass + + +class Service(object): + PAGURE = PagureCI.pagure_name + JENKINS = PagureCI.jenkins_name + + +def get_configs(project_name, service): + """Returns all configurations with given name on a service. + + :raises ConfigNotFound: when no configuration matches + """ + cfg = BASE.query(PagureCI).filter(service == project_name).all() + if len(cfg) == 0: + raise ConfigNotFound(project_name) + return cfg class JenkinsForm(wtf.Form): - + '''Form to configure Jenkins hook''' name = TextField('Name', [validators.Required(), @@ -86,7 +132,8 @@ class JenkinsForm(wtf.Form): validators.Length(max=255)], default='http://jenkins.fedorainfracloud.org/') jenkins_token = TextField('Jenkins token', - [validators.Required()]) + [validators.Required()]) + active = BooleanField('Active',[validators.Optional()]) class Hook(BaseHook): @@ -97,10 +144,10 @@ class Hook(BaseHook): ' the changes made by the pushes to the git repository.' form = JenkinsForm db_object = PagureCI - backref = 'jenkins_hook' + backref = 'pagure_ci_hook' form_fields = [ - 'name', 'pagure_name', 'pagure_url', 'pagure_token', 'jenkins_name', - 'jenkins_url', 'jenkins_token' + 'display_name','name', 'pagure_name', 'pagure_url', 'pagure_token', 'jenkins_name', + 'jenkins_url', 'jenkins_token','active' ] @classmethod @@ -125,7 +172,7 @@ class Hook(BaseHook): if not os.path.exists(hook_file): os.symlink( hook_file, - os.path.join(repopath, 'hooks', 'post-receive.irc') + os.path.join(repopath, 'hooks', 'jenkins_hook.py') ) @classmethod diff --git a/pagure/lib/pagure_ci.py b/pagure/lib/pagure_ci.py new file mode 100644 index 0000000..b6e2723 --- /dev/null +++ b/pagure/lib/pagure_ci.py @@ -0,0 +1,132 @@ +# -*- coding: utf-8 -*- +import os +import flask +from sqlalchemy.orm import scoped_session, sessionmaker +from sqlalchemy import create_engine +from pagure.hooks import jenkins_hook +from pagure.lib import model + +import json +import logging + +import requests +import jenkins + +os.environ.setdefault('INTEGRATOR_SETTINGS', '/etc/poormanci.conf') + +app = flask.Flask(__name__) +app.config.from_object('pagure.default_config') +app.config.from_envvar('INTEGRATOR_SETTINGS', silent=True) +app.logger.setLevel(logging.INFO) + +PAGURE_URL = '{base}api/0/{repo}/pull-request/{pr}/flag' +JENKINS_TRIGGER_URL = '{base}job/{project}/buildWithParameters' + + +db_session = None + +def connect_db(): + global db_session + engine = create_engine(app.config['DB_URL'], convert_unicode=True) + db_session = scoped_session(sessionmaker(autocommit=False, + autoflush=False, + bind=engine)) + model.BASE.query = db_session.query_property() + + + +def process_pr(logger, cfg, pr_id, repo, branch): + post_data(logger, + JENKINS_TRIGGER_URL.format(base=cfg.jenkins_url, project=cfg.jenkins_name), + {'token': cfg.jenkins_token, + 'cause': pr_id, + 'REPO': repo, + 'BRANCH': branch}) + + +def process_build(logger, cfg, build_id): + # Get details from Jenkins + jenk = jenkins.Jenkins(cfg.jenkins_url) + build_info = jenk.get_build_info(cfg.jenkins_name, build_id) + result = build_info['result'] + url = build_info['url'] + + pr_id = None + + for action in build_info['actions']: + for cause in action.get('causes', []): + try: + pr_id = int(cause['note']) + except (KeyError, ValueError): + continue + + if not pr_id: + logger.info('Not a PR check') + return + + # Comment in Pagure + logger.info('Updating %s PR %d: %s', cfg.pagure_name, pr_id, result) + try: + post_flag(logger, cfg.display_name, cfg.pagure_url, cfg.pagure_token, + cfg.pagure_name, pr_id, result, url) + except KeyError as exc: + logger.warning('Unknown build status', exc_info=exc) + + +def post_flag(logger, name, base, token, repo, pr, result, url): + comment, percent = { + 'SUCCESS': ('Build successful', 100), + 'FAILURE': ('Build failed', 0), + }[result] + payload = { + 'username': name, + 'percent': percent, + 'comment': comment, + 'url': url, + } + post_data(logger, PAGURE_URL.format(base=base, repo=repo, pr=pr), payload, + headers={'Authorization': 'token ' + token}) + + +def post_data(logger, *args, **kwargs): + resp = requests.post(*args, **kwargs) + logger.debug('Received response status %s', resp.status_code) + if resp.status_code < 200 or resp.status_code >= 300: + logger.error('Network request failed: %d: %s', resp.status_code, resp.text) + + +@app.route('/hooks//build-finished', methods=['POST']) +def hook_finished(token): + try: + data = json.loads(flask.request.get_data()) + cfg = jenkins_hook.get_configs(data['name'], jenkins_hook.Service.JENKINS)[0] + build_id = data['build']['number'] + if token != cfg.hook_token: + raise ValueError('Token mismatch') + except (TypeError, ValueError, KeyError, models.ConfigNotFound) as exc: + app.logger.error('Error processing jenkins notification', exc_info=exc) + return ('Bad request...\n', 400, {'Content-Type': 'text/plain'}) + app.logger.info('Received jenkins notification') + process_build(app.logger, cfg, build_id) + return ('', 204) + + +@app.errorhandler(403) +def forbidden(e): + return flask.render_template('forbidden.html'), 403 + + +def cleanup_url(url): + """Make sure there is trailing slash.""" + return url.rstrip('/') + '/' + + +@app.before_request +def before_request(): + if db_session is None: + connect_db() + + +@app.teardown_appcontext +def shutdown_session(exception=None): + db_session.remove() diff --git a/pagure/ui/plugins.py b/pagure/ui/plugins.py index 88c9c8a..c7f5a17 100644 --- a/pagure/ui/plugins.py +++ b/pagure/ui/plugins.py @@ -9,6 +9,7 @@ """ import flask +import uuid from sqlalchemy.exc import SQLAlchemyError from straight.plugin import load @@ -94,12 +95,18 @@ def view_plugin(repo, plugin, username=None, full=True): else: dbobj = plugin.db_object() + print dir(dbobj) + + form = plugin.form(obj=dbobj) for field in plugin.form_fields: fields.append(getattr(form, field)) - + print "validate:", form.validate_on_submit(), dir(form), form.errors if form.validate_on_submit(): form.populate_obj(obj=dbobj) + if dbobj.__tablename__ == 'hook_pagure_ci': + dbobj.hook_token = uuid.uuid4().hex + if new: dbobj.project_id = repo.id SESSION.add(dbobj) @@ -121,6 +128,7 @@ def view_plugin(repo, plugin, username=None, full=True): username=username, plugin=plugin, form=form, + dbobj=dbobj, fields=fields) if form.active.data: @@ -154,4 +162,5 @@ def view_plugin(repo, plugin, username=None, full=True): username=username, plugin=plugin, form=form, + dbobj=dbobj, fields=fields) From ebf17a164d6a4cd6eb8955a16b77c7e93dd6ace5 Mon Sep 17 00:00:00 2001 From: farhaanbukhsh Date: Jul 26 2016 07:41:45 +0000 Subject: [PATCH 4/25] Fist cut for having CI work from inside Pagure --- diff --git a/pagure/consumer.py b/pagure/consumer.py index b01496b..738ead4 100644 --- a/pagure/consumer.py +++ b/pagure/consumer.py @@ -41,10 +41,10 @@ class Integrator(fedmsg.consumers.FedmsgConsumer): pr_id = msg['pullrequest']['id'] project = msg['pullrequest']['project']['name'] branch = msg['pullrequest']['branch_from'] - print project, jenkins_hook.Service.PAGURE + for cfg in jenkins_hook.get_configs(project, jenkins_hook.Service.PAGURE): repo = msg['pullrequest'].get('remote_git') or get_repo(cfg, msg) - + print repo self.log.info("Trigger on %s PR #%s from %s: %s", project, pr_id, repo, branch) diff --git a/pagure/hooks/jenkins_hook.py b/pagure/hooks/jenkins_hook.py index 902c87d..348ee44 100644 --- a/pagure/hooks/jenkins_hook.py +++ b/pagure/hooks/jenkins_hook.py @@ -19,7 +19,7 @@ from sqlalchemy.orm import backref from sqlalchemy.ext.declarative import declarative_base from pagure.hooks import BaseHook, RequiredIf -from pagure.lib.model import BASE, Project, User +from pagure.lib.model import BASE, Project from pagure import get_repo_path @@ -79,11 +79,6 @@ class PagureCI(BASE): def __repr__(self): return ''.format(self) -def init_db(db): - from sqlalchemy import create_engine - engine = create_engine(db, convert_unicode=True) - BASE.metadata.create_all(bind=engine) - class ConfigNotFound(Exception): pass @@ -98,7 +93,7 @@ def get_configs(project_name, service): :raises ConfigNotFound: when no configuration matches """ - cfg = BASE.query(PagureCI).filter(service == project_name).all() + cfg = BASE.metadata.bind.query(PagureCI).filter(service == project_name).all() if len(cfg) == 0: raise ConfigNotFound(project_name) return cfg diff --git a/pagure/lib/pagure_ci.py b/pagure/lib/pagure_ci.py index b6e2723..d4ddf9d 100644 --- a/pagure/lib/pagure_ci.py +++ b/pagure/lib/pagure_ci.py @@ -5,6 +5,7 @@ from sqlalchemy.orm import scoped_session, sessionmaker from sqlalchemy import create_engine from pagure.hooks import jenkins_hook from pagure.lib import model +from pagure import APP import json import logging @@ -67,7 +68,7 @@ def process_build(logger, cfg, build_id): # Comment in Pagure logger.info('Updating %s PR %d: %s', cfg.pagure_name, pr_id, result) try: - post_flag(logger, cfg.display_name, cfg.pagure_url, cfg.pagure_token, + post_flag(logger, "Jenkins", cfg.pagure_url, cfg.pagure_token, cfg.pagure_name, pr_id, result, url) except KeyError as exc: logger.warning('Unknown build status', exc_info=exc) @@ -94,23 +95,6 @@ def post_data(logger, *args, **kwargs): if resp.status_code < 200 or resp.status_code >= 300: logger.error('Network request failed: %d: %s', resp.status_code, resp.text) - -@app.route('/hooks//build-finished', methods=['POST']) -def hook_finished(token): - try: - data = json.loads(flask.request.get_data()) - cfg = jenkins_hook.get_configs(data['name'], jenkins_hook.Service.JENKINS)[0] - build_id = data['build']['number'] - if token != cfg.hook_token: - raise ValueError('Token mismatch') - except (TypeError, ValueError, KeyError, models.ConfigNotFound) as exc: - app.logger.error('Error processing jenkins notification', exc_info=exc) - return ('Bad request...\n', 400, {'Content-Type': 'text/plain'}) - app.logger.info('Received jenkins notification') - process_build(app.logger, cfg, build_id) - return ('', 204) - - @app.errorhandler(403) def forbidden(e): return flask.render_template('forbidden.html'), 403 diff --git a/pagure/ui/app.py b/pagure/ui/app.py index a395e4a..7e4996f 100644 --- a/pagure/ui/app.py +++ b/pagure/ui/app.py @@ -13,6 +13,10 @@ from math import ceil from sqlalchemy.exc import SQLAlchemyError +from pagure.hooks import jenkins_hook +import json +from pagure.lib import pagure_ci + import pagure.exceptions import pagure.lib import pagure.lib.git @@ -667,3 +671,21 @@ def ssh_hostkey(): return flask.render_template( 'doc_ssh_keys.html', ) + + +@APP.route('/hooks//build-finished', methods=['POST']) +def hook_finished(token): + print "Hello" + try: + data = json.loads(flask.request.get_data()) + cfg = jenkins_hook.get_configs(data['name'], jenkins_hook.Service.JENKINS)[0] + build_id = data['build']['number'] + print data , cfg, build_id + if token != cfg.hook_token: + raise ValueError('Token mismatch') + except (TypeError, ValueError, KeyError, jenkins_hook.ConfigNotFound) as exc: + APP.logger.error('Error processing jenkins notification', exc_info=exc) + return ('Bad request...\n', 400, {'Content-Type': 'text/plain'}) + APP.logger.info('Received jenkins notification') + pagure_ci.process_build(APP.logger, cfg, build_id) + return ('', 204) From 4c78f42adc549c1c0868d2fc53231d42c3cce502 Mon Sep 17 00:00:00 2001 From: farhaanbukhsh Date: Jul 26 2016 07:41:45 +0000 Subject: [PATCH 5/25] Add Pagure CI hook --- diff --git a/pagure/consumer.py b/pagure/consumer.py index 738ead4..5725240 100644 --- a/pagure/consumer.py +++ b/pagure/consumer.py @@ -44,7 +44,6 @@ class Integrator(fedmsg.consumers.FedmsgConsumer): for cfg in jenkins_hook.get_configs(project, jenkins_hook.Service.PAGURE): repo = msg['pullrequest'].get('remote_git') or get_repo(cfg, msg) - print repo self.log.info("Trigger on %s PR #%s from %s: %s", project, pr_id, repo, branch) diff --git a/pagure/hooks/jenkins_hook.py b/pagure/hooks/jenkins_hook.py index 348ee44..d22f05d 100644 --- a/pagure/hooks/jenkins_hook.py +++ b/pagure/hooks/jenkins_hook.py @@ -9,6 +9,7 @@ """ import os +import uuid import sqlalchemy as sa import pygit2 @@ -29,26 +30,28 @@ class PagureCI(BASE): id = sa.Column(sa.Integer, primary_key=True) project_id = sa.Column( - sa.Integer, + sa.Integer, sa.ForeignKey('projects.id', onupdate='CASCADE'), nullable=False, unique=False, index=True) active = sa.Column(sa.Boolean, nullable=False, default=False) - + display_name = sa.Column(sa.String(64), nullable=False, default='Jenkins') name = sa.Column(sa.String(64)) pagure_name = sa.Column(sa.String(255)) - pagure_url = sa.Column(sa.String(255)) + pagure_url = sa.Column(sa.String(255), nullable=False, + default='https://pagure.io/') pagure_token = sa.Column(sa.String(64)) jenkins_name = sa.Column(sa.String(255)) - jenkins_url = sa.Column(sa.String(255)) + jenkins_url = sa.Column(sa.String(255), nullable=False, + default='http://jenkins.fedorainfracloud.org/') jenkins_token = sa.Column(sa.String(64)) hook_token = sa.Column(sa.String(64), - nullable=True, - unique=True, - index=True) + nullable=True, + unique=True, + index=True) project = relation( 'Project', @@ -58,27 +61,14 @@ class PagureCI(BASE): 'hook_pagure_ci', cascade="delete, delete-orphan", single_parent=True) ) - def __init__(self, name = None, display_name = None, owner = None, - pagure_name = None, pagure_url = None, pagure_token = None, - jenkins_name = None, jenkins_url = None, jenkins_token = None, - hook_token = None, active = False): - self.name = name - self.display_name = display_name - self.owner = owner - self.pagure_name = pagure_name - self.pagure_url = pagure_url - self.pagure_token = pagure_token - - self.jenkins_name = jenkins_name - self.jenkins_url = jenkins_url - self.jenkins_token = jenkins_token - - self.hook_token = hook_token - self.active = active + + def __init__(self): + self.hook_token = uuid.uuid4().hex def __repr__(self): return ''.format(self) + class ConfigNotFound(Exception): pass @@ -93,7 +83,8 @@ def get_configs(project_name, service): :raises ConfigNotFound: when no configuration matches """ - cfg = BASE.metadata.bind.query(PagureCI).filter(service == project_name).all() + cfg = BASE.metadata.bind.query(PagureCI).filter( + service == project_name).all() if len(cfg) == 0: raise ConfigNotFound(project_name) return cfg @@ -128,21 +119,21 @@ class JenkinsForm(wtf.Form): default='http://jenkins.fedorainfracloud.org/') jenkins_token = TextField('Jenkins token', [validators.Required()]) - active = BooleanField('Active',[validators.Optional()]) + active = BooleanField('Active', [validators.Optional()]) class Hook(BaseHook): ''' Jenkins hooks. ''' - name = 'Jenkins Hook' + name = 'Pagure CI' description = 'This hook help to set up CI for the project'\ ' the changes made by the pushes to the git repository.' form = JenkinsForm db_object = PagureCI backref = 'pagure_ci_hook' form_fields = [ - 'display_name','name', 'pagure_name', 'pagure_url', 'pagure_token', 'jenkins_name', - 'jenkins_url', 'jenkins_token','active' + 'display_name', 'name', 'pagure_name', 'pagure_url', 'pagure_token', 'jenkins_name', + 'jenkins_url', 'jenkins_token', 'active' ] @classmethod @@ -160,7 +151,7 @@ class Hook(BaseHook): repo_obj = pygit2.Repository(repopath) # Configure the hook - #repo_obj.config.set_multivar(dbobj) + # repo_obj.config.set_multivar(dbobj) # Install the hook itself hook_file = os.path.join(hook_files, 'jenkins_hook.py') @@ -181,5 +172,5 @@ class Hook(BaseHook): repopath = get_repo_path(project) #hook_path = os.path.join(repopath, 'hooks', 'post-receive.irc') - #if os.path.exists(hook_path): - #os.unlink(hook_path) + # if os.path.exists(hook_path): + # os.unlink(hook_path) diff --git a/pagure/lib/pagure_ci.py b/pagure/lib/pagure_ci.py index d4ddf9d..8157c5f 100644 --- a/pagure/lib/pagure_ci.py +++ b/pagure/lib/pagure_ci.py @@ -15,10 +15,9 @@ import jenkins os.environ.setdefault('INTEGRATOR_SETTINGS', '/etc/poormanci.conf') -app = flask.Flask(__name__) -app.config.from_object('pagure.default_config') -app.config.from_envvar('INTEGRATOR_SETTINGS', silent=True) -app.logger.setLevel(logging.INFO) + +APP.config.from_envvar('INTEGRATOR_SETTINGS', silent=True) +APP.logger.setLevel(logging.INFO) PAGURE_URL = '{base}api/0/{repo}/pull-request/{pr}/flag' JENKINS_TRIGGER_URL = '{base}job/{project}/buildWithParameters' @@ -28,7 +27,7 @@ db_session = None def connect_db(): global db_session - engine = create_engine(app.config['DB_URL'], convert_unicode=True) + engine = create_engine(APP.config['DB_URL'], convert_unicode=True) db_session = scoped_session(sessionmaker(autocommit=False, autoflush=False, bind=engine)) @@ -68,7 +67,7 @@ def process_build(logger, cfg, build_id): # Comment in Pagure logger.info('Updating %s PR %d: %s', cfg.pagure_name, pr_id, result) try: - post_flag(logger, "Jenkins", cfg.pagure_url, cfg.pagure_token, + post_flag(logger, cfg.display_name, cfg.pagure_url, cfg.pagure_token, cfg.pagure_name, pr_id, result, url) except KeyError as exc: logger.warning('Unknown build status', exc_info=exc) @@ -95,22 +94,33 @@ def post_data(logger, *args, **kwargs): if resp.status_code < 200 or resp.status_code >= 300: logger.error('Network request failed: %d: %s', resp.status_code, resp.text) -@app.errorhandler(403) -def forbidden(e): - return flask.render_template('forbidden.html'), 403 +@APP.route('/hooks//build-finished', methods=['POST']) +def hook_finished(token): + try: + data = json.loads(flask.request.get_data()) + cfg = jenkins_hook.get_configs(data['name'], jenkins_hook.Service.JENKINS)[0] + build_id = data['build']['number'] + if token != cfg.hook_token: + raise ValueError('Token mismatch') + except (TypeError, ValueError, KeyError, jenkins_hook.ConfigNotFound) as exc: + APP.logger.error('Error processing jenkins notification', exc_info=exc) + return ('Bad request...\n', 400, {'Content-Type': 'text/plain'}) + APP.logger.info('Received jenkins notification') + process_build(APP.logger, cfg, build_id) + return ('', 204) def cleanup_url(url): """Make sure there is trailing slash.""" return url.rstrip('/') + '/' -@app.before_request +@APP.before_request def before_request(): if db_session is None: connect_db() -@app.teardown_appcontext +@APP.teardown_appcontext def shutdown_session(exception=None): db_session.remove() diff --git a/pagure/templates/plugin.html b/pagure/templates/plugin.html index 0d6d3ff..2fbdc1f 100644 --- a/pagure/templates/plugin.html +++ b/pagure/templates/plugin.html @@ -1,32 +1,27 @@ -{% from "_formhelper.html" import render_field_in_row %} - -{% if full %} -{% extends "repo_master.html" %} - -{% block title %}{{ select.capitalize() }} {{ plugin.name }} - {{ repo.name }}{% endblock %} -{% set tag = "home" %} -{% endif %} - - -{% block repo %} -{% if full %} -

{{ plugin.name }} settings

+{% from "_formhelper.html" import render_field_in_row %} {% if full %} {% extends "repo_master.html" %} {% block title %}{{ select.capitalize() }} {{ plugin.name }} - {{ repo.name }}{% endblock %} {% set tag = "home" %} {% endif %} {% block repo %} {% +if full %} +

{{ plugin.name }} settings

{% endif %} -
- {{ plugin.description | markdown | noJS | safe }} + {{ plugin.description | markdown | noJS | safe }} {% if post_token and (plugin.name == 'Pagure CI') %} +
+ + +
+ {% endif %} - {% for field in fields %} - {{ render_field_in_row(field) }} - {% endfor %} + {% for field in fields %} {{ render_field_in_row(field) }} {% endfor %}

{{ form.csrf_token }}

-
+ + {% endblock %} diff --git a/pagure/ui/app.py b/pagure/ui/app.py index 7e4996f..7aff6de 100644 --- a/pagure/ui/app.py +++ b/pagure/ui/app.py @@ -671,21 +671,3 @@ def ssh_hostkey(): return flask.render_template( 'doc_ssh_keys.html', ) - - -@APP.route('/hooks//build-finished', methods=['POST']) -def hook_finished(token): - print "Hello" - try: - data = json.loads(flask.request.get_data()) - cfg = jenkins_hook.get_configs(data['name'], jenkins_hook.Service.JENKINS)[0] - build_id = data['build']['number'] - print data , cfg, build_id - if token != cfg.hook_token: - raise ValueError('Token mismatch') - except (TypeError, ValueError, KeyError, jenkins_hook.ConfigNotFound) as exc: - APP.logger.error('Error processing jenkins notification', exc_info=exc) - return ('Bad request...\n', 400, {'Content-Type': 'text/plain'}) - APP.logger.info('Received jenkins notification') - pagure_ci.process_build(APP.logger, cfg, build_id) - return ('', 204) diff --git a/pagure/ui/plugins.py b/pagure/ui/plugins.py index c7f5a17..0de4a79 100644 --- a/pagure/ui/plugins.py +++ b/pagure/ui/plugins.py @@ -21,6 +21,8 @@ import pagure.forms from pagure import APP, SESSION, login_required, is_repo_admin from pagure.lib.model import BASE from pagure.exceptions import FileNotFoundException +from pagure.hooks.jenkins_hook import PagureCI + # pylint: disable=E1101 @@ -85,7 +87,15 @@ def view_plugin(repo, plugin, username=None, full=True): plugin = get_plugin(plugin) fields = [] new = True + post_token = None dbobj = plugin.db_object() + + post_token_obj = BASE.metadata.bind.query(PagureCI).filter( + PagureCI.pagure_name == repo.name).first() + + if hasattr(post_token_obj, 'hook_token'): + post_token = getattr(post_token_obj, 'hook_token') + if hasattr(repo, plugin.backref): dbobj = getattr(repo, plugin.backref) # There should always be only one, but let's double check @@ -95,18 +105,13 @@ def view_plugin(repo, plugin, username=None, full=True): else: dbobj = plugin.db_object() - print dir(dbobj) - - form = plugin.form(obj=dbobj) for field in plugin.form_fields: fields.append(getattr(form, field)) - print "validate:", form.validate_on_submit(), dir(form), form.errors + if form.validate_on_submit(): form.populate_obj(obj=dbobj) - if dbobj.__tablename__ == 'hook_pagure_ci': - dbobj.hook_token = uuid.uuid4().hex - + if new: dbobj.project_id = repo.id SESSION.add(dbobj) @@ -128,7 +133,7 @@ def view_plugin(repo, plugin, username=None, full=True): username=username, plugin=plugin, form=form, - dbobj=dbobj, + post_token=post_token, fields=fields) if form.active.data: @@ -162,5 +167,5 @@ def view_plugin(repo, plugin, username=None, full=True): username=username, plugin=plugin, form=form, - dbobj=dbobj, + post_token=post_token, fields=fields) diff --git a/setup.py b/setup.py index 1dbf3c4..02d354b 100644 --- a/setup.py +++ b/setup.py @@ -55,4 +55,8 @@ setup( packages=['pagure'], include_package_data=True, install_requires=get_requirements(), + entry_points=""" + [moksha.consumer] + integrator = pagure.consumer:Integrator + """ ) From b88c3abc7d5115f22a748b56198afa2e2ef3017f Mon Sep 17 00:00:00 2001 From: farhaanbukhsh Date: Jul 26 2016 07:41:45 +0000 Subject: [PATCH 6/25] Fix import --- diff --git a/pagure/ui/app.py b/pagure/ui/app.py index 7aff6de..a395e4a 100644 --- a/pagure/ui/app.py +++ b/pagure/ui/app.py @@ -13,10 +13,6 @@ from math import ceil from sqlalchemy.exc import SQLAlchemyError -from pagure.hooks import jenkins_hook -import json -from pagure.lib import pagure_ci - import pagure.exceptions import pagure.lib import pagure.lib.git From 4d5eaf41e756cd238caab4767ca6de0ba8b074a5 Mon Sep 17 00:00:00 2001 From: farhaanbukhsh Date: Jul 26 2016 07:41:45 +0000 Subject: [PATCH 7/25] Fix the template --- diff --git a/pagure/templates/plugin.html b/pagure/templates/plugin.html index 2fbdc1f..193fc63 100644 --- a/pagure/templates/plugin.html +++ b/pagure/templates/plugin.html @@ -1,20 +1,34 @@ -{% from "_formhelper.html" import render_field_in_row %} {% if full %} {% extends "repo_master.html" %} {% block title %}{{ select.capitalize() }} {{ plugin.name }} - {{ repo.name }}{% endblock %} {% set tag = "home" %} {% endif %} {% block repo %} {% -if full %} -

{{ plugin.name }} settings

+{% from "_formhelper.html" import render_field_in_row %} + +{% if full %} +{% extends "repo_master.html" %} + +{% block title %}{{ select.capitalize() }} {{ plugin.name }} - {{ repo.name }} +{% endblock %} {% set tag = "home" %} +{% endif %} + +{% block repo %} +{% if full %} +

{{ plugin.name }} settings

{% endif %}
- {{ plugin.description | markdown | noJS | safe }} {% if post_token and (plugin.name == 'Pagure CI') %} + {{ plugin.description | markdown | noJS | safe }} + + {% if post_token and (plugin.name == 'Pagure CI') %}
{% endif %} + - {% for field in fields %} {{ render_field_in_row(field) }} {% endfor %} + {% for field in fields %} + {{ render_field_in_row(field) }} + {% endfor %}

From f34f17dac5968cd1bcbf7d567faf69b468438f93 Mon Sep 17 00:00:00 2001 From: farhaanbukhsh Date: Jul 26 2016 07:41:45 +0000 Subject: [PATCH 8/25] Add dependencies --- diff --git a/requirements.txt b/requirements.txt index d79c008..22c4034 100644 --- a/requirements.txt +++ b/requirements.txt @@ -35,3 +35,7 @@ python-fedora # Required only for the `local` authentication backend cryptography py-bcrypt + +#Required for Pagure CI +moksha +python-jenkins From 1a38700f3d44dac1c82a22a01af1c64b8b05b036 Mon Sep 17 00:00:00 2001 From: farhaanbukhsh Date: Jul 26 2016 07:41:45 +0000 Subject: [PATCH 9/25] Fix major issues, restructuring model and improved interaction --- diff --git a/pagure/consumer.py b/pagure/consumer.py deleted file mode 100644 index 5725240..0000000 --- a/pagure/consumer.py +++ /dev/null @@ -1,73 +0,0 @@ -# -*- coding: utf-8 -*- -import fedmsg.consumers -from pagure.hooks import jenkins_hook -from pagure.lib import pagure_ci -from pagure.lib.model import BASE, Project, User -PAGURE_MAIN_REPO = '{base}{name}.git' -PAGURE_FORK_REPO = '{base}forks/{user}/{name}.git' - - -class Integrator(fedmsg.consumers.FedmsgConsumer): - topic = [ - 'io.pagure.prod.pagure.pull-request.comment.added', - 'org.fedoraproject.dev.pagure.pull-request.new', - 'org.fedoraproject.dev.pagure.pull-request.comment.added', - 'io.pagure.prod.pagure.pull-request.new', - 'org.fedoraproject.prod.jenkins.build', - ] - - config_key = 'integrator.enabled' - - def __init__(self, hub): - super(Integrator, self).__init__(hub) - pagure_ci.connect_db() - - def consume(self, msg): - topic, msg = msg['topic'], msg['body'] - self.log.info("Received %r, %r", topic, msg.get('msg_id', None)) - msg = msg['msg'] - try: - if topic.endswith('.pull-request.comment.added'): - if is_rebase(msg): - self.trigger_build(msg) - elif topic.endswith('.pull-request.new'): - self.trigger_build(msg) - else: - self.process_build(msg) - except jenkins_hook.ConfigNotFound as exc: - self.log.info('Unconfigured project %r', str(exc)) - - def trigger_build(self, msg): - pr_id = msg['pullrequest']['id'] - project = msg['pullrequest']['project']['name'] - branch = msg['pullrequest']['branch_from'] - - for cfg in jenkins_hook.get_configs(project, jenkins_hook.Service.PAGURE): - repo = msg['pullrequest'].get('remote_git') or get_repo(cfg, msg) - self.log.info("Trigger on %s PR #%s from %s: %s", - project, pr_id, repo, branch) - - pagure_ci.process_pr(self.log, cfg, pr_id, repo, branch) - - def process_build(self, msg): - for cfg in jenkins_hook.get_configs(msg['project'], jenkins_hook.Service.JENKINS): - pagure_ci.process_build(self.log, cfg, msg['build']) - - -def get_repo(cfg, msg): - url = PAGURE_MAIN_REPO - if msg['pullrequest']['repo_from']['parent']: - url = PAGURE_FORK_REPO - return url.format( - base=cfg.pagure_url, - user=msg['pullrequest']['repo_from']['user']['name'], - name=msg['pullrequest']['repo_from']['name']) - - -def is_rebase(msg): - if msg['pullrequest']['status'] != 'Open': - return False - try: - return msg['pullrequest']['comments'][-1]['notification'] - except (IndexError, KeyError): - return False diff --git a/pagure/hooks/jenkins_hook.py b/pagure/hooks/jenkins_hook.py index d22f05d..1d162ba 100644 --- a/pagure/hooks/jenkins_hook.py +++ b/pagure/hooks/jenkins_hook.py @@ -1,12 +1,5 @@ # -*- coding: utf-8 -*- -""" - (c) 2014 - Copyright Red Hat Inc - - Authors: - Pierre-Yves Chibon - -""" import os import uuid @@ -15,13 +8,13 @@ import sqlalchemy as sa import pygit2 from wtforms import validators, TextField, BooleanField from flask.ext import wtf -from sqlalchemy.orm import relation -from sqlalchemy.orm import backref +from sqlalchemy.orm import relation, backref from sqlalchemy.ext.declarative import declarative_base from pagure.hooks import BaseHook, RequiredIf from pagure.lib.model import BASE, Project from pagure import get_repo_path +from pagure import APP class PagureCI(BASE): @@ -31,7 +24,7 @@ class PagureCI(BASE): id = sa.Column(sa.Integer, primary_key=True) project_id = sa.Column( sa.Integer, - sa.ForeignKey('projects.id', onupdate='CASCADE'), + sa.ForeignKey('projects.id', onupdate='CASCADE',ondelete='CASCADE'), nullable=False, unique=False, index=True) @@ -40,8 +33,7 @@ class PagureCI(BASE): display_name = sa.Column(sa.String(64), nullable=False, default='Jenkins') name = sa.Column(sa.String(64)) pagure_name = sa.Column(sa.String(255)) - pagure_url = sa.Column(sa.String(255), nullable=False, - default='https://pagure.io/') + pagure_token = sa.Column(sa.String(64)) jenkins_name = sa.Column(sa.String(255)) @@ -64,6 +56,7 @@ class PagureCI(BASE): def __init__(self): self.hook_token = uuid.uuid4().hex + self.display_name = 'Jenkins' def __repr__(self): return ''.format(self) @@ -96,17 +89,11 @@ class JenkinsForm(wtf.Form): name = TextField('Name', [validators.Required(), validators.Length(max=64)]) - display_name = TextField('Display name', - [validators.Required(), - validators.Length(max=64)], - default='Jenkins') + pagure_name = TextField('Name of project in Pagure', [validators.Required(), validators.Length(max=255)]) - pagure_url = TextField('Pagure URL', - [validators.Required(), - validators.Length(max=255)], - default='https://pagure.io/') + pagure_token = TextField('Pagure token', [validators.Required()]) @@ -122,7 +109,7 @@ class JenkinsForm(wtf.Form): active = BooleanField('Active', [validators.Optional()]) -class Hook(BaseHook): +class PagureCiHook(BaseHook): ''' Jenkins hooks. ''' name = 'Pagure CI' @@ -130,13 +117,20 @@ class Hook(BaseHook): ' the changes made by the pushes to the git repository.' form = JenkinsForm db_object = PagureCI - backref = 'pagure_ci_hook' + backref = 'hook_pagure_ci' form_fields = [ - 'display_name', 'name', 'pagure_name', 'pagure_url', 'pagure_token', 'jenkins_name', + 'name', 'pagure_name', 'pagure_token', 'jenkins_name', 'jenkins_url', 'jenkins_token', 'active' ] @classmethod + def set_up(cls, project): + ''' Install the generic post-receive hook that allow us to call + multiple post-receive hooks as set per plugin. + ''' + pass + + @classmethod def install(cls, project, dbobj): ''' Method called to install the hook for a project. @@ -144,22 +138,7 @@ class Hook(BaseHook): should be installed ''' - repopath = get_repo_path(project) - - hook_files = os.path.join( - os.path.dirname(os.path.realpath(__file__)), 'files') - repo_obj = pygit2.Repository(repopath) - - # Configure the hook - # repo_obj.config.set_multivar(dbobj) - - # Install the hook itself - hook_file = os.path.join(hook_files, 'jenkins_hook.py') - if not os.path.exists(hook_file): - os.symlink( - hook_file, - os.path.join(repopath, 'hooks', 'jenkins_hook.py') - ) + pass @classmethod def remove(cls, project): @@ -169,8 +148,4 @@ class Hook(BaseHook): should be installed ''' - repopath = get_repo_path(project) - - #hook_path = os.path.join(repopath, 'hooks', 'post-receive.irc') - # if os.path.exists(hook_path): - # os.unlink(hook_path) + pass diff --git a/pagure/lib/pagure_ci.py b/pagure/lib/pagure_ci.py index 8157c5f..ec5bad3 100644 --- a/pagure/lib/pagure_ci.py +++ b/pagure/lib/pagure_ci.py @@ -5,7 +5,7 @@ from sqlalchemy.orm import scoped_session, sessionmaker from sqlalchemy import create_engine from pagure.hooks import jenkins_hook from pagure.lib import model -from pagure import APP +from pagure import APP, SESSION import json import logging @@ -13,9 +13,6 @@ import logging import requests import jenkins -os.environ.setdefault('INTEGRATOR_SETTINGS', '/etc/poormanci.conf') - - APP.config.from_envvar('INTEGRATOR_SETTINGS', silent=True) APP.logger.setLevel(logging.INFO) @@ -23,21 +20,10 @@ PAGURE_URL = '{base}api/0/{repo}/pull-request/{pr}/flag' JENKINS_TRIGGER_URL = '{base}job/{project}/buildWithParameters' -db_session = None - -def connect_db(): - global db_session - engine = create_engine(APP.config['DB_URL'], convert_unicode=True) - db_session = scoped_session(sessionmaker(autocommit=False, - autoflush=False, - bind=engine)) - model.BASE.query = db_session.query_property() - - - def process_pr(logger, cfg, pr_id, repo, branch): post_data(logger, - JENKINS_TRIGGER_URL.format(base=cfg.jenkins_url, project=cfg.jenkins_name), + JENKINS_TRIGGER_URL.format( + base=cfg.jenkins_url, project=cfg.jenkins_name), {'token': cfg.jenkins_token, 'cause': pr_id, 'REPO': repo, @@ -67,7 +53,7 @@ def process_build(logger, cfg, build_id): # Comment in Pagure logger.info('Updating %s PR %d: %s', cfg.pagure_name, pr_id, result) try: - post_flag(logger, cfg.display_name, cfg.pagure_url, cfg.pagure_token, + post_flag(logger, cfg.display_name, APP.config['APP_URL'], cfg.pagure_token, cfg.pagure_name, pr_id, result, url) except KeyError as exc: logger.warning('Unknown build status', exc_info=exc) @@ -92,35 +78,5 @@ def post_data(logger, *args, **kwargs): resp = requests.post(*args, **kwargs) logger.debug('Received response status %s', resp.status_code) if resp.status_code < 200 or resp.status_code >= 300: - logger.error('Network request failed: %d: %s', resp.status_code, resp.text) - - -@APP.route('/hooks//build-finished', methods=['POST']) -def hook_finished(token): - try: - data = json.loads(flask.request.get_data()) - cfg = jenkins_hook.get_configs(data['name'], jenkins_hook.Service.JENKINS)[0] - build_id = data['build']['number'] - if token != cfg.hook_token: - raise ValueError('Token mismatch') - except (TypeError, ValueError, KeyError, jenkins_hook.ConfigNotFound) as exc: - APP.logger.error('Error processing jenkins notification', exc_info=exc) - return ('Bad request...\n', 400, {'Content-Type': 'text/plain'}) - APP.logger.info('Received jenkins notification') - process_build(APP.logger, cfg, build_id) - return ('', 204) - -def cleanup_url(url): - """Make sure there is trailing slash.""" - return url.rstrip('/') + '/' - - -@APP.before_request -def before_request(): - if db_session is None: - connect_db() - - -@APP.teardown_appcontext -def shutdown_session(exception=None): - db_session.remove() + logger.error('Network request failed: %d: %s', + resp.status_code, resp.text) diff --git a/pagure/pagureCI/__init__.py b/pagure/pagureCI/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/pagure/pagureCI/__init__.py diff --git a/pagure/pagureCI/consumer.py b/pagure/pagureCI/consumer.py new file mode 100644 index 0000000..4ef85de --- /dev/null +++ b/pagure/pagureCI/consumer.py @@ -0,0 +1,76 @@ +# -*- coding: utf-8 -*- +import fedmsg.consumers +from pagure.hooks import jenkins_hook +import pagure.lib +from pagure.lib import pagure_ci +from pagure.lib.model import BASE, Project, User +from pagure import APP +PAGURE_MAIN_REPO = '{base}{name}.git' +PAGURE_FORK_REPO = '{base}forks/{user}/{name}.git' + + +class Integrator(fedmsg.consumers.FedmsgConsumer): + topic = [ + 'io.pagure.prod.pagure.pull-request.comment.added', + 'io.pagure.prod.pagure.pull-request.new', + 'org.fedoraproject.prod.jenkins.build', + ] + + config_key = 'integrator.enabled' + + SESSION = None + + def __init__(self, hub): + super(Integrator, self).__init__(hub) + SESSION = pagure.lib.create_session(APP.config['DB_URL']) + + def consume(self, msg): + topic, msg = msg['topic'], msg['body'] + self.log.info("Received %r, %r", topic, msg.get('msg_id', None)) + msg = msg['msg'] + try: + if topic.endswith('.pull-request.comment.added'): + if is_rebase(msg): + self.trigger_build(msg) + elif topic.endswith('.pull-request.new'): + self.trigger_build(msg) + else: + self.process_build(msg) + except jenkins_hook.ConfigNotFound as exc: + self.log.info('Unconfigured project %r', str(exc)) + + def trigger_build(self, msg): + pr_id = msg['pullrequest']['id'] + project = msg['pullrequest']['project']['name'] + branch = msg['pullrequest']['branch_from'] + + for cfg in jenkins_hook.get_configs(project, jenkins_hook.Service.PAGURE): + repo = msg['pullrequest'].get('remote_git') or get_repo(cfg, msg) + self.log.info("Trigger on %s PR #%s from %s: %s", + project, pr_id, repo, branch) + + pagure_ci.process_pr(self.log, cfg, pr_id, repo, branch) + + def process_build(self, msg): + for cfg in jenkins_hook.get_configs(msg['project'], jenkins_hook.Service.JENKINS): + pagure_ci.process_build(self.log, cfg, msg['build']) + + +def get_repo(cfg, msg): + url = PAGURE_MAIN_REPO + if msg['pullrequest']['repo_from']['parent']: + url = PAGURE_FORK_REPO + return url.format( + base=APP.config['APP_URL'], + user=msg['pullrequest']['repo_from']['user']['name'], + name=msg['pullrequest']['repo_from']['name']) + + +def is_rebase(msg): + if msg['pullrequest']['status'] != 'Open': + return False + try: + print msg + return msg['pullrequest']['comments'][-1]['notification'] + except (IndexError, KeyError): + return False diff --git a/pagure/ui/plugins.py b/pagure/ui/plugins.py index 0de4a79..9f3033d 100644 --- a/pagure/ui/plugins.py +++ b/pagure/ui/plugins.py @@ -9,7 +9,6 @@ """ import flask -import uuid from sqlalchemy.exc import SQLAlchemyError from straight.plugin import load @@ -22,6 +21,10 @@ from pagure import APP, SESSION, login_required, is_repo_admin from pagure.lib.model import BASE from pagure.exceptions import FileNotFoundException from pagure.hooks.jenkins_hook import PagureCI +from pagure.hooks import jenkins_hook +from pagure.lib import model, pagure_ci + +import json # pylint: disable=E1101 @@ -90,18 +93,16 @@ def view_plugin(repo, plugin, username=None, full=True): post_token = None dbobj = plugin.db_object() - post_token_obj = BASE.metadata.bind.query(PagureCI).filter( - PagureCI.pagure_name == repo.name).first() - - if hasattr(post_token_obj, 'hook_token'): - post_token = getattr(post_token_obj, 'hook_token') - if hasattr(repo, plugin.backref): dbobj = getattr(repo, plugin.backref) + # There should always be only one, but let's double check if dbobj and len(dbobj) > 0: dbobj = dbobj[0] new = False + # hook_token of pagure shouldn't leak so to put a check on it + if hasattr(dbobj, "hook_token") and plugin.backref == "hook_pagure_ci": + post_token = dbobj.hook_token else: dbobj = plugin.db_object() @@ -169,3 +170,20 @@ def view_plugin(repo, plugin, username=None, full=True): form=form, post_token=post_token, fields=fields) + + +@APP.route('/hooks//build-finished', methods=['POST']) +def hook_finished(token): + try: + data = json.loads(flask.request.get_data()) + cfg = jenkins_hook.get_configs( + data['name'], jenkins_hook.Service.JENKINS)[0] + build_id = data['build']['number'] + if token != cfg.hook_token: + raise ValueError('Token mismatch') + except (TypeError, ValueError, KeyError, jenkins_hook.ConfigNotFound) as exc: + APP.logger.error('Error processing jenkins notification', exc_info=exc) + return ('Bad request...\n', 400, {'Content-Type': 'text/plain'}) + APP.logger.info('Received jenkins notification') + pagure_ci.process_build(APP.logger, cfg, build_id) + return ('', 204) diff --git a/setup.py b/setup.py index 02d354b..544b80c 100644 --- a/setup.py +++ b/setup.py @@ -57,6 +57,6 @@ setup( install_requires=get_requirements(), entry_points=""" [moksha.consumer] - integrator = pagure.consumer:Integrator + integrator = pagure.pagureCI.consumer:Integrator """ ) From 19c574788bdf6d7b4e7c10f7f2e071f141abface Mon Sep 17 00:00:00 2001 From: farhaanbukhsh Date: Jul 26 2016 07:41:45 +0000 Subject: [PATCH 10/25] Fix extra POST request and code cleaning --- diff --git a/pagure/hooks/jenkins_hook.py b/pagure/hooks/jenkins_hook.py index 1d162ba..755f121 100644 --- a/pagure/hooks/jenkins_hook.py +++ b/pagure/hooks/jenkins_hook.py @@ -78,7 +78,7 @@ def get_configs(project_name, service): """ cfg = BASE.metadata.bind.query(PagureCI).filter( service == project_name).all() - if len(cfg) == 0: + if not cfg: raise ConfigNotFound(project_name) return cfg diff --git a/pagure/lib/pagure_ci.py b/pagure/lib/pagure_ci.py index ec5bad3..43d1992 100644 --- a/pagure/lib/pagure_ci.py +++ b/pagure/lib/pagure_ci.py @@ -2,10 +2,14 @@ import os import flask from sqlalchemy.orm import scoped_session, sessionmaker +from sqlalchemy.exc import SQLAlchemyError from sqlalchemy import create_engine + from pagure.hooks import jenkins_hook +import pagure.lib from pagure.lib import model from pagure import APP, SESSION +import pagure.exceptions import json import logging @@ -13,7 +17,6 @@ import logging import requests import jenkins -APP.config.from_envvar('INTEGRATOR_SETTINGS', silent=True) APP.logger.setLevel(logging.INFO) PAGURE_URL = '{base}api/0/{repo}/pull-request/{pr}/flag' @@ -64,14 +67,9 @@ def post_flag(logger, name, base, token, repo, pr, result, url): 'SUCCESS': ('Build successful', 100), 'FAILURE': ('Build failed', 0), }[result] - payload = { - 'username': name, - 'percent': percent, - 'comment': comment, - 'url': url, - } - post_data(logger, PAGURE_URL.format(base=base, repo=repo, pr=pr), payload, - headers={'Authorization': 'token ' + token}) + + pagure_ci_flag(logger, repo=repo, username=name, percent=percent, comment=comment, + url=url, requestid=pr) def post_data(logger, *args, **kwargs): @@ -80,3 +78,38 @@ def post_data(logger, *args, **kwargs): if resp.status_code < 200 or resp.status_code >= 300: logger.error('Network request failed: %d: %s', resp.status_code, resp.text) + + +def pagure_ci_flag(logger, repo, username, percent, comment, url, requestid): + + repo = pagure.lib.get_project(SESSION, repo, user=None) + output = {} + + if repo is None: + raise pagure.exceptions.FileNotFoundException('Repo not found') + + request = pagure.lib.search_pull_requests( + SESSION, project_id=repo.id, requestid=requestid) + + if not request: + raise pagure.exceptions.FileNotFoundException('Request not found') + + try: + message = pagure.lib.add_pull_request_flag( + SESSION, + request=request, + username=username, + percent=percent, + comment=comment, + url=url, + uid=None, + user=repo.user.username, + requestfolder=APP.config['REQUESTS_FOLDER'], + ) + SESSION.commit() + logger.debug('Received response status: %s', message) + output['message'] = message + + except SQLAlchemyError as err: # pragma: no cover + logger.exception(err) + SESSION.rollback() diff --git a/pagure/pagureCI/consumer.py b/pagure/pagureCI/consumer.py index 4ef85de..f45ad93 100644 --- a/pagure/pagureCI/consumer.py +++ b/pagure/pagureCI/consumer.py @@ -4,7 +4,7 @@ from pagure.hooks import jenkins_hook import pagure.lib from pagure.lib import pagure_ci from pagure.lib.model import BASE, Project, User -from pagure import APP +from pagure import APP, SESSION PAGURE_MAIN_REPO = '{base}{name}.git' PAGURE_FORK_REPO = '{base}forks/{user}/{name}.git' @@ -18,11 +18,9 @@ class Integrator(fedmsg.consumers.FedmsgConsumer): config_key = 'integrator.enabled' - SESSION = None def __init__(self, hub): super(Integrator, self).__init__(hub) - SESSION = pagure.lib.create_session(APP.config['DB_URL']) def consume(self, msg): topic, msg = msg['topic'], msg['body'] diff --git a/pagure/templates/plugin.html b/pagure/templates/plugin.html index 193fc63..3015f84 100644 --- a/pagure/templates/plugin.html +++ b/pagure/templates/plugin.html @@ -3,16 +3,16 @@ {% if full %} {% extends "repo_master.html" %} -{% block title %}{{ select.capitalize() }} {{ plugin.name }} - {{ repo.name }} -{% endblock %} {% set tag = "home" %} +{% block title %}{{ select.capitalize() }} {{ plugin.name }} - {{ repo.name }}{% endblock %} +{% set tag = "home" %} {% endif %} {% block repo %} {% if full %} -

{{ plugin.name }} settings

+

{{ plugin.name }} settings

{% endif %} - {{ plugin.description | markdown | noJS | safe }} @@ -35,7 +35,6 @@ {{ form.csrf_token }}

-
- + {% endblock %} diff --git a/requirements.txt b/requirements.txt index 22c4034..de1cd79 100644 --- a/requirements.txt +++ b/requirements.txt @@ -37,5 +37,5 @@ cryptography py-bcrypt #Required for Pagure CI -moksha +fedmsg python-jenkins From cd2b3821e8441d8cc04c7303ba623924189a522f Mon Sep 17 00:00:00 2001 From: farhaanbukhsh Date: Jul 26 2016 07:41:45 +0000 Subject: [PATCH 11/25] Fix plugin template --- diff --git a/pagure/templates/plugin.html b/pagure/templates/plugin.html index 3015f84..9cfc341 100644 --- a/pagure/templates/plugin.html +++ b/pagure/templates/plugin.html @@ -9,7 +9,7 @@ {% block repo %} {% if full %} -

{{ plugin.name }} settings

+

{{ plugin.name }} settings

{% endif %}
- + {% endif %} diff --git a/pagure/ui/plugins.py b/pagure/ui/plugins.py index 2bae4ed..4bbf39d 100644 --- a/pagure/ui/plugins.py +++ b/pagure/ui/plugins.py @@ -91,6 +91,8 @@ def view_plugin(repo, plugin, username=None, full=True): new = True post_token = None dbobj = plugin.db_object() + # Omit trailing '/' + app_url = APP.config['APP_URL'][:-1] if hasattr(repo, plugin.backref): dbobj = getattr(repo, plugin.backref) @@ -133,6 +135,7 @@ def view_plugin(repo, plugin, username=None, full=True): username=username, plugin=plugin, form=form, + app_url=app_url, post_token=post_token, fields=fields) @@ -167,6 +170,7 @@ def view_plugin(repo, plugin, username=None, full=True): username=username, plugin=plugin, form=form, + app_url=app_url, post_token=post_token, fields=fields) From b054813fbcaeebd4201fdc643a633d1362bf969c Mon Sep 17 00:00:00 2001 From: farhaanbukhsh Date: Jul 26 2016 07:41:45 +0000 Subject: [PATCH 14/25] Fix dynamic URL in the template --- diff --git a/pagure/templates/plugin.html b/pagure/templates/plugin.html index 75311b4..7803aa6 100644 --- a/pagure/templates/plugin.html +++ b/pagure/templates/plugin.html @@ -21,7 +21,7 @@
- +
{% endif %} diff --git a/pagure/ui/plugins.py b/pagure/ui/plugins.py index 4bbf39d..dc5ab38 100644 --- a/pagure/ui/plugins.py +++ b/pagure/ui/plugins.py @@ -91,9 +91,7 @@ def view_plugin(repo, plugin, username=None, full=True): new = True post_token = None dbobj = plugin.db_object() - # Omit trailing '/' - app_url = APP.config['APP_URL'][:-1] - + if hasattr(repo, plugin.backref): dbobj = getattr(repo, plugin.backref) @@ -135,7 +133,6 @@ def view_plugin(repo, plugin, username=None, full=True): username=username, plugin=plugin, form=form, - app_url=app_url, post_token=post_token, fields=fields) @@ -170,7 +167,6 @@ def view_plugin(repo, plugin, username=None, full=True): username=username, plugin=plugin, form=form, - app_url=app_url, post_token=post_token, fields=fields) From ae7a980ce1641be15aeace573ff9d36668030913 Mon Sep 17 00:00:00 2001 From: farhaanbukhsh Date: Jul 26 2016 07:41:45 +0000 Subject: [PATCH 15/25] Fix token and some code shedding --- diff --git a/pagure/hooks/jenkins_hook.py b/pagure/hooks/jenkins_hook.py index d4055b8..0309caf 100644 --- a/pagure/hooks/jenkins_hook.py +++ b/pagure/hooks/jenkins_hook.py @@ -2,7 +2,6 @@ import os -import uuid import sqlalchemy as sa import pygit2 @@ -38,10 +37,6 @@ class PagureCI(BASE): jenkins_url = sa.Column(sa.String(255), nullable=False, default='http://jenkins.fedorainfracloud.org/') jenkins_token = sa.Column(sa.String(64)) - hook_token = sa.Column(sa.String(64), - nullable=True, - unique=True, - index=True) project = relation( 'Project', @@ -52,9 +47,6 @@ class PagureCI(BASE): single_parent=True) ) - def __init__(self): - self.hook_token = uuid.uuid4().hex - def __repr__(self): return ''.format(self) diff --git a/pagure/lib/pagure_ci.py b/pagure/lib/pagure_ci.py index 43d1992..a4aa7c2 100644 --- a/pagure/lib/pagure_ci.py +++ b/pagure/lib/pagure_ci.py @@ -56,22 +56,17 @@ def process_build(logger, cfg, build_id): # Comment in Pagure logger.info('Updating %s PR %d: %s', cfg.pagure_name, pr_id, result) try: - post_flag(logger, cfg.display_name, APP.config['APP_URL'], cfg.pagure_token, - cfg.pagure_name, pr_id, result, url) + pagure_ci_flag(logger, + username=cfg.display_name, + repo=cfg.pagure_name, + requestid=pr_id, + result=result, + url=url) + except KeyError as exc: logger.warning('Unknown build status', exc_info=exc) -def post_flag(logger, name, base, token, repo, pr, result, url): - comment, percent = { - 'SUCCESS': ('Build successful', 100), - 'FAILURE': ('Build failed', 0), - }[result] - - pagure_ci_flag(logger, repo=repo, username=name, percent=percent, comment=comment, - url=url, requestid=pr) - - def post_data(logger, *args, **kwargs): resp = requests.post(*args, **kwargs) logger.debug('Received response status %s', resp.status_code) @@ -80,7 +75,12 @@ def post_data(logger, *args, **kwargs): resp.status_code, resp.text) -def pagure_ci_flag(logger, repo, username, percent, comment, url, requestid): +def pagure_ci_flag(logger, repo, username, url, result, requestid): + + comment, percent = { + 'SUCCESS': ('Build successful', 100), + 'FAILURE': ('Build failed', 0), + }[result] repo = pagure.lib.get_project(SESSION, repo, user=None) output = {} diff --git a/pagure/templates/plugin.html b/pagure/templates/plugin.html index 7803aa6..bf10f6a 100644 --- a/pagure/templates/plugin.html +++ b/pagure/templates/plugin.html @@ -17,11 +17,11 @@ ) }}" method="post"> {{ plugin.description | markdown | noJS | safe }} - {% if post_token and (plugin.name == 'Pagure CI') %} + {% if plugin.name == 'Pagure CI' %}
- +
{% endif %} diff --git a/pagure/ui/plugins.py b/pagure/ui/plugins.py index dc5ab38..1274f05 100644 --- a/pagure/ui/plugins.py +++ b/pagure/ui/plugins.py @@ -91,7 +91,7 @@ def view_plugin(repo, plugin, username=None, full=True): new = True post_token = None dbobj = plugin.db_object() - + if hasattr(repo, plugin.backref): dbobj = getattr(repo, plugin.backref) @@ -99,9 +99,6 @@ def view_plugin(repo, plugin, username=None, full=True): if dbobj and len(dbobj) > 0: dbobj = dbobj[0] new = False - # hook_token of pagure shouldn't leak so to put a check on it - if hasattr(dbobj, "hook_token") and plugin.backref == "hook_pagure_ci": - post_token = dbobj.hook_token else: dbobj = plugin.db_object() @@ -133,7 +130,6 @@ def view_plugin(repo, plugin, username=None, full=True): username=username, plugin=plugin, form=form, - post_token=post_token, fields=fields) if form.active.data: @@ -167,19 +163,18 @@ def view_plugin(repo, plugin, username=None, full=True): username=username, plugin=plugin, form=form, - post_token=post_token, fields=fields) -@APP.route('/hooks//build-finished', methods=['POST']) -def hook_finished(token): +@APP.route('/hooks//build-finished', methods=['POST']) +def hook_finished(repo_id): try: data = json.loads(flask.request.get_data()) cfg = jenkins_hook.get_configs( data['name'], jenkins_hook.Service.JENKINS)[0] build_id = data['build']['number'] - if token != cfg.hook_token: - raise ValueError('Token mismatch') + if repo_id != str(cfg.project_id): + raise ValueError('Project ID mismatch') except (TypeError, ValueError, KeyError, jenkins_hook.ConfigNotFound) as exc: APP.logger.error('Error processing jenkins notification', exc_info=exc) return ('Bad request...\n', 400, {'Content-Type': 'text/plain'}) From f0fc746383352a1750e74523defd5e07172b5aa1 Mon Sep 17 00:00:00 2001 From: farhaanbukhsh Date: Jul 26 2016 07:41:45 +0000 Subject: [PATCH 16/25] Fix unused variable --- diff --git a/pagure/ui/plugins.py b/pagure/ui/plugins.py index 1274f05..d1dd5ae 100644 --- a/pagure/ui/plugins.py +++ b/pagure/ui/plugins.py @@ -89,7 +89,6 @@ def view_plugin(repo, plugin, username=None, full=True): plugin = get_plugin(plugin) fields = [] new = True - post_token = None dbobj = plugin.db_object() if hasattr(repo, plugin.backref): From 2d31061b3299bfecafe2a3a0133e86ba585c3876 Mon Sep 17 00:00:00 2001 From: farhaanbukhsh Date: Jul 26 2016 07:41:45 +0000 Subject: [PATCH 17/25] Fix name field not required in form --- diff --git a/pagure/hooks/jenkins_hook.py b/pagure/hooks/jenkins_hook.py index 0309caf..1918c4a 100644 --- a/pagure/hooks/jenkins_hook.py +++ b/pagure/hooks/jenkins_hook.py @@ -30,9 +30,8 @@ class PagureCI(BASE): active = sa.Column(sa.Boolean, nullable=False, default=False) display_name = sa.Column(sa.String(64), nullable=False, default='Jenkins') - name = sa.Column(sa.String(64)) pagure_name = sa.Column(sa.String(255)) - + jenkins_name = sa.Column(sa.String(255)) jenkins_url = sa.Column(sa.String(255), nullable=False, default='http://jenkins.fedorainfracloud.org/') @@ -47,10 +46,6 @@ class PagureCI(BASE): single_parent=True) ) - def __repr__(self): - return ''.format(self) - - class ConfigNotFound(Exception): pass @@ -75,9 +70,6 @@ def get_configs(project_name, service): class JenkinsForm(wtf.Form): '''Form to configure Jenkins hook''' - name = TextField('Name', - [validators.Required(), - validators.Length(max=64)]) pagure_name = TextField('Name of project in Pagure', [validators.Required(), @@ -108,7 +100,7 @@ class PagureCiHook(BaseHook): db_object = PagureCI backref = 'hook_pagure_ci' form_fields = [ - 'name', 'pagure_name', 'jenkins_name', + 'pagure_name', 'jenkins_name', 'jenkins_url', 'jenkins_token', 'active' ] From e0a21ed16c3b7f12a235042631ee30edc8ac51ee Mon Sep 17 00:00:00 2001 From: farhaanbukhsh Date: Jul 26 2016 07:41:45 +0000 Subject: [PATCH 18/25] Add the config file to initialize the consumer When `fedmsg-hubs` is called it looks for an endpoint in `fedmsg.d/` this file tells `fedmsg-hubs` to listen to which consumer and if the consumer is activate. --- diff --git a/fedmsg.d/pagure_ci.py b/fedmsg.d/pagure_ci.py new file mode 100644 index 0000000..9ac9413 --- /dev/null +++ b/fedmsg.d/pagure_ci.py @@ -0,0 +1,3 @@ +config = { + 'integrator.enabled': True, +} From 3f88c94129cd10c864fbf8ae359b58ecf47466cc Mon Sep 17 00:00:00 2001 From: farhaanbukhsh Date: Jul 26 2016 07:41:45 +0000 Subject: [PATCH 19/25] Add doc for development and configuring pagure CI The doc/pagure_ci.rst gives an intro as to how to set it up and pagureCI/readme.rst shows how to set it up for development purpose. The relocation is done because the PagureCI is a service provided by pagure and it should be loosely coupled. --- diff --git a/doc/pagure_ci.rst b/doc/pagure_ci.rst new file mode 100644 index 0000000..3714db0 --- /dev/null +++ b/doc/pagure_ci.rst @@ -0,0 +1,115 @@ +========= +Pagure CI +========= + +Pagure CI is a continuous integration tool using which the PR on the projects can be tested and flaged with the status of the build. + +How to enable Pagure CI +======================= + +* Enable the Fedmsg plugin in pagure project setting . This will emit the message to for consumer to consume it. +* Fill in the Pagure CI form with the required details. + +:: + + Pagure Project Name + Jenkins Project Name + Jenkins Token + Jenkins Url + + All of which are required field. + +* The jenkins token is any string that you give here. The only thing that should be kept in mind that this token should be same through out. + +* This will give a POST URL which will be used for Job Notification in Jenkins + + +Configuring Jenkins +=================== + +Jenkins configuration is the most important part of how the Pagure CI works, after you login to your Jenkins Instance. + +* Go to Manage Jenkins -> Configuire Global Security and under that select 'Project-based Matrix Authorization Strategy' + +* Download the following plugins: + +:: + + Build Authorization Root Plugin + Git Plugins + Notification Plugin + + +* Click on the New Item + +* Select Freestyle Project + +* Click OK and enter the name of the project, make sure the project name you filled in the Pagure CI form should match the name you entered here. + +* Under 'Job Notification' click 'Add Endpoint' + +* Fields in Endpoint will be : + +:: + + FORMAT: JSON + PROTOCOL: HTTP + EVENT: All Event + URL: + TIMEOUT: 3000 + LOG: 1 + +* Tick the build is parameterized + +* From the Add Parameter drop down select String Parameter + +* Two string parameters need to be created REPO and BRANCH + +* Source Code Management select Git and give the URL of the pagure project + +* Under Build Trigger click on Trigger build remotely and give the same token that you gave in the Pagure CI form. + +* Under Build -> Add build step -> Execute Shell + +* In the box given enter the shell steps you want for testing your project. + + +Example Script + +:: + + if [ -n "$REPO" -a -n "$BRANCH" ]; then + git remote rm proposed || true + git remote add proposed "$REPO" + git fetch proposed + git checkout origin/master + git config --global user.email "you@example.com" + git config --global user.name "Your Name" + git merge --no-ff "proposed/$BRANCH" -m "Merge PR" + fi + +How to install Pagure CI +======================== + +Pagure CI requires `fedmsg` to run since it uses a consumer to get messages and take appropriate actions. +The dependency that is required is `fedmdg-hubs`. For that the steps are given. +To install the dependencies required: + + `dnf install fedmsg-hub` + +`fedmsg` apart from the consumer require a file that tells to which cosumer it should listen to. This file basically enable the consumer in PagureCI/. For doing that, we need to place this file in appropriate directory. + + `sudo cp pagure/fedmsg.d/pagure_ci.py /etc/fedmsg.d/` + +Since the deployment is done using rpm, the next step is covered using `setup.py` which binds the consumer with the environment, this is done while building the rpm so if rpm is already built this is not explicitly required. + + `python setup.py install` + +Run the service: + + `sudo systemctl enable fedmsg-hub.service` + + `sudo systemctl start fedmsg-hub.service` + + + diff --git a/pagure/pagureCI/__init__.py b/pagure/pagureCI/__init__.py deleted file mode 100644 index e69de29..0000000 --- a/pagure/pagureCI/__init__.py +++ /dev/null diff --git a/pagure/pagureCI/consumer.py b/pagure/pagureCI/consumer.py deleted file mode 100644 index f45ad93..0000000 --- a/pagure/pagureCI/consumer.py +++ /dev/null @@ -1,74 +0,0 @@ -# -*- coding: utf-8 -*- -import fedmsg.consumers -from pagure.hooks import jenkins_hook -import pagure.lib -from pagure.lib import pagure_ci -from pagure.lib.model import BASE, Project, User -from pagure import APP, SESSION -PAGURE_MAIN_REPO = '{base}{name}.git' -PAGURE_FORK_REPO = '{base}forks/{user}/{name}.git' - - -class Integrator(fedmsg.consumers.FedmsgConsumer): - topic = [ - 'io.pagure.prod.pagure.pull-request.comment.added', - 'io.pagure.prod.pagure.pull-request.new', - 'org.fedoraproject.prod.jenkins.build', - ] - - config_key = 'integrator.enabled' - - - def __init__(self, hub): - super(Integrator, self).__init__(hub) - - def consume(self, msg): - topic, msg = msg['topic'], msg['body'] - self.log.info("Received %r, %r", topic, msg.get('msg_id', None)) - msg = msg['msg'] - try: - if topic.endswith('.pull-request.comment.added'): - if is_rebase(msg): - self.trigger_build(msg) - elif topic.endswith('.pull-request.new'): - self.trigger_build(msg) - else: - self.process_build(msg) - except jenkins_hook.ConfigNotFound as exc: - self.log.info('Unconfigured project %r', str(exc)) - - def trigger_build(self, msg): - pr_id = msg['pullrequest']['id'] - project = msg['pullrequest']['project']['name'] - branch = msg['pullrequest']['branch_from'] - - for cfg in jenkins_hook.get_configs(project, jenkins_hook.Service.PAGURE): - repo = msg['pullrequest'].get('remote_git') or get_repo(cfg, msg) - self.log.info("Trigger on %s PR #%s from %s: %s", - project, pr_id, repo, branch) - - pagure_ci.process_pr(self.log, cfg, pr_id, repo, branch) - - def process_build(self, msg): - for cfg in jenkins_hook.get_configs(msg['project'], jenkins_hook.Service.JENKINS): - pagure_ci.process_build(self.log, cfg, msg['build']) - - -def get_repo(cfg, msg): - url = PAGURE_MAIN_REPO - if msg['pullrequest']['repo_from']['parent']: - url = PAGURE_FORK_REPO - return url.format( - base=APP.config['APP_URL'], - user=msg['pullrequest']['repo_from']['user']['name'], - name=msg['pullrequest']['repo_from']['name']) - - -def is_rebase(msg): - if msg['pullrequest']['status'] != 'Open': - return False - try: - print msg - return msg['pullrequest']['comments'][-1]['notification'] - except (IndexError, KeyError): - return False diff --git a/pagureCI/README.rst b/pagureCI/README.rst new file mode 100644 index 0000000..d79ab58 --- /dev/null +++ b/pagureCI/README.rst @@ -0,0 +1,107 @@ +Pagure CI +========= + +This is to setup Pagure CI for development. It is assumed that all the dependencies +are resolved. It is advised to use a virtual envivironment for development. + + * Run:: + + python setup.py develop + + +Now in pagureCI/consumer.py add the following elements in `topic` list + +:: + + 'org.fedoraproject.dev.pagure.pull-request.new', + 'org.fedoraproject.dev.pagure.pull-request.comment.added', + + +Configuring Jenkins +=================== + +Jenkins configuration is the most important part of how the Pagure CI works, after you login to your Jenkins Instance. + + +* Go to Manage Jenkins -> Configuire Global Security and under that select 'Project-based Matrix Authorization Strategy' + +* Add a user and give all the permission to that user. + +* Download the following plugins: + +:: + + Build Authorization Root Plugin + Git Plugins + Notification Plugin + + +* Click on the New Item + +* Select Freestyle Project + +* Click OK and enter the name of the project, make sure the project name you filled in the Pagure CI form should match the name you entered here. + +* Under 'Job Notification' click 'Add Endpoint' + +* Fields in Endpoint will be : + +:: + + FORMAT: JSON + PROTOCOL: HTTP + EVENT: All Event + URL: + TIMEOUT: 3000 + LOG: 1 + +* Tick the build is parameterized + +* From the Add Parameter drop down select String Parameter + +* Two string parameters need to be created REPO and BRANCH + +* Source Code Management select Git and give the URL of the pagure project + +* Under Build Trigger click on Trigger build remotely and give the same token that you gave in the Pagure CI form. + +* Under Build -> Add build step -> Execute Shell + +* In the box given enter the shell steps you want for testing your project. + + +Example Script + +:: + + if [ -n "$REPO" -a -n "$BRANCH" ]; then + git remote rm proposed || true + git remote add proposed "$REPO" + git fetch proposed + git checkout origin/master + git config --global user.email "you@example.com" + git config --global user.name "Your Name" + git merge --no-ff "proposed/$BRANCH" -m "Merge PR" + fi + +* After all the configuration done, go to the dev instance of pagure running and under project settings in `Plugin` select Pagure CI and fill the appropriate information. Which on submiting should give you a POST url. + +* Copy and paste the URL in the Notification section under the Jenkins project you want the CI to work, + + +Get It Running: +=============== + +In one terminal window run: + +:: + + fedmsg-relay + +Another window: + +:: + + fedmsg-hub + +* Now clone the project locally and make a branch. make some changes and push it to the repo and try to make a PR. You will notice if everything works fine a lot of logs in the server console and `build fail` flag on the PR. Build fail because there is no git server running. diff --git a/pagureCI/__init__.py b/pagureCI/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/pagureCI/__init__.py diff --git a/pagureCI/consumer.py b/pagureCI/consumer.py new file mode 100644 index 0000000..9960416 --- /dev/null +++ b/pagureCI/consumer.py @@ -0,0 +1,76 @@ +# -*- coding: utf-8 -*- +import fedmsg.consumers +from pagure.hooks import jenkins_hook +import pagure.lib +from pagure.lib import pagure_ci +from pagure.lib.model import BASE, Project, User +from pagure import APP, SESSION +PAGURE_MAIN_REPO = '{base}{name}.git' +PAGURE_FORK_REPO = '{base}forks/{user}/{name}.git' + + +class Integrator(fedmsg.consumers.FedmsgConsumer): + topic = [ + 'io.pagure.prod.pagure.pull-request.comment.added', + 'io.pagure.prod.pagure.pull-request.new', + 'org.fedoraproject.dev.pagure.pull-request.new', + 'org.fedoraproject.dev.pagure.pull-request.comment.added', + 'org.fedoraproject.prod.jenkins.build', + ] + + config_key = 'integrator.enabled' + + + def __init__(self, hub): + super(Integrator, self).__init__(hub) + + def consume(self, msg): + topic, msg = msg['topic'], msg['body'] + self.log.info("Received %r, %r", topic, msg.get('msg_id', None)) + msg = msg['msg'] + try: + if topic.endswith('.pull-request.comment.added'): + if is_rebase(msg): + self.trigger_build(msg) + elif topic.endswith('.pull-request.new'): + self.trigger_build(msg) + else: + self.process_build(msg) + except jenkins_hook.ConfigNotFound as exc: + self.log.info('Unconfigured project %r', str(exc)) + + def trigger_build(self, msg): + pr_id = msg['pullrequest']['id'] + project = msg['pullrequest']['project']['name'] + branch = msg['pullrequest']['branch_from'] + + for cfg in jenkins_hook.get_configs(project, jenkins_hook.Service.PAGURE): + repo = msg['pullrequest'].get('remote_git') or get_repo(cfg, msg) + self.log.info("Trigger on %s PR #%s from %s: %s", + project, pr_id, repo, branch) + + pagure_ci.process_pr(self.log, cfg, pr_id, repo, branch) + + def process_build(self, msg): + for cfg in jenkins_hook.get_configs(msg['project'], jenkins_hook.Service.JENKINS): + pagure_ci.process_build(self.log, cfg, msg['build']) + + +def get_repo(cfg, msg): + url = PAGURE_MAIN_REPO + if msg['pullrequest']['repo_from']['parent']: + url = PAGURE_FORK_REPO + return url.format( + base=APP.config['APP_URL'], + user=msg['pullrequest']['repo_from']['user']['name'], + name=msg['pullrequest']['repo_from']['name']) + + +def is_rebase(msg): + if msg['pullrequest']['status'] != 'Open': + return False + try: + print msg + return msg['pullrequest']['comments'][-1]['notification'] + except (IndexError, KeyError): + return False diff --git a/setup.py b/setup.py index 544b80c..efcf7ab 100644 --- a/setup.py +++ b/setup.py @@ -57,6 +57,6 @@ setup( install_requires=get_requirements(), entry_points=""" [moksha.consumer] - integrator = pagure.pagureCI.consumer:Integrator + integrator = pagureCI.consumer:Integrator """ ) From 52f61fdb0d676a108a77e4e751a03da15ad02f08 Mon Sep 17 00:00:00 2001 From: farhaanbukhsh Date: Jul 26 2016 07:41:45 +0000 Subject: [PATCH 20/25] Fix 80 chars limit in thedoc --- diff --git a/doc/pagure_ci.rst b/doc/pagure_ci.rst index 3714db0..ec973cc 100644 --- a/doc/pagure_ci.rst +++ b/doc/pagure_ci.rst @@ -2,12 +2,15 @@ Pagure CI ========= -Pagure CI is a continuous integration tool using which the PR on the projects can be tested and flaged with the status of the build. +Pagure CI is a continuous integration tool using which the PR on the projects +can be tested and flaged with the status of the build. How to enable Pagure CI ======================= -* Enable the Fedmsg plugin in pagure project setting . This will emit the message to for consumer to consume it. +* Enable the Fedmsg plugin in pagure project setting . This will emit the message + to for consumer to consume it. + * Fill in the Pagure CI form with the required details. :: @@ -19,7 +22,8 @@ How to enable Pagure CI All of which are required field. -* The jenkins token is any string that you give here. The only thing that should be kept in mind that this token should be same through out. +* The jenkins token is any string that you give here. The only thing that should be + kept in mind that this token should be same through out. * This will give a POST URL which will be used for Job Notification in Jenkins @@ -27,9 +31,11 @@ How to enable Pagure CI Configuring Jenkins =================== -Jenkins configuration is the most important part of how the Pagure CI works, after you login to your Jenkins Instance. +Jenkins configuration is the most important part of how the Pagure CI works, +after you login to your Jenkins Instance. -* Go to Manage Jenkins -> Configuire Global Security and under that select 'Project-based Matrix Authorization Strategy' +* Go to Manage Jenkins -> Configuire Global Security and under that select + `Project-based Matrix Authorization Strategy` * Download the following plugins: @@ -44,7 +50,8 @@ Jenkins configuration is the most important part of how the Pagure CI works, aft * Select Freestyle Project -* Click OK and enter the name of the project, make sure the project name you filled in the Pagure CI form should match the name you entered here. +* Click OK and enter the name of the project, make sure the project name + you filled in the Pagure CI form should match the name you entered here. * Under 'Job Notification' click 'Add Endpoint' @@ -67,7 +74,8 @@ Jenkins configuration is the most important part of how the Pagure CI works, aft * Source Code Management select Git and give the URL of the pagure project -* Under Build Trigger click on Trigger build remotely and give the same token that you gave in the Pagure CI form. +* Under Build Trigger click on Trigger build remotely and give the same token + that you gave in the Pagure CI form. * Under Build -> Add build step -> Execute Shell @@ -91,17 +99,23 @@ Example Script How to install Pagure CI ======================== -Pagure CI requires `fedmsg` to run since it uses a consumer to get messages and take appropriate actions. -The dependency that is required is `fedmdg-hubs`. For that the steps are given. +Pagure CI requires `fedmsg` to run since it uses a consumer to get messages +and take appropriate actions. The dependency that is required is `fedmdg-hubs`. +For that the steps are given. + To install the dependencies required: `dnf install fedmsg-hub` -`fedmsg` apart from the consumer require a file that tells to which cosumer it should listen to. This file basically enable the consumer in PagureCI/. For doing that, we need to place this file in appropriate directory. +`fedmsg` apart from the consumer require a file that tells to which cosumer +it should listen to. This file basically enable the consumer in PagureCI/. +For doing that, we need to place this file in appropriate directory. `sudo cp pagure/fedmsg.d/pagure_ci.py /etc/fedmsg.d/` -Since the deployment is done using rpm, the next step is covered using `setup.py` which binds the consumer with the environment, this is done while building the rpm so if rpm is already built this is not explicitly required. +Since the deployment is done using rpm, the next step is covered using `setup.py` +which binds the consumer with the environment, this is done while building the rpm +so if rpm is already built this is not explicitly required. `python setup.py install` diff --git a/pagureCI/README.rst b/pagureCI/README.rst index d79ab58..ba59d88 100644 --- a/pagureCI/README.rst +++ b/pagureCI/README.rst @@ -1,8 +1,9 @@ Pagure CI ========= -This is to setup Pagure CI for development. It is assumed that all the dependencies -are resolved. It is advised to use a virtual envivironment for development. +This is to setup Pagure CI for development. It is assumed that all the +dependencies are resolved. It is advised to use a virtual envivironment +for development. * Run:: @@ -20,10 +21,12 @@ Now in pagureCI/consumer.py add the following elements in `topic` list Configuring Jenkins =================== -Jenkins configuration is the most important part of how the Pagure CI works, after you login to your Jenkins Instance. +Jenkins configuration is the most important part of how the Pagure CI works, +after you login to your Jenkins Instance. -* Go to Manage Jenkins -> Configuire Global Security and under that select 'Project-based Matrix Authorization Strategy' +* Go to Manage Jenkins -> Configuire Global Security and under that select + 'Project-based Matrix Authorization Strategy' * Add a user and give all the permission to that user. @@ -40,7 +43,8 @@ Jenkins configuration is the most important part of how the Pagure CI works, aft * Select Freestyle Project -* Click OK and enter the name of the project, make sure the project name you filled in the Pagure CI form should match the name you entered here. +* Click OK and enter the name of the project, make sure the project name you + filled in the Pagure CI form should match the name you entered here. * Under 'Job Notification' click 'Add Endpoint' @@ -63,7 +67,8 @@ Jenkins configuration is the most important part of how the Pagure CI works, aft * Source Code Management select Git and give the URL of the pagure project -* Under Build Trigger click on Trigger build remotely and give the same token that you gave in the Pagure CI form. +* Under Build Trigger click on Trigger build remotely and give the same token + that you gave in the Pagure CI form. * Under Build -> Add build step -> Execute Shell @@ -84,9 +89,12 @@ Example Script git merge --no-ff "proposed/$BRANCH" -m "Merge PR" fi -* After all the configuration done, go to the dev instance of pagure running and under project settings in `Plugin` select Pagure CI and fill the appropriate information. Which on submiting should give you a POST url. +* After all the configuration done, go to the dev instance of pagure running + and under project settings in `Plugin` select Pagure CI and fill the appropriate + information. Which on submiting should give you a POST url. -* Copy and paste the URL in the Notification section under the Jenkins project you want the CI to work, +* Copy and paste the URL in the Notification section under the Jenkins project + you want the CI to work, Get It Running: @@ -104,4 +112,7 @@ Another window: fedmsg-hub -* Now clone the project locally and make a branch. make some changes and push it to the repo and try to make a PR. You will notice if everything works fine a lot of logs in the server console and `build fail` flag on the PR. Build fail because there is no git server running. +* Now clone the project locally and make a branch. make some changes and push it + to the repo and try to make a PR. You will notice if everything works fine a lot + of logs in the server console and `build fail` flag on the PR. + Build fail because there is no git server running. From fbda942bac31fd568fecee77366f7540bcfbee50 Mon Sep 17 00:00:00 2001 From: farhaanbukhsh Date: Jul 26 2016 07:41:45 +0000 Subject: [PATCH 21/25] Add test to test the UI for pagure CI --- diff --git a/tests/test_pagure_flask_ui_plugins_pagure_ci.py b/tests/test_pagure_flask_ui_plugins_pagure_ci.py new file mode 100644 index 0000000..5db71c1 --- /dev/null +++ b/tests/test_pagure_flask_ui_plugins_pagure_ci.py @@ -0,0 +1,241 @@ +# -*- coding: utf-8 -*- + +__requires__ = ['SQLAlchemy >= 0.8'] +import pkg_resources + +import json +import unittest +import shutil +import sys +import os + +import pygit2 +from mock import patch + +sys.path.insert(0, os.path.join(os.path.dirname( + os.path.abspath(__file__)), '..')) + +import pagure.lib +import tests + + +class PagureFlaskPluginPagureCItests(tests.Modeltests): + """ Tests for flask plugins controller of pagure """ + + def setUp(self): + """ Set up the environnment, ran before every tests. """ + super(PagureFlaskPluginPagureCItests, self).setUp() + + pagure.APP.config['TESTING'] = True + pagure.SESSION = self.session + pagure.ui.SESSION = self.session + pagure.ui.app.SESSION = self.session + pagure.ui.plugins.SESSION = self.session + pagure.ui.repo.SESSION = self.session + pagure.ui.filters.SESSION = self.session + + pagure.APP.config['GIT_FOLDER'] = tests.HERE + pagure.APP.config['FORK_FOLDER'] = os.path.join( + tests.HERE, 'forks') + pagure.APP.config['TICKETS_FOLDER'] = os.path.join( + tests.HERE, 'tickets') + pagure.APP.config['DOCS_FOLDER'] = os.path.join( + tests.HERE, 'docs') + self.app = pagure.APP.test_client() + + def test_plugin_pagure_ci(self): + """ Test the pagure ci plugin on/off endpoint. """ + + tests.create_projects(self.session) + + user = tests.FakeUser(username='pingou') + with tests.user_set(pagure.APP, user): + output = self.app.get('/test/settings/Pagure CI') + self.assertEqual(output.status_code, 200) + self.assertIn( + '
\n' + 'test project #1
', output.data) + self.assertTrue('

Pagure CI settings

' in output.data) + self.assertTrue( + '' + in output.data) + self.assertTrue( + '' + in output.data) + self.assertTrue( + '' + in output.data) + self.assertTrue( + '' + in output.data) + self.assertTrue( + '' + in output.data) + + csrf_token = output.data.split( + 'name="csrf_token" type="hidden" value="')[1].split('">')[0] + + data = {} + + output = self.app.post('/test/settings/Pagure CI', data=data) + self.assertEqual(output.status_code, 200) + self.assertIn( + '
\n' + 'test project #1
', output.data) + self.assertTrue('

Pagure CI settings

' in output.data) + self.assertTrue( + '' + in output.data) + self.assertTrue( + '' + in output.data) + self.assertTrue( + '' + in output.data) + self.assertTrue( + '' + in output.data) + self.assertTrue( + '' + in output.data) + + # Activate hook + data = { + 'csrf_token': csrf_token, + 'active': 'y', + 'pagure_name': 'test', + 'jenkins_name': 'jenkins_test', + 'jenkins_url': 'https://jenkins.fedoraproject.org', + 'jenkins_token': 'BEEFCAFE' + } + # No git found + output = self.app.post( + '/test/settings/Pagure CI', data=data, follow_redirects=True) + self.assertEqual(output.status_code, 404) + + tests.create_projects_git(tests.HERE) + + data = {'csrf_token': csrf_token} + # With the git repo + output = self.app.post( + '/test/settings/Pagure CI', data=data, follow_redirects=True) + self.assertEqual(output.status_code, 200) + + self.assertIn( + '
\n' + 'test project #1
', output.data) + self.assertTrue('

Pagure CI settings

' in output.data) + self.assertFalse( + '\n Hook activated' in output.data) + self.assertTrue( + '' + '\nThis field is required.' + in output.data) + self.assertTrue( + '' in output.data) + + output = self.app.get('/test/settings/Pagure CI') + self.assertEqual(output.status_code, 200) + self.assertIn( + '
\n' + 'test project #1
', output.data) + self.assertTrue('

Pagure CI settings

' in output.data) + self.assertTrue( + '' + in output.data) + self.assertTrue( + '' + in output.data) + self.assertTrue( + '' + in output.data) + self.assertTrue( + '' + in output.data) + self.assertTrue( + '' + in output.data) + + # Missing the required + data = {'csrf_token': csrf_token, 'active': 'y'} + + output = self.app.post( + '/test/settings/Pagure CI', data=data, follow_redirects=True) + self.assertEqual(output.status_code, 200) + self.assertIn( + '
\n' + 'test project #1
', output.data) + self.assertTrue('

Pagure CI settings

' in output.data) + self.assertFalse( + '\n Hook activated' in output.data) + self.assertTrue( + '' + '\nThis field is required.' + in output.data) + self.assertTrue( + '' in output.data) + + # Activate hook + data = { + 'csrf_token': csrf_token, + 'active': 'y', + 'pagure_name': 'test', + 'jenkins_name': 'jenkins_test', + 'jenkins_url': 'https://jenkins.fedoraproject.org', + 'jenkins_token': 'BEEFCAFE' + } + + output = self.app.post( + '/test/settings/Pagure CI', data=data, follow_redirects=True) + self.assertEqual(output.status_code, 200) + self.assertIn( + '
\n

Settings for test

', + output.data) + self.assertTrue( + '\n Hook Pagure CI activated' in output.data) + + output = self.app.get('/test/settings/Pagure CI') + self.assertIn( + '
\n' + 'test project #1
', output.data) + self.assertTrue('

Pagure CI settings

' in output.data) + self.assertTrue( + '' + in output.data) + self.assertTrue( + '' + in output.data) + self.assertTrue( + '' + in output.data) + self.assertTrue( + '' + in output.data) + self.assertTrue( + '' + in output.data) + + # De-Activate hook + data = { + 'csrf_token': csrf_token, + 'pagure_name': 'test', + 'jenkins_name': 'jenkins_test', + 'jenkins_url': 'https://jenkins.fedoraproject.org', + 'jenkins_token': 'BEEFCAFE' + } + output = self.app.post( + '/test/settings/Pagure CI', data=data, follow_redirects=True) + self.assertEqual(output.status_code, 200) + self.assertTrue( + '\n Hook Pagure CI inactived' in output.data) + + self.assertIn( + '
\n

Settings for test

', + output.data) + + +if __name__ == '__main__': + SUITE = unittest.TestLoader().loadTestsFromTestCase( + PagureFlaskPluginPagureCItests) + unittest.TextTestRunner(verbosity=2).run(SUITE) From c5c8d46267e92cb73acab37f5eaa835e27fabace Mon Sep 17 00:00:00 2001 From: Farhaan Bukhsh Date: Jul 26 2016 07:41:45 +0000 Subject: [PATCH 22/25] Fix docs and toggle of Pagure CI plugin, add pagure_ci_token Add proper ussage of Pagure CI in docs. Previously there was no way of deactivating the plugin, with the addition of condition and logging the error. pagure_ci_token is a way of authentication added to make sure that the POST request is made from Jenkins only and not from any external source. --- diff --git a/doc/pagure_ci.rst b/doc/pagure_ci.rst index ec973cc..6f76c31 100644 --- a/doc/pagure_ci.rst +++ b/doc/pagure_ci.rst @@ -22,11 +22,13 @@ How to enable Pagure CI All of which are required field. -* The jenkins token is any string that you give here. The only thing that should be - kept in mind that this token should be same through out. +* The jenkins token is any string that you give here. The only thing that should + be kept in mind that this token should be same through out. * This will give a POST URL which will be used for Job Notification in Jenkins +* The POST url will only appear only after you successfully submitted the form. + Configuring Jenkins =================== @@ -37,6 +39,10 @@ after you login to your Jenkins Instance. * Go to Manage Jenkins -> Configuire Global Security and under that select `Project-based Matrix Authorization Strategy` +* Add your username here and make sure to give that username all the permissions. + You should give all the permissions possible so that you save your self from + getting locked in Jenkins. + * Download the following plugins: :: @@ -61,7 +67,7 @@ after you login to your Jenkins Instance. FORMAT: JSON PROTOCOL: HTTP - EVENT: All Event + EVENT: Job Finalized URL: TIMEOUT: 3000 LOG: 1 @@ -72,7 +78,7 @@ after you login to your Jenkins Instance. * Two string parameters need to be created REPO and BRANCH -* Source Code Management select Git and give the URL of the pagure project +* Source Code Management select Git and give the URL of the pagure project * Under Build Trigger click on Trigger build remotely and give the same token that you gave in the Pagure CI form. diff --git a/pagure/hooks/jenkins_hook.py b/pagure/hooks/jenkins_hook.py index 1918c4a..59088b0 100644 --- a/pagure/hooks/jenkins_hook.py +++ b/pagure/hooks/jenkins_hook.py @@ -2,9 +2,11 @@ import os +import uuid import sqlalchemy as sa import pygit2 + from wtforms import validators, TextField, BooleanField from flask.ext import wtf from sqlalchemy.orm import relation, backref @@ -27,11 +29,13 @@ class PagureCI(BASE): nullable=False, unique=False, index=True) + pagure_ci_token = sa.Column(sa.String(64), nullable=True, unique=True, + index=True) active = sa.Column(sa.Boolean, nullable=False, default=False) display_name = sa.Column(sa.String(64), nullable=False, default='Jenkins') pagure_name = sa.Column(sa.String(255)) - + jenkins_name = sa.Column(sa.String(255)) jenkins_url = sa.Column(sa.String(255), nullable=False, default='http://jenkins.fedorainfracloud.org/') @@ -46,6 +50,10 @@ class PagureCI(BASE): single_parent=True) ) + def __init__(self): + self.pagure_ci_token = uuid.uuid4().hex + + class ConfigNotFound(Exception): pass diff --git a/pagure/lib/pagure_ci.py b/pagure/lib/pagure_ci.py index a4aa7c2..8d29540 100644 --- a/pagure/lib/pagure_ci.py +++ b/pagure/lib/pagure_ci.py @@ -23,48 +23,58 @@ PAGURE_URL = '{base}api/0/{repo}/pull-request/{pr}/flag' JENKINS_TRIGGER_URL = '{base}job/{project}/buildWithParameters' +class HookInactive(Exception): + pass + + def process_pr(logger, cfg, pr_id, repo, branch): - post_data(logger, - JENKINS_TRIGGER_URL.format( - base=cfg.jenkins_url, project=cfg.jenkins_name), - {'token': cfg.jenkins_token, - 'cause': pr_id, - 'REPO': repo, - 'BRANCH': branch}) + if cfg.active: + post_data(logger, + JENKINS_TRIGGER_URL.format( + base=cfg.jenkins_url, project=cfg.jenkins_name), + {'token': cfg.jenkins_token, + 'cause': pr_id, + 'REPO': repo, + 'BRANCH': branch}) + else: + raise HookInactive(cfg.pagure_name) def process_build(logger, cfg, build_id): - # Get details from Jenkins - jenk = jenkins.Jenkins(cfg.jenkins_url) - build_info = jenk.get_build_info(cfg.jenkins_name, build_id) - result = build_info['result'] - url = build_info['url'] - - pr_id = None - - for action in build_info['actions']: - for cause in action.get('causes', []): - try: - pr_id = int(cause['note']) - except (KeyError, ValueError): - continue - - if not pr_id: - logger.info('Not a PR check') - return - - # Comment in Pagure - logger.info('Updating %s PR %d: %s', cfg.pagure_name, pr_id, result) - try: - pagure_ci_flag(logger, - username=cfg.display_name, - repo=cfg.pagure_name, - requestid=pr_id, - result=result, - url=url) - - except KeyError as exc: - logger.warning('Unknown build status', exc_info=exc) + if cfg.active: + # Get details from Jenkins + jenk = jenkins.Jenkins(cfg.jenkins_url) + build_info = jenk.get_build_info(cfg.jenkins_name, build_id) + result = build_info['result'] + url = build_info['url'] + + pr_id = None + + for action in build_info['actions']: + for cause in action.get('causes', []): + try: + pr_id = int(cause['note']) + except (KeyError, ValueError): + continue + + if not pr_id: + logger.info('Not a PR check') + return + + # Comment in Pagure + logger.info('Updating %s PR %d: %s', cfg.pagure_name, pr_id, result) + try: + pagure_ci_flag(logger, + username=cfg.display_name, + repo=cfg.pagure_name, + requestid=pr_id, + result=result, + url=url) + + except KeyError as exc: + logger.warning('Unknown build status', exc_info=exc) + else: + raise HookInactive(cfg.pagure_name) def post_data(logger, *args, **kwargs): diff --git a/pagure/templates/plugin.html b/pagure/templates/plugin.html index bf10f6a..d98af1a 100644 --- a/pagure/templates/plugin.html +++ b/pagure/templates/plugin.html @@ -17,11 +17,11 @@ ) }}" method="post"> {{ plugin.description | markdown | noJS | safe }} - {% if plugin.name == 'Pagure CI' %} + {% if pagure_ci_token and (plugin.name == 'Pagure CI') %}
- +
{% endif %} diff --git a/pagure/ui/plugins.py b/pagure/ui/plugins.py index d1dd5ae..06a5d5b 100644 --- a/pagure/ui/plugins.py +++ b/pagure/ui/plugins.py @@ -90,6 +90,7 @@ def view_plugin(repo, plugin, username=None, full=True): fields = [] new = True dbobj = plugin.db_object() + pagure_ci_token = None if hasattr(repo, plugin.backref): dbobj = getattr(repo, plugin.backref) @@ -98,6 +99,9 @@ def view_plugin(repo, plugin, username=None, full=True): if dbobj and len(dbobj) > 0: dbobj = dbobj[0] new = False + # To populate the pagure CI token if generated + if hasattr(dbobj, "pagure_ci_token") and plugin.backref == "hook_pagure_ci": + pagure_ci_token = dbobj.pagure_ci_token else: dbobj = plugin.db_object() @@ -129,6 +133,7 @@ def view_plugin(repo, plugin, username=None, full=True): username=username, plugin=plugin, form=form, + pagure_ci_token=pagure_ci_token, fields=fields) if form.active.data: @@ -162,21 +167,22 @@ def view_plugin(repo, plugin, username=None, full=True): username=username, plugin=plugin, form=form, + pagure_ci_token=pagure_ci_token, fields=fields) -@APP.route('/hooks//build-finished', methods=['POST']) -def hook_finished(repo_id): +@APP.route('/hooks//build-finished', methods=['POST']) +def hook_finished(pagure_ci_token): try: data = json.loads(flask.request.get_data()) cfg = jenkins_hook.get_configs( data['name'], jenkins_hook.Service.JENKINS)[0] build_id = data['build']['number'] - if repo_id != str(cfg.project_id): - raise ValueError('Project ID mismatch') + if pagure_ci_token != cfg.pagure_ci_token: + raise ValueError('Token mismatch') except (TypeError, ValueError, KeyError, jenkins_hook.ConfigNotFound) as exc: APP.logger.error('Error processing jenkins notification', exc_info=exc) - return ('Bad request...\n', 400, {'Content-Type': 'text/plain'}) + flask.abort(400, "Bad Request") APP.logger.info('Received jenkins notification') pagure_ci.process_build(APP.logger, cfg, build_id) return ('', 204) diff --git a/pagureCI/consumer.py b/pagureCI/consumer.py index 9960416..450f385 100644 --- a/pagureCI/consumer.py +++ b/pagureCI/consumer.py @@ -38,6 +38,8 @@ class Integrator(fedmsg.consumers.FedmsgConsumer): self.process_build(msg) except jenkins_hook.ConfigNotFound as exc: self.log.info('Unconfigured project %r', str(exc)) + except pagure_ci.HookInactive as exc: + self.log.info('Hook Inactive for project %r', str(exc)) def trigger_build(self, msg): pr_id = msg['pullrequest']['id'] From 86baf144aa390f61730f5ba868805608835194ce Mon Sep 17 00:00:00 2001 From: Farhaan Bukhsh Date: Jul 26 2016 07:41:45 +0000 Subject: [PATCH 23/25] Fix constant_time comparision --- diff --git a/pagure/ui/plugins.py b/pagure/ui/plugins.py index 06a5d5b..c9d29d9 100644 --- a/pagure/ui/plugins.py +++ b/pagure/ui/plugins.py @@ -24,6 +24,8 @@ from pagure.hooks import jenkins_hook from pagure.lib import model, pagure_ci import json +from kitchen.text.converters import to_bytes +from cryptography.hazmat.primitives import constant_time # pylint: disable=E1101 @@ -173,16 +175,23 @@ def view_plugin(repo, plugin, username=None, full=True): @APP.route('/hooks//build-finished', methods=['POST']) def hook_finished(pagure_ci_token): + """ Flags the Pull-request after getting notification from Jenkins + """ + try: data = json.loads(flask.request.get_data()) cfg = jenkins_hook.get_configs( data['name'], jenkins_hook.Service.JENKINS)[0] build_id = data['build']['number'] - if pagure_ci_token != cfg.pagure_ci_token: - raise ValueError('Token mismatch') + + if not constant_time.bytes_eq( + to_bytes(pagure_ci_token), to_bytes(cfg.pagure_ci_token)): + return ('Token mismatch', 401) + except (TypeError, ValueError, KeyError, jenkins_hook.ConfigNotFound) as exc: APP.logger.error('Error processing jenkins notification', exc_info=exc) flask.abort(400, "Bad Request") + APP.logger.info('Received jenkins notification') pagure_ci.process_build(APP.logger, cfg, build_id) return ('', 204) From 5da92a1622a66b97c847aeb2b47586144020a6bf Mon Sep 17 00:00:00 2001 From: Farhaan Bukhsh Date: Jul 26 2016 07:41:45 +0000 Subject: [PATCH 24/25] Fix Job Notification --- diff --git a/pagureCI/README.rst b/pagureCI/README.rst index ba59d88..9c88d0a 100644 --- a/pagureCI/README.rst +++ b/pagureCI/README.rst @@ -54,7 +54,7 @@ after you login to your Jenkins Instance. FORMAT: JSON PROTOCOL: HTTP - EVENT: All Event + EVENT: Job Finalized URL: TIMEOUT: 3000 LOG: 1 From 89cb8430b387a03cf7bc0da7ed3eef301c73a263 Mon Sep 17 00:00:00 2001 From: Farhaan Bukhsh Date: Jul 26 2016 11:15:55 +0000 Subject: [PATCH 25/25] Fix pagure_ci token to 32 character --- diff --git a/pagure/hooks/jenkins_hook.py b/pagure/hooks/jenkins_hook.py index 59088b0..ad2521c 100644 --- a/pagure/hooks/jenkins_hook.py +++ b/pagure/hooks/jenkins_hook.py @@ -29,7 +29,7 @@ class PagureCI(BASE): nullable=False, unique=False, index=True) - pagure_ci_token = sa.Column(sa.String(64), nullable=True, unique=True, + pagure_ci_token = sa.Column(sa.String(32), nullable=True, unique=True, index=True) active = sa.Column(sa.Boolean, nullable=False, default=False) diff --git a/pagureCI/consumer.py b/pagureCI/consumer.py index 450f385..fcdf347 100644 --- a/pagureCI/consumer.py +++ b/pagureCI/consumer.py @@ -13,8 +13,6 @@ class Integrator(fedmsg.consumers.FedmsgConsumer): topic = [ 'io.pagure.prod.pagure.pull-request.comment.added', 'io.pagure.prod.pagure.pull-request.new', - 'org.fedoraproject.dev.pagure.pull-request.new', - 'org.fedoraproject.dev.pagure.pull-request.comment.added', 'org.fedoraproject.prod.jenkins.build', ]