From 0491b03b6fd74f7016df789aaa25f0db92b414cd Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Mar 13 2018 17:52:30 +0000 Subject: [PATCH 1/30] Move _get_parent_repo_path to pagure.utils as get_parent_repo_path Signed-off-by: Pierre-Yves Chibon --- diff --git a/pagure/ui/fork.py b/pagure/ui/fork.py index 06bdd02..d2188eb 100644 --- a/pagure/ui/fork.py +++ b/pagure/ui/fork.py @@ -33,24 +33,13 @@ import pagure.lib.tasks import pagure.forms from pagure.config import config as pagure_config from pagure.ui import UI_NS -from pagure.utils import login_required, __get_file_in_tree +from pagure.utils import ( + login_required, __get_file_in_tree, get_parent_repo_path) _log = logging.getLogger(__name__) -def _get_parent_repo_path(repo): - """ Return the path of the parent git repository corresponding to the - provided Repository object from the DB. - """ - if repo.parent: - parentpath = os.path.join( - pagure_config['GIT_FOLDER'], repo.parent.path) - else: - parentpath = os.path.join(pagure_config['GIT_FOLDER'], repo.path) - - return parentpath - def _get_parent_request_repo_path(repo): """ Return the path of the parent git repository corresponding to the @@ -326,7 +315,7 @@ def request_pull_to_diff_or_patch( else: repo_from = request.project_from repopath = pagure.utils.get_repo_path(repo_from) - parentpath = _get_parent_repo_path(repo_from) + parentpath = get_parent_repo_path(repo_from) repo_obj = pygit2.Repository(repopath) orig_repo = pygit2.Repository(parentpath) @@ -1105,7 +1094,7 @@ def new_request_pull( repo_obj = flask.g.repo_obj if not project_to: - parentpath = _get_parent_repo_path(repo) + parentpath = get_parent_repo_path(repo) orig_repo = pygit2.Repository(parentpath) else: p_namespace = None diff --git a/pagure/utils.py b/pagure/utils.py index f58057a..c8a10b0 100644 --- a/pagure/utils.py +++ b/pagure/utils.py @@ -338,3 +338,16 @@ def wait_for_task_post(taskid, form, endpoint, initial=False, **kwargs): form_data=form.data, csrf=form.csrf_token, initial=initial) + + +def get_parent_repo_path(repo): + """ Return the path of the parent git repository corresponding to the + provided Repository object from the DB. + """ + if repo.parent: + parentpath = os.path.join( + pagure_config['GIT_FOLDER'], repo.parent.path) + else: + parentpath = os.path.join(pagure_config['GIT_FOLDER'], repo.path) + + return parentpath From 4d06bd725e334af1c28f84e5ec47193be421c450 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Mar 13 2018 17:52:30 +0000 Subject: [PATCH 2/30] Add the possibility to link issues to pull-requests This commit add this linking at the database level as well as in the UI. Fixes https://pagure.io/pagure/issue/2705 Signed-off-by: Pierre-Yves Chibon --- diff --git a/pagure/hooks/files/default_hook.py b/pagure/hooks/files/default_hook.py index 964546f..ceff33f 100644 --- a/pagure/hooks/files/default_hook.py +++ b/pagure/hooks/files/default_hook.py @@ -138,6 +138,10 @@ def inform_pull_request_urls( # Link to existing PRs if there are any seen = len(prs) != 0 for pr in prs: + # Link tickets with pull-requests if the commit mentions it + pagure.lib.tasks.link_pr_to_ticket.delay(pr.uid) + + # Inform the user about the PR print('View pull-request for %s' % refname) print(' %s/%s/pull-request/%s' % ( _config['APP_URL'].rstrip('/'), diff --git a/pagure/lib/__init__.py b/pagure/lib/__init__.py index 9fd79be..ca2c04f 100644 --- a/pagure/lib/__init__.py +++ b/pagure/lib/__init__.py @@ -1739,6 +1739,8 @@ def new_pull_request(session, branch_from, pagure.lib.git.update_git( request, repo=request.project, repofolder=requestfolder) + pagure.lib.tasks.link_pr_to_ticket.delay(request.uid) + log_action(session, 'created', request, user_obj) if notify: @@ -5011,3 +5013,28 @@ def get_project_family(session, project): ) return [parent] + query.all() + + +def link_pr_issue(session, issue, request): + ''' Associate the specified issue with the specified pull-requets. + + :arg session: The SQLAlchemy session to use + :type session: sqlalchemy.orm.session.Session + :arg issue: The issue mentionned in the commits of the pull-requests to + be associated with + :type issue: pagure.lib.model.Issue + :arg request: A pull-request to associate the specified issue with + :type request: pagure.lib.model.PullRequest + + ''' + + + associated_issue = [iss.uid for iss in request.related_issues] + if issue.uid not in associated_issue: + obj = model.PrToIssue( + pull_request_uid=request.uid, + issue_uid=issue.uid + ) + session.add(obj) + session.flush() + session.commit() diff --git a/pagure/lib/model.py b/pagure/lib/model.py index 4725b24..7c0efb4 100644 --- a/pagure/lib/model.py +++ b/pagure/lib/model.py @@ -1309,6 +1309,28 @@ class IssueToIssue(BASE): primary_key=True) +class PrToIssue(BASE): + """ Stores the associations between issues and pull-requests. + + Table -- pr_to_issue + """ + + __tablename__ = 'pr_to_issue' + + pull_request_uid = sa.Column( + sa.String(32), + sa.ForeignKey( + 'pull_requests.uid', ondelete='CASCADE', onupdate='CASCADE', + ), + primary_key=True) + issue_uid = sa.Column( + sa.String(32), + sa.ForeignKey( + 'issues.uid', ondelete='CASCADE', onupdate='CASCADE', + ), + primary_key=True) + + class IssueComment(BASE): """ Stores the comments made on a commit/file. @@ -1762,6 +1784,14 @@ class PullRequest(BASE): viewonly=True ) + related_issues = relation( + "Issue", + secondary="pr_to_issue", + primaryjoin="pull_requests.c.uid==pr_to_issue.c.pull_request_uid", + secondaryjoin="pr_to_issue.c.issue_uid==issues.c.uid", + backref=backref("related_prs", order_by="pull_requests.c.id.desc()") + ) + def __repr__(self): return 'PullRequest(%s, project:%s, user:%s, title:%s)' % ( self.id, self.project.name, self.user.user, self.title diff --git a/pagure/lib/tasks.py b/pagure/lib/tasks.py index a5249b4..a6544d8 100644 --- a/pagure/lib/tasks.py +++ b/pagure/lib/tasks.py @@ -32,9 +32,11 @@ from sqlalchemy.exc import SQLAlchemyError import pagure.lib import pagure.lib.git import pagure.lib.git_auth +import pagure.lib.link import pagure.lib.repo import pagure.utils from pagure.config import config as pagure_config +from pagure.utils import get_parent_repo_path # logging.config.dictConfig(pagure_config.get('LOGGING') or {'version': 1}) _log = logging.getLogger(__name__) @@ -890,3 +892,70 @@ def commits_history_stats(self, repopath): dates[arrow.get(commit.commit_time).date().isoformat()] += 1 return [(key, dates[key]) for key in sorted(dates)] + + +@conn.task(bind=True) +@set_status +def link_pr_to_ticket(self, pr_uid): + """ Link the specified pull-request against the ticket(s) mentioned in + the commits of the pull-request + + """ + _log.info( + 'LINK_PR_TO_TICKET: Linking ticket(s) to PR for: %s' % pr_uid) + + session = pagure.lib.create_session(pagure_config['DB_URL']) + + request = pagure.lib.get_request_by_uid(session, pr_uid) + if not request: + _log.info('LINK_PR_TO_TICKET: Not PR found for: %s' % pr_uid) + session.close() + return + + if request.remote: + repopath = pagure.utils.get_remote_repo_path( + request.remote_git, request.branch_from) + parentpath = pagure.utils.get_repo_path(request.project) + else: + repo_from = request.project_from + repopath = pagure.utils.get_repo_path(repo_from) + parentpath = get_parent_repo_path(repo_from) + + repo_obj = pygit2.Repository(repopath) + orig_repo = pygit2.Repository(parentpath) + + diff_commits = pagure.lib.git.diff_pull_request( + session, request, repo_obj, orig_repo, + requestfolder=pagure_config['REQUESTS_FOLDER'], with_diff=False) + + _log.info( + 'LINK_PR_TO_TICKET: Found %s commits in that PR' % len(diff_commits)) + + name = request.project.name + namespace = request.project.namespace + user = request.project.user.user \ + if request.project.is_fork else None + branch = request.branch_from + + for line in pagure.lib.git.read_git_lines( + ['log', '--no-walk'] + + [c.oid.hex for c in diff_commits] + + ['--'], repopath): + + line = line.strip() + for issue in pagure.lib.link.get_relation( + session, name, user, namespace, line, 'fixes', + include_prs=False): + _log.info( + 'LINK_PR_TO_TICKET: Link ticket %s to PRs %s' % ( + issue, request)) + pagure.lib.link_pr_issue(session, issue, request) + + for issue in pagure.lib.link.get_relation( + session, name, user, namespace, line, 'relates'): + _log.info( + 'LINK_PR_TO_TICKET: Link ticket %s to PRs %s' % ( + issue, request)) + pagure.lib.link_pr_issue(session, issue, request) + + session.close() diff --git a/pagure/templates/issue.html b/pagure/templates/issue.html index 2c0ee45..8099281 100644 --- a/pagure/templates/issue.html +++ b/pagure/templates/issue.html @@ -365,6 +365,32 @@ {% endif %} + {% if issue.related_prs %} +
+
+
+ +
+
    + {% for pr in issue.related_prs %} +
  • + #{{pr.id}} + {{ pr.status if pr.status != 'Open' else 'Last updated' + }} {{ pr.last_updated | humanize }} +
    • + {% endfor %} +
+
+
+
+
+ {% endif %} + {% if repo.issue_keys %}
diff --git a/tests/__init__.py b/tests/__init__.py index e211653..2bfc30f 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -482,31 +482,37 @@ class FakeUser(object): # pylint: disable=too-few-public-methods return self.dic[key] -def create_projects(session): +def create_projects(session, is_fork=False, user_id=1, hook_token_suffix=''): """ Create some projects in the database. """ item = pagure.lib.model.Project( - user_id=1, # pingou + user_id=user_id, # pingou name='test', + is_fork=is_fork, + parent_id=1 if is_fork else None, description='test project #1', - hook_token='aaabbbccc', + hook_token='aaabbbccc' + hook_token_suffix, ) item.close_status = ['Invalid', 'Insufficient data', 'Fixed', 'Duplicate'] session.add(item) item = pagure.lib.model.Project( - user_id=1, # pingou + user_id=user_id, # pingou name='test2', + is_fork=is_fork, + parent_id=2 if is_fork else None, description='test project #2', - hook_token='aaabbbddd', + hook_token='aaabbbddd' + hook_token_suffix, ) item.close_status = ['Invalid', 'Insufficient data', 'Fixed', 'Duplicate'] session.add(item) item = pagure.lib.model.Project( - user_id=1, # pingou + user_id=user_id, # pingou name='test3', + is_fork=is_fork, + parent_id=3 if is_fork else None, description='namespaced test project', - hook_token='aaabbbeee', + hook_token='aaabbbeee' + hook_token_suffix, namespace='somenamespace', ) item.close_status = ['Invalid', 'Insufficient data', 'Fixed', 'Duplicate'] diff --git a/tests/test_pagure_flask_ui_issue_pr_link.py b/tests/test_pagure_flask_ui_issue_pr_link.py new file mode 100644 index 0000000..a82e9a8 --- /dev/null +++ b/tests/test_pagure_flask_ui_issue_pr_link.py @@ -0,0 +1,160 @@ +# -*- coding: utf-8 -*- + +""" + (c) 2018 - Copyright Red Hat Inc + + Authors: + Pierre-Yves Chibon + +""" + +__requires__ = ['SQLAlchemy >= 0.8'] +import pkg_resources + +import datetime +import json +import unittest +import re +import shutil +import sys +import tempfile +import time +import os + +import pygit2 +from mock import ANY, patch, MagicMock + +sys.path.insert(0, os.path.join(os.path.dirname( + os.path.abspath(__file__)), '..')) + +import pagure +import pagure.lib +import tests +from pagure.lib.repo import PagureRepo + + +class PagureFlaskPrIssueLinkTest(tests.Modeltests): + """ Tests pagure when linking PRs to tickets """ + + maxDiff = None + + def setUp(self): + """ Set up the environnment, ran before every tests. """ + super(PagureFlaskPrIssueLinkTest, self).setUp() + + tests.create_projects(self.session) + tests.create_projects( + self.session, is_fork=True, user_id=2, hook_token_suffix='bar') + tests.create_projects_git(os.path.join(self.path, 'repos'), bare=True) + tests.create_projects_git(os.path.join( + self.path, 'repos', 'forks', 'foo'), bare=True) + + repo = pagure.lib.get_authorized_project(self.session, 'test') + + # Create issues to play with + msg = pagure.lib.new_issue( + session=self.session, + repo=repo, + title=u'tést íssüé', + content='We should work on this', + user='pingou', + ticketfolder=None + ) + self.session.commit() + self.assertEqual(msg.title, u'tést íssüé') + + msg = pagure.lib.new_issue( + session=self.session, + repo=repo, + title=u'tést íssüé #2', + content='We should still work on this', + user='foo', + ticketfolder=None + ) + self.session.commit() + self.assertEqual(msg.title, u'tést íssüé #2') + + # Add a commit to the fork + + newpath = tempfile.mkdtemp(prefix='pagure-fork-test') + repopath = os.path.join(newpath, 'test') + clone_repo = pygit2.clone_repository(os.path.join( + self.path, 'repos', 'forks', 'foo', 'test.git'), repopath) + + # Create a file in that git repo + with open(os.path.join(repopath, 'sources'), 'w') as stream: + stream.write('foo\n bar') + clone_repo.index.add('sources') + clone_repo.index.write() + + try: + com = repo.revparse_single('HEAD') + prev_commit = [com.oid.hex] + except: + prev_commit = [] + + # Commits the files added + tree = clone_repo.index.write_tree() + author = pygit2.Signature( + 'Alice Author', 'alice@authors.tld') + committer = pygit2.Signature( + 'Cecil Committer', 'cecil@committers.tld') + clone_repo.create_commit( + 'refs/heads/master', # the name of the reference to update + author, + committer, + 'Add sources file for testing\n\n Relates to #2', + # binary string representing the tree object ID + tree, + # list of binary strings representing parents of the new commit + prev_commit + ) + refname = 'refs/heads/master:refs/heads/master' + ori_remote = clone_repo.remotes[0] + PagureRepo.push(ori_remote, refname) + + # Create the corresponding PR + + repo = pagure.lib.get_authorized_project(self.session, 'test') + fork_repo = pagure.lib.get_authorized_project( + self.session, 'test', user='foo') + + request = pagure.lib.new_pull_request( + self.session, + branch_from='master', + repo_to=repo, + branch_to='master', + title='test PR', + user='foo', + requestfolder=None, + initial_comment=None, + repo_from=fork_repo, + ) + self.session.commit() + + pagure.lib.tasks.link_pr_to_ticket(request.uid) + self.assertEqual(request.id, 3) + + def test_ticket_no_link(self): + """ Test that no Related PR(s) block is showing in the issue page. + """ + output = self.app.get('/test/issue/1') + self.assertEqual(output.status_code, 200) + self.assertNotIn( + u'Related PR(s)', + output.data.decode('utf-8')) + + def test_ticket_link(self): + """ Test that no Related PR(s) block is showing in the issue page. + """ + time.sleep(1) + output = self.app.get('/test/issue/2') + print output.data.decode('utf-8') + self.assertEqual(output.status_code, 200) + self.assertIn( + u'Related PR(s)', + output.data.decode('utf-8')) + + +if __name__ == '__main__': + unittest.main(verbosity=2) From db117c387c4638233298d213eafaa4a6ebb523f9 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Mar 13 2018 17:52:30 +0000 Subject: [PATCH 3/30] Add an alembic migration to create the pr_to_issue table Signed-off-by: Pierre-Yves Chibon --- diff --git a/alembic/versions/369deb8c8b63_add_the_pr_to_issue_table.py b/alembic/versions/369deb8c8b63_add_the_pr_to_issue_table.py new file mode 100644 index 0000000..43c8977 --- /dev/null +++ b/alembic/versions/369deb8c8b63_add_the_pr_to_issue_table.py @@ -0,0 +1,42 @@ +"""Add the pr_to_issue table + +Revision ID: 369deb8c8b63 +Revises: 22fb5256f555 +Create Date: 2018-03-12 11:38:00.955252 + +""" + +# revision identifiers, used by Alembic. +revision = '369deb8c8b63' +down_revision = '22fb5256f555' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + ''' Create the pr_to_issue table. ''' + + op.create_table( + 'pr_to_issue', + sa.Column( + 'pull_request_uid', + sa.String(32), + sa.ForeignKey( + 'pull_requests.uid', ondelete='CASCADE', onupdate='CASCADE', + ), + primary_key=True), + sa.Column( + 'issue_uid', + sa.String(32), + sa.ForeignKey( + 'issues.uid', ondelete='CASCADE', onupdate='CASCADE', + ), + primary_key=True) + ) + + +def downgrade(): + ''' Drop the pr_to_issue table. ''' + + op.drop_table('pr_to_issue') From f8a671591314e673d5b9325e368e92238cc75af7 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Mar 13 2018 17:52:30 +0000 Subject: [PATCH 4/30] Install git before running it Signed-off-by: Pierre-Yves Chibon --- diff --git a/.cico.pipeline b/.cico.pipeline index 366146f..1a9ba16 100644 --- a/.cico.pipeline +++ b/.cico.pipeline @@ -25,7 +25,7 @@ node('pagure') { try { stage('Pre Setup Node'){ // Install EPEL - onmyduffynode 'yum -y install epel-release' + onmyduffynode 'yum -y install epel-release git' } stage('Clone Test Suite') { From 3b03a3823e7eebf6bd216890f7e2c243242b9762 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Mar 13 2018 17:52:30 +0000 Subject: [PATCH 5/30] Adjust argument name Signed-off-by: Pierre-Yves Chibon --- diff --git a/.cico.pipeline b/.cico.pipeline index 1a9ba16..5031f27 100644 --- a/.cico.pipeline +++ b/.cico.pipeline @@ -33,7 +33,7 @@ node('pagure') { } stage('Run Test Suite') { - timeout(6, 'HOURS') { + timeout(time: 6, unit: 'HOURS') { onmyduffynode 'cd pagure && sh ./run_ci_tests.sh' } } From 016a7ff1d0ef1ef90e2ecc6d6b7eb75e8aea808d Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Mar 13 2018 17:52:30 +0000 Subject: [PATCH 6/30] Let's do some debugging Signed-off-by: Pierre-Yves Chibon --- diff --git a/.cico.pipeline b/.cico.pipeline index 5031f27..7e95ba4 100644 --- a/.cico.pipeline +++ b/.cico.pipeline @@ -34,6 +34,7 @@ node('pagure') { stage('Run Test Suite') { timeout(time: 6, unit: 'HOURS') { + onmyduffynode 'cd pagure && ls -la' onmyduffynode 'cd pagure && sh ./run_ci_tests.sh' } } From 8dbdb1c0320db00ad276ce0bc3c5282d332092ca Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Mar 13 2018 17:52:30 +0000 Subject: [PATCH 7/30] Install some of the required dependencies in the run_ci_tests script Signed-off-by: Pierre-Yves Chibon --- diff --git a/run_ci_tests.sh b/run_ci_tests.sh index c2ff8d6..057ab7a 100755 --- a/run_ci_tests.sh +++ b/run_ci_tests.sh @@ -1,3 +1,5 @@ +yum install python-virtualenv libgit2 pygit2 + set -e if [ -n "$REPO" -a -n "$BRANCH" ]; then From 7ad3b99cd166b4142cc49e43ad2a4a475f2bf70e Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Mar 13 2018 17:52:30 +0000 Subject: [PATCH 8/30] Let's not ask questions Signed-off-by: Pierre-Yves Chibon --- diff --git a/run_ci_tests.sh b/run_ci_tests.sh index 057ab7a..79503ec 100755 --- a/run_ci_tests.sh +++ b/run_ci_tests.sh @@ -1,4 +1,4 @@ -yum install python-virtualenv libgit2 pygit2 +yum install -y python-virtualenv libgit2 pygit2 set -e From 6ca3d9095a06da604aa7b4d8a5433a53eaac0b14 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Mar 13 2018 17:52:30 +0000 Subject: [PATCH 9/30] Remove debugging statement Signed-off-by: Pierre-Yves Chibon --- diff --git a/.cico.pipeline b/.cico.pipeline index 7e95ba4..5031f27 100644 --- a/.cico.pipeline +++ b/.cico.pipeline @@ -34,7 +34,6 @@ node('pagure') { stage('Run Test Suite') { timeout(time: 6, unit: 'HOURS') { - onmyduffynode 'cd pagure && ls -la' onmyduffynode 'cd pagure && sh ./run_ci_tests.sh' } } From 88e95c7f39d100fbd3bc643b4dfed75a1595a113 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Mar 13 2018 17:52:30 +0000 Subject: [PATCH 10/30] Install gcc as well Signed-off-by: Pierre-Yves Chibon --- diff --git a/run_ci_tests.sh b/run_ci_tests.sh index 79503ec..ae72bd5 100755 --- a/run_ci_tests.sh +++ b/run_ci_tests.sh @@ -1,4 +1,4 @@ -yum install -y python-virtualenv libgit2 pygit2 +yum install -y python-virtualenv libgit2 pygit2 gcc set -e From 4c91d872e3f7006b819ef6e41943175967591f85 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Mar 13 2018 17:52:30 +0000 Subject: [PATCH 11/30] Ignore removing cffi Signed-off-by: Pierre-Yves Chibon --- diff --git a/run_ci_tests.sh b/run_ci_tests.sh index ae72bd5..35d88ff 100755 --- a/run_ci_tests.sh +++ b/run_ci_tests.sh @@ -38,7 +38,7 @@ then pip install psycopg2 pip install python-openid python-openid-teams python-openid-cla - pip uninstall cffi -y +# pip uninstall cffi -y else source pagureenv-$DATE-$HASH/bin/activate fi From 71710e0f09de8329ac17a5bdb892acbfafb51323 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Mar 13 2018 17:52:30 +0000 Subject: [PATCH 12/30] Fix package name Signed-off-by: Pierre-Yves Chibon --- diff --git a/run_ci_tests.sh b/run_ci_tests.sh index 35d88ff..cf73081 100755 --- a/run_ci_tests.sh +++ b/run_ci_tests.sh @@ -1,4 +1,4 @@ -yum install -y python-virtualenv libgit2 pygit2 gcc +yum install -y python-virtualenv libgit2 python-pygit2 gcc set -e From f0c17a596fcfc420c22fab8b4489229c067a37ea Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Mar 13 2018 17:52:30 +0000 Subject: [PATCH 13/30] Add missing dependency Signed-off-by: Pierre-Yves Chibon --- diff --git a/requirements.txt b/requirements.txt index b2a8708..7ab025b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,6 +2,7 @@ # Use this file by running "$ pip install -r requirements.txt" alembic arrow +bcrypt binaryornot bleach blinker From 63476af5acdb19e8bffd470c43e9aed58d73eb6e Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Mar 13 2018 17:52:30 +0000 Subject: [PATCH 14/30] Simplify the script, we always start from scratch Signed-off-by: Pierre-Yves Chibon --- diff --git a/run_ci_tests.sh b/run_ci_tests.sh index cf73081..4f44da5 100755 --- a/run_ci_tests.sh +++ b/run_ci_tests.sh @@ -18,30 +18,21 @@ git log -2 fi -DATE=`date +%Y%m%d` -HASH=`sha1sum requirements.txt | awk '{print $1}'` - -if [ ! -d pagureenv-$DATE-$HASH ]; -then - rm -rf pagureenv*; - virtualenv pagureenv-$DATE-$HASH --system-site-packages - source pagureenv-$DATE-$HASH/bin/activate - - pip install pip --upgrade - # Needed within the venv - pip install nose --upgrade - pip install --upgrade --force-reinstall python-fedora 'setuptools>=17.1' pygments - pip install -r tests_requirements.txt - pip install -r requirements-ev.txt # We have one test on the SSE server - sed -i -e 's|pygit2 >= 0.20.1||' requirements.txt - pip install -r requirements.txt - pip install psycopg2 - pip install python-openid python-openid-teams python-openid-cla +virtualenv pagureenv --system-site-packages +source pagureenv/bin/activate + +pip install pip --upgrade +# Needed within the venv +pip install nose --upgrade +pip install --upgrade --force-reinstall python-fedora 'setuptools>=17.1' pygments +pip install -r tests_requirements.txt +pip install -r requirements-ev.txt # We have one test on the SSE server +sed -i -e 's|pygit2 >= 0.20.1||' requirements.txt +pip install -r requirements.txt +pip install psycopg2 +pip install python-openid python-openid-teams python-openid-cla # pip uninstall cffi -y -else - source pagureenv-$DATE-$HASH/bin/activate -fi trap deactivate SIGINT SIGTERM EXIT From e957309b8a50646a5f8eddc1fd03e25921f9452f Mon Sep 17 00:00:00 2001 From: Brian Stinson Date: Mar 13 2018 17:52:30 +0000 Subject: [PATCH 15/30] pass the REPO and BRANCH parameters to the duffy node --- diff --git a/.cico.pipeline b/.cico.pipeline index 5031f27..5c35bd9 100644 --- a/.cico.pipeline +++ b/.cico.pipeline @@ -1,7 +1,7 @@ def onmyduffynode(script){ ansiColor('xterm'){ timestamps{ - sh 'ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -l root ${DUFFY_NODE}.ci.centos.org -t "' + script + '"' + sh 'ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -l root ${DUFFY_NODE}.ci.centos.org -t REPO=${REPO} BRANCH=${BRANCH} "' + script + '"' } } } @@ -12,6 +12,13 @@ def syncfromduffynode(rsyncpath){ node('pagure') { + properties([ + parameters([ + string(defaultValue: "", description: "", name: "REPO"), + string(defaultValue: "", description: "", name: "BRANCH"), + ]) + ]) + stage('Allocate Node'){ env.CICO_API_KEY = readFile("${env.HOME}/duffy.key").trim() duffy_rtn=sh( From bf8ab5ee3ca2ecfc08b49a75962ae881fe036450 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Mar 13 2018 17:52:30 +0000 Subject: [PATCH 16/30] Install python-cryptography Signed-off-by: Pierre-Yves Chibon --- diff --git a/run_ci_tests.sh b/run_ci_tests.sh index 4f44da5..4428b97 100755 --- a/run_ci_tests.sh +++ b/run_ci_tests.sh @@ -1,4 +1,4 @@ -yum install -y python-virtualenv libgit2 python-pygit2 gcc +yum install -y python-virtualenv libgit2 python-pygit2 gcc python-cryptography set -e From 4d1841002b6577d81d8f3386a1d2a00756dbded6 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Mar 13 2018 17:52:30 +0000 Subject: [PATCH 17/30] Include redis to run the tests Signed-off-by: Pierre-Yves Chibon --- diff --git a/run_ci_tests.sh b/run_ci_tests.sh index 4428b97..de2730f 100755 --- a/run_ci_tests.sh +++ b/run_ci_tests.sh @@ -1,4 +1,7 @@ -yum install -y python-virtualenv libgit2 python-pygit2 gcc python-cryptography +yum install -y python-virtualenv \ + gcc python-cryptography \ + libgit2 python-pygit2 \ + redis set -e From 4eb5e61d12b01760fb3dad19f3e00f3fc2cc4a83 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Mar 13 2018 17:52:30 +0000 Subject: [PATCH 18/30] Flake8 fixes Signed-off-by: Pierre-Yves Chibon --- diff --git a/pagure/lib/__init__.py b/pagure/lib/__init__.py index ca2c04f..4138119 100644 --- a/pagure/lib/__init__.py +++ b/pagure/lib/__init__.py @@ -5028,7 +5028,6 @@ def link_pr_issue(session, issue, request): ''' - associated_issue = [iss.uid for iss in request.related_issues] if issue.uid not in associated_issue: obj = model.PrToIssue( diff --git a/pagure/lib/tasks.py b/pagure/lib/tasks.py index a6544d8..979b7fe 100644 --- a/pagure/lib/tasks.py +++ b/pagure/lib/tasks.py @@ -935,7 +935,6 @@ def link_pr_to_ticket(self, pr_uid): namespace = request.project.namespace user = request.project.user.user \ if request.project.is_fork else None - branch = request.branch_from for line in pagure.lib.git.read_git_lines( ['log', '--no-walk'] diff --git a/pagure/ui/fork.py b/pagure/ui/fork.py index d2188eb..d1f810a 100644 --- a/pagure/ui/fork.py +++ b/pagure/ui/fork.py @@ -40,7 +40,6 @@ from pagure.utils import ( _log = logging.getLogger(__name__) - def _get_parent_request_repo_path(repo): """ Return the path of the parent git repository corresponding to the provided Repository object from the DB. From a4e97467e440bad929b20fcc58b571fa5d0faf5d Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Mar 13 2018 17:52:30 +0000 Subject: [PATCH 19/30] Include fedmsg for the tests Signed-off-by: Pierre-Yves Chibon --- diff --git a/tests_requirements.txt b/tests_requirements.txt index a5d0566..22fc1d0 100644 --- a/tests_requirements.txt +++ b/tests_requirements.txt @@ -3,6 +3,7 @@ mock==1.1.2 nose>=0.10.4 nosexcover flake8 +fedmsg # optional dependency in the code but tested in the tests # Seems that mock doesn't list this one funcsigs From 706bc356a9890ed049345f5f431f7e6b5ac3851e Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Mar 14 2018 09:19:22 +0000 Subject: [PATCH 20/30] Install all of fedmsg for the tests Signed-off-by: Pierre-Yves Chibon --- diff --git a/tests_requirements.txt b/tests_requirements.txt index 22fc1d0..b65675d 100644 --- a/tests_requirements.txt +++ b/tests_requirements.txt @@ -3,7 +3,7 @@ mock==1.1.2 nose>=0.10.4 nosexcover flake8 -fedmsg # optional dependency in the code but tested in the tests +fedmsg[all] # optional dependency in the code but tested in the tests # Seems that mock doesn't list this one funcsigs From a2414239c4c81d5e84b8733b3032764c60591098 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Mar 14 2018 09:35:18 +0000 Subject: [PATCH 21/30] small flake8 fix Signed-off-by: Pierre-Yves Chibon --- diff --git a/pagure/api/project.py b/pagure/api/project.py index 6a48491..5e4e9c6 100644 --- a/pagure/api/project.py +++ b/pagure/api/project.py @@ -1307,6 +1307,7 @@ def api_new_branch(repo, username=None, namespace=None): jsonout = flask.jsonify(output) return jsonout + @API.route('//c//flag') @API.route('///c//flag') @API.route('/fork///c//flag') From a25901212606870bfbb69796409b50b6ab70cf3a Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Mar 14 2018 09:39:43 +0000 Subject: [PATCH 22/30] Install some more of the dependencies needed Signed-off-by: Pierre-Yves Chibon --- diff --git a/run_ci_tests.sh b/run_ci_tests.sh index de2730f..451b68a 100755 --- a/run_ci_tests.sh +++ b/run_ci_tests.sh @@ -1,7 +1,7 @@ yum install -y python-virtualenv \ gcc python-cryptography \ libgit2 python-pygit2 \ - redis + redis swig openssl-devel m2crypto set -e From 2e422af8c10fa2b7638bbaa4efc5b8370d194c2d Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Mar 14 2018 11:52:59 +0000 Subject: [PATCH 23/30] Install fedmsg as rpm Signed-off-by: Pierre-Yves Chibon --- diff --git a/run_ci_tests.sh b/run_ci_tests.sh index 451b68a..287aa85 100755 --- a/run_ci_tests.sh +++ b/run_ci_tests.sh @@ -1,7 +1,7 @@ yum install -y python-virtualenv \ gcc python-cryptography \ libgit2 python-pygit2 \ - redis swig openssl-devel m2crypto + redis swig openssl-devel m2crypto fedmsg set -e From 420388731d0fcc6610c25233a733052ac13a988d Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Mar 14 2018 13:43:04 +0000 Subject: [PATCH 24/30] Double the size of allowed file descriptors Signed-off-by: Pierre-Yves Chibon --- diff --git a/run_ci_tests.sh b/run_ci_tests.sh index 287aa85..42a9e04 100755 --- a/run_ci_tests.sh +++ b/run_ci_tests.sh @@ -3,6 +3,8 @@ yum install -y python-virtualenv \ libgit2 python-pygit2 \ redis swig openssl-devel m2crypto fedmsg +sysctl -w fs.file-max=2048 + set -e if [ -n "$REPO" -a -n "$BRANCH" ]; then From 33432c5a3e9aa3756f545b5e28547e0ed5ca863f Mon Sep 17 00:00:00 2001 From: Brian Stinson Date: Mar 14 2018 14:22:21 +0000 Subject: [PATCH 25/30] fix envvar in the clone step --- diff --git a/.cico.pipeline b/.cico.pipeline index 5c35bd9..f96350b 100644 --- a/.cico.pipeline +++ b/.cico.pipeline @@ -36,7 +36,7 @@ node('pagure') { } stage('Clone Test Suite') { - onmyduffynode "git clone -b \"${env.BRANCH_NAME}\" --single-branch --depth 1 https://pagure.io/pagure.git" + onmyduffynode "git clone -b \"${env.BRANCH}\" --single-branch --depth 1 https://pagure.io/pagure.git" } stage('Run Test Suite') { From db898ab9e6ebb43dc7aabe42a509f944c1e7ea58 Mon Sep 17 00:00:00 2001 From: Brian Stinson Date: Mar 14 2018 14:22:21 +0000 Subject: [PATCH 26/30] if this is a PR, we don't have a BRANCH_NAME so start from master Merges https://pagure.io/pagure/pull-request/3066 --- diff --git a/.cico.pipeline b/.cico.pipeline index f96350b..f6a09f4 100644 --- a/.cico.pipeline +++ b/.cico.pipeline @@ -36,7 +36,11 @@ node('pagure') { } stage('Clone Test Suite') { - onmyduffynode "git clone -b \"${env.BRANCH}\" --single-branch --depth 1 https://pagure.io/pagure.git" + if (env.BRANCH_NAME}{ + onmyduffynode "git clone -b \"${env.BRANCH_NAME}\" --single-branch --depth 1 https://pagure.io/pagure.git" + } else { + onmyduffynode "git clone --single-branch --depth 1 https://pagure.io/pagure.git" + } } stage('Run Test Suite') { From 1514bbc9b25a27870ebb339fef1537f296f2a8ea Mon Sep 17 00:00:00 2001 From: Brian Stinson Date: Mar 14 2018 14:27:20 +0000 Subject: [PATCH 27/30] bracket typo --- diff --git a/.cico.pipeline b/.cico.pipeline index f6a09f4..ae00242 100644 --- a/.cico.pipeline +++ b/.cico.pipeline @@ -36,7 +36,7 @@ node('pagure') { } stage('Clone Test Suite') { - if (env.BRANCH_NAME}{ + if (env.BRANCH_NAME){ onmyduffynode "git clone -b \"${env.BRANCH_NAME}\" --single-branch --depth 1 https://pagure.io/pagure.git" } else { onmyduffynode "git clone --single-branch --depth 1 https://pagure.io/pagure.git" From b25423e1204462e1033c891a9da71f8768db85b6 Mon Sep 17 00:00:00 2001 From: Brian Stinson Date: Mar 14 2018 14:44:36 +0000 Subject: [PATCH 28/30] sync the out files back into the jenkins workspace --- diff --git a/.cico.pipeline b/.cico.pipeline index ae00242..aa7588a 100644 --- a/.cico.pipeline +++ b/.cico.pipeline @@ -53,6 +53,10 @@ node('pagure') { currentBuild.result = "FAILED" throw e } finally { + stage('Sync Artifacts'){ + syncfromduffynode('*.out') + } + stage('Deallocate Node'){ sh 'cico node done ${SSID}' } From 61188f67f9679c3b4c62d7be2ce0b9a1ec35e016 Mon Sep 17 00:00:00 2001 From: Brian Stinson Date: Mar 14 2018 14:46:59 +0000 Subject: [PATCH 29/30] the .out files will land in the pagure directory --- diff --git a/.cico.pipeline b/.cico.pipeline index aa7588a..743f60a 100644 --- a/.cico.pipeline +++ b/.cico.pipeline @@ -54,7 +54,7 @@ node('pagure') { throw e } finally { stage('Sync Artifacts'){ - syncfromduffynode('*.out') + syncfromduffynode('pagure/*.out') } stage('Deallocate Node'){ From 053794a7a8c457ae82bdfa0608ad61334490992e Mon Sep 17 00:00:00 2001 From: Brian Stinson Date: Mar 14 2018 14:50:30 +0000 Subject: [PATCH 30/30] actually archive the artifacts --- diff --git a/.cico.pipeline b/.cico.pipeline index 743f60a..1fdbfb7 100644 --- a/.cico.pipeline +++ b/.cico.pipeline @@ -60,5 +60,9 @@ node('pagure') { stage('Deallocate Node'){ sh 'cico node done ${SSID}' } + + stage('Archive Artifacts'){ + archiveArtifacts artifacts: '*.out' + } } }