From 47039090db3f5ee0e7f5bdbbfb28f02c719de98b Mon Sep 17 00:00:00 2001 From: Ryan Lerch Date: Feb 06 2018 11:59:16 +0000 Subject: Add new pullrequests widget to replace pagurepr and githubpr This adds a new widget called pullrequests that replaces the githubpr and pagurepr widgets. It consolodates these into a single widget that draws the repos from the team hub config --- diff --git a/hubs/default_config.py b/hubs/default_config.py index 3d2b33d..5ffd641 100644 --- a/hubs/default_config.py +++ b/hubs/default_config.py @@ -61,7 +61,6 @@ WIDGETS = [ 'hubs.widgets.library:Library', 'hubs.widgets.weeklyactivity:WeeklyActivity', 'hubs.widgets.feed:Feed', - 'hubs.widgets.github_pr:GitHubPRs', 'hubs.widgets.githubissues:GitHubIssues', 'hubs.widgets.halp:Halp', 'hubs.widgets.irc:IRC', @@ -69,7 +68,7 @@ WIDGETS = [ 'hubs.widgets.mailinglist:MailingList', 'hubs.widgets.memberships:Memberships', 'hubs.widgets.my_hubs:MyHubs', - 'hubs.widgets.pagure_pr:PagurePRs', + 'hubs.widgets.pullrequests:PullRequests', 'hubs.widgets.pagureissues:PagureIssues', 'hubs.widgets.repositories:Repositories', 'hubs.widgets.sticky:Sticky', diff --git a/hubs/tests/views/test_api_hub_widget.py b/hubs/tests/views/test_api_hub_widget.py index 02130af..8016d83 100644 --- a/hubs/tests/views/test_api_hub_widget.py +++ b/hubs/tests/views/test_api_hub_widget.py @@ -235,7 +235,7 @@ class TestAPIHubWidget(APPTest): self.hub = Hub.by_name("ralph", "user") self.widget = Widget( hub=self.hub, - plugin='pagure_pr', + plugin='pullrequests', index=42, _config=json.dumps({}), ) @@ -248,11 +248,11 @@ class TestAPIHubWidget(APPTest): response = self.check_url(self.url, user=user) response_data = json.loads(response.get_data(as_text=True)) self.assertEqual(response_data["status"], "OK") - self.assertEqual(response_data["data"]["name"], "pagure_pr") + self.assertEqual(response_data["data"]["name"], "pullrequests") def test_get_logged_out(self): self.hub.config["visibility"] = "private" - widget = widget_instance("ralph", "pagure_pr") + widget = widget_instance("ralph", "pullrequests") self.session.commit() response = self.check_url( "/api/hubs/%s/widgets/%s/" % (self.hub.id, widget.idx), code=403) diff --git a/hubs/tests/widgets/test_github_pr.py b/hubs/tests/widgets/test_github_pr.py deleted file mode 100644 index ea175b0..0000000 --- a/hubs/tests/widgets/test_github_pr.py +++ /dev/null @@ -1,64 +0,0 @@ -from __future__ import unicode_literals - -from . import WidgetTest - - -class TestGithubPr(WidgetTest): - - plugin = "github_pr" - initial_widget_config = { - "organization": "fedora-infra", - "display_number": 1, - } - - def populate(self): - super(TestGithubPr, self).populate() - self._add_widget_under_test() - - def _get_should_invalidate_result(self, msg): - func = self.widget.module.get_cached_functions()['GetPRs'] - return func(self.widget).should_invalidate(msg) - - def test_should_invalidate_wrong_topic(self): - msg = {'topic': 'hubs.widget.update.WRONG.TOPIC'} - self.assertFalse(self._get_should_invalidate_result(msg)) - - def test_should_invalidate_good_match(self): - msg = { - 'topic': 'org.fedoraproject.tests.github.pull_request.closed', - 'msg': { - 'repository': { - 'owner': { - 'login': 'fedora-infra', - } - }, - 'display_number': 1, - } - } - self.assertTrue(self._get_should_invalidate_result(msg)) - msg = { - 'topic': 'org.fedoraproject.tests.github.pull_request.closed', - 'msg': { - 'repository': { - 'owner': { - 'name': 'fedora-infra', - } - }, - 'display_number': 1, - } - } - self.assertTrue(self._get_should_invalidate_result(msg)) - - def test_should_invalidate_wrong_org(self): - msg = { - 'topic': 'org.fedoraproject.tests.github.pull_request.closed', - 'msg': { - 'repository': { - 'owner': { - 'login': 'not-fedora-infra', - } - }, - 'display_number': 1, - } - } - self.assertFalse(self._get_should_invalidate_result(msg)) diff --git a/hubs/tests/widgets/test_pagure_pr.py b/hubs/tests/widgets/test_pagure_pr.py deleted file mode 100644 index 88ed547..0000000 --- a/hubs/tests/widgets/test_pagure_pr.py +++ /dev/null @@ -1,60 +0,0 @@ -from __future__ import unicode_literals - -from . import WidgetTest - - -class TestPagurePr(WidgetTest): - - plugin = "pagure_pr" - initial_widget_config = { - "repo": "fedora-hubs", - } - - def populate(self): - super(TestPagurePr, self).populate() - self._add_widget_under_test() - - def _get_should_invalidate_result(self, msg): - func = self.widget.module.get_cached_functions()['GetPRs'] - return func(self.widget).should_invalidate(msg) - - def test_should_invalidate_wrong_topic(self): - msg = {'topic': 'hubs.widget.update.WRONG.TOPIC'} - self.assertFalse(self._get_should_invalidate_result(msg)) - - def test_should_invalidate_good_match(self): - msg = { - 'topic': 'tests.pagure.pull-request.new', - 'msg': { - "pullrequest": { - "project": { - "name": "fedora-hubs", - }, - }, - }, - } - self.assertTrue(self._get_should_invalidate_result(msg)) - msg = { - 'topic': 'tests.pagure.pull-request.closed', - 'msg': { - "pullrequest": { - "project": { - "name": "fedora-hubs", - }, - }, - }, - } - self.assertTrue(self._get_should_invalidate_result(msg)) - - def test_should_invalidate_wrong_repo(self): - msg = { - 'topic': 'tests.pagure.pull-request.new', - 'msg': { - "pullrequest": { - "project": { - "name": "not-fedora-hubs", - }, - }, - }, - } - self.assertFalse(self._get_should_invalidate_result(msg)) diff --git a/hubs/tests/widgets/test_pullrequests.py b/hubs/tests/widgets/test_pullrequests.py new file mode 100644 index 0000000..bc6273c --- /dev/null +++ b/hubs/tests/widgets/test_pullrequests.py @@ -0,0 +1,89 @@ +from __future__ import unicode_literals + +from . import WidgetTest + + +class TestPullRequests(WidgetTest): + + plugin = "pullrequests" + initial_widget_config = { + "repo": "fedora-hubs", + } + + def populate(self): + super(TestPullRequests, self).populate() + self._add_widget_under_test() + + def _get_pagure_should_invalidate_result(self, msg): + f = self.widget.module.get_cached_functions()['GetPagurePullRequests'] + return f(self.widget).should_invalidate(msg) + + def _get_github_should_invalidate_result(self, msg): + f = self.widget.module.get_cached_functions()['GetGithubPullRequests'] + return f(self.widget).should_invalidate(msg) + + def test_should_invalidate_wrong_topic(self): + msg = {'topic': 'hubs.widget.update.WRONG.TOPIC'} + self.assertFalse(self._get_pagure_should_invalidate_result(msg)) + self.assertFalse(self._get_github_should_invalidate_result(msg)) + + def test_should_invalidate_good_match_pagure(self): + self.widget.hub.config["pagure"] = ["fedora-hubs"] + msg = { + 'topic': 'tests.pagure.pull-request.new', + 'msg': { + "pullrequest": { + "project": { + "name": "fedora-hubs", + }, + }, + }, + } + self.assertTrue(self._get_pagure_should_invalidate_result(msg)) + msg = { + 'topic': 'tests.pagure.pull-request.closed', + 'msg': { + "pullrequest": { + "project": { + "name": "fedora-hubs", + }, + }, + }, + } + self.assertTrue(self._get_pagure_should_invalidate_result(msg)) + + def test_should_invalidate_wrong_repo_pagure(self): + msg = { + 'topic': 'tests.pagure.pull-request.new', + 'msg': { + "pullrequest": { + "project": { + "name": "not-fedora-hubs", + }, + }, + }, + } + self.assertFalse(self._get_pagure_should_invalidate_result(msg)) + + def test_should_invalidate_good_match_github(self): + self.widget.hub.config["github"] = ["fedora-infra/bodhi"] + msg = { + 'topic': 'org.fedoraproject.tests.github.pull_request.opened', + 'msg': { + 'repository': { + 'fullname': "fedora-infra/bodhi" + }, + 'display_number': 1, + } + } + self.assertTrue(self._get_github_should_invalidate_result(msg)) + msg = { + 'topic': 'org.fedoraproject.tests.github.pull_request.closed', + 'msg': { + 'repository': { + 'fullname': "fedora-infra/bodhi" + }, + 'display_number': 1, + } + } + self.assertTrue(self._get_github_should_invalidate_result(msg)) diff --git a/hubs/utils/github.py b/hubs/utils/github.py index 57da0d9..40bebcd 100644 --- a/hubs/utils/github.py +++ b/hubs/utils/github.py @@ -34,20 +34,27 @@ def github_repos(token, username): yield result['name'] -def github_pulls(token, username, repo): - log.info("Finding github pull requests for %r %r" % (username, repo)) - tmpl = "https://api.github.com/repos/{username}/{repo}/" + \ +def github_pulls(repo, token=None): + log.info("Finding github pull requests for %r" % (repo)) + tmpl = "https://api.github.com/repos/{repo}/" + \ "pulls?per_page=100" - url = tmpl.format(username=username, repo=repo) - auth = dict(access_token=token) - for result in _github_results(url, auth): + url = tmpl.format(repo=repo) + if token: + token = dict(access_token=token) + for result in _github_results(url, token): yield dict( - user=result['user']['login'], - title=result['title'], - url=result['html_url'], - number=result['number'], - assignee=result['assignee'], - timestamp=arrow.get(result['created_at']).timestamp, + pr_project_url='/'.join(["https://github.com", repo]), + pr_url='/'.join(["https://github.com", + repo, "pull", str(result['number'])]), + pr_project_name=repo, + pr_id=result['number'], + pr_title=result['title'][:40], + pr_title_full=result['title'], + pr_openedby=result['user']['login'], + pr_assignee=result['assignee'], + pr_repo=repo, + pr_hosting_type="Github", + pr_opened_date=arrow.get(result['created_at']), ) diff --git a/hubs/utils/pagure.py b/hubs/utils/pagure.py index 12dbdf9..dc89c03 100644 --- a/hubs/utils/pagure.py +++ b/hubs/utils/pagure.py @@ -1,5 +1,8 @@ from __future__ import unicode_literals +import requests +import arrow + PAGURE_URL = "https://pagure.io" @@ -11,3 +14,31 @@ def msg2projectname(msg): if "pagure.pull-request" in msg["topic"]: return msg["msg"]["pullrequest"]["project"]["fullname"] return msg["msg"]["project"]["fullname"] + + +def pagure_pulls(repo): + url = '/'.join([PAGURE_URL, "api", + "0", repo, "pull-requests"]) + response = requests.get(url) + data = response.json() + + for request in data['requests']: + pr_project_user = None + if request['project']['parent']: + pr_project_user = request['project']['user']['username'] + + yield dict( + pr_project_url="/".join([PAGURE_URL, repo]), + pr_url="/".join([PAGURE_URL, repo, + "pull-request", str(request['id'])]), + pr_project_name=request['project']['name'], + pr_project_user=pr_project_user, + pr_id=request['id'], + pr_title=request['title'][:40], + pr_title_full=request['title'], + pr_openedby=request['user']['name'], + pr_assignee=request['assignee'], + pr_repo=repo, + pr_hosting_type="Pagure", + pr_opened_date=arrow.get(int(request['date_created'])), + ) diff --git a/hubs/widgets/github_pr/__init__.py b/hubs/widgets/github_pr/__init__.py deleted file mode 100644 index 84c76e7..0000000 --- a/hubs/widgets/github_pr/__init__.py +++ /dev/null @@ -1,96 +0,0 @@ -from __future__ import unicode_literals - -import logging - -from hubs.utils import validators -from hubs.utils.fedmsg import get_fedmsg_config -from hubs.utils.github import github_repos, github_pulls -from hubs.widgets.base import Widget -from hubs.widgets.view import RootWidgetView -from hubs.widgets.caching import CachedFunction - - -log = logging.getLogger(__name__) -fedmsg_config = get_fedmsg_config() - - -# TODO: use a checkbox set to select which repos to use from the hub's -# configuration. - - -class GitHubPRs(Widget): - - name = "github_pr" - label = "Github: Pull Requests" - position = "right" - parameters = [ - dict( - name="organization", - label="Organization", - default=None, - validator=validators.GithubOrganization, - help="Github Organization or username", - ), dict( - name="display_number", - label="Number of tickets", - default=6, - validator=validators.Integer, - help="How many pull requests to display at max.", - )] - - -class BaseView(RootWidgetView): - - def get_context(self, instance, *args, **kwargs): - get_prs = GetPRs(instance) - org = instance.config["organization"] - context = dict( - organization=org, - display_number=instance.config["display_number"], - ) - context.update(get_prs()) - return context - - -class GetPRs(CachedFunction): - - def execute(self): - org = self.instance.config["organization"] - display_number = self.instance.config["display_number"] - log.info("Getting GH prs for %r, (%r)" % (org, display_number)) - token = fedmsg_config.get('github.oauth_token') - pulls = [] - displayed_number = 0 - more = 0 - if token: - repos = github_repos(token, org) - pulls = sum([ - list(github_pulls(token, org, repo)) - for repo in repos - ], []) - # Reverse-sort by time (newest-first) - pulls.sort(key=lambda x: x['timestamp'], reverse=True) - - # Some hints for display - len_pulls = len(pulls) - displayed_number = min(display_number, len_pulls) - more = max(len_pulls - displayed_number, 0) - - return dict( - more=more, - pulls=pulls, - ) - - def should_invalidate(self, message): - try: - category = message["topic"].split('.')[3] - except IndexError: - return False - if category != "github": - return False - owner = message['msg']['repository']['owner'] - if 'login' in owner: - owner = owner['login'] - else: - owner = owner['name'] - return owner == self.instance.config['organization'] diff --git a/hubs/widgets/github_pr/templates/root.html b/hubs/widgets/github_pr/templates/root.html deleted file mode 100644 index 4ad7357..0000000 --- a/hubs/widgets/github_pr/templates/root.html +++ /dev/null @@ -1,50 +0,0 @@ -
- - Organization Github Page - -
-{% if pulls %} -
- -{% endif %} diff --git a/hubs/widgets/pagure_pr/__init__.py b/hubs/widgets/pagure_pr/__init__.py deleted file mode 100644 index 9988b57..0000000 --- a/hubs/widgets/pagure_pr/__init__.py +++ /dev/null @@ -1,83 +0,0 @@ -from __future__ import unicode_literals - -from hubs.utils import validators, pagure -from hubs.widgets.base import Widget -from hubs.widgets.view import RootWidgetView -from hubs.widgets.caching import CachedFunction - -import requests - -# TODO: use a checkbox set to select which repos to use from the hub's -# configuration. - - -class PagurePRs(Widget): - - name = "pagure_pr" - label = "Pagure: Newest Open Pull Requests" - position = "right" - parameters = [ - dict( - name="repo", - label="Repository", - default=None, - validator=validators.PagureRepo, - help="Pagure repo name.", - )] - - -class BaseView(RootWidgetView): - - def get_context(self, instance, *args, **kwargs): - get_prs = GetPRs(instance) - context = dict( - repo=instance.config["repo"], - ) - context.update(get_prs()) - return context - - -class GetPRs(CachedFunction): - - def execute(self): - repo = self.instance.config["repo"] - url = '/'.join([pagure.PAGURE_URL, "api", "0", repo, "pull-requests"]) - response = requests.get(url) - try: - data = response.json() - except ValueError: - return dict(all_pr=[], total_req=0) - total_req = data['total_requests'] - all_pr = list() - - for request in data['requests']: - pr_project_user = None - if request['project']['parent']: - pr_project_user = request['project']['user']['username'] - - all_pr.append( - dict( - pr_project_name=request['project']['name'], - pr_project_user=pr_project_user, - pr_id=request['id'], - pr_title=request['title'][:45], - pr_title_full=request['title'], - pr_openedby=request['user']['name'], - pr_assignee=request['assignee'], - ) - ) - - return dict( - all_pr=all_pr, - total_req=total_req, - ) - - def should_invalidate(self, message): - if (".pagure.pull-request.new" not in message["topic"] - and ".pagure.pull-request.closed" not in message["topic"]): - return False - try: - project = message['msg']['pullrequest']['project']['name'] - except KeyError: - return False - return (project == self.instance.config['repo']) diff --git a/hubs/widgets/pagure_pr/templates/root.html b/hubs/widgets/pagure_pr/templates/root.html deleted file mode 100644 index 75cb916..0000000 --- a/hubs/widgets/pagure_pr/templates/root.html +++ /dev/null @@ -1,52 +0,0 @@ - - -
- -
- - All Pull-Requests - -
diff --git a/hubs/widgets/pullrequests/__init__.py b/hubs/widgets/pullrequests/__init__.py new file mode 100644 index 0000000..9373eae --- /dev/null +++ b/hubs/widgets/pullrequests/__init__.py @@ -0,0 +1,109 @@ +from __future__ import unicode_literals + +from hubs.utils import pagure +from hubs.utils.github import github_pulls +from hubs.utils.fedmsg import get_fedmsg_config +from hubs.widgets.base import Widget +from hubs.widgets.view import RootWidgetView +from hubs.widgets.caching import CachedFunction + +fedmsg_config = get_fedmsg_config() + + +class PullRequests(Widget): + + name = "pullrequests" + label = "Latest Pull Requests" + position = "right" + hub_types = ["team"] + + def get_template_environment(self): + env = super(PullRequests, self).get_template_environment() + # Add a filter + env.filters['humanize'] = lambda d: d.humanize() + return env + + +class BaseView(RootWidgetView): + + def get_context(self, instance, *args, **kwargs): + if (not instance.hub.config["pagure"] and + not instance.hub.config["github"]): + return dict(no_repos_defined=True) + get_pagure_prs = GetPagurePullRequests(instance) + get_github_prs = GetGithubPullRequests(instance) + + all_pr = get_pagure_prs() + get_github_prs() + all_pr = sorted(all_pr, + key=lambda x: x["pr_opened_date"], + reverse=True) + return dict(all_pr=all_pr) + + +class GetPagurePullRequests(CachedFunction): + def execute(self): + all_pr = list() + for repo in self.instance.hub.config["pagure"]: + try: + pulls = pagure.pagure_pulls(repo) + except ValueError: + continue + + for pr in pulls: + all_pr.append(pr) + + all_pr = sorted(all_pr, + key=lambda x: x["pr_opened_date"], + reverse=True) + + return all_pr + + def should_invalidate(self, message): + pagure_repos = self.instance.hub.config["pagure"] + if ( + ".pagure.pull-request.new" not in message["topic"] or + ".pagure.pull-request.closed" not in message["topic"]): + try: + repo = message['msg']['pullrequest']['project']['name'] + except KeyError: + return False + return (repo in pagure_repos) + elif message["topic"].endswith('.hubs.hub.updated'): + if "pagure" in message["msg"]["changed_keys"]: + hub_id = message["msg"]["hub_id"] + return hub_id == self.instance.hub.id + else: + return False + else: + return False + + +class GetGithubPullRequests(CachedFunction): + def execute(self): + all_pr = list() + + token = fedmsg_config.get('github.oauth_token') + for repo in self.instance.hub.config["github"]: + for pr in github_pulls(repo, token=token): + all_pr.append(pr) + + return all_pr + + def should_invalidate(self, message): + github_repos = self.instance.hub.config["github"] + if ( + ".github.pull_request.opened" in message["topic"] or + ".github.pull_request.closed" in message["topic"]): + try: + repo = message['msg']['repository']['fullname'] + except KeyError: + return False + return (repo in github_repos) + elif message["topic"].endswith('.hubs.hub.updated'): + if "github" in message["msg"]["changed_keys"]: + hub_id = message["msg"]["hub_id"] + return hub_id == self.instance.hub.id + else: + return False + else: + return False diff --git a/hubs/widgets/pullrequests/templates/root.html b/hubs/widgets/pullrequests/templates/root.html new file mode 100644 index 0000000..e81e860 --- /dev/null +++ b/hubs/widgets/pullrequests/templates/root.html @@ -0,0 +1,37 @@ +{% if no_repos_defined %} +

+ You must configure a software development platform in the hub configuration in order to use this widget. +

+{% else %} + {% if not all_pr %} +
No active pull requests
+ {% else %} + + {% endif %} +{% endif %} +