From 8ada93e0a01771440362751a87fc6714b4571cdc Mon Sep 17 00:00:00 2001 From: Dominik Perpeet Date: Sep 18 2019 21:48:44 +0000 Subject: [PATCH 1/4] Update JIRA label for fedora-ci/AtomicCi --- diff --git a/ansible/files/sync2jira.py b/ansible/files/sync2jira.py new file mode 100644 index 0000000..288cd3f --- /dev/null +++ b/ansible/files/sync2jira.py @@ -0,0 +1,180 @@ +# This file is part of sync2jira. +# Copyright (C) 2016 Red Hat, Inc. +# +# sync2jira is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# sync2jira is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with sync2jira; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110.15.0 USA +# +# Authors: Ralph Bean + +config = { + 'sync2jira': { + 'initialize': True, + 'testing': False, + + # We don't need legacy mode anymore. Not for a long time. Let's + # remove it soon. + 'legacy_matching': False, + + #'jira': { # See /etc/fedmsg.d/sync2jira-credentials.py }, + 'filters': { + 'pagure': { + # Atomic Host repos + 'atomic-wg': { 'status': 'Open', 'tags': ['jira',], }, + # A subset of the fedora-infra issues should go to FACTORY. + 'fedora-infrastructure': { + 'status': 'Open', + 'tags': [ + 'mbs', + 'waiverdb', + 'greenwave', + 'odcs', + 'freshmaker', + 'factory2', + ], + }, + }, + 'github': { + # Only sync multi-type tickets from bodhi. + 'fedora-infra/bodhi': { 'state': 'open', 'milestone': 4, }, + + # Atomic Host repos + 'projectatomic/rpm-ostree': { 'state': 'open', 'labels': 'jira', }, + 'ostreedev/ostree': { 'state': 'open', 'labels': 'jira', }, + 'projectatomic/atomic-host-tests': { 'state': 'open', 'labels': 'jira', }, + 'projectatomic/papr': { 'state': 'open', 'labels': 'jira', }, + }, + }, + 'map': { + 'pagure': { + 'pungi': { 'project': 'COMPOSE', 'component': 'Pungi', }, + 'rpkg': { 'project': 'COMPOSE', 'component': 'rpkg', }, + 'fedpkg': { 'project': 'COMPOSE', 'component': 'fedpkg', }, + 'fedpkg-minimal': { 'project': 'COMPOSE', 'component': 'fedpkg-minimal', }, + + 'koji': { 'project': 'BREW', 'component': 'Koji', }, + + 'sync-to-jira': { 'project': 'FACTORY', 'component': None, }, + 'fm-orchestrator': { 'project': 'FACTORY', 'component': 'MBS', }, + 'waiverdb': { 'project': 'FACTORY', 'component': 'WaiverDB', }, + 'freshmaker': { 'project': 'FACTORY', 'component': 'Freshmaker', }, + 'greenwave': { 'project': 'FACTORY', 'component': 'Greenwave', }, + 'odcs': { 'project': 'FACTORY', 'component': 'ODCS', }, + 'fedrepo_req': { 'project': 'FACTORY', 'component': None, }, + + # A subset of the fedora-infra issues should go to FACTORY. + # See the `filter` above to see which are synced. + 'fedora-infrastructure': { 'project': 'FACTORY', 'component': None, 'labels': ['ops', 'fedora'] }, + + # Unwanted until we have two-way sync. + #'releng': { 'project': 'RCM', 'component': 'Fedora', }, + #'pungi-fedora': { 'project': 'RCM', 'component': 'Fedora', }, + + 'modularity': { 'project': 'RHELPLAN', 'component': None, 'labels': 'appstream', }, + 'modularity/fedmod': { 'project': 'RHELPLAN', 'component': None, 'labels': 'appstream', }, + 'fedora-ci/AtomicCi': { 'project': 'RHELPLAN', 'component': None, 'labels': 'sst_phoebe_testing_process_rhel8'}, + + # Atomic Host repos + 'atomic-wg': {'project': 'RHELPLAN', 'component': None, 'labels': ['sst_atomic_host', 'upstream'],}, + }, + 'github': { + 'nirzari/review-rot': {'project': 'DEVOPSA', 'component': None, }, + 'fedora-infra/pdc-updater': { 'project': 'FACTORY', 'component': 'pdc-updater', }, + 'release-engineering/resultsdb-updater': {'project': 'FACTORY', 'component': 'resultsdb'}, + 'release-engineering/fedmsg_meta_umb': {'project': 'FACTORY', 'component': 'UMB'}, + 'release-engineering/mbs-messaging-umb': {'project': 'FACTORY', 'component': 'MBS'}, + + # Some modularity repos. + 'fedora-modularity/pdc-updater': { 'project': 'FACTORY', 'component': 'pdc-updater', }, + 'fedora-modularity/product-definition-center': { 'project': 'FACTORY', 'component': 'PDC', }, + 'fedora-modularity/BPO': { 'project': 'FACTORY', 'component': None, }, + + 'fedora-infra/bodhi': { 'project': 'FACTORY', 'component': None, }, + + 'fedora-modularity/f27-content-tracking': { 'project': 'RHELPLAN', 'component': None, 'labels': 'appstream', }, + 'fedora-modularity/dependency-report-scripts': { 'project': 'RHELPLAN', 'component': None, 'labels': 'appstream', }, + + # Handy utils + #'ralphbean/finishline': { 'project': 'FACTORY', 'component': None, }, + + 'product-definition-center/product-definition-center': { + 'project': 'PDC', 'component': 'PDC Github', }, + 'product-definition-center/pdc-client': { + 'project': 'PDC', 'component': 'PDC Github', }, + 'product-definition-center/pdc-ruby-gem': { + 'project': 'PDC', 'component': 'PDC Github', }, + + 'Commonjava/indy': {'project': 'NOS', 'component': 'Indy'}, + 'Commonjava/indy-docker': {'project': 'NOS', 'component': 'Indy'}, + 'Commonjava/jhttpc': {'project': 'NOS', 'component': 'Indy'}, + 'Commonjava/partyline': {'project': 'NOS', 'component': 'Indy'}, + + 'Commonjava/cartographer': {'project': 'NOS', 'component': 'Cartographer'}, + 'Commonjava/galley': {'project': 'NOS', 'component': 'Galley'}, + + 'project-ncl/causeway': {'project': 'NOS', 'component': 'BrewBridge'}, + + 'release-engineering/pom-manipulation-ext': {'project': 'NOS', 'component': 'PME'}, + 'release-engineering/offliner': {'project': 'NOS', 'component': 'Offliner'}, + 'release-engineering/kojiji': {'project': 'NOS', 'component': 'Kojiji'}, + 'release-engineering/koji-dojo': {'project': 'NOS', 'component': 'Koji-dojo'}, + + # Atomic Host repos + 'projectatomic/rpm-ostree': {'project': 'RHELPLAN', 'component': None, 'labels': ['sst_atomic_host', 'upstream'],}, + 'projectatomic/atomic-host-tests': {'project': 'RHELPLAN', 'component': None, 'labels': ['sst_atomic_host', 'upstream'],}, + 'projectatomic/papr': {'project': 'RHELPLAN', 'component': None, 'labels': ['sst_atomic_host', 'upstream'],}, + 'ostreedev/ostree': {'project': 'RHELPLAN', 'component': None, 'labels': ['sst_atomic_host', 'upstream'],}, + }, + }, + }, + 'logging': dict( + version=1, + loggers={ + 'sync2jira': { + "level": "INFO", + "propagate": True, + "handlers": ['console'], + }, + }, + ), +} + + +def _link_field_to_dict(field): + """ Utility for ripping apart github's Link header field. + It's kind of ugly. + """ + if not field: + return dict() + return dict([ + ( + part.split('; ')[1][5:-1], + part.split('; ')[0][1:-1], + ) for part in field.split(', ') + ]) + + +import requests +url = 'https://api.github.com/orgs/modularity-modules/repos' + +while url: + response = requests.get(url) + if not bool(response): + print("!! WARNING. Failed to get %r %r" % (url, response)) + break + for repo in response.json(): + config['sync2jira']['map']['github'][repo['full_name']] = { + 'project': 'RHELPLAN', 'component': None, 'labels': 'appstream', + } + links = _link_field_to_dict(response.headers.get('link')) + url = links.get('next') From 56f55bcde6879617016b3bf2bf51d90d9cf67e59 Mon Sep 17 00:00:00 2001 From: Dusty Mabe Date: Sep 18 2019 21:49:19 +0000 Subject: [PATCH 2/4] Add upstream fedora CoreOS repo syncing --- diff --git a/ansible/files/sync2jira.py b/ansible/files/sync2jira.py index 288cd3f..5776a31 100644 --- a/ansible/files/sync2jira.py +++ b/ansible/files/sync2jira.py @@ -53,6 +53,11 @@ config = { 'ostreedev/ostree': { 'state': 'open', 'labels': 'jira', }, 'projectatomic/atomic-host-tests': { 'state': 'open', 'labels': 'jira', }, 'projectatomic/papr': { 'state': 'open', 'labels': 'jira', }, + + # CoreOS repos + 'coreos/bugs': {'state': 'open', 'labels': 'jira', }, + 'openshift/os': {'state': 'open', 'labels': 'jira', }, + 'coreos/fedora-coreos-tracker': {'state': 'open', 'labels': 'jira', }, }, }, 'map': { @@ -130,10 +135,15 @@ config = { 'release-engineering/koji-dojo': {'project': 'NOS', 'component': 'Koji-dojo'}, # Atomic Host repos - 'projectatomic/rpm-ostree': {'project': 'RHELPLAN', 'component': None, 'labels': ['sst_atomic_host', 'upstream'],}, - 'projectatomic/atomic-host-tests': {'project': 'RHELPLAN', 'component': None, 'labels': ['sst_atomic_host', 'upstream'],}, - 'projectatomic/papr': {'project': 'RHELPLAN', 'component': None, 'labels': ['sst_atomic_host', 'upstream'],}, - 'ostreedev/ostree': {'project': 'RHELPLAN', 'component': None, 'labels': ['sst_atomic_host', 'upstream'],}, + 'projectatomic/rpm-ostree': {'project': 'COREOS', 'component': None, 'labels': ['sst_rhcos', 'sst_coreosct', 'upstream'],}, + 'projectatomic/atomic-host-tests': {'project': 'COREOS', 'component': None, 'labels': ['sst_rhcos', 'sst_coreosct', 'upstream'],}, + 'projectatomic/papr': {'project': 'COREOS', 'component': None, 'labels': ['sst_rhcos', 'sst_coreosct', 'upstream'],}, + 'ostreedev/ostree': {'project': 'COREOS', 'component': None, 'labels': ['sst_rhcos', 'sst_coreosct', 'upstream'],}, + + # CoreOS repos + 'coreos/bugs': {'project': 'COREOS', 'component': None, 'labels': ['sst_rhcos', 'sst_coreosct', 'upstream', 'upstream_cl', 'cl'],}, + 'openshift/os': {'project': 'COREOS', 'component': None, 'labels': ['sst_rhcos', 'sst_coreosct', 'upstream','upstream_cos'],}, + 'coreos/fedora-coreos-tracker': {'project': 'COREOS', 'component': None, 'labels': ['sst_rhcos', 'sst_coreosct', 'upstream','upstream_cos'],}, }, }, }, From b7c7223499077ef1f0ddcc31682ec7a6b2d9548c Mon Sep 17 00:00:00 2001 From: Bohdan Iakymets Date: Sep 18 2019 21:50:48 +0000 Subject: [PATCH 3/4] Refactored status transition and updated README --- diff --git a/sync2jira/downstream.py b/sync2jira/downstream.py index 250cd11..7daa964 100644 --- a/sync2jira/downstream.py +++ b/sync2jira/downstream.py @@ -560,7 +560,6 @@ def _create_jira_issue(client, issue, config): custom_fields = issue.downstream.get('custom_fields', {}) default_type = issue.downstream.get('type', "Bug") - # Build the description of the JIRA issue if 'description' in issue.downstream.get('updates', {}): description = "Upstream description: {quote}%s{quote}" % issue.content From 3031433203cfb227cf8b819a98715c6b35bfb9bd Mon Sep 17 00:00:00 2001 From: sidpremkumar Date: Sep 18 2019 21:51:20 +0000 Subject: [PATCH 4/4] Fix issue when GitHub full name is unknown --- diff --git a/sync2jira/downstream.py b/sync2jira/downstream.py index 7daa964..c1b7783 100644 --- a/sync2jira/downstream.py +++ b/sync2jira/downstream.py @@ -17,1162 +17,382 @@ # # Authors: Ralph Bean -import operator - import logging -import re - -import arrow -import jira.client -from jira import JIRAError -from datetime import datetime -import jinja2 -import pypandoc - -from sync2jira.intermediary import Issue -from sync2jira.mailer import send_mail - -# The date the service was upgraded -# This is used to ensure legacy comments are not touched -UPDATE_DATE = datetime(2019, 7, 9, 18, 18, 36, 480291) - -log = logging.getLogger(__name__) - -remote_link_title = "Upstream issue" -duplicate_issues_subject = 'FYI: Duplicate Sync2jira Issues' - -jira_cache = {} - - -def check_jira_status(client): - """ - Function tests the status of the JIRA server. - - - :param jira.client.JIRA client: JIRA client - :return: True/False if the server is up - :rtype: Bool - """ - # Search for any issue remote title - ret = client.search_issues("issueFunction in linkedIssuesOfRemote('*')") - if len(ret) < 1: - # If we did not find anything return false - return False - return True - - -def _comment_format(comment): - """ - Function to format JIRA comments. - - :param dict comment: Upstream comment - :returns: Comments formatted - :rtype: String - """ - pretty_date = comment['date_created'].strftime("%a %b %d") - return "[%s] Upstream, %s wrote [%s]:\n\n{quote}\n%s\n{quote}" % ( - comment['id'], comment['author'], pretty_date, comment['body']) - - -def _comment_format_legacy(comment): - """ - Legacy function to format JIRA comments. - This is still used to match comments so no - duplicates are created. - - :param dict comment: Upstream comment - :returns: Comments formatted - :rtype: String - """ - return "Upstream, %s wrote:\n\n{quote}\n%s\n{quote}" % ( - comment['name'], comment['body']) - - -def _get_jira_client(issue, config): - """ - Function to match and create JIRA client. - - :param sync2jira.intermediary.Issue issue: Issue object - :param dict config: Config dict - :returns: Matching JIRA client - :rtype: jira.client.JIRA - """ - # The name of the jira instance to use is stored under the 'map' - # key in the config where each upstream is mapped to jira projects. - # It is conveniently added to the Issue object from intermediary.py - # so we can use it here: - - if not isinstance(issue, Issue): - log.error("passed in issue is not an Issue instance") - log.error("It is a %s" % type(issue).__name__) - raise TypeError("Got %s, expected Issue" % type(issue).__name__) - - # Use the Jira instance set in the issue config. If none then - # use the configured default jira instance. - jira_instance = issue.downstream.get('jira_instance', False) - if not jira_instance: - jira_instance = config['sync2jira'].get('default_jira_instance', False) - if not jira_instance: - log.error(" No jira_instance for issue and there is no default in the config") - raise Exception - - client = jira.client.JIRA(**config['sync2jira']['jira'][jira_instance]) - return client - - -def _matching_jira_issue_query(client, issue, config, free=False): - """ - API calls that find matching JIRA tickets if any are present. - - :param jira.client.JIRA client: JIRA client - :param sync2jira.intermediary.Issue issue: Issue object - :param Dict config: Config dict - :param Bool free: Free tag to add 'statusCategory != Done' to query - :returns: results: Returns a list of matching JIRA issues if any are found - :rtype: List - """ - # Searches for any remote link to the issue.url - query = 'issueFunction in linkedIssuesOfRemote("%s") and ' \ - 'issueFunction in linkedIssuesOfRemote("%s")' % ( - remote_link_title, issue.url) - if free: - query += ' and statusCategory != Done' - # Query the JIRA client and store the results - results_of_query = client.search_issues(query) - if len(results_of_query) > 1: - # Sometimes if an issue gets dropped it is created with the url: pagure.com/something/issue/5 - # Then when that issue is dropped and another one is created is is created with the same - # url : pagure.com/something/issue/5. - # We need to ensure that we are not catching a dropped issue - # Loop through the results of the query and make sure the ids match - final_results = [] - - for result in results_of_query: - description = result.fields.description or "" - summary = result.fields.summary or "" - if issue.id in description or issue.title == summary: - search = check_comments_for_duplicate(client, result, - find_username(issue, config)) - if search is True: - final_results.append(result) - else: - # Else search returned a linked issue - final_results.append(search) - # If that's not the case, check if they have the same upstream title - # Upstream username/repo can change if repos are merged - elif re.search(r"\[[a-zA-Z0-9!@#$%^&*()_+\-=\[\]{};':\\|,.<>\/?]*\] " - + issue.upstream_title, - result.fields.summary): - search = check_comments_for_duplicate(client, result, - find_username(issue, config)) - if search is True: - # We went through all the comments and didn't find anything - # that indicated it was a duplicate - log.warning(' Matching downstream issue %s to upstream issue %s' % - (result.fields.summary, issue.title)) - final_results.append(result) - else: - # Else search returned a linked issue - final_results.append(search) - if not final_results: - # Just return the most updated issue - results_of_query.sort(key=lambda x: datetime.strptime( - x.fields.updated, '%Y-%m-%dT%H:%M:%S.%f+0000')) - final_results.append(results_of_query[0]) - - # Return the final_results - log.debug("Found %i results for query %r", len(final_results), query) - - # Alert the owner - if issue.downstream.get('owner'): - alert_user_of_duplicate_issues(issue, final_results, - results_of_query, - config, client) - return final_results - else: - return results_of_query - - -def alert_user_of_duplicate_issues(issue, final_result, results_of_query, - config, client): - """ - Alerts owner of duplicate downstream issues. - - :param sync2jira.intermediate.Issue issue: Upstream Issue object - :param List final_result: Issue selected by matching algorithm - :param List results_of_query: Result of JQL query - :param Dict config: Config dict - :param jira.client.JIRA client: JIRA client - :returns: Nothing - """ - # First remove final_result from results_of_query - results_of_query.remove(final_result[0]) - - # Check that all duplicate issues are closed - updated_results = [] - for result in results_of_query: - if result.fields.status.name != 'Closed': - updated_results.append(result) - if not updated_results: - # Nothing to alert the owner of - return - - # Get base URL - jira_instance = issue.downstream.get('jira_instance', False) - if not jira_instance: - jira_instance = config['sync2jira'].get('default_jira_instance', False) - if not jira_instance: - log.error(" No jira_instance for issue and there is no default in the config") - raise Exception - base_url = config['sync2jira']['jira'][jira_instance]['options']['server'] + '/browse/' - - # Format the updated results - template_ready = [] - for update in updated_results: - url = base_url + update.key - new_entry = {'url': url, 'title': update.key} - template_ready.append(new_entry) - - # Get owner name and email from Jira - ret = client.search_users(issue.downstream.get('owner')) - if len(ret) > 1: - log.warning(' Found multiple users for username %s' % issue.downstream.get('owner')) - found = False - for person in ret: - if person.key == issue.downstream.get('owner'): - ret = [person] - found = True - break - if not found: - log.warning(' Could not find JIRA user for username %s' % issue.downstream.get('owner')) - if not ret: - message = ' No owner could be found for username %s' % issue.downstream.get('owner') - log.warning(message.strip()) - raise ValueError(message) - user = {'name': ret[0].displayName, 'email': ret[0].emailAddress} +try: + from urllib.parse import urlencode # py3 + string_type = str +except ImportError: + from urllib import urlencode # py2 + import types + string_type = types.StringTypes - # Format selected issue - selected_issue = {'url': base_url + final_result[0].key, - 'title': final_result[0].key} +import requests +from github import Github - # Get admin information - admins = [] - admin_template = [] - for admin in config['sync2jira']['admins']: - ret = client.search_users(list(admin.keys())[0]) - if len(ret) > 1: - log.warning(' Found multiple users for admin %s' % list(admin.keys())[0]) - found = False - for person in ret: - if person.key == issue.downstream.get('owner'): - ret = [person] - found = True - break - if not found: - log.warning(' Could not find JIRA user for admin %s' % list(admin.keys())[0]) - if not ret: - message = ' No admin could be found for username %s' % list(admin.keys())[0] - log.warning(message.strip()) - raise ValueError(message) - admins.append(ret[0].emailAddress) - admin_template.append({'name': ret[0].displayName, 'email': ret[0].emailAddress}) +import sync2jira.intermediary as i - # Create and send email - templateLoader = jinja2.FileSystemLoader(searchpath='sync2jira/') - templateEnv = jinja2.Environment(loader=templateLoader) - template = templateEnv.get_template('email_template.jinja') - html_text = template.render(user=user, - admins=admin_template, - issue=issue, - selected_issue=selected_issue, - duplicate_issues=template_ready) - - # Send mail - send_mail(recipients=[user['email']], - cc=admins, - subject=duplicate_issues_subject, - text=html_text) - log.info(' Alerted %s about %s duplicate issue(s)' % - (user['email'], len(template_ready))) - - -def find_username(issue, config): - """ - Finds JIRA username for an issue object. - - :param sync2jira.intermediary.Issue issue: Issue object - :param Dict config: Config dict - :returns: Username string - :rtype: String - """ - jira_instance = issue.downstream.get('jira_instance', False) - if not jira_instance: - jira_instance = config['sync2jira'].get('default_jira_instance', False) - if not jira_instance: - log.error(" No jira_instance for issue and there is no default in the config") - raise Exception - return config['sync2jira']['jira'][jira_instance]['basic_auth'][0] - - -def check_comments_for_duplicate(client, result, username): - """ - Checks comment of JIRA issue to see if it has been - marked as a duplicate. - - :param jira.client.JIRA client: JIRA client) - :param jira.resource.Issue result: JIRA issue - :param string username: Username of JIRA user - :returns: True if duplicate comment was not found or JIRA issue if \ - we were able to find it - :rtype: Bool or jira.resource.Issue - """ - for comment in client.comments(result): - search = re.search(r'Marking as duplicate of (\w*)-(\d*)', - comment.body) - if search and comment.author.name == username: - issue_id = search.groups()[0] + '-' + search.groups()[1] - return client.issue(issue_id) - return True - - -def _find_comment_in_jira(comment, j_comments): - """ - Helper function to filter out comments that are matching. - - :param Dict comment: Individual comment from upstream - :param List j_comments: Comments from JIRA downstream - :returns: Item/None - :rtype: jira.resource.Comment/None - """ - formatted_comment = _comment_format(comment) - legacy_formatted_comment = _comment_format_legacy(comment) - for item in j_comments: - if item.raw['body'] == legacy_formatted_comment: - # If the comment is in the legacy comment format - # return the item - return item - if str(comment['id']) in item.raw['body']: - # The comment id's match, if they dont have the same body, - # we need to edit the comment - if item.raw['body'] != formatted_comment: - # We need to update the comment - item.update(body=formatted_comment) - log.info(' Updated one comment') - # Now we can just return the item - return item - else: - # Else they are equal and we can return the item - return item - if comment['date_created'] < UPDATE_DATE: - # If the comments date is prior to the update_date - # We should not try to touch the comment - return item - return None +log = logging.getLogger(__name__) -def _comment_matching(g_comments, j_comments): - """ - Function to filter out comments that are matching. - :param List g_comments: Comments from Issue object - :param List j_comments: Comments from JIRA downstream - :returns: Returns a list of comments that are not matching - :rtype: List +def handle_github_message(msg, config): """ - return list( - filter( - lambda x: _find_comment_in_jira(x, j_comments) is None or x['changed'] is not None, - g_comments - ) - ) + Handle GitHub message from FedMsg. - -def _get_existing_jira_issue(client, issue, config): + :param Dict msg: FedMsg Message + :param Dict config: Config File + :returns: Issue object + :rtype: sync2jira.intermediary.Issue """ - Get a jira issue by the linked remote issue. \ - This is the new supported way of doing this. + owner = msg['msg']['repository']['owner']['login'] + repo = msg['msg']['repository']['name'] + upstream = '{owner}/{repo}'.format(owner=owner, repo=repo) + mapped_repos = config['sync2jira']['map']['github'] - :param jira.client.JIRA client: JIRA client - :param sync2jira.intermediary.Issue issue: Issue object - :param Dict config: Config dict - :returns: Returns a list of matching JIRA issues if any are found - :rtype: List - """ - results = _matching_jira_issue_query(client, issue, config) - if results: - return results[0] - else: + if upstream not in mapped_repos: + log.info("%r not in github map: %r", upstream, mapped_repos.keys()) return None - -def _get_existing_jira_issue_legacy(client, issue, config): - """ - This is our old way of matching issues: use the special url field. - This will be phased out and removed in a future release. - - """ - - kwargs = dict(issue.downstream.items()) - kwargs["External issue URL"] = "%s" % issue.url - kwargs = sorted(kwargs.items(), key=operator.itemgetter(0)) - - query = " AND ".join([ - "=".join(["'%s'" % k, "'%s'" % v]) for k, v in kwargs - if v is not None - ]) + " AND (resolution is null OR resolution = Duplicate)" - results = client.search_issues(query) - if results: - return results[0] - else: + _filter = config['sync2jira']\ + .get('filters', {})\ + .get('github', {})\ + .get(upstream, {}) + + for key, expected in _filter.items(): + # special handling for label: we look for it in the list of msg labels + if key == 'labels': + actual = [label['name'] for label in msg['msg']['issue']['labels']] + if expected not in actual: + log.info("Label %s not set on issue", expected) + return None + else: + # direct comparison + actual = msg['msg']['issue'].get(key) + if actual != expected: + log.info("Actual %r %r != expected %r", key, actual, expected) + return None + + if 'pull_request' in msg['msg']['issue']: + log.info("%r is a pull request. Ignoring.", msg['msg']['issue'].get('html_url')) return None + # Initialize Github object so we can get their full name (instead of their username) + # And get comments if needed + github_client = Github(config['sync2jira']['github_token']) -def _attach_link(client, downstream, remote_link): - """ - Attaches the upstream link to the JIRA ticket. - - :param jira.client.JIRA client: JIRA client - :param jira.resources.Issue downstream: Response from creating the JIRA ticket - :param str remote_link: Remote link - :return: downstream: Response from creating the JIRA ticket - :rtype: jira.resources.Issue - """ - log.info(" Attaching tracking link %r to %r", remote_link, downstream.key) - modified_desc = downstream.fields.description + " " - - # This is crazy. Querying for application links requires admin perms which - # we don't have, so duckpunch the client to think it has already made the - # query. - client._applicationlinks = [] # pylint: disable=protected-access - - # Add the link. - client.add_remote_link(downstream.id, remote_link) - - # Finally, after we've added the link we have to edit the issue so that it - # gets re-indexed, otherwise our searches won't work. Also, Handle some - # weird API changes here... - log.debug(" Modifying desc of %r to trigger re-index.", downstream.key) - downstream.update({'description': modified_desc}) - - return downstream - - -def _upgrade_jira_issue(client, downstream, issue, config): - """ - Given an old legacy-style downstream issue... - ...upgrade it to a new-style issue. - Simply mark it with an external-url field value. - """ - log.info(" Upgrading %r %r issue for %r", downstream.key, issue.downstream, issue) - if config['sync2jira']['testing']: - log.info(" Testing flag is true. Skipping actual upgrade.") - return - - # Do it! - remote_link = dict(url=issue.url, title=remote_link_title) - _attach_link(client, downstream, remote_link) - - -def assign_user(client, issue, downstream, remove_all=False): - """ - Attempts to assigns a JIRA issue to the correct - user based on the issue. - - :param jira.client.JIRA client: JIRA Client - :param sync2jira.intermediary.Issue issue: Issue object - :param jira.resources.Issue downstream: JIRA issue object - :param Bool remove_all: Flag to indicate if we should reset the assignees in the JIRA issue - :returns: Nothing - """ - # If removeAll flag, then we need to reset the assignees - if remove_all: - # Update the issue to have no assignees - downstream.update(assignee={'name': ''}) - # Then we're done! And we can go back ! - return - - # JIRA only supports one assignee - # If we have more than one assignee (i.e. from Github) - # assign the issue to the first user (i.e. issue.assignee[0]) - - # First we need to find the user - # Make API call to get a list of users - users = client.search_assignable_users_for_issues( - issue.assignee[0]['fullname'], - project=issue.downstream['project']) - # Loop through the query - for user in users: - if user.displayName == issue.assignee[0]['fullname']: - # Then we can assign the issue to the user - client.assign_issue(downstream.id, user.key) - return - # If there is an owner, assign it to them - if issue.downstream.get('owner'): - client.assign_issue(downstream.id, issue.downstream.get('owner')) - log.warning('Assigned %s to owner: %s' % - (issue.title, issue.downstream.get('owner'))) - return - log.warning('Was not able to assign user %s' % issue.assignee[0]['fullname']) - - -def _change_status(client, downstream, status, issue): - """ - Change status of JIRA issue. - - - :param jira.client.JIRA client: JIRA client - :param jira.resources.Issue downstream: JIRA issue object - :param String status: Title of status to which issue should be move - :param sync2jira.intermediary.Issue issue: Issue object - """ - transitions = client.transitions(downstream) - id = '' - for t in transitions: - if t['name'] and status.upper() == str(t['name']).upper(): - id = int(t['id']) - break - if id: - try: - client.transition_issue(downstream, id) - log.info(' Updated downstream to %s status for issue %s' % (status, issue.title)) - except JIRAError: - log.error(' Updating downstream issue failed for %s: %s' % (status, issue.title)) + # If there are no comments just make an empty array + if msg['msg']['issue']['comments'] == 0: + msg['msg']['issue']['comments'] = [] else: - log.warning(' Could not update JIRA %s for %s' % (status, issue.title)) - - -def _create_jira_issue(client, issue, config): - """ - Create a JIRA issue and adds all relevant - information in the issue to the JIRA issue. - - :param jira.client.JIRA client: JIRA client - :param sync2jira.intermediary.Issue issue: Issue object - :param Dict config: Config dict - :returns: Returns JIRA issue that was created - :rtype: jira.resources.Issue - """ - log.info(" Creating %r issue for %r", issue.downstream, issue) - if config['sync2jira']['testing']: - log.info(" Testing flag is true. Skipping actual creation.") - return - - custom_fields = issue.downstream.get('custom_fields', {}) - default_type = issue.downstream.get('type', "Bug") - # Build the description of the JIRA issue - if 'description' in issue.downstream.get('updates', {}): - description = "Upstream description: {quote}%s{quote}" % issue.content + # We have multiple comments and need to make api call to get them + repo = github_client.get_repo(upstream) + comments = [] + github_issue = repo.get_issue(number=msg['msg']['issue']['number']) + for comment in github_issue.get_comments(): + # First make API call to get the users name + comments.append({ + 'author': comment.user.name or comment.user.login, + 'name': comment.user.login, + 'body': comment.body, + 'id': comment.id, + 'date_created': comment.created_at, + 'changed': None + }) + # Assign the message with the newly formatted comments :) + msg['msg']['issue']['comments'] = comments + + # Search for the user + reporter = github_client.get_user(msg['msg']['issue']['user']['login']) + # Update the reporter field in the message (to match Pagure format) + if reporter.name: + msg['msg']['issue']['user']['fullname'] = reporter.name else: - description = '' - - if issue.reporter: - # Add to the description - description = '[%s] Upstream Reporter: %s \n %s' % ( - issue.id, - issue.reporter['fullname'], - description - ) - - kwargs = dict( - summary=issue.title, - description=description, - issuetype=dict(name="Story" if "RFE" in issue.title else default_type), - ) - if issue.downstream['project']: - kwargs['project'] = dict(key=issue.downstream['project']) - if issue.downstream.get('component'): - # TODO - make this a list in the config - kwargs['components'] = [dict(name=issue.downstream['component'])] - - for key, custom_field in custom_fields.items(): - kwargs[key] = custom_field - - # Add labels if needed - if 'labels' in issue.downstream.keys(): - kwargs['labels'] = issue.downstream['labels'] - - log.info(" Creating issue.") - downstream = client.create_issue(**kwargs) - - # Add QA field if present - if issue.downstream.get('qa-contact', None): - # Fetch all fields - all_fields = client.fields() - # Make a map from field name -> field id - name_map = {field['name']: field['id'] for field in all_fields} - # Try to get and update the custom field - custom_field = name_map.get('QA Contact', None) - if custom_field: - downstream.update({custom_field: issue.downstream.get('qa-contact')}) - - remote_link = dict(url=issue.url, title=remote_link_title) - _attach_link(client, downstream, remote_link) - - default_status = issue.downstream.get('default_status', None) - if default_status is not None: - _change_status(client, downstream, default_status, issue) - - # Update relevant information (i.e. tags, assignees etc.) if the - # User opted in - _update_jira_issue(downstream, issue, client) - - return downstream + msg['msg']['issue']['user']['fullname'] = msg['msg']['issue']['user']['login'] + + # Now do the same thing for the assignees + assignees = [] + for person in msg['msg']['issue']['assignees']: + assignee = github_client.get_user(person['login']) + if assignee.name: + assignees.append({'fullname': assignee.name}) + else: + # Just add their username + assignees.append({'fullname': person['login']}) + # Update the assignee field in the message (to match Pagure format) + msg['msg']['issue']['assignees'] = assignees -def _label_matching(jira_labels, issue_labels): - """ - Filters through jira_labels to ensure no duplicate labels are present and - no jira_labels are removed. + # Update the label field in the message (to match Pagure format) + if msg['msg']['issue']['labels']: + # loop through all the labels on Github and add them + # to the new label list and then reassign the message + new_label = [] + for label in msg['msg']['issue']['labels']: + new_label.append(label['name']) + msg['msg']['issue']['labels'] = new_label - :param List jira_labels: Existing JIRA labels - :param List issue_labels: Upstream labels - :returns: Updated filtered labels - :rtype: List - """ - # We want to get the union of the jira_labels and the issue_labels - # i.e. all the labels in jira_labels and no duplicates from issue_labels - updated_labels = list(set(jira_labels).union(set(issue_labels))) - # Return our labels - return updated_labels + # Update the milestone field in the message (to match Pagure format) + if msg['msg']['issue']['milestone']: + msg['msg']['issue']['milestone'] = msg['msg']['issue']['milestone']['title'] + return i.Issue.from_github(upstream, msg['msg']['issue'], config) -def _update_jira_issue(existing, issue, client): - """ - Updates an existing JIRA issue (i.e. tags, assignee, comments etc). - :param jira.resources.Issue existing: Existing JIRA issue that was found - :param sync2jira.intermediary.Issue issue: Upstream issue we're pulling data from - :param jira.client.JIRA client: JIRA Client - :returns: Nothing +def handle_pagure_message(msg, config): """ - # Start with comments - # Only synchronize comments for listings that op-in - log.info(" Updating information for upstream issue: %s" % issue.title) - - # Get a list of what the user wants to update for the upstream issue - updates = issue.downstream.get('updates', {}) - # Update relevant data if needed - - # If the user has specified nothing - if not updates: - return - - # Only synchronize comments for listings that op-in - if 'comments' in updates: - log.info(" Looking for new comments") - _update_comments(client, existing, issue) - - # Only synchronize tags for listings that op-in - if any('tags' in item for item in updates): - log.info(" Looking for new tags") - _update_tags(updates, existing, issue) - - # Only synchronize fixVersion for listings that op-in - if any('fixVersion' in item for item in updates) and issue.fixVersion: - log.info(" Looking for new fixVersions") - _update_fixVersion(updates, existing, issue) - - # Only synchronize assignee for listings that op-in - if any('assignee' in item for item in updates): - log.info(" Looking for new assignee(s)") - _update_assignee(client, existing, issue, updates) + Handle Pagure message from FedMsg. - # Only synchronize descriptions for listings that op-in - if 'description' in updates: - log.info(" Looking for new description") - _update_description(existing, issue) - - # Only synchronize title for listings that op-in - if 'title' in updates: - # Update the title if needed - if issue.title != existing.fields.summary: - log.info(" Looking for new title") - _update_title(issue, existing) - - # Only synchronize transition (status) for listings that op-in - if any('transition' in item for item in updates): - log.info(" Looking for new transition(s)") - _update_transition(client, existing, issue) - - log.info(' Done updating %s!' % issue.title) - - -def _update_transition(client, existing, issue): + :param Dict msg: FedMsg Message + :param Dict config: Config File + :returns: Issue object + :rtype: sync2jira.intermediary.Issue """ - Helper function to update the transition of a downstream JIRA issue. + upstream = msg['msg']['project']['name'] + ns = msg['msg']['project'].get('namespace') or None + if ns: + upstream = '{ns}/{upstream}'.format(ns=ns, upstream=upstream) + mapped_repos = config['sync2jira']['map']['pagure'] - :param jira.client.JIRA client: JIRA client - :param jira.resource.Issue existing: Existing JIRA issue - :param sync2jira.intermediary.Issue issue: Upstream issue - :returns: Nothing - """ - # Update the issue status in the JIRA description - # Format the status - today = datetime.today() - formatted_status = "[%s] Upstream issue status: %s" % ( - today.strftime("%a %b %y"), issue.status) - new_description = existing.fields.description - # Check if the issue has the issue status line - if "] Upstream issue status:" in existing.fields.description: - # Use pattern matching to find and update the status - new_description = re.sub( - r"\[.*\] Upstream issue status: .*", - formatted_status, - new_description) - else: - # We can just add this line to the very top - new_description = formatted_status + '\n' + new_description - # Now we can update the JIRA issue (always need to update this - # as there is a timestamp involved) - data = {'description': new_description} - existing.update(data) - log.info(' Updated transition') + if upstream not in mapped_repos: + log.info("%r not in pagure map: %r", upstream, mapped_repos.keys()) + return None - # If the user just inputted True, only update the description - # If the user added a custom closed status, attempt to close the - # downstream JIRA ticket + _filter = config['sync2jira']\ + .get('filters', {})\ + .get('pagure', {}) \ + .get(upstream, {}) + + if _filter: + for key, expected in _filter.items(): + # special handling for tag: we look for it in the list of msg tags + if key == 'tags': + actual = msg['msg']['issue'].get('tags', []) + msg['msg'].get('tags', []) + + # Some messages send tags as strings, others as dicts. Handle both. + actual = \ + [tag['name'] for tag in actual if isinstance(tag, dict)] + \ + [tag for tag in actual if isinstance(tag, string_type)] + + intersection = set(actual) & set(expected) + if not intersection: + log.info("None of %r in %r on issue.", expected, actual) + return None + else: + # direct comparison + actual = msg['msg']['issue'].get(key) + if actual != expected: + log.info("Actual %r %r != expected %r", key, actual, expected) + return None - # First get the closed status from the config file + # If this is a dropped issue upstream try: - # For python 3 > - closed_status = list(filter(lambda d: "transition" in d, issue.downstream.get('updates', {})))[0]['transition'] - except ValueError: - # for python 2.7 - closed_status = (filter(lambda d: "transition" in d, issue.downstream.get('updates', {})))[0]['transition'] - if closed_status is not True and issue.status == 'Closed' \ - and existing.fields.status.name.upper() != closed_status.upper(): - # Now we need to update the status of the JIRA issue - # First add a comment indicating the change (in case it doesn't go through) - hyperlink = f"[Upstream issue|{issue.url}]" - comment_body = f"{hyperlink} closed. Attempting transition to {closed_status}." - client.add_comment(existing, comment_body) - # Ensure that closed_status is a valid choice - # Find all possible transactions (i.e. change states) we could `do - _change_status(client, existing, closed_status, issue) - - -def _update_title(issue, existing): - """ - Helper function to sync upstream/downstream title. + if msg['topic'] == 'io.pagure.prod.pagure.issue.drop': + msg['msg']['issue']['status'] = 'Dropped' + except KeyError: + # Otherwise do nothing + pass - :param sync2jira.intermediary.Issue issue: Upstream issue - :param jira.resource.Issue existing: Existing JIRA issue - :returns: Nothing - """ - new_description = existing.fields.description - if not new_description: - new_description = '' + # If this is a tag edit upstream + try: + # Add all updated tags to the tags on the issue + for tag in msg['msg']['tags']: + msg['msg']['issue']['tags'].append(tag) + except KeyError: + # Otherwise do nothing + pass + + # If this is a comment edit + try: + # Add it to the comments on the issue + msg['msg']['issue']['comments'].append(msg['msg']['comment']) + except KeyError: + # Otherwise do nothing + pass - if '] Upstream Reporter:' not in new_description: - # We have to add the issue ID to the description so we can find it again - if '] Upstream issue status:' in new_description: - # We have to use regex to update the upstream reporter - today = datetime.today() - new_description = re.sub( - r'\[[\w\W]*\] Upstream issue status: %s' % issue.status, - '[%s] Upstream issue status: %s\n[%s] Upstream Reporter: %s' - % (today.strftime("%a %b %y - %H:%M"), - issue.status, issue.id, issue.reporter['fullname']), - new_description) - else: - # We can just add it to the top - new_description = '[%s] Upstream Reporter: %s' % \ - (issue.id, issue.reporter['fullname']) + new_description - # Update the description - # Now that we've updated the description (i.e. added - # issue.id) we can delete the link in the description if its still there. - new_description = re.sub( - r'%s' % issue.url, - r'', - new_description - ) + # Format the assignee field to match github (i.e. in a list) + msg['msg']['issue']['assignee'] = [msg['msg']['issue']['assignee']] - # Now we can update the JIRA issue if we need to - if new_description != existing.fields.description: - data = {'description': new_description} - existing.update(data) - log.info(' Updated description') - # Then we can update the title - data = {'summary': issue.title} - existing.update(data) - log.info(' Updated title') + return i.Issue.from_pagure(upstream, msg['msg']['issue'], config) -def _update_comments(client, existing, issue): +def pagure_issues(upstream, config): """ - Helper function to sync comments between existing JIRA issue and upstream issue. + Creates a Generator for all Pagure issues in upstream repo. - :param jira.client.JIRA client: JIRA client - :param jira.resource.Issue existing: Existing JIRA issue - :param sync2jira.intermediary.Issue issue: Upstream issue - :returns: Nothing + :param String upstream: Upstream Repo + :param Dict config: Config Dict + :returns: Pagure Issue object generator + :rtype: sync2jira.intermediary.Issue """ - # First get all existing comments - comments = client.comments(existing) - # Remove any comments that have already been added - comments_d = _comment_matching(issue.comments, comments) - # Loop through the comments that remain - for comment in comments_d: - # Format and add them - comment_body = _comment_format(comment) - client.add_comment(existing, comment_body) - if len(comments_d) > 0: - log.info(" Comments synchronization done on %i comments." % len(comments_d)) - + base = config['sync2jira'].get('pagure_url', 'https://pagure.io') + url = base + '/api/0/' + upstream + '/issues' -def _update_fixVersion(updates, existing, issue): - """ - Helper function to sync comments between existing JIRA issue and upstream issue. + params = config['sync2jira']\ + .get('filters', {})\ + .get('pagure', {}) \ + .get(upstream, {}) - :param List updates: Downstream updates requested by the user - :param jira.resource.Issue existing: Existing JIRA issue - :param sync2jira.intermediary.Issue issue: Upstream issue - :returns: Nothing - """ - fix_version = [] - # If we are not supposed to overwrite JIRA content - try: - # For python 3 > - if not bool(list(filter(lambda d: "fixVersion" in d, updates))[0]['fixVersion']['overwrite']): - # We need to make sure we're not deleting any fixVersions on JIRA - # Get all fixVersions for the issue - for version in existing.fields.fixVersions: - fix_version.append({'name': version.name}) - except ValueError: - # for python 2.7 - if not bool((filter(lambda d: "fixVersion" in d, updates))[0]['fixVersion']['overwrite']): - # We need to make sure we're not deleting any fixVersions on JIRA - # Get all fixVersions for the issue - for version in existing.fields.fixVersions: - fix_version.append({'name': version.name}) + response = requests.get(url, params=params) + if not bool(response): + try: + reason = response.json() + except Exception: + reason = response.text + raise IOError("response: %r %r %r" % (response, reason, response.request.url)) + data = response.json()['issues'] - # Github and Pagure do not allow for multiple fixVersions (milestones) - # But JIRA does, that is why we're looping here. Hopefully one - # Day Github/Pagure will support multiple fixVersions :0 - for version in issue.fixVersion: - if version is not None: - # Update the fixVersion only if it's already not in JIRA - result = filter(lambda v: v['name'] == str(version), fix_version) - # If we have a result skip, if not then add it to fix_version - if not result or not list(result): - fix_version.append({'name': version}) + # Reformat the assignee value so that it is enclosed within an array + # We do this because Github supports multiple assignees, but JIRA doesn't :( + # Hopefully in the future it will support multiple assignees, thus enclosing + # the assignees in a list prepares for that support + for issue in data: + issue['assignee'] = [issue['assignee']] - # We don't want to make an API call if the labels are the same - jira_labels = [] - for label in existing.fields.fixVersions: - jira_labels.append({'name': label.name}) - res = [i for i in jira_labels if i not in fix_version] + \ - [j for j in fix_version if j not in jira_labels] - if res: - data = {'fixVersions': fix_version} - # If the fixVersion is not in JIRA, it will throw an error - try: - existing.update(data) - log.info(' Updated %s fixVersion(s)' % len(fix_version)) - except JIRAError: - log.warning(' Error updating the fixVersion. %s is an invalid fixVersion.' % issue.fixVersion) + issues = (i.Issue.from_pagure(upstream, issue, config) for issue in data) + for issue in issues: + yield issue -def _update_assignee(client, existing, issue, updates): +def github_issues(upstream, config): """ - Helper function update existing JIRA assignee from downstream issue. + Creates a Generator for all GitHub issues in upstream repo. - :param jira.client.JIRA client: JIRA client - :param jira.resource.Issue existing: Existing JIRA issue - :param sync2jira.intermediary.Issue issue: Upstream issue - :param List updates: Downstream updates requested by the user - :returns: Nothing + :param String upstream: Upstream Repo + :param Dict config: Config Dict + :returns: Pagure Issue object generator + :rtype: sync2jira.intermediary.Issue """ - # First check if overwrite is set to True - try: - # For python 3 > - overwrite = bool(list(filter(lambda d: "assignee" in d, updates))[0]['assignee']['overwrite']) - except ValueError: - # for python 2.7 - overwrite = bool((filter(lambda d: "assignee" in d, updates))[0]['assignee']['overwrite']) - - # First check if the issue is already assigned to the same person - update = False - if issue.assignee and issue.assignee[0]: - try: - update = issue.assignee[0]['fullname'] != existing.fields.assignee.displayName - except AttributeError: - update = True - - if not overwrite: - # Only assign if the existing JIRA issue doesn't have an assignee - # And the issue has an assignee - if not existing.fields.assignee and issue.assignee: - if issue.assignee[0]: - # Update the assignee - assign_user(client, issue, existing) - log.info(' Updated assignee') - return + token = config['sync2jira'].get('github_token') + if not token: + headers = {} + log.warning('No github_token found. We will be rate-limited...') else: - # Update the assignee if we have someone to assignee it too - if update: - assign_user(client, issue, existing) - log.info(' Updated assignee') + headers = {'Authorization': 'token ' + token} + + _filter = config['sync2jira']\ + .get('filters', {})\ + .get('github', {})\ + .get(upstream, {}) + + url = 'https://api.github.com/repos/%s/issues' % upstream + if _filter: + url += '?' + urlencode(_filter) + + issues = _get_all_github_issues(url, headers) + + # Initialize Github object so we can get their full name (instead of their username) + # And get comments if needed + github_client = Github(config['sync2jira']['github_token']) + + # We need to format everything to a standard to we can create an issue object + final_issues = [] + for issue in issues: + # Update comments: + # If there are no comments just make an empty array + if issue['comments'] == 0: + issue['comments'] = [] else: - if existing.fields.assignee: - # Else we should remove all assignees - # Set removeAll flag to true - assign_user(client, issue, existing, remove_all=True) - log.info(' Updated assignee') - - -def _update_tags(updates, existing, issue): - """ - Helper function to sync tags between upstream issue and downstream JIRA issue. - - :param List updates: Downstream updates requested by the user - :param jira.resource.Issue existing: Existing JIRA issue - :param sync2jira.intermediary.Issue issue: Upstream issue - :returns: Nothing - """ - # First get all existing tags on the issue - updated_labels = issue.tags - - # Ensure no duplicates if overwrite is set to false - try: - # for python 3 > - if not bool(list(filter(lambda d: "tags" in d, updates))[0]['tags']['overwrite']): - updated_labels = _label_matching(updated_labels, existing.fields.labels) - except ValueError: - # for python 2.7 - if not bool(filter(lambda d: "tags" in d, updates)[0]['tags']['overwrite']): - updated_labels = _label_matching(updated_labels, existing.fields.labels) - - # Ensure that the tags are all valid - updated_labels = verify_tags(updated_labels) - - # Now we can update the JIRA if labels are different - if sorted(updated_labels) != sorted(existing.fields.labels): - data = {'labels': updated_labels} - existing.update(data) - log.info(' Updated %s tag(s)' % len(updated_labels)) - + # We have multiple comments and need to make api call to get them + repo = github_client.get_repo(upstream) + comments = [] + github_issue = repo.get_issue(number=issue['number']) + for comment in github_issue.get_comments(): + # First make API call to get the users name + comments.append({ + 'author': comment.user.name or comment.user.login, + 'name': comment.user.login, + 'body': comment.body, + 'id': comment.id, + 'date_created': comment.created_at, + 'changed': None + }) + # Assign the message with the newly formatted comments :) + issue['comments'] = comments + + # Update reporter: + # Search for the user + reporter = github_client.get_user(issue['user']['login']) + # Update the reporter field in the message (to match Pagure format) + if reporter.name: + issue['user']['fullname'] = reporter.name + else: + # Just assign it to their username + issue['user']['fullname'] = issue['user']['login'] + + # Update assignee(s): + assignees = [] + for person in issue['assignees']: + assignee = github_client.get_user(person['login']) + if assignee.name: + assignees.append({'fullname': assignee.name}) + else: + # Just add their username + assignees.append({'fullname': person['login']}) -def _update_description(existing, issue): - """ - Helper function to sync description between upstream issue and downstream JIRA issue. + # Update the assignee field in the message (to match Pagure format) + issue['assignees'] = assignees - :param jira.resource.Issue existing: Existing JIRA issue - :param sync2jira.intermediary.Issue issue: Upstream issue - :returns: Nothing - """ - new_description = existing.fields.description - if not new_description: - new_description = '' - if 'Upstream description' in new_description: - # If we just need to update the content of the description - new_description = re.sub( - r"Upstream description:(\r\n*|\r*|\n*|.*){quote}((?s).*){quote}", - r"Upstream description: {quote}%s{quote}" % issue.content, - new_description) - elif '] Upstream issue status:' in new_description and '] Upstream Reporter:' in new_description: - # We need to add a description field - today = datetime.today() - new_description = re.sub(r'\[[\w\W]*\] Upstream issue status: %s\n\[%s\] Upstream Reporter: %s' % ( - issue.status, issue.id, issue.reporter['fullname']), - r'[%s] Upstream issue status: %s\n[%s] Upstream Reporter: %s\n' - r'Upstream description: {quote}%s{quote}' % ( - today.strftime("%a %b %y - %H:%M"), issue.status, issue.id, - issue.reporter['fullname'], - issue.content), - new_description) + # Update label(s): + if issue['labels']: + # loop through all the labels on Github and add them + # to the new label list and then reassign the message + new_label = [] + for label in issue['labels']: + new_label.append(label['name']) + issue['labels'] = new_label - elif '] Upstream issue status:' in new_description and '] Upstream Reporter:' not in new_description: - # We need to add a upstream reporter and description field - today = datetime.today() - new_description = re.sub(r'\[[\w\W]*\] Upstream issue status: %s' % issue.status, - r'[%s] Upstream issue status: %s\n' - r'[%s] Upstream Reporter: %s\n' - r'Upstream description: {quote}%s{quote}' % - (today.strftime("%a %b %y - %H:%M"), issue.status, issue.id, - issue.reporter['fullname'], issue.content), - new_description) - elif '] Upstream issue status:' not in new_description and '] Upstream Reporter:' in new_description: - # We need to just add the description field - new_description = re.sub( - r'\[%s\] Upstream Reporter: %s [\w\W]*' % (issue.id, issue.reporter['fullname']), - r'[%s] Upstream Reporter: %s \nUpstream description: {quote} %s {quote}' % - (issue.id, issue.reporter['fullname'], issue.content), new_description) - else: - # Just add reporter and description to the top - upstream_reporter = '[%s] Upstream Reporter: %s' % ( - issue.id, - issue.reporter['fullname'] - ) - upstream_description = "%s \nUpstream description: " \ - "{quote}%s{quote}" % \ - (upstream_reporter, issue.content) - new_description = '%s \n %s' % \ - (upstream_description, new_description) - # Now that we've updated the description (i.e. added - # issue.id) we can delete the link in the description if its still there. - new_description = re.sub( - r'%s' % issue.url, - r'', - new_description - ) + # Update milestone: + if issue['milestone']: + issue['milestone'] = issue['milestone']['title'] - # Now we can update the JIRA issue if we need to - if new_description != existing.fields.description: - data = {'description': new_description} - existing.update(data) - log.info(' Updated description') + final_issues.append(issue) + final_issues = list(( + i.Issue.from_github(upstream, issue, config) for issue in final_issues + if 'pull_request' not in issue # We don't want to copy these around + )) + for issue in final_issues: + yield issue -def verify_tags(tags): - """ - Helper function to ensure tag are JIRA ready :). - :param List tags: Input tags - :returns: Updates tags - :rtype: List - """ - updated_tags = [] - for tag in tags: - updated_tags.append(tag.replace(" ", "_")) - return updated_tags +def _get_all_github_issues(url, headers): + """ Pagination utility. Obnoxious. """ + link = dict(next=url) + while 'next' in link: + response = _fetch_github_data(link['next'], headers) + for issue in response.json(): + comments = _fetch_github_data(issue['comments_url'], headers) + issue['comments'] = comments.json() + yield issue + link = _github_link_field_to_dict(response.headers.get('link', None)) -def sync_with_jira(issue, config): +def _github_link_field_to_dict(field): """ - Attempts to sync a upstream issue with JIRA (i.e. by finding - an existing issue or creating a new one). - - :param sync2jira.intermediary.Issue issue: Issue object - :param Dict config: Config dict - :returns: Nothing + Utility for ripping apart github's Link header field. + It's kind of ugly. """ - log.info(" Considering upstream %s, %s", issue.url, issue.title) - - # Create a client connection for this issue - client = _get_jira_client(issue, config) - - # Check the status of the JIRA client - if not config['sync2jira']['develop'] and not check_jira_status(client): - log.warning(' The JIRA server looks like its down. Shutting down...') - raise JIRAError - - if issue.downstream.get('updates', None): - if issue.source == 'github' and issue.content and \ - 'github_markdown' in issue.downstream['updates']: - issue.content = pypandoc.convert_text(issue.content, 'plain', format='md') + if not field: + return dict() + return dict([ + ( + part.split('; ')[1][5:-1], + part.split('; ')[0][1:-1], + ) for part in field.split(', ') + ]) - # First, check to see if we have a matching issue using the new method. - # If we do, then just bail out. No sync needed. - log.info(" Looking for matching downstream issue via new method.") - existing = _get_existing_jira_issue(client, issue, config) - if existing: - # If we found an existing JIRA issue already - log.info(" Found existing, matching downstream %r.", existing.key) - if config['sync2jira']['testing']: - log.info(" Testing flag is true. Skipping actual update.") - return - # Update relevant metadata (i.e. tags, assignee, etc) - _update_jira_issue(existing, issue, client) - return - - # If we're *not* configured to do legacy matching (upgrade mode) then there - # is nothing left to do than to but to create the issue and return. - if not config['sync2jira'].get('legacy_matching', True): - log.debug(" Legacy matching disabled.") - _create_jira_issue(client, issue, config) - return - - # Otherwise, if we *are* configured to do legacy matching, then try and - # find this issue the old way. - # - If we can't find it, create it. - # - If we can find it, upgrade it to the new method. - log.info(" Looking for matching downstream issue via legacy method.") - match = _get_existing_jira_issue_legacy(client, issue, config) - if not match: - _create_jira_issue(client, issue, config) - else: - _upgrade_jira_issue(client, match, issue, config) - -def _close_as_duplicate(client, duplicate, keeper, config): +def _fetch_github_data(url, headers): """ - Helper function to close an issue as a duplicate. - - :param jira.client client: JIRA Client - :param jira.resources.Issue duplicate: Duplicate JIRA Issue - :param jira.resources.Issue keeper: JIRA issue to keep - :param Dict config: Config dict - :returns: Nothing + Helper function to gather GitHub data """ - log.info(" Closing %s as duplicate of %s", duplicate.permalink(), keeper.permalink()) - if config['sync2jira']['testing']: - log.info(" Testing flag is true. Skipping actual delete.") - return - - # Find the id of some dropped or done state. - transitions = client.transitions(duplicate) - transitions = dict([(t['name'], t['id']) for t in transitions]) - closed = None - preferences = ['Dropped', 'Reject', 'Done', 'Closed', 'Closed (2)', ] - for preference in preferences: - if preference in transitions: - closed = transitions[preference] - break - - text = 'Marking as duplicate of %s' % keeper.key - if any([text in comment.body for comment in client.comments(duplicate)]): - log.info(" Skipping comment. Already present.") - else: - client.add_comment(duplicate, text) - - text = '%s is a duplicate.' % duplicate.key - if any([text in comment.body for comment in client.comments(keeper)]): - log.info(" Skipping comment. Already present.") - else: - client.add_comment(keeper, text) - - if closed: + response = requests.get(url, headers=headers) + if not bool(response): try: - client.transition_issue(duplicate, closed, resolution={'name': 'Duplicate'}) - except Exception as e: - if "Field 'resolution' cannot be set" in e.response.text: - # Try closing without a specific resolution. - try: - client.transition_issue(duplicate, closed) - except Exception: - log.exception("Failed to close %r", duplicate.permalink()) - else: - log.exception("Failed to close %r", duplicate.permalink()) - else: - log.warning(" Unable to find close transition for %r" % duplicate.key) - - -def close_duplicates(issue, config): - """ - Function to close duplicate JIRA issues. - - :param sync2jira.intermediary.Issue issue: Upstream Issue - :param Dict config: Config dict - :returns: Nothing - """ - # Create a client connection for this issue - client = _get_jira_client(issue, config) - - # Check the status of the JIRA client - if not config['sync2jira']['develop'] and not check_jira_status(client): - log.warning(' The JIRA server looks like its down. Shutting down...') - raise JIRAError - - log.info("Looking for dupes of upstream %s, %s", issue.url, issue.title) - results = _matching_jira_issue_query(client, issue, config, free=True) - if len(results) <= 1: - log.info(" No duplicates found.") - return - - results = sorted(results, key=lambda x: arrow.get(x.fields.created)) - keeper, duplicates = results[0], results[1:] - for duplicate in duplicates: - _close_as_duplicate(client, duplicate, keeper, config) + reason = response.json() + except Exception: + reason = response.text + raise IOError("response: %r %r %r" % (response, reason, response.request.url)) + return response \ No newline at end of file