From 7fd15ec300fa54d4c810b84e6a3c6c51082237f9 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Jan 22 2017 11:09:51 +0000 Subject: [PATCH 1/7] A first stab at using remote links for this. --- diff --git a/ansible/files/sync2jira.py b/ansible/files/sync2jira.py index fbdde9b..a57c608 100644 --- a/ansible/files/sync2jira.py +++ b/ansible/files/sync2jira.py @@ -22,8 +22,8 @@ config = { 'initialize': True, 'testing': False, - # Disable old-school mode. - 'legacy_matching': False, + # Temporarily enable old-school mode. + 'legacy_matching': True, # With this, you can specify the custom field key for an "external url" # See https://pagure.io/sync-to-jira/pull-request/3 diff --git a/sync2jira/downstream.py b/sync2jira/downstream.py index d68005c..7444c31 100644 --- a/sync2jira/downstream.py +++ b/sync2jira/downstream.py @@ -18,7 +18,6 @@ # Authors: Ralph Bean import distutils.version -import json import operator import logging @@ -30,32 +29,34 @@ jira_version = distutils.version.LooseVersion(jira.version.__version__) log = logging.getLogger(__name__) +remote_link_title = "upstream issue linked by sync2jira" jira_cache = {} -def get_existing_jira_issues_legacy(downstream, config): - """ This is our old way of matching issues: get all, search by title. - This will be phased out and removed in a future release. - """ - key = json.dumps(downstream) - if not key in jira_cache: - kwargs = sorted(downstream.items(), key=operator.itemgetter(0)) - client = jira.client.JIRA(**config['sync2jira']['jira']) - query = " AND ".join([ - "=".join([k, v]) for k, v in kwargs - if v is not None - ]) + " AND (resolution is null OR resolution = Duplicate)" - results = client.search_issues(query) - # TODO -- handle pagination here... - jira_cache[key] = results - return jira_cache[key] +def get_existing_jira_issue(issue, config): + """ Get a jira issue by the linked remote issue. + This is the new supported way of doing this. + """ + client = jira.client.JIRA(**config['sync2jira']['jira']) + query = ( + 'issueFunction in linkedIssuesOfRemote("%s") AND ' + 'issueFunction in linkedIssuesOfRemote("%s")') % ( + issue.url, remote_link_title) + client = jira.client.JIRA(**config['sync2jira']['jira']) + results = client.search_issues(query) + log.debug("Found %i results for query %r" % (len(results), query)) + if results: + return results[0] + else: + return None -def get_existing_jira_issue(issue, config): - """ This is the supported way of matching issues. - Use the upstream url to uniquely grab individual downstream issues. +def get_existing_jira_issue_legacy(issue, config): + """ This is our old way of matching issues: use the special url field. + + This will be phased out and removed in a future release. """ kwargs = dict(issue.downstream.items()) @@ -86,15 +87,10 @@ def upgrade_jira_issue(downstream, issue, config): return # Do it! - external_url_field = config['sync2jira']['jira_opts']['external_url_field'] - # Handle some weird API changes here... - if jira_version < distutils.version.LooseVersion('0.39'): - # This is the old busted way - # https://github.com/pycontribs/jira/issues/65 - downstream.update(fields=dict(fields={external_url_field: issue.url})) - else: - # This is the new, normal way. - downstream.update({external_url_field: issue.url}) + remote_link = dict(url=issue.url, title=remote_link_title) + log.info("Attaching tracking link %r" % remote_link) + client = jira.client.JIRA(**config['sync2jira']['jira']) + client.add_remote_link(downstream, remote_link) def create_jira_issue(issue, config): @@ -117,7 +113,12 @@ def create_jira_issue(issue, config): external_url_field = config['sync2jira']['jira_opts']['external_url_field'] kwargs[external_url_field] = issue.url - return client.create_issue(**kwargs) + log.info("Creating issue.") + downstream = client.create_issue(**kwargs) + remote_link = dict(url=issue.url, title=remote_link_title) + log.info("Attaching tracking link %r" % remote_link) + client.add_remote_link(downstream, remote_link) + return downstream def sync_with_jira(issue, config): @@ -141,13 +142,8 @@ def sync_with_jira(issue, config): # - If we can't find it, create it. # - If we can find it, upgrade it to the new method. log.info(" Looking for matching downstream issue via legacy method.") - existing_issues = get_existing_jira_issues_legacy(issue.downstream, config) - existing_summaries = [i.fields.summary for i in existing_issues] - if issue.title not in existing_summaries: + match = get_existing_jira_issue_legacy(issue, config) + if not match: create_jira_issue(issue, config) else: - downstream = [ - i for i in existing_issues - if i.fields.summary == issue.title - ][0] - upgrade_jira_issue(downstream, issue, config) + upgrade_jira_issue(match.id, issue, config) diff --git a/sync2jira/main.py b/sync2jira/main.py index 645ec20..897ffcf 100644 --- a/sync2jira/main.py +++ b/sync2jira/main.py @@ -95,6 +95,7 @@ def listen(config): def initialize(config): log.info("Running initialization to sync all issues from upstream to jira") + log.info(" Testing flag is %r" % config['sync2jira']['testing']) mapping = config['sync2jira']['map'] for upstream in mapping.get('pagure', {}).keys(): for issue in u.pagure_issues(upstream, config): diff --git a/tests/test_downstream.py b/tests/test_downstream.py index dd97b3f..b257f9b 100644 --- a/tests/test_downstream.py +++ b/tests/test_downstream.py @@ -20,22 +20,20 @@ class TestDownstream(unittest.TestCase): @mock.patch('jira.client.JIRA') def test_get_existing_legacy(self, client): + class MockIssue(object): + downstream = {'key': 'value'} + url = 'wat' + issue = MockIssue() config = self.config.copy() # Ensure that we get results back from the jira client. target1 = "target1" - client.return_value.search_issues = mock.MagicMock(return_value=target1) - result = d.get_existing_jira_issues_legacy({'key': 'value'}, config) + client.return_value.search_issues = mock.MagicMock(return_value=[target1]) + result = d.get_existing_jira_issue_legacy(issue, config) eq_(result, target1) - # Ensure that caching works. - target2 = "target2" - client.return_value.search_issues.return_value = target2 - result = d.get_existing_jira_issues_legacy({'key': 'value'}, config) - eq_(result, target1) # Really, target1, because caching. - - # Make sure we called search issues really only once client.return_value.search_issues.assert_called_once_with( - 'key=value AND (resolution is null OR resolution = Duplicate)', + "'External issue URL'='wat' AND key=value AND " + "(resolution is null OR resolution = Duplicate)", ) @mock.patch('jira.client.JIRA') @@ -54,8 +52,8 @@ class TestDownstream(unittest.TestCase): eq_(result, target1) client.return_value.search_issues.assert_called_once_with( - '\'External issue URL\'=\'http://threebean.org\' AND key=value AND ' - '(resolution is null OR resolution = Duplicate)', + 'issueFunction in linkedIssuesOfRemote("http://threebean.org") AND ' + 'issueFunction in linkedIssuesOfRemote("upstream issue linked by sync2jira")' ) @mock.patch('jira.client.JIRA') @@ -68,11 +66,14 @@ class TestDownstream(unittest.TestCase): downstream = mock.MagicMock() issue = MockIssue() + client_obj = mock.MagicMock() + client.return_value = client_obj d.upgrade_jira_issue(downstream, issue, config) - - downstream.update.assert_called_once_with( - dict(customfield_10400='http://threebean.org'), - ) + remote = { + 'url': 'http://threebean.org', + 'title': 'upstream issue linked by sync2jira', + } + client_obj.add_remote_link.assert_called_once_with(downstream, remote) @mock.patch('jira.client.JIRA') From 88fa7d1185dc43439e66d58e95c002be40b080de Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Jan 22 2017 17:18:46 +0000 Subject: [PATCH 2/7] Use a tag suffix to identify "our" tasks. --- diff --git a/sync2jira/downstream.py b/sync2jira/downstream.py index 7444c31..6422777 100644 --- a/sync2jira/downstream.py +++ b/sync2jira/downstream.py @@ -29,7 +29,8 @@ jira_version = distutils.version.LooseVersion(jira.version.__version__) log = logging.getLogger(__name__) -remote_link_title = "upstream issue linked by sync2jira" +remote_link_title = "upstream issue" +tag = "#sync2jira" jira_cache = {} @@ -40,13 +41,10 @@ def get_existing_jira_issue(issue, config): This is the new supported way of doing this. """ client = jira.client.JIRA(**config['sync2jira']['jira']) - query = ( - 'issueFunction in linkedIssuesOfRemote("%s") AND ' - 'issueFunction in linkedIssuesOfRemote("%s")') % ( - issue.url, remote_link_title) + query = 'issueFunction in linkedIssuesOfRemote("%s")' % (issue.url + tag) client = jira.client.JIRA(**config['sync2jira']['jira']) results = client.search_issues(query) - log.debug("Found %i results for query %r" % (len(results), query)) + log.info("Found %i results for query %r" % (len(results), query)) if results: return results[0] else: @@ -87,7 +85,7 @@ def upgrade_jira_issue(downstream, issue, config): return # Do it! - remote_link = dict(url=issue.url, title=remote_link_title) + remote_link = dict(url=issue.url + tag, title=remote_link_title) log.info("Attaching tracking link %r" % remote_link) client = jira.client.JIRA(**config['sync2jira']['jira']) client.add_remote_link(downstream, remote_link) @@ -115,7 +113,7 @@ def create_jira_issue(issue, config): log.info("Creating issue.") downstream = client.create_issue(**kwargs) - remote_link = dict(url=issue.url, title=remote_link_title) + remote_link = dict(url=issue.url + tag, title=remote_link_title) log.info("Attaching tracking link %r" % remote_link) client.add_remote_link(downstream, remote_link) return downstream From 28f44bfec4c67ca55f1ec07cf7665cb85b5bfecc Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Jan 22 2017 17:19:02 +0000 Subject: [PATCH 3/7] Disable querying for application link types, which requires admin perms. --- diff --git a/sync2jira/downstream.py b/sync2jira/downstream.py index 6422777..e0fbb30 100644 --- a/sync2jira/downstream.py +++ b/sync2jira/downstream.py @@ -88,6 +88,7 @@ def upgrade_jira_issue(downstream, issue, config): remote_link = dict(url=issue.url + tag, title=remote_link_title) log.info("Attaching tracking link %r" % remote_link) client = jira.client.JIRA(**config['sync2jira']['jira']) + client._applicationlinks = [] # Crazy. client.add_remote_link(downstream, remote_link) From 13c0a41e48c6495b45c8da3acef1669f5e88da3b Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Feb 03 2017 20:28:34 +0000 Subject: [PATCH 4/7] Switch back away from trying to use a tag link. --- diff --git a/sync2jira/downstream.py b/sync2jira/downstream.py index e0fbb30..e404f78 100644 --- a/sync2jira/downstream.py +++ b/sync2jira/downstream.py @@ -29,8 +29,7 @@ jira_version = distutils.version.LooseVersion(jira.version.__version__) log = logging.getLogger(__name__) -remote_link_title = "upstream issue" -tag = "#sync2jira" +remote_link_title = "Upstream issue" jira_cache = {} @@ -41,7 +40,9 @@ def get_existing_jira_issue(issue, config): This is the new supported way of doing this. """ client = jira.client.JIRA(**config['sync2jira']['jira']) - query = 'issueFunction in linkedIssuesOfRemote("%s")' % (issue.url + tag) + query = 'issueFunction in linkedIssuesOfRemote("%s") and ' \ + 'issueFunction in linkedIssuesOfRemote("%s")' % ( + remote_link_title, issue.url) client = jira.client.JIRA(**config['sync2jira']['jira']) results = client.search_issues(query) log.info("Found %i results for query %r" % (len(results), query)) @@ -85,7 +86,11 @@ def upgrade_jira_issue(downstream, issue, config): return # Do it! - remote_link = dict(url=issue.url + tag, title=remote_link_title) + remote_link = dict( + url=issue.url, + title=remote_link_title, + icon={"url16x16": ""}, + ) log.info("Attaching tracking link %r" % remote_link) client = jira.client.JIRA(**config['sync2jira']['jira']) client._applicationlinks = [] # Crazy. @@ -114,8 +119,9 @@ def create_jira_issue(issue, config): log.info("Creating issue.") downstream = client.create_issue(**kwargs) - remote_link = dict(url=issue.url + tag, title=remote_link_title) + remote_link = dict(url=issue.url, title=remote_link_title) log.info("Attaching tracking link %r" % remote_link) + client._applicationlinks = [] # Crazy. client.add_remote_link(downstream, remote_link) return downstream From f997d33ca5207a34d34b51a47549f2abb0503c8e Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Feb 03 2017 21:21:15 +0000 Subject: [PATCH 5/7] Try editing the description after attaching an issue due to caching/reindexing. --- diff --git a/sync2jira/downstream.py b/sync2jira/downstream.py index e404f78..a31dfa1 100644 --- a/sync2jira/downstream.py +++ b/sync2jira/downstream.py @@ -73,6 +73,33 @@ def get_existing_jira_issue_legacy(issue, config): else: return None +def _attach_link(config, downstream, remote_link): + log.info("Attaching tracking link %r" % remote_link) + modified_desc = downstream.description + " " + client = jira.client.JIRA(**config['sync2jira']['jira']) + + # This is crazy. Querying for application links requires admin perms which + # we don't have, so duckpunch the client to think it has already made the + # query. + client._applicationlinks = [] # Crazy. + + # Add the link. + client.add_remote_link(downstream, remote_link) + + # Finally, after we've added the link we have to edit the issue so that it + # gets re-indexed, otherwise our searches won't work. Also, Handle some + # weird API changes here... + log.debug(" Modifying description to trigger re-index.") + if jira_version < distutils.version.LooseVersion('0.39'): + # This is the old busted way + # https://github.com/pycontribs/jira/issues/65 + downstream.update(fields=dict(fields={'description': modified_desc})) + else: + # This is the new, normal way. + downstream.update({'description': modified_desc}) + + return downstream + def upgrade_jira_issue(downstream, issue, config): """ Given an old legacy-style downstream issue... @@ -86,15 +113,8 @@ def upgrade_jira_issue(downstream, issue, config): return # Do it! - remote_link = dict( - url=issue.url, - title=remote_link_title, - icon={"url16x16": ""}, - ) - log.info("Attaching tracking link %r" % remote_link) - client = jira.client.JIRA(**config['sync2jira']['jira']) - client._applicationlinks = [] # Crazy. - client.add_remote_link(downstream, remote_link) + remote_link = dict(url=issue.url, title=remote_link_title) + _attach_link(config, downstream, remote_link) def create_jira_issue(issue, config): @@ -119,10 +139,9 @@ def create_jira_issue(issue, config): log.info("Creating issue.") downstream = client.create_issue(**kwargs) + remote_link = dict(url=issue.url, title=remote_link_title) - log.info("Attaching tracking link %r" % remote_link) - client._applicationlinks = [] # Crazy. - client.add_remote_link(downstream, remote_link) + _attach_link(config, downstream, remote_link) return downstream From 92f99b6935ba62d907d7f18722afe8c3e69d534e Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Feb 10 2017 22:35:02 +0000 Subject: [PATCH 6/7] Remove unused customfield_10400. --- diff --git a/ansible/files/sync2jira.py b/ansible/files/sync2jira.py index a57c608..1fe5978 100644 --- a/ansible/files/sync2jira.py +++ b/ansible/files/sync2jira.py @@ -25,12 +25,6 @@ config = { # Temporarily enable old-school mode. 'legacy_matching': True, - # With this, you can specify the custom field key for an "external url" - # See https://pagure.io/sync-to-jira/pull-request/3 - 'jira_opts': { - 'external_url_field': 'customfield_10400', - }, - #'jira': { # See /etc/fedmsg.d/sync2jira-credentials.py }, 'map': { 'pagure': { diff --git a/fedmsg.d/sync2jira.py b/fedmsg.d/sync2jira.py index 0de012d..53feba4 100644 --- a/fedmsg.d/sync2jira.py +++ b/fedmsg.d/sync2jira.py @@ -22,12 +22,6 @@ config = { 'initialize': True, 'testing': True, - # With this, you must specify the custom field key for an "external url" - # See https://pagure.io/sync-to-jira/pull-request/3 - 'jira_opts': { - 'external_url_field': 'customfield_10400', - }, - 'jira': { 'options': { 'server': 'https://some_jira_server_somewhere.com', diff --git a/sync2jira/downstream.py b/sync2jira/downstream.py index a31dfa1..8fda87e 100644 --- a/sync2jira/downstream.py +++ b/sync2jira/downstream.py @@ -134,9 +134,6 @@ def create_jira_issue(issue, config): if issue.downstream['component']: kwargs['components'] = [dict(name=issue.downstream['component'])] # TODO - make this a list in the config - external_url_field = config['sync2jira']['jira_opts']['external_url_field'] - kwargs[external_url_field] = issue.url - log.info("Creating issue.") downstream = client.create_issue(**kwargs) diff --git a/tests/test_downstream.py b/tests/test_downstream.py index b257f9b..10ff35a 100644 --- a/tests/test_downstream.py +++ b/tests/test_downstream.py @@ -12,9 +12,6 @@ class TestDownstream(unittest.TestCase): 'jira': { # Nothing, really.. }, - 'jira_opts': { - 'external_url_field': 'url_field', - }, }, } @@ -117,7 +114,6 @@ class TestDownstream(unittest.TestCase): url = 'http://threebean.org' config['sync2jira']['testing'] = False - config['sync2jira']['jira_opts'] = dict(external_url_field='customfield_10400') result = d.create_jira_issue(MockIssue(), config) eq_(result, target1) client.return_value.create_issue.assert_called_with( @@ -126,5 +122,4 @@ class TestDownstream(unittest.TestCase): project={'key': 'awesome'}, description='http://threebean.org', summary='A title, a title...', - customfield_10400='http://threebean.org', ) From 20a99e58a27741eec250d832ea24375173112038 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Feb 10 2017 22:35:39 +0000 Subject: [PATCH 7/7] Cleanup after getting upgrades working. --- diff --git a/sync2jira/downstream.py b/sync2jira/downstream.py index 8fda87e..e62f2d2 100644 --- a/sync2jira/downstream.py +++ b/sync2jira/downstream.py @@ -74,8 +74,9 @@ def get_existing_jira_issue_legacy(issue, config): return None def _attach_link(config, downstream, remote_link): - log.info("Attaching tracking link %r" % remote_link) - modified_desc = downstream.description + " " + log.info(" Attaching tracking link %r to %r" % ( + remote_link, downstream.key)) + modified_desc = downstream.fields.description + " " client = jira.client.JIRA(**config['sync2jira']['jira']) # This is crazy. Querying for application links requires admin perms which @@ -84,12 +85,12 @@ def _attach_link(config, downstream, remote_link): client._applicationlinks = [] # Crazy. # Add the link. - client.add_remote_link(downstream, remote_link) + client.add_remote_link(downstream.id, remote_link) # Finally, after we've added the link we have to edit the issue so that it # gets re-indexed, otherwise our searches won't work. Also, Handle some # weird API changes here... - log.debug(" Modifying description to trigger re-index.") + log.debug(" Modifying desc of %r to trigger re-index." % downstream.key) if jira_version < distutils.version.LooseVersion('0.39'): # This is the old busted way # https://github.com/pycontribs/jira/issues/65 @@ -107,7 +108,8 @@ def upgrade_jira_issue(downstream, issue, config): Simply mark it with an external-url field value. """ - log.info(" Upgrading %r issue for %r" % (issue.downstream, issue)) + log.info(" Upgrading %r %r issue for %r" % ( + downstream.key, issue.downstream, issue)) if config['sync2jira']['testing']: log.info(" Testing flag is true. Skipping actual upgrade.") return @@ -147,8 +149,9 @@ def sync_with_jira(issue, config): # First, check to see if we have a matching issue using the new method. # If we do, then just bail out. No sync needed. - if get_existing_jira_issue(issue, config): - log.info(" Found existing, matching issue downstream.") + existing = get_existing_jira_issue(issue, config) + if existing: + log.info(" Found existing, matching downstream %r." % existing.key) return # If we're *not* configured to do legacy matching (upgrade mode) then there @@ -167,4 +170,4 @@ def sync_with_jira(issue, config): if not match: create_jira_issue(issue, config) else: - upgrade_jira_issue(match.id, issue, config) + upgrade_jira_issue(match, issue, config) diff --git a/tests/test_downstream.py b/tests/test_downstream.py index 10ff35a..f74717b 100644 --- a/tests/test_downstream.py +++ b/tests/test_downstream.py @@ -49,8 +49,8 @@ class TestDownstream(unittest.TestCase): eq_(result, target1) client.return_value.search_issues.assert_called_once_with( - 'issueFunction in linkedIssuesOfRemote("http://threebean.org") AND ' - 'issueFunction in linkedIssuesOfRemote("upstream issue linked by sync2jira")' + 'issueFunction in linkedIssuesOfRemote("Upstream issue") and ' + 'issueFunction in linkedIssuesOfRemote("http://threebean.org")' ) @mock.patch('jira.client.JIRA') @@ -68,15 +68,18 @@ class TestDownstream(unittest.TestCase): d.upgrade_jira_issue(downstream, issue, config) remote = { 'url': 'http://threebean.org', - 'title': 'upstream issue linked by sync2jira', + 'title': 'Upstream issue', } - client_obj.add_remote_link.assert_called_once_with(downstream, remote) + client_obj.add_remote_link.assert_called_once_with(downstream.id, remote) @mock.patch('jira.client.JIRA') def test_create_jira_issue(self, client): config = self.config.copy() - target1 = "target1" + target1 = mock.Mock() + target1.id = "target1 id" + target1.key = "target key" + target1.fields.description = "description" client.return_value.create_issue = mock.MagicMock(return_value=target1) class MockIssue(object): @@ -96,13 +99,15 @@ class TestDownstream(unittest.TestCase): project={'key': 'awesome'}, description='http://threebean.org', summary='A title, a title...', - url_field='http://threebean.org', ) @mock.patch('jira.client.JIRA') def test_create_jira_issue_with_custom_url(self, client): config = self.config.copy() - target1 = "target1" + target1 = mock.Mock() + target1.id = "target1 id" + target1.key = "target key" + target1.fields.description = "description" client.return_value.create_issue = mock.MagicMock(return_value=target1) class MockIssue(object):