From 3c50caa886a58a042b11891f19acabddedac4b2a Mon Sep 17 00:00:00 2001 From: Bohdan Iakymets Date: Jul 09 2019 15:48:22 +0000 Subject: Add more sync features * Comment Edits * Tags (Labels) * Fix Version (Milestone) * Assignee * Description * Reporter (In JIRA description) * Closed/Open status * Auto transitions --- diff --git a/.gitignore b/.gitignore index 3a95df6..ef75c1b 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,10 @@ fedmsg.d/zz_sync2jira.py .eggs .coverage .tox +.idea dist ansible/deploy.retry rpmbuild-output +.idea/* +venv/* +venv3/* diff --git a/README.rst b/README.rst index 51c2706..74ae335 100644 --- a/README.rst +++ b/README.rst @@ -19,6 +19,35 @@ scraped and added to Jira if they are absent. If the ``testing`` option is set to ``True``, then the script will perform a "dry run" and not actually add any new issues to Jira. +What To Sync +____________ + +Each project is accompanied by an 'updates' array as seen below:: + + 'Demo_project': {'project': 'PROJECT', 'component': 'COMP', + 'updates': [...], 'owner': 'project_owner_username'}, + + +The following can be added to the updates array to specify what to sync with downstream +JIRA issues:: + + 'comments' :: Sync comments and comment edits + {'tags': {'overwrite': True/False}} :: Sync tags, do/don't overwrite downstream tags + {'fixVersion'; {'overwrite': True/False}} :: Sync fixVersion (downstream milestone), + do/don't overwrite downstream fixVersion + 'assignee' :: Sync assignee (for Github only the first assignee will sync) + 'description' :: Sync description + 'title' :: Sync title + {'transition': True/'CUSTOM_TRANSITION'} :: Sync status (open/closed), Sync only status/ + Attempt to transition JIRA ticket to + CUSTOM_TRANSITION on upstream closure + +Note: Overwrite set to True will ensure that upstream issue fields will clear downstream +issue fields, overwrite set to False will never delete downstream issue fields only append. + +The optional owner field can be used to specify a username that should be used if +the program cannot find a matching downstream user to assigne an issue too. + Development ----------- diff --git a/fedmsg.d/sync2jira.py b/fedmsg.d/sync2jira.py index 8ab7fe2..3c81ee7 100644 --- a/fedmsg.d/sync2jira.py +++ b/fedmsg.d/sync2jira.py @@ -28,37 +28,38 @@ config = { # Don't actually make changes to JIRA... 'testing': True, + # Your Github token + 'github_token': 'YOUR_TOKEN', + 'legacy_matching': False, - #'default_jira_instance': 'example', - #'jira': { - # 'example': { - # 'options': { - # 'server': 'https://some_jira_server_somewhere.com', - # 'verify': True, - # }, - # 'basic_auth': ('YOUR_USERNAME', 'YOUR_PASSWORD'), - # }, - #}, + 'default_jira_instance': 'example', + 'jira': { + 'example': { + 'options': { + 'server': 'https://some_jira_server_somewhere.com', + 'verify': True, + }, + 'basic_auth': ('YOU_USERNAME', 'YOUR_PASSWORD'), + }, + }, 'map': { 'pagure': { - 'pungi': { 'project': 'COMPOSE', 'component': 'Pungi', }, - #'koji': { 'project': 'BREW', 'component': None, }, + 'Demo_project': {'project': 'FACTORY', 'component': 'gitbz', + 'updates': [...]}, + # 'koji': { 'project': 'BREW', 'component': None, }, }, 'github': { - 'fedora-infra/bodhi': { 'project': 'FACTORY', 'component': None, }, - #'product-definition-center/product-definition-center': { - # 'project': 'PDC', 'component': 'General', }, - #'product-definition-center/pdc-client': { - # 'project': 'PDC', 'component': 'General', }, + 'GITHUB_USERNAME/Demo_project': {'project': 'FACTORY', 'component': 'gitbz', + 'updates': [...]}, }, }, 'filters': { 'github': { # Only sync multi-type tickets from bodhi. - 'fedora-infra/bodhi': { 'state': 'open', 'milestone': 4, }, + 'fedora-infra/bodhi': {'state': 'open', 'milestone': 4, }, }, - }, + } }, } diff --git a/openshift/sync2jira-dev-pipeline-template.yml b/openshift/sync2jira-dev-pipeline-template.yml index e1e39b6..201dbbf 100644 --- a/openshift/sync2jira-dev-pipeline-template.yml +++ b/openshift/sync2jira-dev-pipeline-template.yml @@ -221,11 +221,11 @@ objects: stage('Run unit tests') { steps { // run unit tests - sh 'nosetests-3 -v --with-xunit --xunit-file=xunit-tests.xml' + sh 'tox --result-json=xunit-tests.json' } post { always { - junit 'xunit-tests.xml' + junit 'xunit-tests.json' } } } diff --git a/requirements.txt b/requirements.txt index 76faa48..4ce4996 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,5 @@ jira -requests +requests==2.14.0 fedmsg +PyGithub +urllib3 \ No newline at end of file diff --git a/sync2jira/downstream.py b/sync2jira/downstream.py index 4167650..fe4bbe6 100644 --- a/sync2jira/downstream.py +++ b/sync2jira/downstream.py @@ -20,12 +20,20 @@ import operator import logging +import re import arrow import jira.client +from jira import JIRAError +from datetime import datetime from sync2jira.intermediary import Issue +# The date the service was upgraded +# This is used to ensure legacy comments are not touched +# UPDATE_DATE = datetime.(2019, 0o6, 23) +UPDATE_DATE = datetime(2019, 7, 8, 11, 37, 26, 312400) + log = logging.getLogger(__name__) remote_link_title = "Upstream issue" @@ -34,12 +42,43 @@ jira_cache = {} def _comment_format(comment): + """ + Function to format JIRA comments + Args: + comment (dict): Upstream comment + Returns: + Response (str): Comments formatted + """ + pretty_date = comment['date_created'].strftime("%a %b %y - %H:%M") + + return "[%s] Upstream, %s wrote [%s]:\n\n{quote}\n%s\n{quote}" % ( + comment['id'], comment['author'], + pretty_date, comment['body']) + + +def _comment_format_legacy(comment): + """ + Legacy function to format JIRA comments. + This is still used to match comments so no + duplicates are created + Args: + comment (dict): Upstream comment + Returns: + Response (str): Comments formatted + """ return "Upstream, %s wrote:\n\n{quote}\n%s\n{quote}" % ( - comment['author'], comment['body']) + comment['name'], comment['body']) def _get_jira_client(issue, config): - + """ + Function to match and create JIRA client + Args: + issue (sync2jira.intermediary.Issue): Issue object + config (dict): Config dict + Returns: + client (jira.client.JIRA): Matching JIRA client + """ # The name of the jira instance to use is stored under the 'map' # key in the config where each upstream is mapped to jira projects. # It is conveniently added to the Issue object from intermediary.py @@ -48,7 +87,7 @@ def _get_jira_client(issue, config): if not isinstance(issue, Issue): log.error("passed in issue is not an Issue instance") log.error("It is a %s" % (type(issue).__name__)) - raise + raise TypeError("Got %s, expected Issue" % type(issue).__name__) # Use the Jira instance set in the issue config. If none then # use the configured default jira instance. @@ -56,33 +95,96 @@ def _get_jira_client(issue, config): if not jira_instance: jira_instance = config['sync2jira'].get('default_jira_instance', False) if not jira_instance: - log.error("No jira_instance for issue and there is no default in the config") - raise + log.error(" No jira_instance for issue and there is no default in the config") + raise Exception client = jira.client.JIRA(**config['sync2jira']['jira'][jira_instance]) return client def _matching_jira_issue_query(client, issue, config, free=False): + """ + API calls that find matching JIRA tickets if any are present + Args: + client (jira.client.JIRA): JIRA client + issue (sync2jira.intermediary.Issue): Issue object + config (dict): Config dict + free (Bool): Free tag to add 'statusCategory != Done' to query + Returns: + results (lst): Returns a list of matching JIRA issues if any are found + """ + # Searches for any remote link to the issue.url query = 'issueFunction in linkedIssuesOfRemote("%s") and ' \ 'issueFunction in linkedIssuesOfRemote("%s")' % ( remote_link_title, issue.url) if free: query += ' and statusCategory != Done' - results = client.search_issues(query) - log.debug("Found %i results for query %r", len(results), query) - return results + # Query the JIRA client and store the results + results_of_query = client.search_issues(query) + if len(results_of_query) > 1: + # Sometimes if an issue gets dropped it is created with the url: pagure.com/something/issue/5 + # Then when that issue is dropped and another one is created is is created with the same + # url : pagure.com/something/issue/5. + # We need to ensure that we are not catching a dropped issue + # Loop through the results of the query and make sure the ids + final_results = [] + for result in results_of_query: + # If the queried JIRA issue has the id of the upstream issue or the same title + if issue.id in result.fields.description or issue.title == result.fields.summary: + # Add it to the final results + final_results.append(result) + + # Return the final_results + log.debug("Found %i results for query %r", len(final_results), query) + return final_results + else: + return results_of_query def _find_comment_in_jira(comment, j_comments): - formated_comment = _comment_format(comment) + """ + Helper function to filter out comments that are matching + Args: + comment (dict): Individual comment from upstream + j_comments (lst(jira.resources.Comment)): Comments from JIRA downstream + Returns: + Item/None (jira.resource.Comment) + """ + formatted_comment = _comment_format(comment) + legacy_formatted_comment = _comment_format_legacy(comment) for item in j_comments: - if item.raw['body'] == formated_comment: + if item.raw['body'] == legacy_formatted_comment: + # If the comment is in the legacy comment format + # return the item + return item + if str(comment['id']) in item.raw['body']: + # The comment id's match, if they dont have the same body, + # we need to edit the comment + if item.raw['body'] != formatted_comment: + # We need to update the comment + item.update(body=formatted_comment) + log.info(' Updated one comment') + # Now we can just return the item + return item + else: + # Else they are equal and we can return the item + return item + if comment['date_created'] < UPDATE_DATE: + # If the comments date is prior to the update_date + # We should not try to touch the comment return item return None def _comment_matching(g_comments, j_comments): + """ + Function to filter out comments that are matching + Args: + g_comments (lst(dict)): Comments from Issue object + j_comments (lst(jira.resources.Comment)): Comments from JIRA downstream + Returns: + response (lst): Returns a list of comments that are not matching + """ return list( filter( lambda x: _find_comment_in_jira(x, j_comments) is None or x['changed'] is not None, @@ -92,9 +194,15 @@ def _comment_matching(g_comments, j_comments): def _get_existing_jira_issue(client, issue, config): - """ Get a jira issue by the linked remote issue. - + """ + Get a jira issue by the linked remote issue. This is the new supported way of doing this. + Args: + client (jira.client.JIRA): JIRA client + issue (sync2jira.intermediary.Issue): Issue object + config (dict): Config dict + Returns: + response (lst): Returns a list of matching JIRA issues if any are found """ results = _matching_jira_issue_query(client, issue, config) if results: @@ -125,7 +233,20 @@ def _get_existing_jira_issue_legacy(client, issue, config): def _attach_link(client, downstream, remote_link): - log.info(" Attaching tracking link %r to %r", remote_link, downstream.key) + """ + Attaches the upstream link to the JIRA ticket + Args: + client (jira.client.JIRA): JIRA client + downstream (jira.resources.Issue): Response from + creating the + JIRA ticket + remote_link(): + Returns: + downstream (jira.resources.Issue): Response from + creating the + JIRA ticket + """ + log.info(" Attaching tracking link %r to %r", remote_link, downstream.key) modified_desc = downstream.fields.description + " " # This is crazy. Querying for application links requires admin perms which @@ -161,17 +282,86 @@ def _upgrade_jira_issue(client, downstream, issue, config): _attach_link(client, downstream, remote_link) +def assign_user(client, issue, downstream, remove_all=False): + """ + Attempts to assigns a JIRA issue to the correct + user based on the issue + Args: + client (jira.client.JIRA): JIRA Client + issue (sync2jira.intermediary.Issue): Issue object + downstream (jira.resources.Issue): JIRA issue object + remove_all (bool): Flag to indicate if we should reset + the assignees in the JIRA issue + Returns: + Nothing + """ + # If removeAll flag, then we need to reset the assignees + if remove_all: + # Update the issue to have no assignees + downstream.update(assignee={'name': ''}) + # Then we're done! And we can go back ! + return + + # JIRA only supports one assignee + # If we have more than one assignee (i.e. from Github) + # assign the issue to the first user (i.e. issue.assignee[0]) + + # First we need to find the user + # Make API call to get a list of users + users = client.search_assignable_users_for_issues( + issue.assignee[0]['fullname'], + project=issue.downstream['project']) + # Loop through the query + for user in users: + if user.displayName == issue.assignee[0]['fullname']: + # Then we can assign the issue to the user + client.assign_issue(downstream.id, user.key) + return + # If there is an owner, assign it to them + if issue.downstream.get('owner'): + client.assign_issue(downstream.id, issue.downstream.get('owner')) + log.warning('Assigned %s to owner: %s' % + (issue.title, issue.downstream.get('owner'))) + return + log.warning('Was not able to assign user %s' % issue.assignee[0]['fullname']) + + def _create_jira_issue(client, issue, config): - log.info(" Creating %r issue for %r", issue.downstream, issue) + """ + Create a JIRA issue and adds all relevant + information in the issue to the JIRA issue + Args: + client (jira.client.JIRA): JIRA client + issue (sync2jira.intermediary.Issue): Issue object + config (dict): Config dict + Returns: + downstream (jira.resources.Issue): Returns JIRA issue + that was created + """ + log.info(" Creating %r issue for %r", issue.downstream, issue) if config['sync2jira']['testing']: log.info(" Testing flag is true. Skipping actual creation.") return custom_fields = issue.downstream.get('custom_fields', {}) default_type = issue.downstream.get('type', "Bug") + # Build the description of the JIRA issue + if 'description' in issue.downstream.get('updates'): + description = "Upstream description: {quote}%s{quote}" % issue.content + else: + description = '' + + if issue.reporter: + # Add to the description + description = '[%s] Upstream Reporter: %s \n %s' % ( + issue.id, + issue.reporter['fullname'], + description + ) + kwargs = dict( summary=issue.title, - description=issue.url, + description=description, issuetype=dict(name="Story" if "RFE" in issue.title else default_type), ) if issue.downstream['project']: @@ -179,30 +369,427 @@ def _create_jira_issue(client, issue, config): if issue.downstream.get('component'): # TODO - make this a list in the config kwargs['components'] = [dict(name=issue.downstream['component'])] - if issue.downstream.get('labels'): - labels = issue.downstream['labels'] - if not isinstance(labels, list): - labels = [labels] - kwargs['labels'] = labels for key, custom_field in custom_fields.items(): kwargs[key] = custom_field - log.info("Creating issue.") + log.info(" Creating issue.") downstream = client.create_issue(**kwargs) remote_link = dict(url=issue.url, title=remote_link_title) _attach_link(client, downstream, remote_link) - # Only synchronize comments for listings that op-in - if issue.downstream.get('comments'): - log.info(" Adding %i comments." % len(issue.comments)) - for comment in issue.comments: - client.add_comment(downstream, _comment_format(comment)) + # Update relevant information (i.e. tags, assignees etc.) if the + # User opted in + _update_jira_issue(downstream, issue, client) return downstream +def _label_matching(jira_labels, issue_labels): + """ + Filters through jira_labels to ensure no duplicate labels are present and + no jira_labels are removed + Args: + jira_labels (lst(str)): Existing JIRA labels + issue_labels (lst(str): Upstream labels + Returns: + updated_labels (lst(str)): Updated filtered labels + """ + # We want to get the union of the jira_labels and the issue_labels + # i.e. all the labels in jira_labels and no duplicates from issue_labels + updated_labels = list(set(jira_labels).union(set(issue_labels))) + # Return our labels + return updated_labels + + +def _update_jira_issue(existing, issue, client): + """ + Updates an existing JIRA issue (i.e. tags, assignee, comments etc) + Args: + existing (jira.resources.Issue): Existing JIRA issue that was found + issue (sync2jira.intermediary.Issue): Upstream issue we're pulling data from + client (jira.client.JIRA): JIRA Client + Returns: + Nothing + """ + # Start with comments + # Only synchronize comments for listings that op-in + log.info(" Updating information for upstream issue: %s" % issue.title) + + # Get a list of what the user wants to update for the upstream issue + updates = issue.downstream.get('updates') + # Update relevant data if needed + + # If the user has specified nothing + if not updates: + return + + # Only synchronize comments for listings that op-in + if 'comments' in updates: + log.info(" Looking for new comments") + _update_comments(client, existing, issue) + + # Only synchronize tags for listings that op-in + if any('tags' in item for item in updates): + log.info(" Looking for new tags") + _update_tags(updates, existing, issue) + + # Only synchronize fixVersion for listings that op-in + if any('fixVersion' in item for item in updates) and issue.fixVersion: + log.info(" Looking for new fixVersions") + _update_fixVersion(updates, existing, issue) + + # Only synchronize assignee for listings that op-in + if 'assignee' in updates: + log.info(" Looking for new assignee(s)") + _update_assignee(client, existing, issue) + + # Only synchronize descriptions for listings that op-in + if 'description' in updates: + log.info(" Looking for new description") + _update_description(existing, issue) + + # Only synchronize title for listings that op-in + if 'title' in updates: + # Update the title if needed + if issue.title != existing.fields.summary: + data = {'summary': issue.title} + existing.update(data) + log.info(' Updated title') + + # Only synchronize transition (status) for listings that op-in + if any('transition' in item for item in updates): + log.info(" Looking for new transition(s)") + _update_transition(client, existing, issue) + + log.info(' Done updating %s!' % issue.title) + + +def _update_transition(client, existing, issue): + """ + Helper function to update the transition of a downstream JIRA issue + Args: + client (jira.client.JIRA): JIRA client + existing (jira.resource.Issue): Existing JIRA issue + issue (sync2jira.intermediary.Issue): Upstream issue + Returns: + Nothing + """ + # Update the issue status in the JIRA description + # Format the status + today = datetime.today() + formatted_status = "[%s] Upstream issue status: %s" % ( + today.strftime("%a %b %y - %H:%M"), issue.status) + new_description = existing.fields.description + # Check if the issue has the issue status line + if "] Upstream issue status:" in existing.fields.description: + # Use pattern matching to find and update the status + new_description = re.sub( + r"\[.*\] Upstream issue status: .*", + formatted_status, + new_description) + else: + # We can just add this line to the very top + new_description = formatted_status + '\n' + new_description + # Now we can update the JIRA issue (always need to update this + # as there is a timestamp involved) + data = {'description': new_description} + existing.update(data) + log.info(' Updated transition') + + # If the user just inputted True, only update the description + # If the user added a custom closed status, attempt to close the + # downstream JIRA ticket + + # First get the closed status from the config file + try: + # For python 3 > + closed_status = list(filter(lambda d: "transition" in d, issue.downstream.get('updates')))[0]['transition'] + except ValueError: + # for python 2.7 + closed_status = (filter(lambda d: "transition" in d, issue.downstream.get('updates')))[0]['transition'] + + if closed_status is not True and issue.status == 'Closed' \ + and existing.fields.status.name.upper() != closed_status.upper(): + # Now we need to update the status of the JIRA issue\ + # Ensure that closed_status is a valid choice + # Find all possible transactions (i.e. change states) we could `do + transactions = client.transitions(existing) + found = False + for t in transactions: + if t['name'] and str(t['name']).upper() == closed_status.upper(): + # We can perform this transition (i.e. change state) + try: + client.transition_issue(existing, int(t['id'])) + log.info(' Updated downstream closed status for issue %s' % issue.title) + except JIRAError: + log.error(' Updating downstream issue failed for closed_status: %s' % closed_status) + found = True + if not found: + log.warning(' Could not update JIRA closed_status for %s' % issue.title) + + +def _update_comments(client, existing, issue): + """ + Helper function to sync comments between existing JIRA issue and upstream issue + Args: + client (jira.client.JIRA): JIRA client + existing (jira.resource.Issue): Existing JIRA issue + issue (sync2jira.intermediary.Issue): Upstream issue + Returns: + Nothing + """ + # First get all existing comments + comments = client.comments(existing) + # Remove any comments that have already been added + comments_d = _comment_matching(issue.comments, comments) + # Loop through the comments that remain + for comment in comments_d: + # Format and add them + comment_body = _comment_format(comment) + client.add_comment(existing, comment_body) + if len(comments_d) > 0: + log.info(" Comments synchronization done on %i comments." % len(comments_d)) + + +def _update_fixVersion(updates, existing, issue): + """ + Helper function to sync comments between existing JIRA issue and upstream issue + Args: + updates (list): Downstream updates requested by the user + existing (jira.resource.Issue): Existing JIRA issue + issue (sync2jira.intermediary.Issue): Upstream issue + Returns: + Nothing + """ + fix_version = [] + # If we are not supposed to overwrite JIRA content + try: + # For python 3 > + if not bool(list(filter(lambda d: "fixVersion" in d, updates))[0]['fixVersion']['overwrite']): + # We need to make sure we're not deleting any fixVersions on JIRA + # Get all fixVersions for the issue + for version in existing.fields.fixVersions: + fix_version.append({'name': version.name}) + except ValueError: + # for python 2.7 + if not bool((filter(lambda d: "fixVersion" in d, updates))[0]['fixVersion']['overwrite']): + # We need to make sure we're not deleting any fixVersions on JIRA + # Get all fixVersions for the issue + for version in existing.fields.fixVersions: + fix_version.append({'name': version.name}) + + # Github and Pagure do not allow for multiple fixVersions (milestones) + # But JIRA does, that is why we're looping here. Hopefully one + # Day Github/Pagure will support multiple fixVersions :0 + for version in issue.fixVersion: + if version is not None: + # Update the fixVersion only if it's already not in JIRA + result = filter(lambda v: v['name'] == str(version), fix_version) + # If we have a result skip, if not then add it to fix_version + if not result or not list(result): + fix_version.append({'name': version}) + + # We don't want to make an API call if the labels are the same + jira_labels = [] + for label in existing.fields.fixVersions: + jira_labels.append({'name': label.name}) + res = [i for i in jira_labels if i not in fix_version] + \ + [j for j in fix_version if j not in jira_labels] + if res: + data = {'fixVersions': fix_version} + # If the fixVersion is not in JIRA, it will throw an error + try: + existing.update(data) + log.info(' Updated %s fixVersion(s)' % len(fix_version)) + except JIRAError: + log.warning(' Error updating the fixVersion. %s is an invalid fixVersion.' % issue.fixVersion) + + +def _update_assignee(client, existing, issue): + """ + Helper function update existing JIRA assignee from downstream issue + Args: + client (jira.client.JIRA): JIRA client + existing (jira.resource.Issue): Existing JIRA issue + issue (sync2jira.intermediary.Issue): Upstream issue + Returns: + Nothing + """ + # Update the assignee + if issue.assignee and issue.assignee[0]: + # Check if the issue is already assigned to reduce API calls + try: + update = issue.assignee[0]['fullname'] != existing.fields.assignee.displayName + except AttributeError: + update = True + if update: + # If we have someone to assign it too + assign_user(client, issue, existing) + log.info(' Updated assignee') + else: + if existing.fields.assignee: + # Else we should remove all assignees + # Set removeAll flag to true + assign_user(client, issue, existing, remove_all=True) + log.info(' Updated assignee') + + +def _update_tags(updates, existing, issue): + """ + Helper function to sync tags between upstream issue and downstream JIRA issue + Args: + updates (list): Downstream updates requested by the user + existing (jira.resource.Issue): Existing JIRA issue + issue (sync2jira.intermediary.Issue): Upstream issue + Returns: + Nothing + """ + # First get all existing tags on the issue + updated_labels = issue.tags + + # Ensure no duplicates if overwrite is set to false + try: + # for python 3 > + if not bool(list(filter(lambda d: "tags" in d, updates))[0]['tags']['overwrite']): + updated_labels = _label_matching(updated_labels, existing.fields.labels) + except ValueError: + # for python 2.7 + if not bool(filter(lambda d: "tags" in d, updates)[0]['tags']['overwrite']): + updated_labels = _label_matching(updated_labels, existing.fields.labels) + + # Ensure that the tags are all valid + updated_labels = verify_tags(updated_labels) + + # Now we can update the JIRA if labels are different + if sorted(updated_labels) != sorted(existing.fields.labels): + data = {'labels': updated_labels} + existing.update(data) + log.info(' Updated %s tag(s)' % len(updated_labels)) + + +def _update_description(existing, issue): + new_description = existing.fields.description + if 'Upstream description' in new_description: + # If we just need to update the content of the description + new_description = re.sub( + r"Upstream description:(\r\n*|\r*|\n*|.*){quote}((?s).*){quote}", + r"Upstream description: {quote}%s{quote}" % issue.content, + new_description) + elif '] Upstream issue status:' in new_description and '] Upstream Reporter:' in new_description: + # We need to add a description field + # Need to find the 2nd new line as we already have + # Reporter and Status + index1 = new_description.find('\n') + index2 = new_description[index1+1:].find('\n') + new_description = "%s \nUpstream description: " \ + "{quote}%s{quote}\n%s" % \ + (new_description[:index1+index2+1], + issue.content, + new_description[index1+index2+1:]) + + elif '] Upstream issue status:' in new_description: + # We need to add a upstream reporter and description field + upstream_reporter = '[%s] Upstream Reporter: %s' % ( + issue.id, + issue.reporter['fullname'] + ) + upstream_description = "%s \nUpstream description: " \ + "{quote}%s{quote}" % \ + (upstream_reporter, issue.content) + # Add it to the old description + index = new_description.find('\n') + new_description = (new_description[:index] + + '\n' + + upstream_description + + new_description[index+1:]) + else: + # Just add reporter and description to the top + upstream_reporter = '[%s] Upstream Reporter: %s' % ( + issue.id, + issue.reporter['fullname'] + ) + upstream_description = "%s \nUpstream description: " \ + "{quote}%s{quote}" % \ + (upstream_reporter, issue.content) + new_description = '%s \n %s' % \ + (upstream_description, new_description) + # Now that we've updated the description (i.e. added + # issue.id) we can delete the link in the description if its still there. + new_description = re.sub( + r'\[%s\]' % issue.url, + r'', + new_description + ) + + # Now we can update the JIRA issue if we need to + if new_description != existing.fields.description: + data = {'description': new_description} + existing.update(data) + log.info(' Updated description') + + +def verify_tags(tags): + """ + Helper function to ensure tag are JIRA ready :) + Args: + tags (lst(str)): Input tags + Returns: + updated_tags (lst(str)): Updates tags + """ + updated_tags = [] + for tag in tags: + updated_tags.append(tag.replace(" ", "_")) + return updated_tags + + +def sync_with_jira(issue, config): + """ + Attempts to sync a upstream issue with JIRA (i.e. by finding + an existing issue or creating a new one) + Args: + issue (sync2jira.intermediary.Issue): Issue object + config (dict): Config dict + Returns: + Nothing + """ + + log.info(" Considering upstream %s, %s", issue.url, issue.title) + + # Create a client connection for this issue + client = _get_jira_client(issue, config) + + # First, check to see if we have a matching issue using the new method. + # If we do, then just bail out. No sync needed. + log.info(" Looking for matching downstream issue via new method.") + existing = _get_existing_jira_issue(client, issue, config) + if existing: + # If we found an existing JIRA issue already + log.info(" Found existing, matching downstream %r.", existing.key) + # Update relevant metadata (i.e. tags, assignee, etc) + _update_jira_issue(existing, issue, client) + return + + # If we're *not* configured to do legacy matching (upgrade mode) then there + # is nothing left to do than to but to create the issue and return. + if not config['sync2jira'].get('legacy_matching', True): + log.debug(" Legacy matching disabled.") + _create_jira_issue(client, issue, config) + return + + # Otherwise, if we *are* configured to do legacy matching, then try and + # find this issue the old way. + # - If we can't find it, create it. + # - If we can find it, upgrade it to the new method. + log.info(" Looking for matching downstream issue via legacy method.") + match = _get_existing_jira_issue_legacy(client, issue, config) + if not match: + _create_jira_issue(client, issue, config) + else: + _upgrade_jira_issue(client, match, issue, config) + + def _close_as_duplicate(client, duplicate, keeper, config): log.info(" Closing %s as duplicate of %s", duplicate.permalink(), keeper.permalink()) if config['sync2jira']['testing']: @@ -244,7 +831,7 @@ def _close_as_duplicate(client, duplicate, keeper, config): else: log.exception("Failed to close %r", duplicate.permalink()) else: - log.warn(" Unable to find close transition for %r" % duplicate.key) + log.warning(" Unable to find close transition for %r" % duplicate.key) def close_duplicates(issue, config): @@ -261,48 +848,3 @@ def close_duplicates(issue, config): keeper, duplicates = results[0], results[1:] for duplicate in duplicates: _close_as_duplicate(client, duplicate, keeper, config) - - -def sync_with_jira(issue, config): - log.info("Considering upstream %s, %s", issue.url, issue.title) - - # Create a client connection for this issue - client = _get_jira_client(issue, config) - - # First, check to see if we have a matching issue using the new method. - # If we do, then just bail out. No sync needed. - log.info(" Looking for matching downstream issue via new method.") - existing = _get_existing_jira_issue(client, issue, config) - if existing: - log.info(" Found existing, matching downstream %r.", existing.key) - - # Only synchronize comments for listings that op-in - if not issue.downstream.get('comments'): - return - - log.info(" Looking for new comments") - comments = client.comments(existing) - comments_d = _comment_matching(issue.comments, comments) - for comment in comments_d: - comment_body = _comment_format(comment) - client.add_comment(existing, comment_body) - log.info(" Comments synchronization done on %i comments." % len(comments_d)) - return - - # If we're *not* configured to do legacy matching (upgrade mode) then there - # is nothing left to do than to but to create the issue and return. - if not config['sync2jira'].get('legacy_matching', True): - log.debug(" Legacy matching disabled.") - _create_jira_issue(client, issue, config) - return - - # Otherwise, if we *are* configured to do legacy matching, then try and - # find this issue the old way. - # - If we can't find it, create it. - # - If we can find it, upgrade it to the new method. - log.info(" Looking for matching downstream issue via legacy method.") - match = _get_existing_jira_issue_legacy(client, issue, config) - if not match: - _create_jira_issue(client, issue, config) - else: - _upgrade_jira_issue(client, match, issue, config) diff --git a/sync2jira/intermediary.py b/sync2jira/intermediary.py index 5a10f2b..66287c7 100644 --- a/sync2jira/intermediary.py +++ b/sync2jira/intermediary.py @@ -16,17 +16,31 @@ # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110.15.0 USA # # Authors: Ralph Bean +from datetime import datetime class Issue(object): - def __init__(self, source, title, url, upstream, comments, config): + def __init__(self, source, title, url, upstream, comments, + config, tags, fixVersion, priority, content, + reporter, assignee, status, id, downstream=None): self.source = source self._title = title self.url = url self.upstream = upstream self.comments = comments - self.downstream = config['sync2jira']['map'][self.source][upstream] + self.tags = tags + self.fixVersion = fixVersion + self.priority = priority + self.content = content + self.reporter = reporter + self.assignee = assignee + self.status = status + self.id = id + if not downstream: + self.downstream = config['sync2jira']['map'][self.source][upstream] + else: + self.downstream = downstream @property def title(self): @@ -37,11 +51,21 @@ class Issue(object): base = config['sync2jira'].get('pagure_url', 'https://pagure.io') comments = [] for comment in issue['comments']: + # Only add comments that are not Metadata updates + if '**Metadata Update' in comment['comment']: + continue + # Else add the comment + # Convert the date to datetime + comment['date_created'] = datetime.utcfromtimestamp(float(comment['date_created'])) comments.append({ 'author': comment['user']['name'], 'body': comment['comment'], + 'name': comment['user']['name'], + 'id': comment['id'], + 'date_created': comment['date_created'], 'changed': None }) + return Issue( source='pagure', title=issue['title'], @@ -49,6 +73,14 @@ class Issue(object): upstream=upstream, config=config, comments=comments, + tags=issue['tags'], + fixVersion=[issue['milestone']], + priority=issue['priority'], + content=issue['content'], + reporter=issue['user'], + assignee=issue['assignee'], + status=issue['status'], + id=issue['date_created'] ) @classmethod @@ -56,17 +88,37 @@ class Issue(object): comments = [] for comment in issue['comments']: comments.append({ - 'author': comment['user']['login'], + 'author': comment['author'], + 'name': comment['name'], 'body': comment['body'], - 'changed': comment['changed'] if 'changed' in comment else None + 'id': comment['id'], + 'date_created': comment['date_created'], + 'changed': None }) + + # Reformat the state field + if issue['state']: + if issue['state'] == 'open': + issue['state'] = 'Open' + elif issue['state'] == 'closed': + issue['state'] = 'Closed' + + # TODO: Priority is broken return Issue( source='github', title=issue['title'], url=issue['html_url'], - comments=comments, upstream=upstream, config=config, + comments=comments, + tags=issue['labels'], + fixVersion=[issue['milestone']], + priority=None, + content=issue['body'], + reporter=issue['user'], + assignee=issue['assignees'], + status=issue['state'], + id=issue['id'] ) def __repr__(self): diff --git a/sync2jira/main.py b/sync2jira/main.py index 09883c1..60bc13c 100644 --- a/sync2jira/main.py +++ b/sync2jira/main.py @@ -33,7 +33,7 @@ import sync2jira.downstream as d log = logging.getLogger('sync2jira.main') - +remote_link_title = "Upstream issue" handlers = { # Example: https://apps.fedoraproject.org/datagrepper/id?id=2016-895ed21e-5d53-4fde-86ac-64dab36a14ad&is_raw=true&size=extra-large @@ -42,16 +42,38 @@ handlers = { 'github.issue.reopened': u.handle_github_message, # Example: https://apps.fedoraproject.org/datagrepper/id?id=2017-a053e0c2-f514-47d6-8cb2-f7b2858f7052&is_raw=true&size=extra-large 'github.issue.labeled': u.handle_github_message, - 'github.issue.comment': u.handle_github_comment, + 'github.issue.assigned': u.handle_github_message, + 'github.issue.unassigned': u.handle_github_message, + 'github.issue.closed': u.handle_github_message, + 'github.issue.comment': u.handle_github_message, + 'github.issue.unlabeled': u.handle_github_message, + 'github.issue.milestoned': u.handle_github_message, + 'github.issue.demilestoned': u.handle_github_message, + 'github.issue.edited': u.handle_github_message, # Example: https://apps.fedoraproject.org/datagrepper/id?id=2016-d578d8f6-0c4c-493d-9535-4e138a03e197&is_raw=true&size=extra-large 'pagure.issue.new': u.handle_pagure_message, # Example: https://apps.fedoraproject.org/datagrepper/id?id=2017-c2e81259-8576-41a9-83c6-6db2cbcf67d3&is_raw=true&size=extra-large 'pagure.issue.tag.added': u.handle_pagure_message, - 'pagure.issue.comment.added': u.handle_pagure_comment, + 'pagure.issue.comment.added': u.handle_pagure_message, + 'pagure.issue.comment.edited': u.handle_pagure_message, + 'pagure.issue.assigned.added': u.handle_pagure_message, + 'pagure.issue.assigned.reset': u.handle_pagure_message, + 'pagure.issue.edit': u.handle_pagure_message, + 'pagure.issue.drop': u.handle_pagure_message, + 'pagure.issue.tag.removed': u.handle_pagure_message, } def load_config(loader=fedmsg.config.load_config): + """ + Generates and validates the config file + that will be used by fedmsg and JIRA client + Args: + loader (function): Function to set up runtime config + Returns: + config (dict): The config dict to be used + later in the program + """ config = loader() # Force some vars that we like @@ -87,32 +109,53 @@ def load_config(loader=fedmsg.config.load_config): def listen(config): + """ + Listens to activity on upstream repos on pagure and github + via fedmsg, and syncs new issues there to the JIRA instance + defined in 'fedmsg.d/sync2jira.py' + Args: + config (dict): Config dict + Returns: + Nothing + """ if not config['sync2jira'].get('listen'): log.info("`listen` is disabled. Exiting.") return - log.info("Waiting for a relevant fedmsg message to arrive...") + log.info(" Waiting for a relevant fedmsg message to arrive...") for _, _, topic, msg in fedmsg.tail_messages(**config): idx = msg['msg_id'] suffix = ".".join(topic.split('.')[3:]) log.debug("Encountered %r %r %r", suffix, topic, idx) - if suffix not in handlers: continue - log.info("Handling %r %r %r", suffix, topic, idx) + log.info(" Handling %r %r %r", suffix, topic, idx) - issue = handlers[suffix](msg, config) + if 'pagure' in suffix: + issue = u.handle_pagure_message(msg, config) + elif 'github' in suffix: + issue = u.handle_github_message(msg, config) if not issue: - log.warning("%s, %s yielded no Issue object.", suffix, idx) + log.warning(" %s, %s yielded no Issue object.", suffix, idx) continue d.sync_with_jira(issue, config) def initialize(config): - log.info("Running initialization to sync all issues from upstream to jira") + """ + Initial initialization needed to sync any upstream + repo with JIRA. Goes through all issues and + checks if they're already on JIRA / Need to be + created + Args: + config (dict): Config dict for JIRA + Returns: + Nothing + """ + log.info(" Running initialization to sync all issues from upstream to jira") log.info(" Testing flag is %r", config['sync2jira']['testing']) mapping = config['sync2jira']['map'] for upstream in mapping.get('pagure', {}).keys(): @@ -122,7 +165,7 @@ def initialize(config): except Exception: log.error("Failed on %r", issue) raise - log.info("Done with pagure initialization.") + log.info(" Done with pagure initialization.") for upstream in mapping.get('github', {}).keys(): for issue in u.github_issues(upstream, config): @@ -131,10 +174,18 @@ def initialize(config): except Exception: log.error("Failed on %r", issue) raise - log.info("Done with github initialization.") + log.info(" Done with github initialization.") def main(): + """ + Main function to check for initial sync + and listen for fedmgs + Args: + Nothing + Returns: + Nothing + """ config = load_config() logging.basicConfig(level=logging.INFO) warnings.simplefilter("ignore") diff --git a/sync2jira/upstream.py b/sync2jira/upstream.py index df60cdb..3bf6ecc 100644 --- a/sync2jira/upstream.py +++ b/sync2jira/upstream.py @@ -28,6 +28,7 @@ except ImportError: string_type = types.StringTypes import requests +from github import Github import sync2jira.intermediary as i @@ -48,7 +49,7 @@ def handle_github_message(msg, config): _filter = config['sync2jira']\ .get('filters', {})\ .get('github', {})\ - .get(upstream, {'state': 'open'}) + .get(upstream, {}) for key, expected in _filter.items(): # special handling for label: we look for it in the list of msg labels @@ -64,6 +65,58 @@ def handle_github_message(msg, config): log.info("Actual %r %r != expected %r", key, actual, expected) return None + # Initialize Github object so we can get their full name (instead of their username) + # And get comments if needed + github_client = Github(config['sync2jira']['github_token']) + + # If there are no comments just make an empty array + if msg['msg']['issue']['comments'] == 0: + msg['msg']['issue']['comments'] = [] + else: + # We have multiple comments and need to make api call to get them + repo = github_client.get_repo(upstream) + comments = [] + github_issue = repo.get_issue(number=msg['msg']['issue']['number']) + for comment in github_issue.get_comments(): + # First make API call to get the users name + comments.append({ + 'author': comment.user.name, + 'name': comment.user.login, + 'body': comment.body, + 'id': comment.id, + 'date_created': comment.created_at, + 'changed': None + }) + # Assign the message with the newly formatted comments :) + msg['msg']['issue']['comments'] = comments + + # Search for the user + reporter = github_client.get_user(msg['msg']['issue']['user']['login']) + # Update the reporter field in the message (to match Pagure format) + msg['msg']['issue']['user']['fullname'] = reporter.name + + # Now do the same thing for the assignees + assignees = [] + for person in msg['msg']['issue']['assignees']: + assignee = github_client.get_user(person['login']) + assignees.append({'fullname': assignee.name}) + + # Update the assignee field in the message (to match Pagure format) + msg['msg']['issue']['assignees'] = assignees + + # Update the label field in the message (to match Pagure format) + if msg['msg']['issue']['labels']: + # loop through all the labels on Github and add them + # to the new label list and then reassign the message + new_label = [] + for label in msg['msg']['issue']['labels']: + new_label.append(label['name']) + msg['msg']['issue']['labels'] = new_label + + # Update the milestone field in the message (to match Pagure format) + if msg['msg']['issue']['milestone']: + msg['msg']['issue']['milestone'] = msg['msg']['issue']['milestone']['title'] + return i.Issue.from_github(upstream, msg['msg']['issue'], config) @@ -80,29 +133,58 @@ def handle_pagure_message(msg, config): _filter = config['sync2jira']\ .get('filters', {})\ - .get('pagure', {})\ - .get(upstream, {'status': 'Open'}) - - for key, expected in _filter.items(): - # special handling for tag: we look for it in the list of msg tags - if key == 'tags': - actual = msg['msg']['issue'].get('tags', []) + msg['msg'].get('tags', []) - - # Some messages send tags as strings, others as dicts. Handle both. - actual = \ - [tag['name'] for tag in actual if isinstance(tag, dict)] + \ - [tag for tag in actual if isinstance(tag, string_type)] - - intersection = set(actual) & set(expected) - if not intersection: - log.info("None of %r in %r on issue.", expected, actual) - return None - else: - # direct comparison - actual = msg['msg']['issue'].get(key) - if actual != expected: - log.info("Actual %r %r != expected %r", key, actual, expected) - return None + .get('pagure', {}) \ + .get(upstream, {}) + + if _filter: + for key, expected in _filter.items(): + # special handling for tag: we look for it in the list of msg tags + if key == 'tags': + actual = msg['msg']['issue'].get('tags', []) + msg['msg'].get('tags', []) + + # Some messages send tags as strings, others as dicts. Handle both. + actual = \ + [tag['name'] for tag in actual if isinstance(tag, dict)] + \ + [tag for tag in actual if isinstance(tag, string_type)] + + intersection = set(actual) & set(expected) + if not intersection: + log.info("None of %r in %r on issue.", expected, actual) + return None + else: + # direct comparison + actual = msg['msg']['issue'].get(key) + if actual != expected: + log.info("Actual %r %r != expected %r", key, actual, expected) + return None + + # If this is a dropped issue upstream + try: + if msg['topic'] == 'io.pagure.prod.pagure.issue.drop': + msg['msg']['issue']['status'] = 'Dropped' + except KeyError: + # Otherwise do nothing + pass + + # If this is a tag edit upstream + try: + # Add all updated tags to the tags on the issue + for tag in msg['msg']['tags']: + msg['msg']['issue']['tags'].append(tag) + except KeyError: + # Otherwise do nothing + pass + + # If this is a comment edit + try: + # Add it to the comments on the issue + msg['msg']['issue']['comments'].append(msg['msg']['comment']) + except KeyError: + # Otherwise do nothing + pass + + # Format the assignee field to match github (i.e. in a list) + msg['msg']['issue']['assignee'] = [msg['msg']['issue']['assignee']] return i.Issue.from_pagure(upstream, msg['msg']['issue'], config) @@ -113,8 +195,8 @@ def pagure_issues(upstream, config): params = config['sync2jira']\ .get('filters', {})\ - .get('pagure', {})\ - .get(upstream, {'status': 'Open'}) + .get('pagure', {}) \ + .get(upstream, {}) response = requests.get(url, params=params) if not bool(response): @@ -124,6 +206,14 @@ def pagure_issues(upstream, config): reason = response.text raise IOError("response: %r %r %r" % (response, reason, response.request.url)) data = response.json()['issues'] + + # Reformat the assignee value so that it is enclosed within an array + # We do this because Github supports multiple assignees, but JIRA doesn't :( + # Hopefully in the future it will support multiple assignees, thus enclosing + # the assignees in a list prepares for that support + for issue in data: + issue['assignee'] = [issue['assignee']] + issues = (i.Issue.from_pagure(upstream, issue, config) for issue in data) for issue in issues: yield issue @@ -140,111 +230,78 @@ def github_issues(upstream, config): _filter = config['sync2jira']\ .get('filters', {})\ .get('github', {})\ - .get(upstream, {'state': 'open'}) + .get(upstream, {}) + url = 'https://api.github.com/repos/%s/issues' % upstream - url += '?' + urlencode(_filter) + if _filter: + url += '?' + urlencode(_filter) issues = _get_all_github_issues(url, headers) - issues = list(( - i.Issue.from_github(upstream, issue, config) for issue in issues - if 'pull_request' not in issue # We don't want to copy these around - )) - for issue in issues: - yield issue - -def handle_github_comment(msg, config): - owner = msg['msg']['repository']['owner']['login'] - repo = msg['msg']['repository']['name'] - upstream = '{owner}/{repo}'.format(owner=owner, repo=repo) - mapped_repos = config['sync2jira']['map']['github'] - - token = config['sync2jira'].get('github_token') - if not token: - headers = {} - log.warning('No github_token found. We will be rate-limited...') - else: - headers = {'Authorization': 'token ' + token} - - if upstream not in mapped_repos: - log.info("%r not in github map: %r", upstream, mapped_repos.keys()) - return None - - _filter = config['sync2jira']\ - .get('filters', {})\ - .get('github', {})\ - .get(upstream, {'state': 'open'}) - - for key, expected in _filter.items(): - # special handling for label: we look for it in the list of msg labels - if key == 'labels': - actual = [label['name'] for label in msg['msg']['issue']['labels']] - if expected not in actual: - log.info("Label %s not set on issue", expected) - return None - else: - # direct comparison - actual = msg['msg']['issue'].get(key) - if actual != expected: - log.info("Actual %r %r != expected %r", key, actual, expected) - return None + # Initialize Github object so we can get their full name (instead of their username) + # And get comments if needed + github_client = Github(config['sync2jira']['github_token']) - issue = msg['msg']['issue'] - - if 'pull_request' in issue: - log.info("%r is a pull request. Ignoring.", issue.get('html_url')) - return None - - comment = msg['msg']['comment'] - log.info("Fetching comments for: %s", issue['url']) - issue['comments'] = _fetch_github_data("%s/comments" % (issue['url']), headers).json() - if msg['msg']['action'] == 'edited': - for c in issue['comments']: - if c['body'] == comment['body']: - c['body'] = msg['msg']['changes']['body']['from'] - c['changed'] = comment['body'] - return i.Issue.from_github(upstream, issue, config) - - -def handle_pagure_comment(msg, config): - upstream = msg['msg']['project']['name'] - ns = msg['msg']['project'].get('namespace') or None - if ns: - upstream = '{ns}/{upstream}'.format(ns=ns, upstream=upstream) - mapped_repos = config['sync2jira']['map']['pagure'] - - if upstream not in mapped_repos: - log.info("%r not in pagure map: %r", upstream, mapped_repos.keys()) - return None - - _filter = config['sync2jira']\ - .get('filters', {})\ - .get('pagure', {})\ - .get(upstream, {'status': 'Open'}) - - for key, expected in _filter.items(): - # special handling for tag: we look for it in the list of msg tags - if key == 'tags': - actual = msg['msg']['issue'].get('tags', []) + msg['msg'].get('tags', []) - - # Some messages send tags as strings, others as dicts. Handle both. - actual = \ - [tag['name'] for tag in actual if isinstance(tag, dict)] + \ - [tag for tag in actual if isinstance(tag, string_type)] - - intersection = set(actual) & set(expected) - if not intersection: - log.info("None of %r in %r on issue.", expected, actual) - return None + # We need to format everything to a standard to we can create an issue object + final_issues = [] + for issue in issues: + # Update comments: + # If there are no comments just make an empty array + if issue['comments'] == 0: + issue['comments'] = [] else: - # direct comparison - actual = msg['msg']['issue'].get(key) - if actual != expected: - log.info("Actual %r %r != expected %r", key, actual, expected) - return None - - issue = msg['msg']['issue'] - return i.Issue.from_pagure(upstream, issue, config) + # We have multiple comments and need to make api call to get them + repo = github_client.get_repo(upstream) + comments = [] + github_issue = repo.get_issue(number=issue['number']) + for comment in github_issue.get_comments(): + # First make API call to get the users name + comments.append({ + 'author': comment.user.name, + 'name': comment.user.login, + 'body': comment.body, + 'id': comment.id, + 'date_created': comment.created_at, + 'changed': None + }) + # Assign the message with the newly formatted comments :) + issue['comments'] = comments + + # Update reporter: + # Search for the user + reporter = github_client.get_user(issue['user']['login']) + # Update the reporter field in the message (to match Pagure format) + issue['user']['fullname'] = reporter.name + + # Update assignee(s): + assignees = [] + for person in issue['assignees']: + assignee = github_client.get_user(person['login']) + assignees.append({'fullname': assignee.name}) + # Update the assignee field in the message (to match Pagure format) + issue['assignees'] = assignees + + # Update label(s): + if issue['labels']: + # loop through all the labels on Github and add them + # to the new label list and then reassign the message + new_label = [] + for label in issue['labels']: + new_label.append(label['name']) + issue['labels'] = new_label + + # Update milestone: + if issue['milestone']: + issue['milestone'] = issue['milestone']['title'] + + final_issues.append(issue) + + final_issues = list(( + i.Issue.from_github(upstream, issue, config) for issue in final_issues + if 'pull_request' not in issue # We don't want to copy these around + )) + for issue in final_issues: + yield issue def _get_all_github_issues(url, headers): @@ -259,17 +316,6 @@ def _get_all_github_issues(url, headers): link = _github_link_field_to_dict(response.headers.get('link', None)) -def _fetch_github_data(url, headers): - response = requests.get(url, headers=headers) - if not bool(response): - try: - reason = response.json() - except Exception: - reason = response.text - raise IOError("response: %r %r %r" % (response, reason, response.request.url)) - return response - - def _github_link_field_to_dict(field): """ Utility for ripping apart github's Link header field. It's kind of ugly. @@ -283,3 +329,14 @@ def _github_link_field_to_dict(field): part.split('; ')[0][1:-1], ) for part in field.split(', ') ]) + + +def _fetch_github_data(url, headers): + response = requests.get(url, headers=headers) + if not bool(response): + try: + reason = response.json() + except Exception: + reason = response.text + raise IOError("response: %r %r %r" % (response, reason, response.request.url)) + return response diff --git a/test-requirements.txt b/test-requirements.txt index a678696..819b6c3 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,2 +1,3 @@ nose mock +pytest diff --git a/tests/test_downstream.py b/tests/test_downstream.py index 7d64c8f..c32d054 100644 --- a/tests/test_downstream.py +++ b/tests/test_downstream.py @@ -1,28 +1,169 @@ -from nose.tools import eq_ - import mock import unittest +try: + # Python 3.3 > + from unittest.mock import MagicMock # noqa: F401 +except ImportError: + from mock import MagicMock # noqa: F401 + import sync2jira.downstream as d +from sync2jira.intermediary import Issue + +from nose.tools import eq_ import jira.client +from jira import JIRAError + +PATH = 'sync2jira.downstream.' class TestDownstream(unittest.TestCase): - config = { - 'sync2jira': { - 'jira': { - # Nothing, really.. + """ + This class tests the downstream.py file under sync2jira + """ + def setUp(self): + """ + Setting up the testing environment + """ + # Mock Config dict + self.mock_config = { + 'sync2jira': { + 'jira': { + 'mock_jira_instance': {'mock_jira': 'mock_jira'} + }, + 'testing': {}, + 'legacy_matching': False }, - }, - } + } + + # Mock sync2jira.intermediary.Issue + self.mock_issue = MagicMock() + self.mock_issue.assignee = [{'fullname': 'mock_user'}] + self.mock_issue.downstream = { + 'project': 'mock_project', + 'custom_fields': {'somecustumfield': 'somecustumvalue'}, + 'type': 'Fix', + 'updates': [ + 'comments', + {'tags': {'overwrite': False}}, + {'fixVersion': {'overwrite': False}}, + 'assignee', 'description', 'title', + {'transition': 'CUSTOM TRANSITION'} + ], + 'owner': 'mock_owner' + } + self.mock_issue.content = 'mock_content' + self.mock_issue.reporter = {'fullname': 'mock_user'} + self.mock_issue.url = 'mock_url' + self.mock_issue.title = 'mock_title' + self.mock_issue.comments = 'mock_comments' + self.mock_issue.tags = ['tag1', 'tag2'] + self.mock_issue.fixVersion = ['fixVersion3', 'fixVersion4'] + self.mock_issue.fixVersion = ['fixVersion3', 'fixVersion4'] + self.mock_issue.assignee = [{'fullname': 'mock_assignee'}] + self.mock_issue.status = 'Open' + self.mock_issue.id = '1234' + + # Mock issue updates + self.mock_updates = [ + 'comments', + {'tags': {'overwrite': False}}, + {'fixVersion': {'overwrite': False}}, + 'assignee', 'description', 'title', + {'transition': 'CUSTOM TRANSITION'}, + ] + + # Mock Jira transition + self.mock_transition = [{ + 'name': 'custom_closed_status', + 'id': 1234 + }] + + # Mock jira.resources.Issue + self.mock_downstream = MagicMock() + self.mock_downstream.id = 1234 + self.mock_downstream.fields.labels = ['tag3', 'tag4'] + mock_version1 = MagicMock() + mock_version1.name = 'fixVersion3' + mock_version2 = MagicMock() + mock_version2.name = 'fixVersion4' + self.mock_downstream.fields.fixVersions = [mock_version1, mock_version2] + self.mock_downstream.update.return_value = True + self.mock_downstream.fields.description = "This is an existing description" + + # Mock datetime.today() + self.mock_today = MagicMock() + self.mock_today.strftime.return_value = 'mock_today' + + @mock.patch('jira.client.JIRA') + def test_get_jira_client_not_issue(self, + mock_client): + """ + This tests '_get_jira_client' function where the passed in + argument is not an Issue instance + """ + # Call the function + with self.assertRaises(Exception): + d._get_jira_client( + issue='string', + config=self.mock_config + ) + + # Assert everything was called correctly + mock_client.assert_not_called() + + @mock.patch('jira.client.JIRA') + def test_get_jira_client_not_instance(self, + mock_client): + """ + This tests '_get_jira_client' function there is no JIRA instance + """ + # Set up return values + self.mock_issue.downstream = {} + self.mock_config['sync2jira']['default_jira_instance'] = {} + + # Call the function + with self.assertRaises(Exception): + d._get_jira_client( + issue=self.mock_issue, + config=self.mock_config + ) + + # Assert everything was called correctly + mock_client.assert_not_called() + + @mock.patch('jira.client.JIRA') + def test_get_jira_client(self, + mock_client): + """ + This tests '_get_jira_client' function where everything goes smoothly + """ + # Set up return values + mock_issue = MagicMock(spec=Issue) + mock_issue.downstream = {'jira_instance': 'mock_jira_instance'} + mock_client.return_value = 'Successful call!' + + # Call the function + + response = d._get_jira_client( + issue=mock_issue, + config=self.mock_config + ) + + # Assert everything was called correctly + mock_client.assert_called_with(mock_jira='mock_jira') + self.assertEqual('Successful call!', response) @mock.patch('jira.client.JIRA') def test_get_existing_legacy(self, client): + """ + This tests '_get_existing_jira_issue_legacy' function + """ class MockIssue(object): downstream = {'key': 'value'} url = 'wat' issue = MockIssue() - config = self.config.copy() + config = self.mock_config # Ensure that we get results back from the jira client. target1 = "target1" client.return_value.search_issues = mock.MagicMock(return_value=[target1]) @@ -36,19 +177,22 @@ class TestDownstream(unittest.TestCase): @mock.patch('jira.client.JIRA') def test_get_existing_newstyle(self, client): - config = self.config.copy() + config = self.mock_config class MockIssue(object): downstream = {'key': 'value'} title = 'A title, a title...' url = 'http://threebean.org' - # Ensure that we get results back from the jira client. - target1 = "target1" + issue = MockIssue() - client.return_value.search_issues = mock.MagicMock(return_value=[target1]) + mock_results_of_query = MagicMock() + mock_results_of_query.fields.summary = 'A title, a title...' + + client.return_value.search_issues.return_value = [mock_results_of_query] result = d._get_existing_jira_issue(jira.client.JIRA(), issue, config) - eq_(result, target1) + # Ensure that we get the mock_result_of_query as a result + self.assertEqual(result, mock_results_of_query) client.return_value.search_issues.assert_called_once_with( 'issueFunction in linkedIssuesOfRemote("Upstream issue") and ' @@ -57,7 +201,7 @@ class TestDownstream(unittest.TestCase): @mock.patch('jira.client.JIRA') def test_upgrade_oldstyle_jira_issue(self, client): - config = self.config.copy() + config = self.mock_config class MockIssue(object): downstream = {'key': 'value'} @@ -69,6 +213,7 @@ class TestDownstream(unittest.TestCase): client_obj = mock.MagicMock() client.return_value = client_obj d._upgrade_jira_issue(jira.client.JIRA(), downstream, issue, config) + remote = { 'url': 'http://threebean.org', 'title': 'Upstream issue', @@ -76,62 +221,785 @@ class TestDownstream(unittest.TestCase): client_obj.add_remote_link.assert_called_once_with(downstream.id, remote) @mock.patch('jira.client.JIRA') - def test_create_jira_issue(self, client): - config = self.config.copy() - target1 = mock.Mock() - target1.id = "target1 id" - target1.key = "target key" - target1.fields.description = "description" - client.return_value.create_issue = mock.MagicMock(return_value=target1) + def test_assign_user(self, mock_client): + """ + Test 'assign_user' function where remove_all flag is False + """ + # Set up return values + mock_user = MagicMock() + mock_user.displayName = 'mock_assignee' + mock_user.key = 'mock_user_key' + mock_client.search_assignable_users_for_issues.return_value = [mock_user] + mock_client.assign_issue.return_value = True - class MockIssue(object): - downstream = { - 'project': 'awesome', - 'component': 'alsoawesome', - } - title = 'A title, a title...' - url = 'http://threebean.org' - comments = [{ - 'author': 'Ralph', - 'body': 'Super duper.', - }] + # Call the assign user function + d.assign_user( + issue=self.mock_issue, + downstream=self.mock_downstream, + client=mock_client + ) - config['sync2jira']['testing'] = False - result = d._create_jira_issue(jira.client.JIRA(), MockIssue(), config) - eq_(result, target1) - client.return_value.create_issue.assert_called_with( - components=[{'name': 'alsoawesome'}], - issuetype={'name': 'Bug'}, - project={'key': 'awesome'}, - description='http://threebean.org', - summary='A title, a title...', + # Assert that all calls mocked were called properly + mock_client.assign_issue.assert_called_with(1234, 'mock_user_key') + mock_client.search_assignable_users_for_issues.assert_called_with( + 'mock_assignee', + project='mock_project' ) @mock.patch('jira.client.JIRA') - def test_create_jira_issue_with_custom_url(self, client): - config = self.config.copy() - target1 = mock.Mock() - target1.id = "target1 id" - target1.key = "target key" - target1.fields.description = "description" - client.return_value.create_issue = mock.MagicMock(return_value=target1) + def test_assign_user_with_owner(self, mock_client): + """ + Test 'assign_user' function where remove_all flag is False + """ + # Set up return values + mock_user = MagicMock() + mock_user.displayName = 'mock_assignee' + mock_user.key = 'mock_user_key' + mock_client.search_assignable_users_for_issues.return_value = [] + mock_client.assign_issue.return_value = True - class MockIssue(object): - downstream = { - 'project': 'awesome', - 'component': 'alsoawesome', + # Call the assign user function + d.assign_user( + issue=self.mock_issue, + downstream=self.mock_downstream, + client=mock_client + ) + + # Assert that all calls mocked were called properly + mock_client.assign_issue.assert_called_with(1234, 'mock_owner') + mock_client.search_assignable_users_for_issues.assert_called_with( + 'mock_assignee', + project='mock_project' + ) + + @mock.patch('jira.client.JIRA') + def test_assign_user_without_owner(self, mock_client): + """ + Test 'assign_user' function where remove_all flag is False + """ + # Set up return values + mock_user = MagicMock() + mock_user.displayName = 'mock_assignee' + mock_user.key = 'mock_user_key' + mock_client.search_assignable_users_for_issues.return_value = [] + mock_client.assign_issue.return_value = True + self.mock_issue.downstream.pop('owner') + + # Call the assign user function + d.assign_user( + issue=self.mock_issue, + downstream=self.mock_downstream, + client=mock_client + ) + + # Assert that all calls mocked were called properly + mock_client.assign_issue.assert_not_called() + mock_client.search_assignable_users_for_issues.assert_called_with( + 'mock_assignee', + project='mock_project' + ) + + @mock.patch('jira.client.JIRA') + def test_assign_user_remove_all(self, mock_client): + """ + Test 'assign_user' function where remove_all flag is True + """ + # Call the assign user function + d.assign_user( + issue=self.mock_issue, + downstream=self.mock_downstream, + client=mock_client, + remove_all=True + ) + + # Assert that all calls mocked were called properly + self.mock_downstream.update.assert_called_with(assignee={'name': ''}) + mock_client.assign_issue.assert_not_called() + mock_client.search_assignable_users_for_issues.assert_not_called() + + @mock.patch(PATH + '_update_jira_issue') + @mock.patch(PATH + '_attach_link') + @mock.patch('jira.client.JIRA') + def test_create_jira_issue(self, + mock_client, + mock_attach_link, + mock_update_jira_issue): + """ + Tests '_create_jira_issue' function + """ + # Set up return values + mock_client.create_issue.return_value = self.mock_downstream + + # Call the function + response = d._create_jira_issue( + client=mock_client, + issue=self.mock_issue, + config=self.mock_config + ) + + # Assert everything was called correctly + mock_client.create_issue.assert_called_with( + issuetype={'name': 'Fix'}, + project={'key': 'mock_project'}, + somecustumfield='somecustumvalue', + description='[1234] Upstream Reporter: mock_user \n Upstream description: {quote}mock_content{quote}', + summary='mock_title' + ) + mock_attach_link.assert_called_with( + mock_client, + self.mock_downstream, + { + 'url': 'mock_url', + 'title': 'Upstream issue' } - title = 'A title, a title...' - url = 'http://threebean.org' - comments = [] + ) + mock_update_jira_issue.assert_called_with( + self.mock_downstream, + self.mock_issue, + mock_client + ) + self.assertEqual(response, self.mock_downstream) - config['sync2jira']['testing'] = False - result = d._create_jira_issue(jira.client.JIRA(), MockIssue(), config) - eq_(result, target1) - client.return_value.create_issue.assert_called_with( - components=[{'name': 'alsoawesome'}], - issuetype={'name': 'Bug'}, - project={'key': 'awesome'}, - description='http://threebean.org', - summary='A title, a title...', + @mock.patch(PATH + '_get_jira_client') + @mock.patch(PATH + '_get_existing_jira_issue') + @mock.patch(PATH + '_update_jira_issue') + @mock.patch(PATH + '_create_jira_issue') + @mock.patch('jira.client.JIRA') + @mock.patch(PATH + '_get_existing_jira_issue_legacy') + def test_sync_with_jira_matching(self, + mock_existing_jira_issue_legacy, + mock_client, + mock_create_jira_issue, + mock_update_jira_issue, + mock_existing_jira_issue, + mock_get_jira_client): + """ + Tests 'sync_with_jira' function where we do find a matching issue + This assumes we're not using the legacy matching anymore + """ + # Set up return values + mock_get_jira_client.return_value = mock_client + mock_existing_jira_issue.return_value = self.mock_downstream + + # Call the function + d.sync_with_jira( + issue=self.mock_issue, + config=self.mock_config + ) + + # Assert all calls were made correctly + mock_get_jira_client.assert_called_with(self.mock_issue, self.mock_config) + mock_update_jira_issue.assert_called_with(self.mock_downstream, self.mock_issue, mock_client) + mock_create_jira_issue.assert_not_called() + mock_existing_jira_issue_legacy.assert_not_called() + + @mock.patch(PATH + '_get_jira_client') + @mock.patch(PATH + '_get_existing_jira_issue') + @mock.patch(PATH + '_update_jira_issue') + @mock.patch(PATH + '_create_jira_issue') + @mock.patch('jira.client.JIRA') + @mock.patch(PATH + '_get_existing_jira_issue_legacy') + def test_sync_with_jira_no_matching(self, + mock_existing_jira_issue_legacy, + mock_client, + mock_create_jira_issue, + mock_update_jira_issue, + mock_existing_jira_issue, + mock_get_jira_client): + """ + Tests 'sync_with_jira' function where we do NOT find a matching issue + This assumes we're not using the legacy matching anymore + """ + # Set up return values + mock_get_jira_client.return_value = mock_client + mock_existing_jira_issue.return_value = None + + # Call the function + d.sync_with_jira( + issue=self.mock_issue, + config=self.mock_config + ) + + # Assert all calls were made correctly + mock_get_jira_client.assert_called_with(self.mock_issue, self.mock_config) + mock_update_jira_issue.assert_not_called() + mock_create_jira_issue.assert_called_with(mock_client, self.mock_issue, self.mock_config) + mock_existing_jira_issue_legacy.assert_not_called() + + @mock.patch(PATH + '_update_description') + @mock.patch(PATH + '_update_comments') + @mock.patch(PATH + '_update_tags') + @mock.patch(PATH + '_update_fixVersion') + @mock.patch(PATH + '_update_transition') + @mock.patch('jira.client.JIRA') + def test_update_jira_issue(self, + mock_client, + mock_update_transition, + mock_update_fixVersion, + mock_update_tags, + mock_update_comments, + mock_update_description): + """ + This tests '_update_jira_issue' function + """ + # Call the function + d._update_jira_issue( + existing=self.mock_downstream, + issue=self.mock_issue, + client=mock_client + ) + + # Assert all calls were made correctly + mock_update_comments.assert_called_with( + mock_client, + self.mock_downstream, + self.mock_issue + ) + mock_update_tags.assert_called_with( + self.mock_updates, + self.mock_downstream, + self.mock_issue + ) + mock_update_fixVersion.assert_called_with( + self.mock_updates, + self.mock_downstream, + self.mock_issue + ) + mock_update_description.assert_called_with( + self.mock_downstream, + self.mock_issue + ) + self.mock_downstream.update.assert_called_with({ + 'summary': 'mock_title' + }) + mock_update_transition.assert_called_with( + mock_client, + self.mock_downstream, + self.mock_issue + ) + + @mock.patch(PATH + 'datetime') + @mock.patch('jira.client.JIRA') + def test_update_transition_JIRAError(self, + mock_client, + mock_datetime): + """ + This function tests the '_update_transition' function where Upstream issue status + s not in existing.fields.description and transitioning the issue throws an error + """ + # Set up return values + self.mock_issue.status = 'Closed' + self.mock_downstream.fields.description = '' + mock_datetime.today.return_value = self.mock_today + mock_client.transitions.return_value = [{'name': 'CUSTOM TRANSITION', 'id': '1234'}] + mock_client.transition_issue.side_effect = JIRAError + + # Call the function + d._update_transition( + client=mock_client, + existing=self.mock_downstream, + issue=self.mock_issue + ) + + # Assert all calls were made correctly + self.mock_today.strftime.assert_called_with("%a %b %y - %H:%M") + self.mock_downstream.update.assert_called_with({'description': '[mock_today] Upstream issue status: Closed\n'}) + mock_client.transitions.assert_called_with(self.mock_downstream) + mock_client.transition_issue.asert_called_with(self.mock_downstream, 1234) + + @mock.patch(PATH + 'datetime') + @mock.patch('jira.client.JIRA') + def test_update_transition_not_found(self, + mock_client, + mock_datetime): + """ + This function tests the '_update_transition' function where Upstream issue status + s not in existing.fields.description and we can't find the appropriate closed status + """ + # Set up return values + self.mock_issue.status = 'Closed' + self.mock_issue.downstream['transition'] = 'bad_transition' + self.mock_downstream.fields.description = '' + mock_datetime.today.return_value = self.mock_today + mock_client.transitions.return_value = [{'name': 'CUSTOM TRANSITION', 'id': '1234'}] + + # Call the function + d._update_transition( + client=mock_client, + existing=self.mock_downstream, + issue=self.mock_issue + ) + + # Assert all calls were made correctly + self.mock_today.strftime.assert_called_with("%a %b %y - %H:%M") + self.mock_downstream.update.assert_called_with({'description': '[mock_today] Upstream issue status: Closed\n'}) + mock_client.transitions.assert_called_with(self.mock_downstream) + mock_client.transition_issue.asert_called_with(self.mock_downstream, 1234) + + @mock.patch(PATH + 'datetime') + @mock.patch('jira.client.JIRA') + def test_update_transition_successful(self, + mock_client, + mock_datetime): + """ + This function tests the '_update_transition' function where everything goes smoothly! + """ + # Set up return values + self.mock_issue.status = 'Closed' + self.mock_downstream.fields.description = '[test] Upstream issue status: Open' + mock_datetime.today.return_value = self.mock_today + mock_client.transitions.return_value = [{'name': 'CUSTOM TRANSITION', 'id': '1234'}] + + # Call the function + d._update_transition( + client=mock_client, + existing=self.mock_downstream, + issue=self.mock_issue + ) + + # Assert all calls were made correctly + self.mock_today.strftime.assert_called_with("%a %b %y - %H:%M") + self.mock_downstream.update.assert_called_with({'description': '[mock_today] Upstream issue status: Closed'}) + mock_client.transitions.assert_called_with(self.mock_downstream) + mock_client.transition_issue.asert_called_with(self.mock_downstream, 1234) + + @mock.patch(PATH + '_comment_format') + @mock.patch(PATH + '_comment_matching') + @mock.patch('jira.client.JIRA') + def test_update_comments(self, + mock_client, + mock_comment_matching, + mock_comment_format): + """ + This function tests the 'update_comments' function + """ + # Set up return values + mock_client.comments.return_value = 'mock_comments' + mock_comment_matching.return_value = ['mock_comments_d'] + mock_comment_format.return_value = 'mock_comment_body' + + # Call the function + d._update_comments( + client=mock_client, + existing=self.mock_downstream, + issue=self.mock_issue + ) + + # Assert all calls were made correctly + mock_client.comments.assert_called_with(self.mock_downstream) + mock_comment_matching.assert_called_with(self.mock_issue.comments, 'mock_comments') + mock_comment_format.assert_called_with('mock_comments_d') + mock_client.add_comment.assert_called_with(self.mock_downstream, 'mock_comment_body') + + def test_update_fixVersion_JIRAError(self): + """ + This function tests the 'update_fixVersion' function where updating the downstream + issue throws an error + """ + # Set up return values + self.mock_downstream.update.side_effect = JIRAError + self.mock_downstream.fields.fixVersions = [] + + # Call the function + d._update_fixVersion( + updates=self.mock_updates, + existing=self.mock_downstream, + issue=self.mock_issue + ) + # Assert all calls were made correctly + self.mock_downstream.update.assert_called_with( + {'fixVersions': [{'name': 'fixVersion3'}, {'name': 'fixVersion4'}]}) + + def test_update_fixVersion_no_api_call(self): + """ + This function tests the 'update_fixVersion' function existing labels are the same + and thus no API call should be made + """ + # Set up return values + self.mock_downstream.update.side_effect = JIRAError + + # Call the function + d._update_fixVersion( + updates=self.mock_updates, + existing=self.mock_downstream, + issue=self.mock_issue + ) + # Assert all calls were made correctly + self.mock_downstream.update.assert_not_called() + + def test_update_fixVersion_successful(self): + """ + This function tests the 'update_fixVersion' function where everything goes smoothly! + """ + # Set up return values + self.mock_downstream.fields.fixVersions = [] + + # Call the function + d._update_fixVersion( + updates=self.mock_updates, + existing=self.mock_downstream, + issue=self.mock_issue + ) + # Assert all calls were made correctly + self.mock_downstream.update.assert_called_with( + {'fixVersions': [{'name': 'fixVersion3'}, {'name': 'fixVersion4'}]}) + + @mock.patch(PATH + 'assign_user') + @mock.patch('jira.client.JIRA') + def test_update_assignee_assignee(self, + mock_client, + mock_assign_user): + """ + This function tests the 'update_assignee' function where issue.assignee exists + """ + # Call the function + d._update_assignee( + client=mock_client, + existing=self.mock_downstream, + issue=self.mock_issue + ) + + # Assert all calls were made correctly + mock_assign_user.assert_called_with( + mock_client, + self.mock_issue, + self.mock_downstream + ) + + @mock.patch(PATH + 'assign_user') + @mock.patch('jira.client.JIRA') + def test_update_assignee_no_assignee(self, + mock_client, + mock_assign_user): + """ + This function tests the '_update_assignee' function where issue.assignee does not exist + """ + # Set up return values + self.mock_issue.assignee = None + + # Call the function + d._update_assignee( + client=mock_client, + existing=self.mock_downstream, + issue=self.mock_issue + ) + + # Assert all calls were made correctly + mock_assign_user.assert_called_with( + mock_client, + self.mock_issue, + self.mock_downstream, + remove_all=True + ) + + @mock.patch(PATH + 'verify_tags') + @mock.patch(PATH + '_label_matching') + def test_update_tags(self, + mock_label_matching, + mock_verify_tags): + """ + This function tests the '_update_tags' function + """ + # Set up return values + mock_label_matching.return_value = 'mock_updated_labels' + mock_verify_tags.return_value = 'mock_verified_tags' + + # Call the function + d._update_tags( + updates=self.mock_updates, + existing=self.mock_downstream, + issue=self.mock_issue + ) + + # Assert all calls were made correctly + mock_label_matching.assert_called_with( + self.mock_issue.tags, + self.mock_downstream.fields.labels + ) + mock_verify_tags.assert_called_with('mock_updated_labels') + self.mock_downstream.update.assert_called_with({'labels': 'mock_verified_tags'}) + + @mock.patch(PATH + 'verify_tags') + @mock.patch(PATH + '_label_matching') + def test_update_tags_no_api_call(self, + mock_label_matching, + mock_verify_tags): + """ + This function tests the '_update_tags' function where the existing tags are the same + as the new ones + """ + # Set up return values + mock_label_matching.return_value = 'mock_updated_labels' + mock_verify_tags.return_value = ['tag3', 'tag4'] + + # Call the function + d._update_tags( + updates=self.mock_updates, + existing=self.mock_downstream, + issue=self.mock_issue + ) + + # Assert all calls were made correctly + mock_label_matching.assert_called_with( + self.mock_issue.tags, + self.mock_downstream.fields.labels + ) + mock_verify_tags.assert_called_with('mock_updated_labels') + self.mock_downstream.update.assert_not_called() + + def test_update_description_update(self): + """ + This function tests '_update_description' where we just have to update the contents of the description + """ + # Set up return values + self.mock_downstream.fields.description = 'Upstream description: {quote} test {quote}' + + # Call the function + d._update_description( + existing=self.mock_downstream, + issue=self.mock_issue + ) + + # Assert all calls were made correctly + self.mock_downstream.update.assert_called_with( + {'description': 'Upstream description: {quote}mock_content{quote}'}) + + def test_update_description_add_field(self): + """ + This function tests '_update_description' where we just have to add a description field + """ + # Set up return values + self.mock_downstream.fields.description = '[123] Upstream Reporter: mock_user \n' \ + 'Upstream description: {quote} test {quote}' + + # Call the function + d._update_description( + existing=self.mock_downstream, + issue=self.mock_issue + ) + + # Assert all calls were made correctly + self.mock_downstream.update.assert_called_with( + {'description': '[123] Upstream Reporter: mock_user \n' + 'Upstream description: {quote}mock_content{quote}'}) + + def test_update_description_add_reporter(self): + """ + This function tests '_update_description' where we have to add a description and upstream reporter field + """ + # Set up return values + self.mock_downstream.fields.description = '[123] Upstream issue status: Open\n' + + # Call the function + d._update_description( + existing=self.mock_downstream, + issue=self.mock_issue + ) + # Assert all calls were made correctly + self.mock_downstream.update.assert_called_with({'description': '[123] Upstream issue status: Open\n[1234] Upstream Reporter: mock_user \nUpstream description: {quote}mock_content{quote}'}) + + def test_update_description_add_reporter_no_status(self): + """ + This function tests '_update_description' where we have to add reporter and description without status + """ + # Set up return values + self.mock_downstream.fields.description = '' + + # Call the function + d._update_description( + existing=self.mock_downstream, + issue=self.mock_issue + ) + + # Assert all calls were made correctly + self.mock_downstream.update.assert_called_with( + {'description': '[1234] Upstream Reporter: mock_user \n' + 'Upstream description: {quote}mock_content{quote} \n '}) + + def test_update_description_add_description(self): + """ + This function tests '_update_description' where we have a reporter and status already + """ + # Set up return values + self.mock_downstream.fields.description = '[123] Upstream issue status: Open\n' \ + '[123] Upstream Reporter: mock_user\n' + + # Call the function + d._update_description( + existing=self.mock_downstream, + issue=self.mock_issue + ) + + # Assert all calls were made correctly + self.mock_downstream.update.assert_called_with( + {'description': '[123] Upstream issue status: Open\n' + '[123] Upstream Reporter: mock_user \n' + 'Upstream description: {quote}mock_content{quote}\n\n'}) + + def test_verify_tags(self): + """ + This function tests 'verify_tags' function + """ + # Call the function + response = d.verify_tags( + tags=['this is a tag'] + ) + + # Assert everything was called correctly + self.assertEqual(response, ['this_is_a_tag']) + + @mock.patch(PATH + '_get_jira_client') + @mock.patch(PATH + '_matching_jira_issue_query') + @mock.patch(PATH + '_close_as_duplicate') + @mock.patch('jira.client.JIRA') + def test_close_duplicates_no_matching(self, + mock_client, + mock_close_as_duplicate, + mock_matching_jira_issue_query, + mock_get_jira_client): + """ + This tests 'close_duplicates' function where len(results) <= 1 + """ + # Set up return values + mock_get_jira_client.return_value = mock_client + mock_matching_jira_issue_query.return_value = ['only_one_response'] + + # Call the function + response = d.close_duplicates( + issue=self.mock_issue, + config=self.mock_config + ) + + # Assert everything was called correctly + mock_get_jira_client.assert_called_with(self.mock_issue, self.mock_config) + mock_matching_jira_issue_query.assert_called_with( + mock_client, + self.mock_issue, + self.mock_config, + free=True + ) + mock_close_as_duplicate.assert_not_called() + self.assertEqual(None, response) + + @mock.patch(PATH + '_get_jira_client') + @mock.patch(PATH + '_matching_jira_issue_query') + @mock.patch(PATH + '_close_as_duplicate') + @mock.patch('jira.client.JIRA') + def test_close_duplicates(self, + mock_client, + mock_close_as_duplicate, + mock_matching_jira_issue_query, + mock_get_jira_client): + """ + This tests 'close_duplicates' function where len(results) > 1 + """ + # Set up return values + mock_get_jira_client.return_value = mock_client + mock_item = MagicMock() + mock_item.fields.created = 1 + mock_matching_jira_issue_query.return_value = [mock_item, mock_item, mock_item] + + # Call the function + response = d.close_duplicates( + issue=self.mock_issue, + config=self.mock_config + ) + + # Assert everything was called correctly + mock_get_jira_client.assert_called_with(self.mock_issue, self.mock_config) + mock_matching_jira_issue_query.assert_called_with( + mock_client, + self.mock_issue, + self.mock_config, + free=True + ) + mock_close_as_duplicate.assert_called_with( + mock_client, + mock_item, + mock_item, + self.mock_config + ) + self.assertEqual(None, response) + + @mock.patch('jira.client.JIRA') + def test_close_as_duplicate_errors(self, + mock_client): + """ + This tests '_close_as_duplicate' function where client.transition_issue throws an exception + """ + # Set up return values + class HTTPExceptionHelper(): + text = "Field 'resolution' cannot be set" + + class HTTPException(Exception): + response = HTTPExceptionHelper + + mock_duplicate = MagicMock() + mock_duplicate.permalink.return_value = 'mock_url' + mock_duplicate.key = 'mock_key' + mock_keeper = MagicMock() + mock_keeper.key = 'mock_key' + mock_keeper.permalink.return_value = 'mock_url' + mock_client.transitions.return_value = [{'name': 'Dropped', 'id': '1234'}] + mock_client.comments.return_value = [] + mock_client.transition_issue.side_effect = HTTPException + + # Call the function + d._close_as_duplicate( + client=mock_client, + duplicate=mock_duplicate, + keeper=mock_keeper, + config=self.mock_config + ) + + # Assert everything was called correctly + mock_client.comments.assert_any_call(mock_keeper) + mock_client.comments.assert_any_call(mock_duplicate) + mock_client.transitions.assert_called_with(mock_duplicate) + mock_client.add_comment.assert_any_call(mock_duplicate, 'Marking as duplicate of mock_key') + mock_client.add_comment.assert_any_call(mock_keeper, 'mock_key is a duplicate.') + mock_client.transition_issue.assert_any_call( + mock_duplicate, + '1234', + resolution={'name': 'Duplicate'} + ) + mock_client.transition_issue.assert_any_call( + mock_duplicate, + '1234' + ) + + @mock.patch('jira.client.JIRA') + def test_close_as_duplicate(self, + mock_client): + """ + This tests '_close_as_duplicate' function where everything goes smoothly + """ + # Set up return values + mock_duplicate = MagicMock() + mock_duplicate.permalink.return_value = 'mock_url' + mock_duplicate.key = 'mock_key' + mock_keeper = MagicMock() + mock_keeper.key = 'mock_key' + mock_keeper.permalink.return_value = 'mock_url' + mock_client.transitions.return_value = [{'name': 'Dropped', 'id': '1234'}] + mock_client.comments.return_value = [] + + # Call the function + d._close_as_duplicate( + client=mock_client, + duplicate=mock_duplicate, + keeper=mock_keeper, + config=self.mock_config + ) + + # Assert everything was called correctly + mock_client.comments.assert_any_call(mock_keeper) + mock_client.comments.assert_any_call(mock_duplicate) + mock_client.transitions.assert_called_with(mock_duplicate) + mock_client.add_comment.assert_any_call(mock_duplicate, 'Marking as duplicate of mock_key') + mock_client.add_comment.assert_any_call(mock_keeper, 'mock_key is a duplicate.') + mock_client.transition_issue.assert_called_with( + mock_duplicate, + '1234', + resolution={'name': 'Duplicate'} ) diff --git a/tests/test_intermediary.py b/tests/test_intermediary.py index 4ef85f2..b521339 100644 --- a/tests/test_intermediary.py +++ b/tests/test_intermediary.py @@ -1,69 +1,130 @@ from nose.tools import eq_ - +import mock import unittest import sync2jira.intermediary as i +PATH = 'sync2jira.intermediary.' class TestIntermediary(unittest.TestCase): - config = { - 'sync2jira': { - 'map': { - 'pagure': { - 'koji': {'project': 'RCMPROJ', 'component': 'Brew'}, - }, - 'github': { - 'product-definition-center/product-definition-center': { - 'project': 'PDC', 'component': 'General', }, - }, - }, - 'jira': { - # Nothing, really.. - }, - }, - } - - def test_issue_repr(self): - issue = i.Issue('pagure', 'title', 'url', 'koji', [], self.config) - eq_(repr(issue), '') - - def test_issue_title(self): - issue = i.Issue('pagure', 'title', 'url', 'koji', [], self.config) - eq_(issue.title, '[koji] title') + """ + This class tests the downstream.py file under sync2jira + """ + def setUp(self): + self.mock_config = { + 'sync2jira': { + 'pagure_url': 'dummy_pagure_url', + 'map': { + 'pagure': { + 'pagure': 'mock_downstream' + }, + 'github': { + 'github': 'mock_downstream' + } + } + } + } - def test_issue_from_pagure(self): - upstream_issue = { - 'title': 'title', - 'id': 21, + @mock.patch(PATH + 'datetime') + def test_from_pagure(self, + mock_datetime): + """ + This tests the 'from_pagure' function under the Issue class + """ + # Set up return values + mock_datetime.utcfromtimestamp.return_value = 'mock_date' + mock_issue = { 'comments': [{ - 'user': {'name': 'Ralph'}, - 'comment': 'This is fine.', + 'date_created': '1234', + 'user': { + 'name': 'mock_name' + }, + 'comment': 'mock_body', + 'id': '1234', }], + 'title': 'mock_title', + 'id': 1234, + 'tags': 'mock_tags', + 'milestone': 'mock_milestone', + 'priority': 'mock_priority', + 'content': 'mock_content', + 'user': 'mock_reporter', + 'assignee': 'mock_assignee', + 'status': 'mock_status', + 'date_created': 'mock_date' } - issue = i.Issue.from_pagure('koji', upstream_issue, self.config) - eq_(issue.title, '[koji] title') - eq_(issue.url, 'https://pagure.io/koji/issue/21') - eq_(issue.comments, [{ - 'author': 'Ralph', - 'body': 'This is fine.', - 'changed': None - }]) - def test_issue_from_github(self): - upstream_issue = { - 'title': 'title', - 'html_url': 'Some github url', + # Call the function + response = i.Issue.from_pagure( + upstream='pagure', + issue=mock_issue, + config=self.mock_config + ) + + # Assert that we made the calls correctly + self.assertEqual(response.source, 'pagure') + self.assertEqual(response.title, '[pagure] mock_title') + self.assertEqual(response.url, 'dummy_pagure_url/pagure/issue/1234') + self.assertEqual(response.upstream, 'pagure') + self.assertEqual(response.comments, [{'body': 'mock_body', 'name': 'mock_name', + 'author': 'mock_name', 'changed': None, + 'date_created': 'mock_date', 'id': '1234'}]) + self.assertEqual(response.tags, 'mock_tags') + self.assertEqual(response.fixVersion, ['mock_milestone']) + self.assertEqual(response.priority, 'mock_priority') + self.assertEqual(response.content, 'mock_content') + self.assertEqual(response.reporter, 'mock_reporter') + self.assertEqual(response.assignee, 'mock_assignee') + self.assertEqual(response.status, 'mock_status') + self.assertEqual(response.id, 'mock_date') + self.assertEqual(response.downstream, 'mock_downstream') + + def test_from_github(self): + """ + This tests the 'from_github' function under the Issue class + """ + # Set up return values + mock_issue = { 'comments': [{ - 'user': {'login': 'ralphbean'}, - 'body': 'This is dandy.', + 'author': 'mock_author', + 'name': 'mock_name', + 'body': 'mock_body', + 'id': 'mock_id', + 'date_created': 'mock_date' }], + 'title': 'mock_title', + 'html_url': 'mock_url', + 'id': 1234, + 'labels': 'mock_tags', + 'milestone': 'mock_milestone', + 'priority': 'mock_priority', + 'body': 'mock_content', + 'user': 'mock_reporter', + 'assignees': 'mock_assignee', + 'state': 'open', + 'date_created': 'mock_date' } - repo = 'product-definition-center/product-definition-center' - issue = i.Issue.from_github(repo, upstream_issue, self.config) - eq_(issue.title, '[product-definition-center/product-definition-center] title') - eq_(issue.url, 'Some github url') - eq_(issue.comments, [{ - 'author': 'ralphbean', - 'body': 'This is dandy.', - 'changed': None - }]) + + # Call the function + response = i.Issue.from_github( + upstream='github', + issue=mock_issue, + config=self.mock_config + ) + + # Assert that we made the calls correctly + self.assertEqual(response.source, 'github') + self.assertEqual(response.title, '[github] mock_title') + self.assertEqual(response.url, 'mock_url') + self.assertEqual(response.upstream, 'github') + self.assertEqual(response.comments, [{'body': 'mock_body', 'name': 'mock_name', 'author': 'mock_author', + 'changed': None, 'date_created': 'mock_date', 'id': 'mock_id'}]) + self.assertEqual(response.tags, 'mock_tags') + self.assertEqual(response.fixVersion, ['mock_milestone']) + self.assertEqual(response.priority, None) + self.assertEqual(response.content, 'mock_content') + self.assertEqual(response.reporter, 'mock_reporter') + self.assertEqual(response.assignee, 'mock_assignee') + self.assertEqual(response.status, 'Open') + self.assertEqual(response.id, 1234) + self.assertEqual(response.downstream, 'mock_downstream') diff --git a/tests/test_main.py b/tests/test_main.py index c92d450..82c6328 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -1,9 +1,47 @@ +import mock import unittest +try: + # Python 3.3 > + from unittest.mock import MagicMock # noqa: F401 +except ImportError: + from mock import MagicMock # noqa: F401 import sync2jira.main as m +PATH = 'sync2jira.main.' + + class TestMain(unittest.TestCase): + """ + This class tests the main.py file under sync2jira + """ + def setUp(self): + """ + Set up the testing environment + """ + # Mock Config dict + self.mock_config = { + 'sync2jira': { + 'jira': { + 'mock_jira_instance': {'mock_jira': 'mock_jira'} + }, + 'testing': {}, + 'legacy_matching': False, + 'map': { + 'pagure': {'key_pagure': 'value1'}, + 'github': {'key_github': 'value1'} + }, + 'initialize': True, + 'listen': True + }, + } + + # Mock Fedmsg Message + self.mock_message = { + 'msg_id': 'mock_id' + } + def _check_for_exception(self, loader, target, exc=ValueError): try: m.load_config(loader) @@ -30,3 +68,202 @@ class TestMain(unittest.TestCase): def test_config_validate_all_good(self): loader = lambda: {'sync2jira': {'map': {'pagure': {}}, 'jira': {}}} m.load_config(loader) # ahhh, no exception. + + @mock.patch(PATH + 'u') + @mock.patch(PATH + 'd') + @mock.patch(PATH + 'load_config') + def test_close_duplicates(self, + mock_load_config, + mock_d, + mock_u): + """ + This tests the 'close_duplicates' function where everything goes smoothly + """ + # Set up return values + mock_load_config.return_value = self.mock_config + mock_u.pagure_issues.return_value = ['mock_issue_github'] + mock_u.github_issues.return_value = ['mock_issue_pagure'] + + # Call the function + m.close_duplicates() + + # Assert everything was called correctly + mock_load_config.assert_called_once() + mock_u.pagure_issues.assert_called_with('key_pagure', self.mock_config) + mock_u.github_issues.assert_called_with('key_github', self.mock_config) + mock_d.close_duplicates.assert_any_call('mock_issue_github', self.mock_config) + mock_d.close_duplicates.assert_any_call('mock_issue_pagure', self.mock_config) + + @mock.patch(PATH + 'u') + @mock.patch(PATH + 'd') + @mock.patch(PATH + 'load_config') + def test_close_duplicates_errors(self, + mock_load_config, + mock_d, + mock_u): + """ + This tests the 'close_duplicates' function where closing duplicates raises an exception + """ + # Set up return values + mock_load_config.return_value = self.mock_config + mock_u.pagure_issues.return_value = ['mock_issue'] + mock_u.github_issues.return_value = ['mock_issue'] + mock_d.close_duplicates.side_effect = Exception() + + # Call the function + with self.assertRaises(Exception): + m.close_duplicates() + + # Assert everything was called correctly + mock_load_config.assert_called_once() + mock_u.pagure_issues.assert_called_with('key_pagure', self.mock_config) + mock_u.github_issues.assert_not_called() + mock_d.close_duplicates.assert_called_with('mock_issue', self.mock_config) + + @mock.patch(PATH + 'load_config') + @mock.patch(PATH + 'u') + def test_list_managed(self, + mock_u, + mock_load_config): + """ + This tests the 'list_managed' function + """ + # Set up return values + mock_load_config.return_value = self.mock_config + + # Call the function + m.list_managed() + + # Assert everything was called correctly + mock_load_config.assert_called_once() + mock_u.pagure_issues.assert_called_with('key_pagure', self.mock_config) + mock_u.github_issues.assert_called_with('key_github', self.mock_config) + + @mock.patch(PATH + 'initialize') + @mock.patch(PATH + 'load_config') + @mock.patch(PATH + 'listen') + def test_main(self, + mock_listen, + mock_load_config, + mock_initialize): + """ + This tests the 'main' function + """ + # Set up return values + mock_load_config.return_value = self.mock_config + + # Call the function + m.main() + + # Assert everything was called correctly + mock_load_config.assert_called_once() + mock_listen.assert_called_with(self.mock_config) + mock_listen.assert_called_with(self.mock_config) + mock_initialize.assert_called_with(self.mock_config) + + @mock.patch(PATH + 'u') + @mock.patch(PATH + 'd') + def test_initialize(self, + mock_d, + mock_u): + """ + This tests 'initialize' function where everything goes smoothly! + """ + # Set up return values + mock_u.pagure_issues.return_value = ['mock_issue_pagure'] + mock_u.github_issues.return_value = ['mock_issue_github'] + + # Call the function + m.initialize(self.mock_config) + + # Assert everything was called correctly + mock_u.pagure_issues.assert_called_with('key_pagure', self.mock_config) + mock_u.github_issues.assert_called_with('key_github', self.mock_config) + mock_d.sync_with_jira.assert_any_call('mock_issue_pagure', self.mock_config) + mock_d.sync_with_jira.assert_any_call('mock_issue_github', self.mock_config) + + @mock.patch(PATH + 'u') + @mock.patch(PATH + 'd') + def test_initialize_errors(self, + mock_d, + mock_u): + """ + This tests 'initialize' function where syncing with JIRA throws an exception + """ + # Set up return values + mock_u.pagure_issues.return_value = ['mock_issue_pagure'] + mock_u.github_issues.return_value = ['mock_issue_github'] + mock_d.sync_with_jira.side_effect = Exception() + + # Call the function + with self.assertRaises(Exception): + m.initialize(self.mock_config) + + # Assert everything was called correctly + mock_u.pagure_issues.assert_called_with('key_pagure', self.mock_config) + mock_d.sync_with_jira.assert_any_call('mock_issue_pagure', self.mock_config) + + @mock.patch(PATH + 'u') + @mock.patch(PATH + 'd') + @mock.patch(PATH + 'fedmsg') + def test_listen_no_handlers(self, + mock_fedmsg, + mock_d, + mock_u): + """ + Test 'listen' function where suffix is not in handlers + """ + # Set up return values + mock_fedmsg.tail_messages.return_value = [("dummy", "dummy", "mock_topic", self.mock_message)] + + # Call the function + m.listen(self.mock_config) + + # Assert everything was called correctly + mock_d.sync_with_jira.assert_not_called() + mock_u.handle_github_message.assert_not_called() + mock_u.handle_pagure_message.assert_not_called() + + @mock.patch(PATH + 'u') + @mock.patch(PATH + 'd') + @mock.patch(PATH + 'fedmsg') + def test_listen_no_issue(self, + mock_fedmsg, + mock_d, + mock_u): + """ + Test 'listen' function where the handler returns none + """ + # Set up return values + mock_fedmsg.tail_messages.return_value = [("dummy", "dummy", "d.d.d.pagure.issue.drop", self.mock_message)] + mock_u.handle_pagure_message.return_value = None + + # Call the function + m.listen(self.mock_config) + + # Assert everything was called correctly + mock_d.sync_with_jira.assert_not_called() + mock_u.handle_github_message.assert_not_called() + mock_u.handle_pagure_message.assert_called_with(self.mock_message, self.mock_config) + + @mock.patch(PATH + 'u') + @mock.patch(PATH + 'd') + @mock.patch(PATH + 'fedmsg') + def test_listen(self, + mock_fedmsg, + mock_d, + mock_u): + """ + Test 'listen' function where everything goes smoothly + """ + # Set up return values + mock_fedmsg.tail_messages.return_value = [("dummy", "dummy", "d.d.d.github.issue.comment", self.mock_message)] + mock_u.handle_github_message.return_value = 'dummy_issue' + + # Call the function + m.listen(self.mock_config) + + # Assert everything was called correctly + mock_d.sync_with_jira.assert_called_with('dummy_issue', self.mock_config) + mock_u.handle_github_message.assert_called_with(self.mock_message, self.mock_config) + mock_u.handle_pagure_message.assert_not_called() \ No newline at end of file diff --git a/tests/test_upstream.py b/tests/test_upstream.py index 8e8967d..593ef1e 100644 --- a/tests/test_upstream.py +++ b/tests/test_upstream.py @@ -1,378 +1,576 @@ import mock import unittest -from nose.tools import eq_ +import json +try: + # Python 3.3 > + from unittest.mock import MagicMock # noqa: F401 +except ImportError: + from mock import MagicMock # noqa: F401 + import sync2jira.upstream as u +PATH = 'sync2jira.upstream.' + + class TestUpstream(unittest.TestCase): + """ + This class tests the upstream.py file under sync2jira + """ def setUp(self): - self.config = { + self.mock_config = { 'sync2jira': { 'map': { 'github': { 'org/repo': {}, }, 'pagure': { - 'some_repo': {}, + 'org/repo': {}, }, }, 'jira': { # Nothing, really.. }, - }, - } - - @mock.patch('sync2jira.upstream.i.Issue.from_github') - def test_handle_github_message(self, from_github): - issue_dict = {'state': 'open'} - message = { - 'msg': { - 'repository': { - 'name': 'repo', - 'owner': { - 'login': 'org', - }, - }, - 'issue': issue_dict, - }, - } - u.handle_github_message(message, self.config) - from_github.assert_called_once_with( - 'org/repo', issue_dict, self.config) - - @mock.patch('sync2jira.upstream.i.Issue.from_pagure') - def test_handle_pagure_message(self, from_pagure): - issue_dict = {'status': 'Open'} - message = { - 'msg': { - 'project': { - 'name': 'some_repo', - 'owner': { - 'login': 'org', - }, - }, - 'issue': issue_dict, - }, - } - u.handle_pagure_message(message, self.config) - from_pagure.assert_called_once_with( - 'some_repo', issue_dict, self.config) - - @mock.patch('sync2jira.upstream.i.Issue.from_github') - @mock.patch('sync2jira.upstream._fetch_github_data') - def test_handle_github_comment(self, _fetch_github_data, from_github): - comment = { - 'user': {'login': 'threebean'}, - 'body': 'This is fine.', - } - comments = [comment] - issue_dict = { - 'state': 'open', - 'title': 'title', - 'url': 'some_issue_url', - } - message = { - 'msg': { - 'action': 'created', - 'repository': { - 'name': 'repo', - 'owner': { - 'login': 'org', - }, - }, - 'issue': issue_dict, - 'comment': comment, - }, - } - githubResponse = mock.MagicMock() - githubResponse.json.return_value = comments - _fetch_github_data.return_value = githubResponse - u.handle_github_comment(message, self.config) - - from_github.assert_called_once_with( - 'org/repo', issue_dict, self.config) - - @mock.patch('sync2jira.upstream.i.Issue.from_github') - @mock.patch('sync2jira.upstream._fetch_github_data') - def test_handle_github_change_comment(self, _fetch_github_data, from_github): - comment = { - 'user': {'login': 'threebean'}, - 'body': 'This is fine.', - } - comments = [ - comment, - { - 'user': {'login': 'threebean'}, - 'body': 'Some other text', - } - ] - issue_dict = { - 'state': 'open', - 'title': 'title', - 'url': 'some_issue_url', - } - message = { - 'msg': { - 'action': 'edited', - 'changes': { - 'body': { - 'from': 'Some text', - }, - }, - 'repository': { - 'name': 'repo', - 'owner': { - 'login': 'org', - }, + 'filters': { + 'github': + {'org/repo': {'filter1': 'filter1', 'labels': 'custom_tag'}}, + 'pagure': + {'org/repo': {'filter1': 'filter1', 'tags': ['custom_tag']}}, }, - 'issue': issue_dict, - 'comment': comment, + 'github_token': 'mock_token' }, } - githubResponse = mock.MagicMock() - githubResponse.json.return_value = comments - _fetch_github_data.return_value = githubResponse - u.handle_github_comment(message, self.config) - - from_github.assert_called_once_with( - 'org/repo', issue_dict, self.config) - - - @mock.patch('sync2jira.upstream.i.Issue.from_pagure') - def test_handle_pagure_comment(self, from_pagure): - issue_dict = { - 'status': 'Open', - 'title': 'title', - 'id': 21, - 'comments': [{ - 'user': {'name': 'Ralph'}, - 'comment': 'This is fine.', - }], - } - message = { + # Mock Pagure Message + self.mock_pagure_message = { 'msg': { 'project': { - 'name': 'some_repo', - 'owner': { - 'login': 'org', - }, + 'name': 'org/repo' }, - 'issue': issue_dict, - }, - } - u.handle_pagure_comment(message, self.config) - from_pagure.assert_called_once_with( - 'some_repo', issue_dict, self.config) - - @mock.patch('sync2jira.upstream.i.Issue.from_github') - def test_handle_github_filter_positive(self, from_github): - self.config['sync2jira']['filters'] = { - 'github': { - 'org/repo': { - 'some_value': 'present', + 'issue': { + 'filter1': 'filter1', + 'tags': ['custom_tag'], + 'comments': [], + 'assignee': 'mock_assignee' }, + 'tags': ['new_tag'], + 'comment': 'new_comment', + 'status': 'temp' }, + 'topic': 'io.pagure.prod.pagure.issue.drop', } - issue_dict = { - 'state': 'open', - 'some_value': 'present', - } - message = { + + # Mock Github Comment + self.mock_github_comment = MagicMock() + self.mock_github_comment.user.name = 'mock_username' + self.mock_github_comment.user.login = 'mock_user_login' + self.mock_github_comment.body = 'mock_body' + self.mock_github_comment.id = 'mock_id' + self.mock_github_comment.created_at = 'mock_created_at' + + # Mock Github Message + self.mock_github_message = { 'msg': { 'repository': { - 'name': 'repo', - 'owner': { - 'login': 'org', - }, - }, - 'issue': issue_dict, - }, - } - u.handle_github_message(message, self.config) - from_github.assert_called_once_with( - 'org/repo', issue_dict, self.config) - - @mock.patch('sync2jira.upstream.i.Issue.from_pagure') - def test_handle_pagure_filter_positive(self, from_pagure): - self.config['sync2jira']['filters'] = { - 'pagure': { - 'some_repo': { - 'some_value': 'present', - }, - }, - } - issue_dict = { - 'status': 'Open', - 'some_value': 'present', - } - message = { - 'msg': { - 'project': { - 'name': 'some_repo', 'owner': { - 'login': 'org', + 'login': 'org' }, + 'name': 'repo' }, - 'issue': issue_dict, - }, - } - u.handle_pagure_message(message, self.config) - from_pagure.assert_called_once_with( - 'some_repo', issue_dict, self.config) - - @mock.patch('sync2jira.upstream.i.Issue.from_github') - def test_handle_github_filter_negative(self, from_github): - self.config['sync2jira']['filters'] = { - 'github': { - 'org/repo': { - 'some_value': 'present', - }, - }, - } - issue_dict = { - 'state': 'open', - 'some_value': 'absent', - } - message = { - 'msg': { - 'repository': { - 'name': 'repo', - 'owner': { - 'login': 'org', + 'issue': { + 'filter1': 'filter1', + 'labels': [{'name': 'custom_tag'}], + 'comments': ['some_comments!'], + 'number': 'mock_number', + 'user': { + 'login': 'mock_login' }, - }, - 'issue': issue_dict, - }, + 'assignees': [{'login': 'mock_login'}], + 'milestone': { + 'title': 'mock_milestone' + } + } + } } - u.handle_github_message(message, self.config) - assert not from_github.called, 'should not have been called' - - @mock.patch('sync2jira.upstream.i.Issue.from_pagure') - def test_handle_pagure_filter_negative(self, from_pagure): - self.config['sync2jira']['filters'] = { - 'pagure': { - 'some_repo': { - 'some_value': 'present', - }, + + # Mock github issue + self.mock_github_issue = MagicMock() + self.mock_github_issue.get_comments.return_value = [self.mock_github_comment] + + # Mock Github Issue Raw + self.mock_github_issue_raw = { + 'comments': ['some comment'], + 'number': '1234', + 'user': { + 'login': 'mock_login' }, + 'assignees': [{'login': 'mock_assignee_login'}], + 'labels': [{'name': 'some_label'}], + 'milestone': { + 'title': 'mock_milestone' + } } - issue_dict = { - 'status': 'Open', - 'some_value': 'absent', - } - message = { - 'msg': { - 'project': { - 'name': 'some_repo', - 'owner': { - 'login': 'org', - }, - }, - 'issue': issue_dict, - }, + + # Mock Github Reporter + self.mock_github_person = MagicMock() + self.mock_github_person.name = 'mock_name' + + # Mock Github Repo + self.mock_github_repo = MagicMock() + self.mock_github_repo.get_issue.return_value = self.mock_github_issue + + # Mock Github Client + self.mock_github_client = MagicMock() + self.mock_github_client.get_repo.return_value = self.mock_github_repo + self.mock_github_client.get_user.return_value = self.mock_github_person + + @mock.patch('sync2jira.intermediary.Issue.from_github') + @mock.patch(PATH + 'Github') + @mock.patch(PATH + '_get_all_github_issues') + def test_github_issues(self, + mock_get_all_github_issues, + mock_github, + mock_issue_from_github): + """ + This function tests 'github_issues' function + """ + # Set up return values + mock_github.return_value = self.mock_github_client + mock_get_all_github_issues.return_value = [self.mock_github_issue_raw] + mock_issue_from_github.return_value = 'Successful Call!' + + # Call the function + response = list(u.github_issues( + upstream='org/repo', + config=self.mock_config + )) + + # Assert that calls were made correctly + try: + mock_get_all_github_issues.assert_called_with( + 'https://api.github.com/repos/org/repo/issues?labels=custom_tag&filter1=filter1', + {'Authorization': 'token mock_token'} + ) + except AssertionError: + mock_get_all_github_issues.assert_called_with( + 'https://api.github.com/repos/org/repo/issues?filter1=filter1&labels=custom_tag', + {'Authorization': 'token mock_token'} + ) + self.mock_github_client.get_user.assert_any_call('mock_login') + self.mock_github_client.get_user.assert_any_call('mock_assignee_login') + mock_issue_from_github.assert_called_with( + 'org/repo', + {'labels': ['some_label'], 'number': '1234', 'comments': [ + {'body': 'mock_body', 'name': 'mock_user_login', 'author': 'mock_username', 'changed': None, + 'date_created': 'mock_created_at', 'id': 'mock_id'}], 'assignees': [{'fullname': 'mock_name'}], + 'user': {'login': 'mock_login', 'fullname': 'mock_name'}, 'milestone': 'mock_milestone'}, + self.mock_config + ) + self.mock_github_client.get_repo.assert_called_with('org/repo') + self.mock_github_repo.get_issue.assert_called_with(number='1234') + self.mock_github_issue.get_comments.assert_any_call() + self.assertEqual(response[0], 'Successful Call!') + + @mock.patch('sync2jira.intermediary.Issue.from_github') + @mock.patch(PATH + 'Github') + @mock.patch(PATH + '_get_all_github_issues') + def test_github_issues_no_token(self, + mock_get_all_github_issues, + mock_github, + mock_issue_from_github): + """ + This function tests 'github_issues' function where we have no github token + and no comments + """ + # Set up return values + self.mock_config['sync2jira']['github_token'] = None + self.mock_github_issue_raw['comments'] = 0 + mock_github.return_value = self.mock_github_client + mock_get_all_github_issues.return_value = [self.mock_github_issue_raw] + mock_issue_from_github.return_value = 'Successful Call!' + + # Call the function + response = list(u.github_issues( + upstream='org/repo', + config=self.mock_config + )) + + # Assert that calls were made correctly + try: + mock_get_all_github_issues.assert_called_with( + 'https://api.github.com/repos/org/repo/issues?labels=custom_tag&filter1=filter1', + {} + ) + except AssertionError: + mock_get_all_github_issues.assert_called_with( + 'https://api.github.com/repos/org/repo/issues?filter1=filter1&labels=custom_tag', + {} + ) + self.mock_github_client.get_user.assert_any_call('mock_login') + self.mock_github_client.get_user.assert_any_call('mock_assignee_login') + mock_issue_from_github.assert_called_with( + 'org/repo', + {'labels': ['some_label'], 'number': '1234', 'comments': [], 'assignees': [{'fullname': 'mock_name'}], + 'user': {'login': 'mock_login', 'fullname': 'mock_name'}, 'milestone': 'mock_milestone'}, + self.mock_config + ) + self.assertEqual(response[0], 'Successful Call!') + self.mock_github_client.get_repo.assert_not_called() + self.mock_github_repo.get_issue.assert_not_called() + self.mock_github_issue.get_comments.assert_not_called() + + @mock.patch('sync2jira.intermediary.Issue.from_pagure') + @mock.patch(PATH + 'requests') + def test_pagure_issues_error(self, + mock_requests, + mock_issue_from_pagure): + """ + This function tests 'pagure_issues' function where we get an IOError + """ + # Set up return values + get_return = MagicMock() + get_return.__bool__ = mock.Mock(return_value=False) + get_return.__nonzero__ = get_return.__bool__ + get_return.json.side_effect = Exception() + get_return.text.return_value = { + 'issues': [ + {'assignee': 'mock_assignee'} + ] + } - u.handle_pagure_message(message, self.config) - assert not from_pagure.called, 'should not have been called' - - @mock.patch('sync2jira.upstream.requests') - @mock.patch('sync2jira.upstream.i.Issue.from_pagure') - def test_get_all_pagure_issues(self, from_pagure, requests): - response = mock.MagicMock() - mock_issue = {'comments_url': 'comment_url'} - response.json.return_value = { - 'issues': [mock_issue], + mock_requests.get.return_value = get_return + + # Call the function + with self.assertRaises(IOError): + list(u.pagure_issues( + upstream='org/repo', + config=self.mock_config + )) + + # Assert everything was called correctly + mock_requests.get.assert_called_with( + 'https://pagure.io/api/0/org/repo/issues', + params={'filter1': 'filter1', 'tags': ['custom_tag']} + ) + mock_issue_from_pagure.assert_not_called() + + @mock.patch('sync2jira.intermediary.Issue.from_pagure') + @mock.patch(PATH + 'requests') + def test_pagure_issues(self, + mock_requests, + mock_issue_from_pagure): + """ + This function tests 'pagure_issues' function + """ + # Set up return values + get_return = MagicMock() + get_return.json.return_value = { + 'issues': [ + {'assignee': 'mock_assignee'} + ] + } - requests.get.return_value = response + get_return.request.url = 'mock_url' + mock_requests.get.return_value = get_return + mock_issue_from_pagure.return_value = 'Successful Call!' - generator = u.pagure_issues('some_repo', self.config) - # Step through that... - list(generator) + # Call the function + response = list(u.pagure_issues( + upstream='org/repo', + config=self.mock_config + )) - requests.get.assert_called_once_with( - 'https://pagure.io/api/0/some_repo/issues', - params={'status': 'Open'}, + # Assert everything was called correctly + self.assertEqual(response[0], 'Successful Call!') + mock_requests.get.assert_called_with( + 'https://pagure.io/api/0/org/repo/issues', + params={'filter1': 'filter1', 'tags': ['custom_tag']} ) - from_pagure.assert_called_once_with( - 'some_repo', mock_issue, self.config) - - @mock.patch('sync2jira.upstream.requests') - @mock.patch('sync2jira.upstream.i.Issue.from_github') - def test_get_all_github_issues(self, from_github, requests): - response = mock.MagicMock() - mock_issue = {'comments_url': 'comment_url'} - response.json.return_value = [mock_issue] - requests.get.return_value = response - - generator = u.github_issues('some_repo', self.config) - # Step through that... - list(generator) - - requests.get.assert_any_call( - 'https://api.github.com/repos/some_repo/issues?state=open', - headers={}, + mock_issue_from_pagure.assert_called_with( + 'org/repo', + {'assignee': ['mock_assignee']}, + self.mock_config ) - requests.get.assert_any_call( - 'comment_url', - headers={}, + + @mock.patch('sync2jira.intermediary.Issue.from_pagure') + def test_handle_pagure_message_not_in_mapped(self, + mock_issue_from_pagure): + """ + This function tests 'handle_pagure_message' where upstream is not in mapped repo + """ + # Set up return values + self.mock_pagure_message['msg']['project']['name'] = 'bad_repo' + # Call the function + response = u.handle_pagure_message( + msg=self.mock_pagure_message, + config=self.mock_config ) - from_github.assert_called_once_with( - 'some_repo', mock_issue, self.config) - - @mock.patch('sync2jira.upstream.requests') - @mock.patch('sync2jira.upstream.i.Issue.from_pagure') - def test_get_filtered_pagure_issues(self, from_pagure, requests): - self.config['sync2jira']['filters'] = { - 'pagure': { - 'some_repo': { - 'some_value': 'present', - }, - }, - } - response = mock.MagicMock() - mock_issue = {'comments_url': 'comment_url'} - response.json.return_value = { - 'issues': [mock_issue], - } - requests.get.return_value = response - generator = u.pagure_issues('some_repo', self.config) - # Step through that... - list(generator) + # Assert all calls made correctly + self.assertEqual(None, response) + mock_issue_from_pagure.assert_not_called() + + @mock.patch('sync2jira.intermediary.Issue.from_pagure') + def test_handle_pagure_message_bad_filter(self, + mock_issue_from_pagure): + """ + This function tests 'handle_pagure_message' where comparing the actual vs. filter does not equate + """ + # Set up return values + self.mock_pagure_message['msg']['issue']['filter1'] = 'filter2' + + # Call function + response = u.handle_pagure_message( + msg=self.mock_pagure_message, + config=self.mock_config) + + # Assert that calls were made correctly + mock_issue_from_pagure.assert_not_called() + self.assertEqual(None, response) + + @mock.patch('sync2jira.intermediary.Issue.from_pagure') + def test_handle_pagure_message_bad_tag(self, + mock_issue_from_pagure): + """ + This function tests 'handle_pagure_message' where the tags do not match + """ + # Set up return values + self.mock_pagure_message['msg']['issue']['tags'] = ['bad_tags'] - requests.get.assert_called_once_with( - 'https://pagure.io/api/0/some_repo/issues', - params={'some_value': 'present'}, + # Call function + response = u.handle_pagure_message( + msg=self.mock_pagure_message, + config=self.mock_config) + + # Assert that calls were made correctly + mock_issue_from_pagure.assert_not_called() + self.assertEqual(None, response) + + @mock.patch('sync2jira.intermediary.Issue.from_pagure') + def test_handle_pagure_message_successful(self, + mock_issue_from_pagure): + """ + This function tests 'handle_pagure_message' where everything goes smoothly + and we test edge cases! + """ + # Set up return values + mock_issue_from_pagure.return_value = "Successful Call!" + + # Call the function + response = u.handle_pagure_message( + msg=self.mock_pagure_message, + config=self.mock_config ) - from_pagure.assert_called_once_with( - 'some_repo', mock_issue, self.config) - - @mock.patch('sync2jira.upstream.requests') - @mock.patch('sync2jira.upstream.i.Issue.from_github') - def test_get_filtered_github_issues(self, from_github, requests): - self.config['sync2jira']['filters'] = { - 'github': { - 'some_repo': { - 'some_value': 'present', - }, - }, - } - response = mock.MagicMock() - mock_issue = {'comments_url': 'comment_url'} - response.json.return_value = [mock_issue] - requests.get.return_value = response - - generator = u.github_issues('some_repo', self.config) - # Step through that... - list(generator) - - requests.get.assert_any_call( - 'https://api.github.com/repos/some_repo/issues?some_value=present', - headers={}, + + # Assert that calls were made correctly + mock_issue_from_pagure.assert_called_with( + 'org/repo', + {'status': 'Dropped', 'assignee': ['mock_assignee'], 'filter1': 'filter1', 'comments': ['new_comment'], + 'tags': ['custom_tag', 'new_tag']}, + self.mock_config + ) + self.assertEqual(response, 'Successful Call!') + + @mock.patch(PATH + 'Github') + @mock.patch('sync2jira.intermediary.Issue.from_github') + def test_handle_github_message_not_in_mapped(self, + mock_issue_from_github, + mock_github): + """ + This function tests 'handle_github_message' where upstream is not in mapped repos + """ + # Set up return values + self.mock_github_message['msg']['repository']['owner']['login'] = 'bad_owner' + + # Call the function + response = u.handle_github_message( + msg=self.mock_github_message, + config=self.mock_config + ) + + # Assert that all calls were made correctly + mock_issue_from_github.assert_not_called() + mock_github.assert_not_called() + self.assertEqual(None, response) + + @mock.patch('sync2jira.intermediary.Issue.from_github') + def test_handle_github_message_bad_filter(self, + mock_issue_from_github): + """ + This function tests 'handle_github_message' where comparing the actual vs. filter does not equate + """ + # Set up return values + self.mock_github_message['msg']['issue']['filter1'] = 'filter2' + + # Call function + response = u.handle_github_message( + msg=self.mock_github_message, + config=self.mock_config + ) + # Assert that calls were made correctly + mock_issue_from_github.assert_not_called() + self.assertEqual(None, response) + + @mock.patch('sync2jira.intermediary.Issue.from_github') + def test_handle_github_message_bad_label(self, + mock_issue_from_github): + """ + This function tests 'handle_github_message' where comparing the actual vs. filter does not equate + """ + # Set up return values + self.mock_github_message['msg']['issue']['labels'] = [{'name': 'bad_label'}] + + # Call function + response = u.handle_github_message( + msg=self.mock_github_message, + config=self.mock_config + ) + # Assert that calls were made correctly + mock_issue_from_github.assert_not_called() + self.assertEqual(None, response) + + @mock.patch(PATH + 'Github') + @mock.patch('sync2jira.intermediary.Issue.from_github') + def test_handle_github_message_no_comments(self, + mock_issue_from_github, + mock_github): + """ + This function tests 'handle_github_message' where we have no comments + """ + # Set up return values + mock_issue_from_github.return_value = "Successful Call!" + mock_github.return_value = self.mock_github_client + self.mock_github_message['msg']['issue']['comments'] = 0 + + # Call function + response = u.handle_github_message( + msg=self.mock_github_message, + config=self.mock_config + ) + # Assert that calls were made correctly + mock_issue_from_github.assert_called_with('org/repo', + {'labels': ['custom_tag'], 'number': 'mock_number', + 'comments': [], 'assignees': [{'fullname': 'mock_name'}], + 'filter1': 'filter1', + 'user': {'login': 'mock_login', 'fullname': 'mock_name'}, + 'milestone': 'mock_milestone'}, + self.mock_config) + mock_github.assert_called_with('mock_token') + self.assertEqual('Successful Call!', response) + self.mock_github_client.get_repo.assert_not_called() + self.mock_github_repo.get_issue.assert_not_called() + self.mock_github_issue.get_comments.assert_not_called() + self.mock_github_client.get_user.assert_called_with('mock_login') + + @mock.patch(PATH + 'Github') + @mock.patch('sync2jira.intermediary.Issue.from_github') + def test_handle_github_message_successful(self, + mock_issue_from_github, + mock_github): + """ + This function tests 'handle_github_message' where everything goes smoothly! + """ + # Set up return values + mock_issue_from_github.return_value = "Successful Call!" + mock_github.return_value = self.mock_github_client + + # Call function + response = u.handle_github_message( + msg=self.mock_github_message, + config=self.mock_config ) - requests.get.assert_any_call( - 'comment_url', - headers={}, + + # Assert that calls were made correctly + mock_issue_from_github.assert_called_with('org/repo', + {'labels': ['custom_tag'], 'number': 'mock_number', + 'comments': [{'body': 'mock_body', 'name': 'mock_user_login', + 'author': 'mock_username', 'changed': None, + 'date_created': 'mock_created_at', 'id': 'mock_id'}], + 'assignees': [{'fullname': 'mock_name'}], + 'filter1': 'filter1', 'user': + {'login': 'mock_login', 'fullname': 'mock_name'}, + 'milestone': 'mock_milestone'}, self.mock_config) + mock_github.assert_called_with('mock_token') + self.assertEqual('Successful Call!', response) + self.mock_github_client.get_repo.assert_called_with('org/repo') + self.mock_github_repo.get_issue.assert_called_with(number='mock_number') + self.mock_github_issue.get_comments.assert_any_call() + self.mock_github_client.get_user.assert_called_with('mock_login') + + @mock.patch(PATH + '_fetch_github_data') + @mock.patch(PATH + '_github_link_field_to_dict') + def test_get_all_github_issues(self, + mock_github_link_field_to_dict, + mock_fetch_github_data): + """ + This tests the '_get_all_github_issues' function + """ + # Set up return values + get_return = MagicMock() + get_return.json.return_value = [{'comments_url': 'mock_comments_url'}] + get_return.headers = {'link': 'mock_link'} + mock_fetch_github_data.return_value = get_return + + # Call the function + response = list(u._get_all_github_issues( + url='mock_url', + headers='mock_headers' + )) + + # Assert everything was called correctly + mock_fetch_github_data.assert_any_call('mock_url', 'mock_headers') + mock_fetch_github_data.assert_any_call('mock_comments_url', 'mock_headers') + mock_github_link_field_to_dict.assert_called_with('mock_link') + self.assertEqual('mock_comments_url', response[0]['comments_url']) + + @mock.patch(PATH + 'requests') + def test_fetch_github_data_error(self, + mock_requests): + """ + Tests the '_fetch_github_data' function where we raise an IOError + """ + # Set up return values + get_return = MagicMock() + get_return.__bool__ = mock.Mock(return_value=False) + get_return.__nonzero__ = get_return.__bool__ + get_return.json.side_effect = Exception() + get_return.text.return_value = { + 'issues': [ + {'assignee': 'mock_assignee'} + ] + + } + mock_requests.get.return_value = get_return + + # Call the function + with self.assertRaises(IOError): + u._fetch_github_data( + url='mock_url', + headers='mock_headers' + ) + + # Assert everything was called correctly + mock_requests.get.assert_called_with('mock_url', headers='mock_headers') + + @mock.patch(PATH + 'requests') + def test_fetch_github_data(self, + mock_requests): + """ + Tests the '_fetch_github_data' function where everything goes smoothly! + """ + # Set up return values + get_return = MagicMock() + get_return.__bool__ = mock.Mock(return_value=True) + get_return.__nonzero__ = get_return.__bool__ + mock_requests.get.return_value = get_return + + # Call the function + + response = u._fetch_github_data( + url='mock_url', + headers='mock_headers' ) - from_github.assert_called_once_with( - 'some_repo', mock_issue, self.config) + + # Assert everything was called correctly + mock_requests.get.assert_called_with('mock_url', headers='mock_headers') + self.assertEqual(response, get_return) diff --git a/tox.ini b/tox.ini index dbd99b4..eff3480 100644 --- a/tox.ini +++ b/tox.ini @@ -8,9 +8,11 @@ basepython = deps = -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt + pytest + nose sitepackages = False commands = - nosetests {posargs} + pytest {posargs} [testenv:flake8] basepython = python3