From 194d53008593f76eaa2a5a0a17332e06c3282dea Mon Sep 17 00:00:00 2001 From: sidpremkumar Date: Nov 18 2019 20:09:24 +0000 Subject: Add timestamp to logs and reformat all logs --- diff --git a/sync2jira/downstream.py b/sync2jira/downstream.py index e47f305..796e831 100644 --- a/sync2jira/downstream.py +++ b/sync2jira/downstream.py @@ -114,7 +114,7 @@ def _get_jira_client(issue, config): if not jira_instance: jira_instance = config['sync2jira'].get('default_jira_instance', False) if not jira_instance: - log.error(" No jira_instance for issue and there is no default in the config") + log.error("No jira_instance for issue and there is no default in the config") raise Exception client = jira.client.JIRA(**config['sync2jira']['jira'][jira_instance]) @@ -169,7 +169,7 @@ def _matching_jira_issue_query(client, issue, config, free=False): if search is True: # We went through all the comments and didn't find anything # that indicated it was a duplicate - log.warning(' Matching downstream issue %s to upstream issue %s' % + log.warning('Matching downstream issue %s to upstream issue %s' % (result.fields.summary, issue.title)) final_results.append(result) else: @@ -223,7 +223,7 @@ def alert_user_of_duplicate_issues(issue, final_result, results_of_query, if not jira_instance: jira_instance = config['sync2jira'].get('default_jira_instance', False) if not jira_instance: - log.error(" No jira_instance for issue and there is no default in the config") + log.error("No jira_instance for issue and there is no default in the config") raise Exception base_url = config['sync2jira']['jira'][jira_instance]['options']['server'] + '/browse/' @@ -237,7 +237,7 @@ def alert_user_of_duplicate_issues(issue, final_result, results_of_query, # Get owner name and email from Jira ret = client.search_users(issue.downstream.get('owner')) if len(ret) > 1: - log.warning(' Found multiple users for username %s' % issue.downstream.get('owner')) + log.warning('Found multiple users for username %s' % issue.downstream.get('owner')) found = False for person in ret: if person.key == issue.downstream.get('owner'): @@ -245,9 +245,9 @@ def alert_user_of_duplicate_issues(issue, final_result, results_of_query, found = True break if not found: - log.warning(' Could not find JIRA user for username %s' % issue.downstream.get('owner')) + log.warning('Could not find JIRA user for username %s' % issue.downstream.get('owner')) if not ret: - message = ' No owner could be found for username %s' % issue.downstream.get('owner') + message = 'No owner could be found for username %s' % issue.downstream.get('owner') log.warning(message.strip()) raise ValueError(message) @@ -263,7 +263,7 @@ def alert_user_of_duplicate_issues(issue, final_result, results_of_query, for admin in config['sync2jira']['admins']: ret = client.search_users(list(admin.keys())[0]) if len(ret) > 1: - log.warning(' Found multiple users for admin %s' % list(admin.keys())[0]) + log.warning('Found multiple users for admin %s' % list(admin.keys())[0]) found = False for person in ret: if person.key == issue.downstream.get('owner'): @@ -271,9 +271,9 @@ def alert_user_of_duplicate_issues(issue, final_result, results_of_query, found = True break if not found: - log.warning(' Could not find JIRA user for admin %s' % list(admin.keys())[0]) + log.warning('Could not find JIRA user for admin %s' % list(admin.keys())[0]) if not ret: - message = ' No admin could be found for username %s' % list(admin.keys())[0] + message = 'No admin could be found for username %s' % list(admin.keys())[0] log.warning(message.strip()) raise ValueError(message) admins.append(ret[0].emailAddress) @@ -294,7 +294,7 @@ def alert_user_of_duplicate_issues(issue, final_result, results_of_query, cc=admins, subject=duplicate_issues_subject, text=html_text) - log.info(' Alerted %s about %s duplicate issue(s)' % + log.info('Alerted %s about %s duplicate issue(s)' % (user['email'], len(template_ready))) @@ -311,7 +311,7 @@ def find_username(issue, config): if not jira_instance: jira_instance = config['sync2jira'].get('default_jira_instance', False) if not jira_instance: - log.error(" No jira_instance for issue and there is no default in the config") + log.error("No jira_instance for issue and there is no default in the config") raise Exception return config['sync2jira']['jira'][jira_instance]['basic_auth'][0] @@ -359,7 +359,7 @@ def _find_comment_in_jira(comment, j_comments): if item.raw['body'] != formatted_comment: # We need to update the comment item.update(body=formatted_comment) - log.info(' Updated one comment') + log.info('Updated one comment') # Now we can just return the item return item else: @@ -439,7 +439,7 @@ def _attach_link(client, downstream, remote_link): :return: downstream: Response from creating the JIRA ticket :rtype: jira.resources.Issue """ - log.info(" Attaching tracking link %r to %r", remote_link, downstream.key) + log.info("Attaching tracking link %r to %r", remote_link, downstream.key) modified_desc = downstream.fields.description + " " # This is crazy. Querying for application links requires admin perms which @@ -453,7 +453,7 @@ def _attach_link(client, downstream, remote_link): # Finally, after we've added the link we have to edit the issue so that it # gets re-indexed, otherwise our searches won't work. Also, Handle some # weird API changes here... - log.debug(" Modifying desc of %r to trigger re-index.", downstream.key) + log.debug("Modifying desc of %r to trigger re-index.", downstream.key) downstream.update({'description': modified_desc}) return downstream @@ -465,9 +465,9 @@ def _upgrade_jira_issue(client, downstream, issue, config): ...upgrade it to a new-style issue. Simply mark it with an external-url field value. """ - log.info(" Upgrading %r %r issue for %r", downstream.key, issue.downstream, issue) + log.info("Upgrading %r %r issue for %r", downstream.key, issue.downstream, issue) if config['sync2jira']['testing']: - log.info(" Testing flag is true. Skipping actual upgrade.") + log.info("Testing flag is true. Skipping actual upgrade.") return # Do it! @@ -536,11 +536,11 @@ def _change_status(client, downstream, status, issue): if id: try: client.transition_issue(downstream, id) - log.info(' Updated downstream to %s status for issue %s' % (status, issue.title)) + log.info('Updated downstream to %s status for issue %s' % (status, issue.title)) except JIRAError: - log.error(' Updating downstream issue failed for %s: %s' % (status, issue.title)) + log.error('Updating downstream issue failed for %s: %s' % (status, issue.title)) else: - log.warning(' Could not update JIRA %s for %s' % (status, issue.title)) + log.warning('Could not update JIRA %s for %s' % (status, issue.title)) def _create_jira_issue(client, issue, config): @@ -554,9 +554,9 @@ def _create_jira_issue(client, issue, config): :returns: Returns JIRA issue that was created :rtype: jira.resources.Issue """ - log.info(" Creating %r issue for %r", issue.downstream, issue) + log.info("Creating %r issue for %r", issue.downstream, issue) if config['sync2jira']['testing']: - log.info(" Testing flag is true. Skipping actual creation.") + log.info("Testing flag is true. Skipping actual creation.") return custom_fields = issue.downstream.get('custom_fields', {}) @@ -601,7 +601,7 @@ def _create_jira_issue(client, issue, config): if 'labels' in issue.downstream.keys(): kwargs['labels'] = issue.downstream['labels'] - log.info(" Creating issue.") + log.info("Creating issue.") downstream = client.create_issue(**kwargs) # Add Epic link or QA field if present @@ -670,7 +670,7 @@ def _update_jira_issue(existing, issue, client): """ # Start with comments # Only synchronize comments for listings that op-in - log.info(" Updating information for upstream issue: %s" % issue.title) + log.info("Updating information for upstream issue: %s" % issue.title) # Get a list of what the user wants to update for the upstream issue updates = issue.downstream.get('updates', {}) @@ -682,47 +682,47 @@ def _update_jira_issue(existing, issue, client): # Only synchronize comments for listings that op-in if 'comments' in updates: - log.info(" Looking for new comments") + log.info("Looking for new comments") _update_comments(client, existing, issue) # Only synchronize tags for listings that op-in if any('tags' in item for item in updates): - log.info(" Looking for new tags") + log.info("Looking for new tags") _update_tags(updates, existing, issue) # Only synchronize fixVersion for listings that op-in if any('fixVersion' in item for item in updates) and issue.fixVersion: - log.info(" Looking for new fixVersions") + log.info("Looking for new fixVersions") _update_fixVersion(updates, existing, issue) # Only synchronize assignee for listings that op-in if any('assignee' in item for item in updates): - log.info(" Looking for new assignee(s)") + log.info("Looking for new assignee(s)") _update_assignee(client, existing, issue, updates) # Only synchronize descriptions for listings that op-in if 'description' in updates: - log.info(" Looking for new description") + log.info("Looking for new description") _update_description(existing, issue) # Only synchronize title for listings that op-in if 'title' in updates: # Update the title if needed if issue.title != existing.fields.summary: - log.info(" Looking for new title") + log.info("Looking for new title") _update_title(issue, existing) # Only synchronize url for listings that op-in if 'url' in updates: - log.info(" Looking for new url") + log.info("Looking for new url") _update_url(existing, issue) # Only synchronize transition (status) for listings that op-in if any('transition' in item for item in updates): - log.info(" Looking for new transition(s)") + log.info("Looking for new transition(s)") _update_transition(client, existing, issue) - log.info(' Done updating %s!' % issue.title) + log.info('Done updating %s!' % issue.title) def _update_url(existing, issue): @@ -745,7 +745,7 @@ def _update_url(existing, issue): # Update our issue data = {'description': new_description} existing.update(data) - log.info(' Updated description') + log.info('Updated description') def _update_transition(client, existing, issue): @@ -792,7 +792,7 @@ def _update_transition(client, existing, issue): # as there is a timestamp involved) data = {'description': new_description} existing.update(data) - log.info(' Updated transition') + log.info('Updated transition') # If the user just inputted True, only update the description # If the user added a custom closed status, attempt to close the @@ -857,11 +857,11 @@ def _update_title(issue, existing): if new_description != existing.fields.description: data = {'description': new_description} existing.update(data) - log.info(' Updated description') + log.info('Updated description') # Then we can update the title data = {'summary': issue.title} existing.update(data) - log.info(' Updated title') + log.info('Updated title') def _update_comments(client, existing, issue): @@ -883,7 +883,7 @@ def _update_comments(client, existing, issue): comment_body = _comment_format(comment) client.add_comment(existing, comment_body) if len(comments_d) > 0: - log.info(" Comments synchronization done on %i comments." % len(comments_d)) + log.info("Comments synchronization done on %i comments." % len(comments_d)) def _update_fixVersion(updates, existing, issue): @@ -934,9 +934,9 @@ def _update_fixVersion(updates, existing, issue): # If the fixVersion is not in JIRA, it will throw an error try: existing.update(data) - log.info(' Updated %s fixVersion(s)' % len(fix_version)) + log.info('Updated %s fixVersion(s)' % len(fix_version)) except JIRAError: - log.warning(' Error updating the fixVersion. %s is an invalid fixVersion.' % issue.fixVersion) + log.warning('Error updating the fixVersion. %s is an invalid fixVersion.' % issue.fixVersion) def _update_assignee(client, existing, issue, updates): @@ -972,19 +972,19 @@ def _update_assignee(client, existing, issue, updates): if issue.assignee[0]: # Update the assignee assign_user(client, issue, existing) - log.info(' Updated assignee') + log.info('Updated assignee') return else: # Update the assignee if we have someone to assignee it too if update: assign_user(client, issue, existing) - log.info(' Updated assignee') + log.info('Updated assignee') else: if existing.fields.assignee: # Else we should remove all assignees # Set removeAll flag to true assign_user(client, issue, existing, remove_all=True) - log.info(' Updated assignee') + log.info('Updated assignee') def _update_tags(updates, existing, issue): @@ -1016,7 +1016,7 @@ def _update_tags(updates, existing, issue): if sorted(updated_labels) != sorted(existing.fields.labels): data = {'labels': updated_labels} existing.update(data) - log.info(' Updated %s tag(s)' % len(updated_labels)) + log.info('Updated %s tag(s)' % len(updated_labels)) def _update_description(existing, issue): @@ -1090,14 +1090,14 @@ def _update_description(existing, issue): # issue regarding phantom updates # Get the diff between new_description and existing diff = difflib.unified_diff(existing.fields.description, new_description) - log.info(f" DEBUG: Issue {issue.title}") - log.info(f" DEBUG: Diff: {''.join(diff)}") - log.info(f" DEBUG: Old: {existing.fields.description}") - log.info(f" DEBUG: New: {new_description}") + log.info(f"DEBUG: Issue {issue.title}") + log.info(f"DEBUG: Diff: {''.join(diff)}") + log.info(f"DEBUG: Old: {existing.fields.description}") + log.info(f"DEBUG: New: {new_description}") data = {'description': new_description} existing.update(data) - log.info(' Updated description') + log.info('Updated description') def verify_tags(tags): @@ -1124,14 +1124,14 @@ def sync_with_jira(issue, config): :returns: Nothing """ - log.info(" Considering upstream %s, %s", issue.url, issue.title) + log.info("Considering upstream %s, %s", issue.url, issue.title) # Create a client connection for this issue client = _get_jira_client(issue, config) # Check the status of the JIRA client if not config['sync2jira']['develop'] and not check_jira_status(client): - log.warning(' The JIRA server looks like its down. Shutting down...') + log.warning('The JIRA server looks like its down. Shutting down...') raise JIRAError if issue.downstream.get('updates', None): @@ -1141,13 +1141,13 @@ def sync_with_jira(issue, config): # First, check to see if we have a matching issue using the new method. # If we do, then just bail out. No sync needed. - log.info(" Looking for matching downstream issue via new method.") + log.info("Looking for matching downstream issue via new method.") existing = _get_existing_jira_issue(client, issue, config) if existing: # If we found an existing JIRA issue already - log.info(" Found existing, matching downstream %r.", existing.key) + log.info("Found existing, matching downstream %r.", existing.key) if config['sync2jira']['testing']: - log.info(" Testing flag is true. Skipping actual update.") + log.info("Testing flag is true. Skipping actual update.") return # Update relevant metadata (i.e. tags, assignee, etc) _update_jira_issue(existing, issue, client) @@ -1156,7 +1156,7 @@ def sync_with_jira(issue, config): # If we're *not* configured to do legacy matching (upgrade mode) then there # is nothing left to do than to but to create the issue and return. if not config['sync2jira'].get('legacy_matching', True): - log.debug(" Legacy matching disabled.") + log.debug("Legacy matching disabled.") _create_jira_issue(client, issue, config) return @@ -1164,7 +1164,7 @@ def sync_with_jira(issue, config): # find this issue the old way. # - If we can't find it, create it. # - If we can find it, upgrade it to the new method. - log.info(" Looking for matching downstream issue via legacy method.") + log.info("Looking for matching downstream issue via legacy method.") match = _get_existing_jira_issue_legacy(client, issue, config) if not match: _create_jira_issue(client, issue, config) @@ -1182,9 +1182,9 @@ def _close_as_duplicate(client, duplicate, keeper, config): :param Dict config: Config dict :returns: Nothing """ - log.info(" Closing %s as duplicate of %s", duplicate.permalink(), keeper.permalink()) + log.info("Closing %s as duplicate of %s", duplicate.permalink(), keeper.permalink()) if config['sync2jira']['testing']: - log.info(" Testing flag is true. Skipping actual delete.") + log.info("Testing flag is true. Skipping actual delete.") return # Find the id of some dropped or done state. @@ -1199,13 +1199,13 @@ def _close_as_duplicate(client, duplicate, keeper, config): text = 'Marking as duplicate of %s' % keeper.key if any([text in comment.body for comment in client.comments(duplicate)]): - log.info(" Skipping comment. Already present.") + log.info("Skipping comment. Already present.") else: client.add_comment(duplicate, text) text = '%s is a duplicate.' % duplicate.key if any([text in comment.body for comment in client.comments(keeper)]): - log.info(" Skipping comment. Already present.") + log.info("Skipping comment. Already present.") else: client.add_comment(keeper, text) @@ -1222,7 +1222,7 @@ def _close_as_duplicate(client, duplicate, keeper, config): else: log.exception("Failed to close %r", duplicate.permalink()) else: - log.warning(" Unable to find close transition for %r" % duplicate.key) + log.warning("Unable to find close transition for %r" % duplicate.key) def close_duplicates(issue, config): @@ -1238,13 +1238,13 @@ def close_duplicates(issue, config): # Check the status of the JIRA client if not config['sync2jira']['develop'] and not check_jira_status(client): - log.warning(' The JIRA server looks like its down. Shutting down...') + log.warning('The JIRA server looks like its down. Shutting down...') raise JIRAError log.info("Looking for dupes of upstream %s, %s", issue.url, issue.title) results = _matching_jira_issue_query(client, issue, config, free=True) if len(results) <= 1: - log.info(" No duplicates found.") + log.info("No duplicates found.") return results = sorted(results, key=lambda x: arrow.get(x.fields.created)) diff --git a/sync2jira/main.py b/sync2jira/main.py index e5af266..5657f3e 100644 --- a/sync2jira/main.py +++ b/sync2jira/main.py @@ -35,7 +35,12 @@ import sync2jira.upstream as u import sync2jira.downstream as d from sync2jira.mailer import send_mail - +# Set up our logging +FORMAT = "[%(asctime)s] %(levelname)s: %(message)s" +# FORMAT = '%(asctime)-15s %(clientip)s %(user)-8s %(message)s' +logging.basicConfig(format=FORMAT, level=logging.INFO) +logging.basicConfig(format=FORMAT, level=logging.DEBUG) +logging.basicConfig(format=FORMAT, level=logging.WARNING) log = logging.getLogger('sync2jira.main') # Only allow fedmsg logs that are critical @@ -130,7 +135,7 @@ def listen(config): log.info("`listen` is disabled. Exiting.") return - log.info(" Waiting for a relevant fedmsg message to arrive...") + log.info("Waiting for a relevant fedmsg message to arrive...") for _, _, topic, msg in fedmsg.tail_messages(**config): idx = msg['msg_id'] suffix = ".".join(topic.split('.')[3:]) @@ -139,7 +144,7 @@ def listen(config): if suffix not in handlers: continue - log.debug(" Handling %r %r %r", suffix, topic, idx) + log.debug("Handling %r %r %r", suffix, topic, idx) if 'pagure' in suffix: issue = u.handle_pagure_message(msg, config) @@ -163,8 +168,8 @@ def initialize(config, testing=False): :param Bool testing: Flag to indicate if we are testing. Default false :returns: Nothing """ - log.info(" Running initialization to sync all issues from upstream to jira") - log.info(" Testing flag is %r", config['sync2jira']['testing']) + log.info("Running initialization to sync all issues from upstream to jira") + log.info("Testing flag is %r", config['sync2jira']['testing']) mapping = config['sync2jira']['map'] for upstream in mapping.get('pagure', {}).keys(): for issue in u.pagure_issues(upstream, config): @@ -173,7 +178,7 @@ def initialize(config, testing=False): except Exception: log.error(" Failed on %r", issue) raise - log.info(" Done with pagure initialization.") + log.info("Done with pagure initialization.") for upstream in mapping.get('github', {}).keys(): # Try and except for github API limit @@ -188,7 +193,7 @@ def initialize(config, testing=False): if "API rate limit exceeded" in e.__str__(): # If we've hit out API limit: # Sleep for 1 hour and call our function again - log.info(" Hit Github API limit. Sleeping for 1 hour...") + log.info("Hit Github API limit. Sleeping for 1 hour...") sleep(3600) if not testing: initialize(config) @@ -198,7 +203,7 @@ def initialize(config, testing=False): # Only send the failure email if we are not developing report_failure(config) raise - log.info(" Done with github initialization.") + log.info("Done with github initialization.") def main(): @@ -216,8 +221,8 @@ def main(): try: if config['sync2jira'].get('initialize'): + log.info("Initializing...") initialize(config) - try: listen(config) except KeyboardInterrupt: @@ -281,7 +286,7 @@ def close_duplicates(): """ config = load_config() logging.basicConfig(level=logging.INFO) - log.info(" Testing flag is %r", config['sync2jira']['testing']) + log.info("Testing flag is %r", config['sync2jira']['testing']) mapping = config['sync2jira']['map'] warnings.simplefilter("ignore")