From 32e862322cc556bb400e2b8fbb54a89fd4c11c79 Mon Sep 17 00:00:00 2001 From: Vivek Anand Date: Jul 27 2017 05:29:40 +0000 Subject: [PATCH 1/5] Send email using celery Signed-off-by: Vivek Anand --- diff --git a/pagure/lib/notify.py b/pagure/lib/notify.py index 1337d61..63df844 100644 --- a/pagure/lib/notify.py +++ b/pagure/lib/notify.py @@ -15,21 +15,17 @@ from __future__ import print_function import datetime -import hashlib import json import logging import urlparse import re -import smtplib import time import warnings import flask import pagure -from email.header import Header -from email.mime.text import MIMEText - +from pagure.lib import tasks _log = logging.getLogger(__name__) @@ -224,95 +220,15 @@ def send_email(text, subject, to_mail, :kwarg project_name: if defined, the name of the project ''' - if not to_mail: - return - - from_email = pagure.APP.config.get( - 'FROM_EMAIL', 'pagure@fedoraproject.org') - if user_from: - header = Header(user_from, 'utf-8') - from_email = '%s <%s>' % (header, from_email) - - if project_name is not None: - subject_tag = project_name - else: - subject_tag = 'Pagure' - - smtp = None - for mailto in to_mail.split(','): - msg = MIMEText(text.encode('utf-8'), 'plain', 'utf-8') - msg['Subject'] = header = Header( - '[%s] %s' % (subject_tag, subject), 'utf-8') - msg['From'] = from_email - - if mail_id: - msg['mail-id'] = mail_id - msg['Message-Id'] = '<%s>' % mail_id - - if in_reply_to: - msg['In-Reply-To'] = '<%s>' % in_reply_to - - msg['X-Auto-Response-Suppress'] = 'All' - msg['X-pagure'] = pagure.APP.config['APP_URL'] - if project_name is not None: - msg['X-pagure-project'] = project_name - msg['List-ID'] = project_name - msg['List-Archive'] = _build_url( - pagure.APP.config['APP_URL'], - _fullname_to_url(project_name)) - - # Send the message via our own SMTP server, but don't include the - # envelope header. - if isinstance(mailto, unicode): - mailto = mailto.encode('utf-8') - msg['To'] = mailto - salt = pagure.APP.config.get('SALT_EMAIL') - if isinstance(mail_id, unicode): - mail_id = mail_id.encode('utf-8') - mhash = hashlib.sha512('<%s>%s%s' % (mail_id, salt, mailto)) - msg['Reply-To'] = 'reply+%s@%s' % ( - mhash.hexdigest(), - pagure.APP.config['DOMAIN_EMAIL_NOTIFICATIONS']) - msg['Mail-Followup-To'] = msg['Reply-To'] - if not pagure.APP.config.get('EMAIL_SEND', True): - print('******EMAIL******') - print('From: %s' % from_email) - print('To: %s' % to_mail) - print('Subject: %s' % subject) - print('in_reply_to: %s' % in_reply_to) - print('mail_id: %s' % mail_id) - print('Contents:') - print(text.encode('utf-8')) - print('*****************') - print(msg.as_string()) - print('*****/EMAIL******') - continue - try: - if smtp is None: - if pagure.APP.config['SMTP_SSL']: - smtp = smtplib.SMTP_SSL( - pagure.APP.config['SMTP_SERVER'], - pagure.APP.config['SMTP_PORT']) - else: - smtp = smtplib.SMTP( - pagure.APP.config['SMTP_SERVER'], - pagure.APP.config['SMTP_PORT']) - if pagure.APP.config['SMTP_USERNAME'] \ - and pagure.APP.config['SMTP_PASSWORD']: - smtp.login( - pagure.APP.config['SMTP_USERNAME'], - pagure.APP.config['SMTP_PASSWORD'] - ) - - smtp.sendmail( - from_email, - [mailto], - msg.as_string()) - except smtplib.SMTPException as err: - _log.exception(err) - if smtp: - smtp.quit() - return msg + tasks.send_email.delay( + text=text, + subject=subject, + to_mail=to_mail, + mail_id=mail_id, + in_reply_to=in_reply_to, + project_name=project_name, + user_from=user_from, + ) def notify_new_comment(comment, user=None): diff --git a/pagure/lib/tasks.py b/pagure/lib/tasks.py index 22a376e..d861dce 100644 --- a/pagure/lib/tasks.py +++ b/pagure/lib/tasks.py @@ -20,6 +20,8 @@ from celery.result import AsyncResult import pygit2 import tempfile import six +import hashlib +import smtplib import logging @@ -29,6 +31,9 @@ import pagure.lib import pagure.lib.git import pagure.lib.git_auth +from email.header import Header +from email.mime.text import MIMEText + logging.config.dictConfig(APP.config.get('LOGGING') or {'version': 1}) _log = logging.getLogger(__name__) @@ -544,3 +549,111 @@ def sync_pull_ref(name, namespace, user, requestid): session.remove() gc_clean() + + +@conn.task +def send_email(text, subject, to_mail, + mail_id=None, in_reply_to=None, + project_name=None, user_from=None): # pragma: no cover + ''' Send an email with the specified information. + + :arg text: the content of the email to send + :arg subject: the subject of the email + :arg to_mail: a string representing a list of recipient separated by a + coma + :kwarg mail_id: if defined, the header `mail-id` is set with this value + :kwarg in_reply_to: if defined, the header `In-Reply-To` is set with + this value + :kwarg project_name: if defined, the name of the project + + ''' + import pagure.lib.notify as notify + + if not to_mail: + return + + from_email = pagure.APP.config.get( + 'FROM_EMAIL', 'pagure@fedoraproject.org') + if user_from: + header = Header(user_from, 'utf-8') + from_email = '%s <%s>' % (header, from_email) + + if project_name is not None: + subject_tag = project_name + else: + subject_tag = 'Pagure' + + smtp = None + for mailto in to_mail.split(','): + msg = MIMEText(text.encode('utf-8'), 'plain', 'utf-8') + msg['Subject'] = header = Header( + '[%s] %s' % (subject_tag, subject), 'utf-8') + msg['From'] = from_email + + if mail_id: + msg['mail-id'] = mail_id + msg['Message-Id'] = '<%s>' % mail_id + + if in_reply_to: + msg['In-Reply-To'] = '<%s>' % in_reply_to + + msg['X-Auto-Response-Suppress'] = 'All' + msg['X-pagure'] = pagure.APP.config['APP_URL'] + if project_name is not None: + msg['X-pagure-project'] = project_name + msg['List-ID'] = project_name + msg['List-Archive'] = notify._build_url( + pagure.APP.config['APP_URL'], + notify._fullname_to_url(project_name)) + + # Send the message via our own SMTP server, but don't include the + # envelope header. + if isinstance(mailto, unicode): + mailto = mailto.encode('utf-8') + msg['To'] = mailto + salt = pagure.APP.config.get('SALT_EMAIL') + if isinstance(mail_id, unicode): + mail_id = mail_id.encode('utf-8') + mhash = hashlib.sha512('<%s>%s%s' % (mail_id, salt, mailto)) + msg['Reply-To'] = 'reply+%s@%s' % ( + mhash.hexdigest(), + pagure.APP.config['DOMAIN_EMAIL_NOTIFICATIONS']) + msg['Mail-Followup-To'] = msg['Reply-To'] + if not pagure.APP.config.get('EMAIL_SEND', True): + print('******EMAIL******') + print('From: %s' % from_email) + print('To: %s' % to_mail) + print('Subject: %s' % subject) + print('in_reply_to: %s' % in_reply_to) + print('mail_id: %s' % mail_id) + print('Contents:') + print(text.encode('utf-8')) + print('*****************') + print(msg.as_string()) + print('*****/EMAIL******') + continue + try: + if smtp is None: + if pagure.APP.config['SMTP_SSL']: + smtp = smtplib.SMTP_SSL( + pagure.APP.config['SMTP_SERVER'], + pagure.APP.config['SMTP_PORT']) + else: + smtp = smtplib.SMTP( + pagure.APP.config['SMTP_SERVER'], + pagure.APP.config['SMTP_PORT']) + if pagure.APP.config['SMTP_USERNAME'] \ + and pagure.APP.config['SMTP_PASSWORD']: + smtp.login( + pagure.APP.config['SMTP_USERNAME'], + pagure.APP.config['SMTP_PASSWORD'] + ) + + smtp.sendmail( + from_email, + [mailto], + msg.as_string()) + except smtplib.SMTPException as err: + _log.exception(err) + if smtp: + smtp.quit() From ae33324c4e77154c83196806f7288bb35dd74e0e Mon Sep 17 00:00:00 2001 From: Vivek Anand Date: Jul 27 2017 05:29:40 +0000 Subject: [PATCH 2/5] Refactor code for send_email Signed-off-by: Vivek Anand --- diff --git a/pagure/lib/tasks.py b/pagure/lib/tasks.py index d861dce..4705963 100644 --- a/pagure/lib/tasks.py +++ b/pagure/lib/tasks.py @@ -558,16 +558,24 @@ def send_email(text, subject, to_mail, ''' Send an email with the specified information. :arg text: the content of the email to send + :type: str :arg subject: the subject of the email - :arg to_mail: a string representing a list of recipient separated by a - coma + :type: str + :arg to_mail: a string representing a list of recipients separated by a + comma + :type: str :kwarg mail_id: if defined, the header `mail-id` is set with this value + :type: None or str :kwarg in_reply_to: if defined, the header `In-Reply-To` is set with this value + :type: None or str :kwarg project_name: if defined, the name of the project + :type: None or str + :kwarg user_from: if defined, the email id of the user in the name of whom + the email is being sent. + :type: None or str ''' - import pagure.lib.notify as notify if not to_mail: return @@ -585,75 +593,121 @@ def send_email(text, subject, to_mail, smtp = None for mailto in to_mail.split(','): - msg = MIMEText(text.encode('utf-8'), 'plain', 'utf-8') - msg['Subject'] = header = Header( - '[%s] %s' % (subject_tag, subject), 'utf-8') - msg['From'] = from_email - - if mail_id: - msg['mail-id'] = mail_id - msg['Message-Id'] = '<%s>' % mail_id - - if in_reply_to: - msg['In-Reply-To'] = '<%s>' % in_reply_to - - msg['X-Auto-Response-Suppress'] = 'All' - msg['X-pagure'] = pagure.APP.config['APP_URL'] - if project_name is not None: - msg['X-pagure-project'] = project_name - msg['List-ID'] = project_name - msg['List-Archive'] = notify._build_url( - pagure.APP.config['APP_URL'], - notify._fullname_to_url(project_name)) - - # Send the message via our own SMTP server, but don't include the - # envelope header. - if isinstance(mailto, unicode): - mailto = mailto.encode('utf-8') - msg['To'] = mailto - salt = pagure.APP.config.get('SALT_EMAIL') - if isinstance(mail_id, unicode): - mail_id = mail_id.encode('utf-8') - mhash = hashlib.sha512('<%s>%s%s' % (mail_id, salt, mailto)) - msg['Reply-To'] = 'reply+%s@%s' % ( - mhash.hexdigest(), - pagure.APP.config['DOMAIN_EMAIL_NOTIFICATIONS']) - msg['Mail-Followup-To'] = msg['Reply-To'] + msg, header = _prepare_individual_email( + header=header, + subject=subject, + subject_tag=subject_tag, + from_email=from_email, + mail_id=mail_id, + in_reply_to=in_reply_to, + project_name=project_name, + mailto=mailto, + text=text, + ) + if not pagure.APP.config.get('EMAIL_SEND', True): - print('******EMAIL******') - print('From: %s' % from_email) - print('To: %s' % to_mail) - print('Subject: %s' % subject) - print('in_reply_to: %s' % in_reply_to) - print('mail_id: %s' % mail_id) - print('Contents:') - print(text.encode('utf-8')) - print('*****************') - print(msg.as_string()) - print('*****/EMAIL******') + # This will be viewed on the console where the worker is running + _print_email_to_console( + from_email=from_email, + to_mail=to_mail, + subject=subject, + in_reply_to=in_reply_to, + mail_id=mail_id, + text=text, + msg=msg, + ) continue - try: - if smtp is None: - if pagure.APP.config['SMTP_SSL']: - smtp = smtplib.SMTP_SSL( - pagure.APP.config['SMTP_SERVER'], - pagure.APP.config['SMTP_PORT']) - else: - smtp = smtplib.SMTP( - pagure.APP.config['SMTP_SERVER'], - pagure.APP.config['SMTP_PORT']) - if pagure.APP.config['SMTP_USERNAME'] \ - and pagure.APP.config['SMTP_PASSWORD']: - smtp.login( - pagure.APP.config['SMTP_USERNAME'], - pagure.APP.config['SMTP_PASSWORD'] - ) + # Send it to user if EMAIL_SEND is Enabled + _send_email(smtp=smtp, from_email=from_email, mailto=mailto, msg=msg) + if smtp: + smtp.quit() - smtp.sendmail( - from_email, - [mailto], - msg.as_string()) - except smtplib.SMTPException as err: - _log.exception(err) - if smtp: - smtp.quit() + +def _prepare_individual_email( + header, subject, subject_tag, from_email, + mail_id, in_reply_to, project_name, mailto, text): + ''' Prepares each email before sending or printing to console ''' + + # It's here because we had to import lib.tasks in lib.notify + import pagure.lib.notify as notify + + msg = MIMEText(text.encode('utf-8'), 'plain', 'utf-8') + msg['Subject'] = header = Header( + '[%s] %s' % (subject_tag, subject), 'utf-8') + msg['From'] = from_email + + if mail_id: + msg['mail-id'] = mail_id + msg['Message-Id'] = '<%s>' % mail_id + + if in_reply_to: + msg['In-Reply-To'] = '<%s>' % in_reply_to + + msg['X-Auto-Response-Suppress'] = 'All' + msg['X-pagure'] = pagure.APP.config['APP_URL'] + if project_name is not None: + msg['X-pagure-project'] = project_name + msg['List-ID'] = project_name + msg['List-Archive'] = notify._build_url( + pagure.APP.config['APP_URL'], + notify._fullname_to_url(project_name)) + + # Send the message via our own SMTP server, but don't include the + # envelope header. + if isinstance(mailto, unicode): + mailto = mailto.encode('utf-8') + msg['To'] = mailto + salt = pagure.APP.config.get('SALT_EMAIL') + if isinstance(mail_id, unicode): + mail_id = mail_id.encode('utf-8') + mhash = hashlib.sha512('<%s>%s%s' % (mail_id, salt, mailto)) + msg['Reply-To'] = 'reply+%s@%s' % ( + mhash.hexdigest(), + pagure.APP.config['DOMAIN_EMAIL_NOTIFICATIONS']) + msg['Mail-Followup-To'] = msg['Reply-To'] + return (msg, header) + + +def _print_email_to_console( + from_email, to_mail, subject, in_reply_to, mail_id, text, msg): + ''' Prints the email to be sent in the worker's console, invoked when + EMAIL_SEND is False ''' + + print('******EMAIL******') + print('From: %s' % from_email) + print('To: %s' % to_mail) + print('Subject: %s' % subject) + print('in_reply_to: %s' % in_reply_to) + print('mail_id: %s' % mail_id) + print('Contents:') + print(text.encode('utf-8')) + print('*****************') + print(msg.as_string()) + print('*****/EMAIL******') + + +def _send_email(smtp, from_email, mailto, msg): + ''' Send the prepared mail to the respective users ''' + try: + if smtp is None: + if pagure.APP.config['SMTP_SSL']: + smtp = smtplib.SMTP_SSL( + pagure.APP.config['SMTP_SERVER'], + pagure.APP.config['SMTP_PORT']) + else: + smtp = smtplib.SMTP( + pagure.APP.config['SMTP_SERVER'], + pagure.APP.config['SMTP_PORT']) + if pagure.APP.config['SMTP_USERNAME'] \ + and pagure.APP.config['SMTP_PASSWORD']: + smtp.login( + pagure.APP.config['SMTP_USERNAME'], + pagure.APP.config['SMTP_PASSWORD'] + ) + + smtp.sendmail( + from_email, + [mailto], + msg.as_string()) + except smtplib.SMTPException as err: + _log.exception(err) From ae14103b8b3eac0a0afc403db848844b70752ec1 Mon Sep 17 00:00:00 2001 From: Vivek Anand Date: Jul 27 2017 05:29:40 +0000 Subject: [PATCH 3/5] send_email: 'To' field should contain individual email id in each mail Signed-off-by: Vivek Anand --- diff --git a/pagure/lib/tasks.py b/pagure/lib/tasks.py index 4705963..1d506a8 100644 --- a/pagure/lib/tasks.py +++ b/pagure/lib/tasks.py @@ -609,7 +609,7 @@ def send_email(text, subject, to_mail, # This will be viewed on the console where the worker is running _print_email_to_console( from_email=from_email, - to_mail=to_mail, + mailto=mailto, subject=subject, in_reply_to=in_reply_to, mail_id=mail_id, @@ -619,8 +619,8 @@ def send_email(text, subject, to_mail, continue # Send it to user if EMAIL_SEND is Enabled _send_email(smtp=smtp, from_email=from_email, mailto=mailto, msg=msg) - if smtp: - smtp.quit() + if smtp: + smtp.quit() def _prepare_individual_email( @@ -669,13 +669,13 @@ def _prepare_individual_email( def _print_email_to_console( - from_email, to_mail, subject, in_reply_to, mail_id, text, msg): + from_email, mailto, subject, in_reply_to, mail_id, text, msg): ''' Prints the email to be sent in the worker's console, invoked when EMAIL_SEND is False ''' print('******EMAIL******') print('From: %s' % from_email) - print('To: %s' % to_mail) + print('To: %s' % mailto) print('Subject: %s' % subject) print('in_reply_to: %s' % in_reply_to) print('mail_id: %s' % mail_id) From d06539fa7d11f004cff18b63d92c119508323ebc Mon Sep 17 00:00:00 2001 From: Vivek Anand Date: Jul 27 2017 05:29:40 +0000 Subject: [PATCH 4/5] send_email: move import of lib.notify inside the method Signed-off-by: Vivek Anand --- diff --git a/pagure/lib/task_helpers.py b/pagure/lib/task_helpers.py new file mode 100644 index 0000000..055bf1d --- /dev/null +++ b/pagure/lib/task_helpers.py @@ -0,0 +1,117 @@ +# coding=utf-8 + +import gc +import hashlib +import smtplib +import logging + +from email.header import Header +from email.mime.text import MIMEText + +import pagure +from pagure import APP + + +logging.config.dictConfig(APP.config.get('LOGGING') or {'version': 1}) +_log = logging.getLogger(__name__) + + +def ret(endpoint, **kwargs): + toret = {'endpoint': endpoint} + toret.update(kwargs) + return toret + + +def gc_clean(): + """ Force a run of the garbage collector. """ + # https://pagure.io/pagure/issue/2302 + gc.collect() + + +def _prepare_individual_email( + header, subject, subject_tag, from_email, + mail_id, in_reply_to, project_name, mailto, text): + ''' Prepares each email before sending or printing to console ''' + + from pagure.lib import notify + msg = MIMEText(text.encode('utf-8'), 'plain', 'utf-8') + msg['Subject'] = header = Header( + '[%s] %s' % (subject_tag, subject), 'utf-8') + msg['From'] = from_email + + if mail_id: + msg['mail-id'] = mail_id + msg['Message-Id'] = '<%s>' % mail_id + + if in_reply_to: + msg['In-Reply-To'] = '<%s>' % in_reply_to + + msg['X-Auto-Response-Suppress'] = 'All' + msg['X-pagure'] = pagure.APP.config['APP_URL'] + if project_name is not None: + msg['X-pagure-project'] = project_name + msg['List-ID'] = project_name + msg['List-Archive'] = notify._build_url( + pagure.APP.config['APP_URL'], + notify._fullname_to_url(project_name)) + + # Send the message via our own SMTP server, but don't include the + # envelope header. + if isinstance(mailto, unicode): + mailto = mailto.encode('utf-8') + msg['To'] = mailto + salt = pagure.APP.config.get('SALT_EMAIL') + if isinstance(mail_id, unicode): + mail_id = mail_id.encode('utf-8') + mhash = hashlib.sha512('<%s>%s%s' % (mail_id, salt, mailto)) + msg['Reply-To'] = 'reply+%s@%s' % ( + mhash.hexdigest(), + pagure.APP.config['DOMAIN_EMAIL_NOTIFICATIONS']) + msg['Mail-Followup-To'] = msg['Reply-To'] + return (msg, header) + + +def _print_email_to_console( + from_email, mailto, subject, in_reply_to, mail_id, text, msg): + ''' Prints the email to be sent in the worker's console, invoked when + EMAIL_SEND is False ''' + + print('******EMAIL******') + print('From: %s' % from_email) + print('To: %s' % mailto) + print('Subject: %s' % subject) + print('in_reply_to: %s' % in_reply_to) + print('mail_id: %s' % mail_id) + print('Contents:') + print(text.encode('utf-8')) + print('*****************') + print(msg.as_string()) + print('*****/EMAIL******') + + +def _send_email(smtp, from_email, mailto, msg): + ''' Send the prepared mail to the respective users ''' + try: + if smtp is None: + if pagure.APP.config['SMTP_SSL']: + smtp = smtplib.SMTP_SSL( + pagure.APP.config['SMTP_SERVER'], + pagure.APP.config['SMTP_PORT']) + else: + smtp = smtplib.SMTP( + pagure.APP.config['SMTP_SERVER'], + pagure.APP.config['SMTP_PORT']) + if pagure.APP.config['SMTP_USERNAME'] \ + and pagure.APP.config['SMTP_PASSWORD']: + smtp.login( + pagure.APP.config['SMTP_USERNAME'], + pagure.APP.config['SMTP_PASSWORD'] + ) + + smtp.sendmail( + from_email, + [mailto], + msg.as_string()) + except smtplib.SMTPException as err: + _log.exception(err) + diff --git a/pagure/lib/tasks.py b/pagure/lib/tasks.py index 1d506a8..25b9a42 100644 --- a/pagure/lib/tasks.py +++ b/pagure/lib/tasks.py @@ -8,7 +8,6 @@ """ -import gc import os import os.path import shutil @@ -20,8 +19,6 @@ from celery.result import AsyncResult import pygit2 import tempfile import six -import hashlib -import smtplib import logging @@ -30,9 +27,9 @@ from pagure import APP import pagure.lib import pagure.lib.git import pagure.lib.git_auth +from pagure.lib import task_helpers from email.header import Header -from email.mime.text import MIMEText logging.config.dictConfig(APP.config.get('LOGGING') or {'version': 1}) _log = logging.getLogger(__name__) @@ -60,18 +57,6 @@ def get_result(uuid): return AsyncResult(uuid, conn.backend) -def ret(endpoint, **kwargs): - toret = {'endpoint': endpoint} - toret.update(kwargs) - return toret - - -def gc_clean(): - """ Force a run of the garbage collector. """ - # https://pagure.io/pagure/issue/2302 - gc.collect() - - @conn.task def generate_gitolite_acls(namespace=None, name=None, user=None, group=None): """ Generate the gitolite configuration file either entirely or for a @@ -106,7 +91,7 @@ def generate_gitolite_acls(namespace=None, name=None, user=None, group=None): helper, project, group_obj) helper.generate_acls(project=project, group=group_obj) session.remove() - gc_clean() + task_helpers.gc_clean() @conn.task @@ -227,9 +212,9 @@ def create_project(username, namespace, name, add_readme, user=project.user.user if project.is_fork else None) session.remove() - gc_clean() + task_helpers.gc_clean() - return ret('view_repo', repo=name, namespace=namespace) + return task_helpers.ret('view_repo', repo=name, namespace=namespace) @conn.task @@ -254,7 +239,7 @@ def update_git(name, namespace, user, ticketuid=None, requestuid=None): result = pagure.lib.git._update_git(obj, project, folder) session.remove() - gc_clean() + task_helpers.gc_clean() return result @@ -291,8 +276,13 @@ def update_file_in_git(name, namespace, user, branch, branchto, filename, content, message, userobj, email) session.remove() - return ret('view_commits', repo=project.name, username=user, - namespace=namespace, branchname=branchto) + return task_helpers.ret( + 'view_commits', + repo=project.name, + username=user, + namespace=namespace, + branchname=branchto + ) @conn.task @@ -311,7 +301,8 @@ def delete_branch(name, namespace, user, branchname): _log.exception(err) session.remove() - return ret('view_repo', repo=name, namespace=namespace, username=user) + return task_helpers.ret( + 'view_repo', repo=name, namespace=namespace, username=user) @conn.task @@ -414,15 +405,20 @@ def fork(name, namespace, user_owner, user_forker, editbranch, editfile): namespace=repo_to.namespace, name=repo_to.name, user=repo_to.user.user if repo_to.is_fork else None) - gc_clean() + task_helpers.gc_clean() if editfile is None: - return ret('view_repo', repo=name, namespace=namespace, - username=user_forker) + return task_helpers.ret( + 'view_repo', repo=name, namespace=namespace, username=user_forker) else: - return ret('edit_file', repo=name, namespace=namespace, - username=user_forker, branchname=editbranch, - filename=editfile) + return task_helpers.ret( + 'edit_file', + repo=name, + namespace=namespace, + username=user_forker, + branchname=editbranch, + filename=editfile + ) @conn.task @@ -433,7 +429,7 @@ def pull_remote_repo(remote_git, branch_from): remote_git, clonepath, checkout_branch=branch_from) del repo - gc_clean() + task_helpers.gc_clean() return clonepath @@ -447,7 +443,7 @@ def refresh_pr_cache(name, namespace, user): pagure.lib.reset_status_pull_request(session, project) session.remove() - gc_clean() + task_helpers.gc_clean() @conn.task @@ -467,8 +463,9 @@ def merge_pull_request(name, namespace, user, requestid, user_merger): refresh_pr_cache.delay(name, namespace, user) session.remove() - gc_clean() - return ret('view_repo', repo=name, username=user, namespace=namespace) + task_helpers.gc_clean() + return task_helpers.ret( + 'view_repo', repo=name, username=user, namespace=namespace) @conn.task @@ -493,7 +490,7 @@ def add_file_to_git(name, namespace, user, user_attacher, issueuid, filename): filename) session.remove() - gc_clean() + task_helpers.gc_clean() @conn.task @@ -513,9 +510,10 @@ def project_dowait(name, namespace, user): time.sleep(10) session.remove() - gc_clean() + task_helpers.gc_clean() - return ret('view_repo', repo=name, username=user, namespace=namespace) + return task_helpers.ret( + 'view_repo', repo=name, username=user, namespace=namespace) @conn.task @@ -548,7 +546,7 @@ def sync_pull_ref(name, namespace, user, requestid): pagure.lib.git.update_pull_ref(request, repo_obj) session.remove() - gc_clean() + task_helpers.gc_clean() @conn.task @@ -593,7 +591,7 @@ def send_email(text, subject, to_mail, smtp = None for mailto in to_mail.split(','): - msg, header = _prepare_individual_email( + msg, header = task_helpers._prepare_individual_email( header=header, subject=subject, subject_tag=subject_tag, @@ -607,7 +605,7 @@ def send_email(text, subject, to_mail, if not pagure.APP.config.get('EMAIL_SEND', True): # This will be viewed on the console where the worker is running - _print_email_to_console( + task_helpers._print_email_to_console( from_email=from_email, mailto=mailto, subject=subject, @@ -618,96 +616,8 @@ def send_email(text, subject, to_mail, ) continue # Send it to user if EMAIL_SEND is Enabled - _send_email(smtp=smtp, from_email=from_email, mailto=mailto, msg=msg) + task_helpers._send_email( + smtp=smtp, from_email=from_email, mailto=mailto, msg=msg) if smtp: smtp.quit() - -def _prepare_individual_email( - header, subject, subject_tag, from_email, - mail_id, in_reply_to, project_name, mailto, text): - ''' Prepares each email before sending or printing to console ''' - - # It's here because we had to import lib.tasks in lib.notify - import pagure.lib.notify as notify - - msg = MIMEText(text.encode('utf-8'), 'plain', 'utf-8') - msg['Subject'] = header = Header( - '[%s] %s' % (subject_tag, subject), 'utf-8') - msg['From'] = from_email - - if mail_id: - msg['mail-id'] = mail_id - msg['Message-Id'] = '<%s>' % mail_id - - if in_reply_to: - msg['In-Reply-To'] = '<%s>' % in_reply_to - - msg['X-Auto-Response-Suppress'] = 'All' - msg['X-pagure'] = pagure.APP.config['APP_URL'] - if project_name is not None: - msg['X-pagure-project'] = project_name - msg['List-ID'] = project_name - msg['List-Archive'] = notify._build_url( - pagure.APP.config['APP_URL'], - notify._fullname_to_url(project_name)) - - # Send the message via our own SMTP server, but don't include the - # envelope header. - if isinstance(mailto, unicode): - mailto = mailto.encode('utf-8') - msg['To'] = mailto - salt = pagure.APP.config.get('SALT_EMAIL') - if isinstance(mail_id, unicode): - mail_id = mail_id.encode('utf-8') - mhash = hashlib.sha512('<%s>%s%s' % (mail_id, salt, mailto)) - msg['Reply-To'] = 'reply+%s@%s' % ( - mhash.hexdigest(), - pagure.APP.config['DOMAIN_EMAIL_NOTIFICATIONS']) - msg['Mail-Followup-To'] = msg['Reply-To'] - return (msg, header) - - -def _print_email_to_console( - from_email, mailto, subject, in_reply_to, mail_id, text, msg): - ''' Prints the email to be sent in the worker's console, invoked when - EMAIL_SEND is False ''' - - print('******EMAIL******') - print('From: %s' % from_email) - print('To: %s' % mailto) - print('Subject: %s' % subject) - print('in_reply_to: %s' % in_reply_to) - print('mail_id: %s' % mail_id) - print('Contents:') - print(text.encode('utf-8')) - print('*****************') - print(msg.as_string()) - print('*****/EMAIL******') - - -def _send_email(smtp, from_email, mailto, msg): - ''' Send the prepared mail to the respective users ''' - try: - if smtp is None: - if pagure.APP.config['SMTP_SSL']: - smtp = smtplib.SMTP_SSL( - pagure.APP.config['SMTP_SERVER'], - pagure.APP.config['SMTP_PORT']) - else: - smtp = smtplib.SMTP( - pagure.APP.config['SMTP_SERVER'], - pagure.APP.config['SMTP_PORT']) - if pagure.APP.config['SMTP_USERNAME'] \ - and pagure.APP.config['SMTP_PASSWORD']: - smtp.login( - pagure.APP.config['SMTP_USERNAME'], - pagure.APP.config['SMTP_PASSWORD'] - ) - - smtp.sendmail( - from_email, - [mailto], - msg.as_string()) - except smtplib.SMTPException as err: - _log.exception(err) From 8e0e9b639f81f8d06d54ce0fb1668850e69d8854 Mon Sep 17 00:00:00 2001 From: Vivek Anand Date: Jul 27 2017 05:29:40 +0000 Subject: [PATCH 5/5] send mail: only mail helpers in differnt file Signed-off-by: Vivek Anand --- diff --git a/pagure/lib/mail_helpers.py b/pagure/lib/mail_helpers.py new file mode 100644 index 0000000..1807449 --- /dev/null +++ b/pagure/lib/mail_helpers.py @@ -0,0 +1,105 @@ +# coding=utf-8 + +""" + (c) 2017 - Copyright Red Hat Inc + + Authors: + Pierre-Yves Chibon + Vivek Anand + +""" + +import hashlib +import smtplib + +from email.header import Header +from email.mime.text import MIMEText + +import pagure + + +def _prepare_individual_email( + header, subject, subject_tag, from_email, + mail_id, in_reply_to, project_name, mailto, text): + ''' Prepares each email before sending or printing to console ''' + + from pagure.lib import notify + msg = MIMEText(text.encode('utf-8'), 'plain', 'utf-8') + msg['Subject'] = header = Header( + '[%s] %s' % (subject_tag, subject), 'utf-8') + msg['From'] = from_email + + if mail_id: + msg['mail-id'] = mail_id + msg['Message-Id'] = '<%s>' % mail_id + + if in_reply_to: + msg['In-Reply-To'] = '<%s>' % in_reply_to + + msg['X-Auto-Response-Suppress'] = 'All' + msg['X-pagure'] = pagure.APP.config['APP_URL'] + if project_name is not None: + msg['X-pagure-project'] = project_name + msg['List-ID'] = project_name + msg['List-Archive'] = notify._build_url( + pagure.APP.config['APP_URL'], + notify._fullname_to_url(project_name)) + + # Send the message via our own SMTP server, but don't include the + # envelope header. + if isinstance(mailto, unicode): + mailto = mailto.encode('utf-8') + msg['To'] = mailto + salt = pagure.APP.config.get('SALT_EMAIL') + if isinstance(mail_id, unicode): + mail_id = mail_id.encode('utf-8') + mhash = hashlib.sha512('<%s>%s%s' % (mail_id, salt, mailto)) + msg['Reply-To'] = 'reply+%s@%s' % ( + mhash.hexdigest(), + pagure.APP.config['DOMAIN_EMAIL_NOTIFICATIONS']) + msg['Mail-Followup-To'] = msg['Reply-To'] + return (msg, header) + + +def _print_email_to_console( + from_email, mailto, subject, in_reply_to, mail_id, text, msg): + ''' Prints the email to be sent in the worker's console, invoked when + EMAIL_SEND is False ''' + + print('******EMAIL******') + print('From: %s' % from_email) + print('To: %s' % mailto) + print('Subject: %s' % subject) + print('in_reply_to: %s' % in_reply_to) + print('mail_id: %s' % mail_id) + print('Contents:') + print(text.encode('utf-8')) + print('*****************') + print(msg.as_string()) + print('*****/EMAIL******') + + +def _send_email(smtp, from_email, mailto, msg): + ''' Send the prepared mail to the respective users ''' + + if smtp is None: + if pagure.APP.config['SMTP_SSL']: + smtp = smtplib.SMTP_SSL( + pagure.APP.config['SMTP_SERVER'], + pagure.APP.config['SMTP_PORT']) + else: + smtp = smtplib.SMTP( + pagure.APP.config['SMTP_SERVER'], + pagure.APP.config['SMTP_PORT']) + if pagure.APP.config['SMTP_USERNAME'] \ + and pagure.APP.config['SMTP_PASSWORD']: + smtp.login( + pagure.APP.config['SMTP_USERNAME'], + pagure.APP.config['SMTP_PASSWORD'] + ) + + smtp.sendmail( + from_email, + [mailto], + msg.as_string()) + diff --git a/pagure/lib/task_helpers.py b/pagure/lib/task_helpers.py deleted file mode 100644 index 055bf1d..0000000 --- a/pagure/lib/task_helpers.py +++ /dev/null @@ -1,117 +0,0 @@ -# coding=utf-8 - -import gc -import hashlib -import smtplib -import logging - -from email.header import Header -from email.mime.text import MIMEText - -import pagure -from pagure import APP - - -logging.config.dictConfig(APP.config.get('LOGGING') or {'version': 1}) -_log = logging.getLogger(__name__) - - -def ret(endpoint, **kwargs): - toret = {'endpoint': endpoint} - toret.update(kwargs) - return toret - - -def gc_clean(): - """ Force a run of the garbage collector. """ - # https://pagure.io/pagure/issue/2302 - gc.collect() - - -def _prepare_individual_email( - header, subject, subject_tag, from_email, - mail_id, in_reply_to, project_name, mailto, text): - ''' Prepares each email before sending or printing to console ''' - - from pagure.lib import notify - msg = MIMEText(text.encode('utf-8'), 'plain', 'utf-8') - msg['Subject'] = header = Header( - '[%s] %s' % (subject_tag, subject), 'utf-8') - msg['From'] = from_email - - if mail_id: - msg['mail-id'] = mail_id - msg['Message-Id'] = '<%s>' % mail_id - - if in_reply_to: - msg['In-Reply-To'] = '<%s>' % in_reply_to - - msg['X-Auto-Response-Suppress'] = 'All' - msg['X-pagure'] = pagure.APP.config['APP_URL'] - if project_name is not None: - msg['X-pagure-project'] = project_name - msg['List-ID'] = project_name - msg['List-Archive'] = notify._build_url( - pagure.APP.config['APP_URL'], - notify._fullname_to_url(project_name)) - - # Send the message via our own SMTP server, but don't include the - # envelope header. - if isinstance(mailto, unicode): - mailto = mailto.encode('utf-8') - msg['To'] = mailto - salt = pagure.APP.config.get('SALT_EMAIL') - if isinstance(mail_id, unicode): - mail_id = mail_id.encode('utf-8') - mhash = hashlib.sha512('<%s>%s%s' % (mail_id, salt, mailto)) - msg['Reply-To'] = 'reply+%s@%s' % ( - mhash.hexdigest(), - pagure.APP.config['DOMAIN_EMAIL_NOTIFICATIONS']) - msg['Mail-Followup-To'] = msg['Reply-To'] - return (msg, header) - - -def _print_email_to_console( - from_email, mailto, subject, in_reply_to, mail_id, text, msg): - ''' Prints the email to be sent in the worker's console, invoked when - EMAIL_SEND is False ''' - - print('******EMAIL******') - print('From: %s' % from_email) - print('To: %s' % mailto) - print('Subject: %s' % subject) - print('in_reply_to: %s' % in_reply_to) - print('mail_id: %s' % mail_id) - print('Contents:') - print(text.encode('utf-8')) - print('*****************') - print(msg.as_string()) - print('*****/EMAIL******') - - -def _send_email(smtp, from_email, mailto, msg): - ''' Send the prepared mail to the respective users ''' - try: - if smtp is None: - if pagure.APP.config['SMTP_SSL']: - smtp = smtplib.SMTP_SSL( - pagure.APP.config['SMTP_SERVER'], - pagure.APP.config['SMTP_PORT']) - else: - smtp = smtplib.SMTP( - pagure.APP.config['SMTP_SERVER'], - pagure.APP.config['SMTP_PORT']) - if pagure.APP.config['SMTP_USERNAME'] \ - and pagure.APP.config['SMTP_PASSWORD']: - smtp.login( - pagure.APP.config['SMTP_USERNAME'], - pagure.APP.config['SMTP_PASSWORD'] - ) - - smtp.sendmail( - from_email, - [mailto], - msg.as_string()) - except smtplib.SMTPException as err: - _log.exception(err) - diff --git a/pagure/lib/tasks.py b/pagure/lib/tasks.py index 25b9a42..3d97d88 100644 --- a/pagure/lib/tasks.py +++ b/pagure/lib/tasks.py @@ -8,6 +8,7 @@ """ +import gc import os import os.path import shutil @@ -21,13 +22,14 @@ import tempfile import six import logging +import smtplib import pagure from pagure import APP import pagure.lib import pagure.lib.git import pagure.lib.git_auth -from pagure.lib import task_helpers +from pagure.lib import mail_helpers from email.header import Header @@ -57,6 +59,18 @@ def get_result(uuid): return AsyncResult(uuid, conn.backend) +def ret(endpoint, **kwargs): + toret = {'endpoint': endpoint} + toret.update(kwargs) + return toret + + +def gc_clean(): + """ Force a run of the garbage collector. """ + # https://pagure.io/pagure/issue/2302 + gc.collect() + + @conn.task def generate_gitolite_acls(namespace=None, name=None, user=None, group=None): """ Generate the gitolite configuration file either entirely or for a @@ -91,7 +105,7 @@ def generate_gitolite_acls(namespace=None, name=None, user=None, group=None): helper, project, group_obj) helper.generate_acls(project=project, group=group_obj) session.remove() - task_helpers.gc_clean() + gc_clean() @conn.task @@ -212,9 +226,9 @@ def create_project(username, namespace, name, add_readme, user=project.user.user if project.is_fork else None) session.remove() - task_helpers.gc_clean() + gc_clean() - return task_helpers.ret('view_repo', repo=name, namespace=namespace) + return ret('view_repo', repo=name, namespace=namespace) @conn.task @@ -239,7 +253,7 @@ def update_git(name, namespace, user, ticketuid=None, requestuid=None): result = pagure.lib.git._update_git(obj, project, folder) session.remove() - task_helpers.gc_clean() + gc_clean() return result @@ -276,7 +290,7 @@ def update_file_in_git(name, namespace, user, branch, branchto, filename, content, message, userobj, email) session.remove() - return task_helpers.ret( + return ret( 'view_commits', repo=project.name, username=user, @@ -301,7 +315,7 @@ def delete_branch(name, namespace, user, branchname): _log.exception(err) session.remove() - return task_helpers.ret( + return ret( 'view_repo', repo=name, namespace=namespace, username=user) @@ -405,13 +419,13 @@ def fork(name, namespace, user_owner, user_forker, editbranch, editfile): namespace=repo_to.namespace, name=repo_to.name, user=repo_to.user.user if repo_to.is_fork else None) - task_helpers.gc_clean() + gc_clean() if editfile is None: - return task_helpers.ret( + return ret( 'view_repo', repo=name, namespace=namespace, username=user_forker) else: - return task_helpers.ret( + return ret( 'edit_file', repo=name, namespace=namespace, @@ -429,7 +443,7 @@ def pull_remote_repo(remote_git, branch_from): remote_git, clonepath, checkout_branch=branch_from) del repo - task_helpers.gc_clean() + gc_clean() return clonepath @@ -443,7 +457,7 @@ def refresh_pr_cache(name, namespace, user): pagure.lib.reset_status_pull_request(session, project) session.remove() - task_helpers.gc_clean() + gc_clean() @conn.task @@ -463,8 +477,8 @@ def merge_pull_request(name, namespace, user, requestid, user_merger): refresh_pr_cache.delay(name, namespace, user) session.remove() - task_helpers.gc_clean() - return task_helpers.ret( + gc_clean() + return ret( 'view_repo', repo=name, username=user, namespace=namespace) @@ -490,7 +504,7 @@ def add_file_to_git(name, namespace, user, user_attacher, issueuid, filename): filename) session.remove() - task_helpers.gc_clean() + gc_clean() @conn.task @@ -510,9 +524,9 @@ def project_dowait(name, namespace, user): time.sleep(10) session.remove() - task_helpers.gc_clean() + gc_clean() - return task_helpers.ret( + return ret( 'view_repo', repo=name, username=user, namespace=namespace) @@ -546,7 +560,7 @@ def sync_pull_ref(name, namespace, user, requestid): pagure.lib.git.update_pull_ref(request, repo_obj) session.remove() - task_helpers.gc_clean() + gc_clean() @conn.task @@ -591,7 +605,7 @@ def send_email(text, subject, to_mail, smtp = None for mailto in to_mail.split(','): - msg, header = task_helpers._prepare_individual_email( + msg, header = mail_helpers._prepare_individual_email( header=header, subject=subject, subject_tag=subject_tag, @@ -605,7 +619,7 @@ def send_email(text, subject, to_mail, if not pagure.APP.config.get('EMAIL_SEND', True): # This will be viewed on the console where the worker is running - task_helpers._print_email_to_console( + mail_helpers._print_email_to_console( from_email=from_email, mailto=mailto, subject=subject, @@ -616,8 +630,11 @@ def send_email(text, subject, to_mail, ) continue # Send it to user if EMAIL_SEND is Enabled - task_helpers._send_email( - smtp=smtp, from_email=from_email, mailto=mailto, msg=msg) + try: + mail_helpers._send_email( + smtp=smtp, from_email=from_email, mailto=mailto, msg=msg) + except smtplib.SMTPException as err: + _log.exception(err) if smtp: smtp.quit()