From 7dbc9d08cd7249b9fa06e00133001f5f2417b84e Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Jun 18 2015 11:04:26 +0000 Subject: [PATCH 1/58] Move the JSON representation of comments on issue into their own function --- diff --git a/pagure/lib/model.py b/pagure/lib/model.py index d98cc81..f827292 100644 --- a/pagure/lib/model.py +++ b/pagure/lib/model.py @@ -510,14 +510,7 @@ class Issue(BASE): comments = [] for comment in self.comments: - cmt = { - 'id': comment.id, - 'comment': comment.comment, - 'parent': comment.parent_id, - 'date_created': comment.date_created.strftime('%s'), - 'user': comment.user.to_json(public=public), - } - comments.append(cmt) + comments.append(comment.to_json(public=public)) output['comments'] = comments @@ -597,6 +590,19 @@ class IssueComment(BASE): ''' Return the parent, in this case the issue object. ''' return self.issue + def to_json(self, public=False): + ''' Returns a dictionary representation of the issue. + + ''' + output = { + 'id': self.id, + 'comment': self.comment, + 'parent': self.parent_id, + 'date_created': self.date_created.strftime('%s'), + 'user': self.user.to_json(public=public), + } + return output + class Tag(BASE): """ Stores the tags. From 2d98818416f043fc8298b400b9c4abee83121c23 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Jun 18 2015 11:04:26 +0000 Subject: [PATCH 2/58] Move the conversion from text to html via markdown in the internal library --- diff --git a/pagure/lib/__init__.py b/pagure/lib/__init__.py index cb2c63a..94d47cd 100644 --- a/pagure/lib/__init__.py +++ b/pagure/lib/__init__.py @@ -10,6 +10,7 @@ import datetime +import markdown import os import shutil import tempfile @@ -2176,3 +2177,21 @@ def add_token_to_user(session, project, acls, username): session.commit() return 'Token created' + +def text2markdown(text): + """ Simple text to html converter using the markdown library. + """ + if text: + # Hack to allow blockquotes to be marked by ~~~ + ntext = [] + indent = False + for line in text.split('\n'): + if line.startswith('~~~'): + indent = not indent + continue + if indent: + line = ' %s' % line + ntext.append(line) + return markdown.markdown('\n'.join(ntext)) + + return '' diff --git a/pagure/ui/filters.py b/pagure/ui/filters.py index e49e76f..703dc46 100644 --- a/pagure/ui/filters.py +++ b/pagure/ui/filters.py @@ -15,7 +15,6 @@ import urlparse import arrow import bleach import flask -import markdown from pygments import highlight from pygments.lexers.text import DiffLexer @@ -215,20 +214,7 @@ def markdown_filter(text): """ Template filter converting a string into html content using the markdown library. """ - if text: - # Hack to allow blockquotes to be marked by ~~~ - ntext = [] - indent = False - for line in text.split('\n'): - if line.startswith('~~~'): - indent = not indent - continue - if indent: - line = ' %s' % line - ntext.append(line) - return no_js(markdown.markdown('\n'.join(ntext))) - - return '' + return no_js(pagure.lib.text2markdown(text)) @APP.template_filter('html_diff') From 09fb7e6a367e9feec4514aa523924b62dc782df7 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Jun 18 2015 11:04:26 +0000 Subject: [PATCH 3/58] Create a main REDIS connection to be used throughout the application --- diff --git a/pagure/__init__.py b/pagure/__init__.py index e361219..2174776 100644 --- a/pagure/__init__.py +++ b/pagure/__init__.py @@ -25,6 +25,7 @@ from logging.handlers import SMTPHandler import flask import pygit2 +import redis from flask_fas_openid import FAS from functools import wraps from sqlalchemy.exc import SQLAlchemyError @@ -53,6 +54,10 @@ if 'PAGURE_CONFIG' in os.environ: FAS = FAS(APP) SESSION = pagure.lib.create_session(APP.config['DB_URL']) +REDIS = redis.StrictRedis( + host=APP.config['REDIS_HOST'], + port=APP.config['REDIS_PORT'], + db=APP.config['REDIS_DB']) if not APP.debug: APP.logger.addHandler(pagure.mail_logging.get_mail_handler( diff --git a/pagure/default_config.py b/pagure/default_config.py index 5bb57c2..d7330fa 100644 --- a/pagure/default_config.py +++ b/pagure/default_config.py @@ -47,6 +47,11 @@ MAX_CONTENT_LENGTH = 4 * 1024 * 1024 # 4 megabytes # IP addresses allowed to access the internal endpoints IP_ALLOWED_INTERNAL = ['127.0.0.1', 'localhost', '::1'] +# Redis configuration +REDIS_HOST = 'localhost' +REDIS_PORT = 6379 +REDIS_DB = 'pagure' + # Folder containing to the git repos GIT_FOLDER = os.path.join( os.path.abspath(os.path.dirname(__file__)), From 749d2264443c4312c0281f557b45fc028da287a4 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Jun 18 2015 11:04:26 +0000 Subject: [PATCH 4/58] List python-redis in the list of dependencies --- diff --git a/requirements.txt b/requirements.txt index 87fe8f3..b0801d2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,6 +20,7 @@ python-fedora python-openid python-openid-cla python-openid-teams +python-redis six sqlalchemy >= 0.8 straight.plugin==1.4.0-post-1 From 6c6d892dcd9e32ff80efd17878e79dfa9b12b20f Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Jun 18 2015 11:04:26 +0000 Subject: [PATCH 5/58] Make the application run threaded so that we can use redis --- diff --git a/runserver.py b/runserver.py index 33d6dec..270b1b2 100755 --- a/runserver.py +++ b/runserver.py @@ -14,4 +14,4 @@ if '--profile' in sys.argv: APP.config['PROFILE'] = True APP.wsgi_app = ProfilerMiddleware(APP.wsgi_app, restrictions=[30]) -APP.run() +APP.run(threaded=True) From 8d838d7d6063d7b3941d80eafda47f4f674e999b Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Jun 18 2015 11:04:26 +0000 Subject: [PATCH 6/58] Adjust Issue.to_json() to allow not incorporating the comments in the JSON --- diff --git a/pagure/lib/model.py b/pagure/lib/model.py index f827292..63aa734 100644 --- a/pagure/lib/model.py +++ b/pagure/lib/model.py @@ -489,7 +489,7 @@ class Issue(BASE): ''' Return the list of issue this issue blocks on in simple text. ''' return [issue.id for issue in self.parents] - def to_json(self, public=False): + def to_json(self, public=False, with_comments=True): ''' Returns a dictionary representation of the issue. ''' @@ -509,8 +509,9 @@ class Issue(BASE): } comments = [] - for comment in self.comments: - comments.append(comment.to_json(public=public)) + if with_comments: + for comment in self.comments: + comments.append(comment.to_json(public=public)) output['comments'] = comments From b729b012b1d7928212872bef2620e34217a363ca Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Jun 18 2015 11:04:26 +0000 Subject: [PATCH 7/58] Add a new endpoint streaming information about a given issue This endpoint streams all information about a given issue put in the redis queue and sends it to the browser. The browser can then listen to this stream and adjust the UI on the fly --- diff --git a/pagure/ui/issues.py b/pagure/ui/issues.py index bd80924..cf8ee0e 100644 --- a/pagure/ui/issues.py +++ b/pagure/ui/issues.py @@ -21,8 +21,8 @@ import mimetypes import pagure.doc_utils import pagure.lib import pagure.forms -from pagure import (APP, SESSION, LOG, __get_file_in_tree, cla_required, - is_repo_admin, authenticated) +from pagure import (APP, SESSION, REDIS, LOG, __get_file_in_tree, + cla_required, is_repo_admin, authenticated) # pylint: disable=E1101 @@ -497,6 +497,42 @@ def view_issue(repo, issueid, username=None): ) +@APP.route('//issue//stream') +@APP.route('/fork///issue//stream') +def stream_issue(repo, issueid, username=None): + """ Streams the changes made to an issue live + """ + + repo = pagure.lib.get_project(SESSION, repo, user=username) + + if repo is None: + flask.abort(404, 'Project not found') + + if not repo.settings.get('issue_tracker', True): + flask.abort(404, 'No issue tracker found for this project') + + issue = pagure.lib.search_issues(SESSION, repo, issueid=issueid) + + if issue is None or issue.project != repo: + flask.abort(404, 'Issue not found') + + if issue.private and not is_repo_admin(repo) \ + and (not authenticated() or + not issue.user.user == flask.g.fas_user.username): + flask.abort( + 403, 'This issue is private and you are not allowed to view it') + + pubsub = REDIS.pubsub() + pubsub.subscribe(issue.uid) + def event_stream(pubsub): + for message in pubsub.listen(): + yield 'data: %s\n\n' % message['data'] + + return flask.Response( + event_stream(pubsub), + mimetype="text/event-stream") + + @APP.route('//issue//drop', methods=['POST']) @APP.route('/fork///issue//drop', methods=['POST']) From 72a8e708176131342f7924b6d1b958b488298593 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Jun 18 2015 11:04:26 +0000 Subject: [PATCH 8/58] When adding a comment to an issue, notify redis if asked With this change, when a comment is made on an issue, if there is a redis connection available, it will queue a message information about the change to the channel of this specific issue. --- diff --git a/pagure/lib/__init__.py b/pagure/lib/__init__.py index 94d47cd..e921de3 100644 --- a/pagure/lib/__init__.py +++ b/pagure/lib/__init__.py @@ -8,6 +8,10 @@ """ +try: + import simplejson as json +except ImportError: + import json import datetime import markdown @@ -147,7 +151,7 @@ def search_user(session, username=None, email=None, token=None, pattern=None): def add_issue_comment(session, issue, comment, user, ticketfolder, - notify=True): + notify=True, redis=None): ''' Add a comment to an issue. ''' user_obj = __get_user(session, user) @@ -177,6 +181,15 @@ def add_issue_comment(session, issue, comment, user, ticketfolder, ) ) + if redis: + redis.publish(issue.uid, json.dumps({ + 'comment_id': len(issue.comments), + 'comment_added': text2markdown(issue_comment.comment), + 'comment_user': issue_comment.user.user, + 'avatar_url': avatar_url(issue_comment.user.user, size=16), + 'comment_date': issue_comment.date_created.strftime('%Y-%m-%d %H:%M'), + })) + return 'Comment added' From 26753b7d4a6bd6938a84f7afe60d99ef5babc869 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Jun 18 2015 11:04:26 +0000 Subject: [PATCH 9/58] When adding a tag to an issue, notify redis if asked With this change, when a tag is added on an issue, if there is a redis connection available, it will queue a message information about which tag(s) was/were added to the channel of this specific issue. --- diff --git a/pagure/lib/__init__.py b/pagure/lib/__init__.py index e921de3..ee781fc 100644 --- a/pagure/lib/__init__.py +++ b/pagure/lib/__init__.py @@ -193,7 +193,7 @@ def add_issue_comment(session, issue, comment, user, ticketfolder, return 'Comment added' -def add_issue_tag(session, issue, tags, user, ticketfolder): +def add_issue_tag(session, issue, tags, user, ticketfolder, redis=None): ''' Add a tag to an issue. ''' user_obj = __get_user(session, user) @@ -241,6 +241,9 @@ def add_issue_tag(session, issue, tags, user, ticketfolder): ) ) + if redis: + redis.publish(issue.uid, json.dumps({'added_tags': added_tags})) + if added_tags: return 'Tag added: %s' % ', '.join(added_tags) else: From 9f629a26dc401654a724e3bd0d56a3414fe6a33a Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Jun 18 2015 11:04:26 +0000 Subject: [PATCH 10/58] When removing a tag to an issue, notify redis if asked With this change, when a tag is removied on an issue, if there is a redis connection available, it will queue a message information about which tag(s) was/were removed to the channel of this specific issue. --- diff --git a/pagure/lib/__init__.py b/pagure/lib/__init__.py index ee781fc..5adee6c 100644 --- a/pagure/lib/__init__.py +++ b/pagure/lib/__init__.py @@ -486,7 +486,7 @@ def remove_tags(session, project, tags, ticketfolder, user): return msgs -def remove_tags_issue(session, issue, tags, ticketfolder, user): +def remove_tags_issue(session, issue, tags, ticketfolder, user, redis=None): ''' Removes the specified tag(s) of a issue. ''' user_obj = __get_user(session, user) @@ -514,6 +514,9 @@ def remove_tags_issue(session, issue, tags, ticketfolder, user): ) ) + if redis: + redis.publish(issue.uid, json.dumps({'removed_tags': removed_tags})) + return 'Removed tag: %s' % ', '.join(removed_tags) From 1c0d0c46a08a2dc0c36739b182414338fbd7b618 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Jun 18 2015 11:04:27 +0000 Subject: [PATCH 11/58] Adjust update_tags_issue to forward the potential redis connection specified --- diff --git a/pagure/lib/__init__.py b/pagure/lib/__init__.py index 5adee6c..d2d2018 100644 --- a/pagure/lib/__init__.py +++ b/pagure/lib/__init__.py @@ -1729,7 +1729,7 @@ def avatar_url_from_openid(openid, size=64, default='retro', dns=False): hashhex, query) -def update_tags_issue(session, issue, tags, username, ticketfolder): +def update_tags_issue(session, issue, tags, username, ticketfolder, redis=None): """ Update the tags of a specified issue (adding or removing them). """ @@ -1747,6 +1747,7 @@ def update_tags_issue(session, issue, tags, username, ticketfolder): tags=toadd, user=username, ticketfolder=ticketfolder, + redis=redis, ) ) @@ -1758,6 +1759,7 @@ def update_tags_issue(session, issue, tags, username, ticketfolder): tags=torm, user=username, ticketfolder=ticketfolder, + redis=redis, ) ) session.commit() From ca090711a3b9fd6a2b198f326b8cea154369494d Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Jun 18 2015 11:04:27 +0000 Subject: [PATCH 12/58] When assigning a issue to someone, notify redis if asked With this change, when an issue is assigned to someone, if there is a redis connection available, it will queue a message information about to whom the issue was assigned to the channel of this specific issue. --- diff --git a/pagure/lib/__init__.py b/pagure/lib/__init__.py index d2d2018..22b5807 100644 --- a/pagure/lib/__init__.py +++ b/pagure/lib/__init__.py @@ -250,7 +250,8 @@ def add_issue_tag(session, issue, tags, user, ticketfolder, redis=None): return 'Nothing to add' -def add_issue_assignee(session, issue, assignee, user, ticketfolder): +def add_issue_assignee(session, issue, assignee, user, ticketfolder, + redis=None): ''' Add an assignee to an issue, in other words, assigned an issue. ''' user_obj = __get_user(session, user) @@ -273,6 +274,9 @@ def add_issue_assignee(session, issue, assignee, user, ticketfolder): ) ) + if redis: + redis.publish(issue.uid, json.dumps({'unassigned': '-'})) + return 'Assignee reset' elif assignee is None and issue.assignee is None: return @@ -301,6 +305,10 @@ def add_issue_assignee(session, issue, assignee, user, ticketfolder): ) ) + if redis: + redis.publish(issue.uid, json.dumps( + {'assigned': assignee_obj.to_json(public=True)})) + return 'Issue assigned' From 8a08270bbee9bf0a8963e60b74a10b36e544361c Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Jun 18 2015 11:04:27 +0000 Subject: [PATCH 13/58] Forward the redis connection to the method of the internal library that supports it --- diff --git a/pagure/ui/issues.py b/pagure/ui/issues.py index cf8ee0e..e6f7148 100644 --- a/pagure/ui/issues.py +++ b/pagure/ui/issues.py @@ -128,6 +128,7 @@ def update_issue(repo, issueid, username=None): comment=comment, user=flask.g.fas_user.username, ticketfolder=APP.config['TICKETS_FOLDER'], + redis=REDIS, ) SESSION.commit() if message: @@ -138,7 +139,8 @@ def update_issue(repo, issueid, username=None): messages = pagure.lib.update_tags_issue( SESSION, issue, tags, username=flask.g.fas_user.username, - ticketfolder=APP.config['TICKETS_FOLDER']) + ticketfolder=APP.config['TICKETS_FOLDER'], + redis=REDIS) for message in messages: flask.flash(message) @@ -148,7 +150,9 @@ def update_issue(repo, issueid, username=None): issue=issue, assignee=assignee or None, user=flask.g.fas_user.username, - ticketfolder=APP.config['TICKETS_FOLDER'],) + ticketfolder=APP.config['TICKETS_FOLDER'], + redis=REDIS, + ) if message: SESSION.commit() flask.flash(message) From 1549ec0bea9ff7668d3e7c4932bdf34a88e4f52e Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Jun 18 2015 11:04:27 +0000 Subject: [PATCH 14/58] Add JS logic to the issue page to start supporting eventsource This adds eventsource support for: - Adding/Removing a tag of an issue - Assigning/Resetting the assignee of an issue - Adding a comment to an issue --- diff --git a/pagure/templates/issue.html b/pagure/templates/issue.html index e2d0d9d..a3f8cfe 100644 --- a/pagure/templates/issue.html +++ b/pagure/templates/issue.html @@ -53,11 +53,13 @@
Tags: + {% for tag in issue.tags %} - {{ tag.tag }}{%- if not loop.last -%},{%- endif -%} + repo=repo.name, tags=tag.tag) }}">{{ tag.tag }} + {%- if not loop.last -%},{%- endif -%} {% endfor %} + {% if authenticated and repo_admin %} @@ -66,11 +68,13 @@
Assigned: + {% if issue.assignee %} {{ issue.assignee.username }} {% endif %} + {% if authenticated %} @@ -113,11 +117,13 @@ {{ show_comment(issue, 0, repo, username, issueid, form) }} +
{% if issue.comments %} {% for comment in issue.comments %} {{ show_comment(comment, loop.index, repo, username, issueid, form, repo_admin) }} {% endfor %} {% endif %} +
{% if authenticated and form %} @@ -234,14 +240,14 @@ $(function() { $( ".reply" ).click( function() { - var _section = $(this).parent().parent().parent(); - var _comment = _section.find('.comment_body'); - var _text = _comment.text().split("\n"); - var _output = new Array(); - for (cnt = 0; cnt < _text.length - 1; cnt ++) { - _output[cnt] = '> ' + jQuery.trim(_text[cnt + 1]); - } - $( "#comment" ).val(_output.join("\n")); + var _section = $(this).parent().parent().parent(); + var _comment = _section.find('.comment_body'); + var _text = _comment.text().split("\n"); + var _output = new Array(); + for (cnt = 0; cnt < _text.length - 1; cnt ++) { + _output[cnt] = '> ' + jQuery.trim(_text[cnt + 1]); + } + $( "#comment" ).val(_output.join("\n")); } ); @@ -317,6 +323,85 @@ $(function() { }); }); + +if (!!window.EventSource) { + var source = new EventSource('{{ + url_for("stream_issue", username=username, + repo=repo.name, issueid=issueid) }}'); +} else { + // Result to xhr polling :( +} +source.addEventListener('message', function(e) { + console.log(e.data); + var data = $.parseJSON(e.data); + console.log(data); + if (data.added_tags){ + console.log('adding ' + data.added_tags); + var field = $('#taglist'); + var _data = field.html(); + var _url =''; + for (i=0; i'; + } + } + if (data.removed_tags){ + console.log('removing ' + data.removed_tags); + var field = $('#taglist'); + var _data = field.html(); + var _url =''; + for (i=0; i', ''); + } + field.html(_data); + } + if (data.assigned){ + console.log('assigning ' + data.assigned); + var field = $('#assigneduser'); + var _data = field.html(); + var _url =''; + _data = _url.replace('--', data.assigned.name) + data.assigned.name + ''; + field.html(_data); + field = $('#assignee'); + field.val(data.assigned.name); + } + if (data.unassigned){ + console.log('un-assigning '); + var field = $('#assigneduser'); + field.html(' '); + field = $('#assignee'); + field.val(''); + } + if (data.comment_added){ + console.log('Adding comment ' + data.comment_added); + var field = $('#comments'); + var _data = '
\ +
\ + \ + \ + ' + data.comment_user + '\ + - seconds ago \ + \ + \ +
\ +
\ +

' + data.comment_added + '

\ +
\ +
'; + field.html(field.html() + _data); + } + +}, false); + {% endblock %} From f9140cd79c70f85cb5862d3f0e4cb48758d5bc1f Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Jun 18 2015 11:04:27 +0000 Subject: [PATCH 15/58] Adjust the default REDIS_DB to be 0, a safe bet --- diff --git a/pagure/default_config.py b/pagure/default_config.py index d7330fa..c3e6ee3 100644 --- a/pagure/default_config.py +++ b/pagure/default_config.py @@ -50,7 +50,7 @@ IP_ALLOWED_INTERNAL = ['127.0.0.1', 'localhost', '::1'] # Redis configuration REDIS_HOST = 'localhost' REDIS_PORT = 6379 -REDIS_DB = 'pagure' +REDIS_DB = 0 # Folder containing to the git repos GIT_FOLDER = os.path.join( From 85ee44deba4689c35635b1a16aee1fdbe4a4b80e Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Jun 18 2015 11:04:27 +0000 Subject: [PATCH 16/58] Add redis support to the method handling the issue dependency updates This way we can publish messages via redis to the web UI live --- diff --git a/pagure/lib/__init__.py b/pagure/lib/__init__.py index 22b5807..a74b565 100644 --- a/pagure/lib/__init__.py +++ b/pagure/lib/__init__.py @@ -366,7 +366,8 @@ def add_pull_request_assignee( return 'Request assigned' -def add_issue_dependency(session, issue, issue_blocked, user, ticketfolder): +def add_issue_dependency( + session, issue, issue_blocked, user, ticketfolder, redis=None): ''' Add a dependency between two issues. ''' user_obj = __get_user(session, user) @@ -407,10 +408,23 @@ def add_issue_dependency(session, issue, issue_blocked, user, ticketfolder): ) ) + if redis: + redis.publish(issue.uid, json.dumps({ + 'added_dependency': issue_blocked.id, + 'issue_uid': issue.uid, + 'type': 'children', + })) + redis.publish(issue_blocked.uid, json.dumps({ + 'added_dependency': issue.id, + 'issue_uid': issue_blocked.uid, + 'type': 'parent', + })) + return 'Dependency added' -def remove_issue_dependency(session, issue, issue_blocked, user, ticketfolder): +def remove_issue_dependency( + session, issue, issue_blocked, user, ticketfolder, redis=None): ''' Remove a dependency between two issues. ''' user_obj = __get_user(session, user) @@ -452,6 +466,18 @@ def remove_issue_dependency(session, issue, issue_blocked, user, ticketfolder): ) ) + if redis: + redis.publish(issue.uid, json.dumps({ + 'removed_dependency': child_del, + 'issue_uid': issue.uid, + 'type': 'children', + })) + redis.publish(issue_blocked.uid, json.dumps({ + 'removed_dependency': issue.id, + 'issue_uid': issue_blocked.uid, + 'type': 'parent', + })) + return 'Dependency removed' @@ -1776,7 +1802,7 @@ def update_tags_issue(session, issue, tags, username, ticketfolder, redis=None): def update_dependency_issue( - session, repo, issue, depends, username, ticketfolder): + session, repo, issue, depends, username, ticketfolder, redis=None): """ Update the dependency of a specified issue (adding or removing them) """ @@ -1803,6 +1829,7 @@ def update_dependency_issue( issue_blocked=issue, user=username, ticketfolder=ticketfolder, + redis=redis, ) ) @@ -1824,6 +1851,7 @@ def update_dependency_issue( issue_blocked=issue_depend, user=username, ticketfolder=ticketfolder, + redis=redis, ) ) @@ -1832,7 +1860,7 @@ def update_dependency_issue( def update_blocked_issue( - session, repo, issue, blocks, username, ticketfolder): + session, repo, issue, blocks, username, ticketfolder, redis=None): """ Update the upstream dependency of a specified issue (adding or removing them) @@ -1860,6 +1888,7 @@ def update_blocked_issue( issue_blocked=issue_block, user=username, ticketfolder=ticketfolder, + redis=redis, ) ) session.commit() @@ -1883,6 +1912,7 @@ def update_blocked_issue( issue_blocked=issue, user=username, ticketfolder=ticketfolder, + redis=redis, ) ) From 25e7f33d8604c65a634fb5ff9733ef55b08ca0b9 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Jun 18 2015 11:04:27 +0000 Subject: [PATCH 17/58] Forward the redis connection to the backend to support notifying dependency changes --- diff --git a/pagure/ui/issues.py b/pagure/ui/issues.py index e6f7148..0748381 100644 --- a/pagure/ui/issues.py +++ b/pagure/ui/issues.py @@ -175,7 +175,9 @@ def update_issue(repo, issueid, username=None): messages = pagure.lib.update_dependency_issue( SESSION, repo, issue, depends, username=flask.g.fas_user.username, - ticketfolder=APP.config['TICKETS_FOLDER']) + ticketfolder=APP.config['TICKETS_FOLDER'], + redis=REDIS, + ) for message in messages: flask.flash(message) @@ -183,7 +185,9 @@ def update_issue(repo, issueid, username=None): messages = pagure.lib.update_blocked_issue( SESSION, repo, issue, blocks, username=flask.g.fas_user.username, - ticketfolder=APP.config['TICKETS_FOLDER']) + ticketfolder=APP.config['TICKETS_FOLDER'], + redis=REDIS, + ) for message in messages: flask.flash(message) From 418d03250c6ffd361c55b974845e590bcba52f16 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Jun 18 2015 11:04:27 +0000 Subject: [PATCH 18/58] Update the dependencies sections upon notification via eventsource --- diff --git a/pagure/templates/issue.html b/pagure/templates/issue.html index a3f8cfe..ce4842e 100644 --- a/pagure/templates/issue.html +++ b/pagure/templates/issue.html @@ -81,15 +81,17 @@ {% endif %}
-
+
Blocking: + {% if issue.parents %} {% for ticket in issue.parents %} - {{ ticket.id }}{%- if not loop.last -%},{%- endif -%} + repo=repo.name, issueid=ticket.id) + }}">{{ ticket.id }}{%- if not loop.last -%},{%- endif -%} {% endfor %} {% endif %} + {% if authenticated %} -
+
Depends on: + {% if issue.children %} {% for ticket in issue.children %} - {{ ticket.id }}{%- if not loop.last -%},{%- endif -%} + repo=repo.name, issueid=ticket.id) + }}">{{ ticket.id }}{%- if not loop.last -%},{%- endif -%} {% endfor %} {% endif %} + {% if authenticated %} '; + dep = data.added_dependency; + _data += ' ' + _url.replace('/-1', '/' + dep) + dep + ''; + field.html(_data); + var _curval = field2.val(); + if (_curval && _curval != ',') { + _curval += ','; + } + field2.val(_curval + dep); + } + if (data.removed_dependency){ + console.log('Removing ' + data.removed_dependency); + if (data.issue_uid == "{{ issue.uid }}"){ + if (data.type == "children"){ + var field = $('#dependencies'); + var field2 = $('#depends'); + } else { + var field = $('#blockers'); + var field2 = $('#blocks'); + } + } + var _data = field.html(); + var _url =''; + dep = data.removed_dependency; + _data = _data.replace(_url.replace('/-1', '/' + dep) + dep + '', ''); + _data = _data.replace(',,', ','); + field.html(_data); + field2.val(field2.val().replace(dep, '')); + } if (data.comment_added){ console.log('Adding comment ' + data.comment_added); var field = $('#comments'); From 132abae1995ccffec64c4475a2bd570c5c90cab6 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Jun 18 2015 11:04:27 +0000 Subject: [PATCH 19/58] Adjust the logic to add/remove tags or dependencies This change makes sure we do not delete something we did not want to just because it looked close to something we were looking for. In addition, it makes the UI closer to the usual UI when updated by eventsource --- diff --git a/pagure/templates/issue.html b/pagure/templates/issue.html index ce4842e..a2751b4 100644 --- a/pagure/templates/issue.html +++ b/pagure/templates/issue.html @@ -332,9 +332,22 @@ if (!!window.EventSource) { var source = new EventSource('{{ url_for("stream_issue", username=username, repo=repo.name, issueid=issueid) }}'); -} else { - // Result to xhr polling :( } + +clean_entry= function(text, element){ + var _data = $.trim(text).split(','); + var _out = [] + var y=0; + for (i=0; i<_data.length; i++){ + if ($.trim(_data[i]) == element) { + continue; + } + _out[y] = $.trim(_data[i]); + y+=1; + } + return _out; +} + source.addEventListener('message', function(e) { console.log(e.data); var data = $.parseJSON(e.data); @@ -342,25 +355,37 @@ source.addEventListener('message', function(e) { if (data.added_tags){ console.log('adding ' + data.added_tags); var field = $('#taglist'); + var field2 = $('#tag'); var _data = field.html(); var _url =''; for (i=0; i'; + _data += ',' + _url.replace('--', tag) + tag + ''; + field.html(_data); + var _curval = field2.val(); + if (_curval) { + _curval += ','; + } + field2.val(_curval + tag); } } if (data.removed_tags){ console.log('removing ' + data.removed_tags); var field = $('#taglist'); + var field2 = $('#tag'); var _data = field.html(); + var _data2 = field2.html(); var _url =''; for (i=0; i', ''); + var _turl = _url.replace('=--', '=' + tag) + tag + ''; + _data = clean_entry(_data, _turl).join(); + _data2 = clean_entry(_data2, tag).join(); } field.html(_data); + field2.val(_data2); } if (data.assigned){ console.log('assigning ' + data.assigned); @@ -395,7 +420,7 @@ source.addEventListener('message', function(e) { var _url =''; dep = data.added_dependency; - _data += ' ' + _url.replace('/-1', '/' + dep) + dep + ''; + _data += ',' + _url.replace('/-1', '/' + dep) + dep + ''; field.html(_data); var _curval = field2.val(); if (_curval && _curval != ',') { @@ -414,14 +439,15 @@ source.addEventListener('message', function(e) { var field2 = $('#blocks'); } } - var _data = field.html(); + var dep = data.removed_dependency; + // Set links + var _data = $.trim(field.html()).split(','); var _url =''; - dep = data.removed_dependency; - _data = _data.replace(_url.replace('/-1', '/' + dep) + dep + '', ''); - _data = _data.replace(',,', ','); - field.html(_data); - field2.val(field2.val().replace(dep, '')); + _url = _url.replace('/-1', '/' + dep) + dep + ''; + field.html(clean_entry(field.html(), _url).join()); + // Set the value in the input field + field2.val(clean_entry(field2.val(), dep).join()); } if (data.comment_added){ console.log('Adding comment ' + data.comment_added); From 68664fd65b81430cbb34edb70f74ab3a4b95c9f9 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Jun 18 2015 11:04:27 +0000 Subject: [PATCH 20/58] When editing an issue, send a message to redis about the change --- diff --git a/pagure/lib/__init__.py b/pagure/lib/__init__.py index a74b565..dd0523b 100644 --- a/pagure/lib/__init__.py +++ b/pagure/lib/__init__.py @@ -961,7 +961,8 @@ def new_pull_request(session, repo_from, branch_from, def edit_issue(session, issue, ticketfolder, user, - title=None, content=None, status=None, private=False): + title=None, content=None, status=None, private=False, + redis=None): ''' Edit the specified issue. ''' user_obj = __get_user(session, user) @@ -1002,6 +1003,12 @@ def edit_issue(session, issue, ticketfolder, user, ) ) + if redis: + redis.publish(issue.uid, json.dumps({ + 'fields': edit, + 'issue': issue.to_json(public=True, with_comments=False), + })) + if edit: session.add(issue) session.flush() From 834aec32fdc69e8317fa603b4c660e42aa096949 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Jun 18 2015 11:04:27 +0000 Subject: [PATCH 21/58] Forward the redis connector to the backend when editing an issue --- diff --git a/pagure/ui/issues.py b/pagure/ui/issues.py index 0748381..d9fb3bd 100644 --- a/pagure/ui/issues.py +++ b/pagure/ui/issues.py @@ -166,6 +166,7 @@ def update_issue(repo, issueid, username=None): status=new_status, user=flask.g.fas_user.username, ticketfolder=APP.config['TICKETS_FOLDER'], + redis=REDIS, ) SESSION.commit() if message: @@ -633,6 +634,7 @@ def edit_issue(repo, issueid, username=None): user=flask.g.fas_user.username, ticketfolder=APP.config['TICKETS_FOLDER'], private=private, + redis=REDIS, ) SESSION.commit() flask.flash(message) From 4fd8f436098fab1e6fa42e555b7ae9a3bd13b368 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Jun 18 2015 11:04:27 +0000 Subject: [PATCH 22/58] Expand the JS logic to support updating title, status and report via eventsource --- diff --git a/pagure/templates/issue.html b/pagure/templates/issue.html index a2751b4..5cd8999 100644 --- a/pagure/templates/issue.html +++ b/pagure/templates/issue.html @@ -35,7 +35,8 @@ {% endif %}

- #{{ issueid }} {{ issue.title | noJS }} + #{{ issueid }} {{ + issue.title | noJS }} {% if authenticated and (repo_admin or g.fas_user.username == issue.user.username) %} - @@ -472,6 +473,22 @@ source.addEventListener('message', function(e) {

'; field.html(field.html() + _data); } + if (data.fields){ + console.log('Adjusting issue ' + data.fields); + for (i=0; i' + data.issue.content + '

'); + } + } + } }, false); From 30a0dc02b0d12f1d1b63ab8bef94c18819e74945 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Jun 18 2015 11:04:27 +0000 Subject: [PATCH 23/58] Let's try to close the eventsource connection before closing the page --- diff --git a/pagure/templates/issue.html b/pagure/templates/issue.html index 5cd8999..7d4bd1a 100644 --- a/pagure/templates/issue.html +++ b/pagure/templates/issue.html @@ -329,12 +329,17 @@ $(function() { }); +var source = null; if (!!window.EventSource) { - var source = new EventSource('{{ + source = new EventSource('{{ url_for("stream_issue", username=username, repo=repo.name, issueid=issueid) }}'); } +window.onbeforeunload = function() { + source.close() +}; + clean_entry= function(text, element){ var _data = $.trim(text).split(','); var _out = [] From 1cb412265dc6036c8c8784a62ba003c13c2f6f04 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Jun 18 2015 11:04:27 +0000 Subject: [PATCH 24/58] Only send a message on the redis queue if there is something to announce --- diff --git a/pagure/lib/__init__.py b/pagure/lib/__init__.py index dd0523b..fb2428a 100644 --- a/pagure/lib/__init__.py +++ b/pagure/lib/__init__.py @@ -1003,7 +1003,7 @@ def edit_issue(session, issue, ticketfolder, user, ) ) - if redis: + if redis and edit: redis.publish(issue.uid, json.dumps({ 'fields': edit, 'issue': issue.to_json(public=True, with_comments=False), From 4b921c554ab0695ca5195118031e08ccb5ed4b19 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Jun 18 2015 11:04:27 +0000 Subject: [PATCH 25/58] Add a global configuration boolean to activate or not redis/eventsource integration --- diff --git a/pagure/default_config.py b/pagure/default_config.py index c3e6ee3..220ce52 100644 --- a/pagure/default_config.py +++ b/pagure/default_config.py @@ -48,6 +48,7 @@ MAX_CONTENT_LENGTH = 4 * 1024 * 1024 # 4 megabytes IP_ALLOWED_INTERNAL = ['127.0.0.1', 'localhost', '::1'] # Redis configuration +REDIS_EVENTSOURCE = True REDIS_HOST = 'localhost' REDIS_PORT = 6379 REDIS_DB = 0 diff --git a/pagure/templates/issue.html b/pagure/templates/issue.html index 7d4bd1a..a674602 100644 --- a/pagure/templates/issue.html +++ b/pagure/templates/issue.html @@ -328,7 +328,7 @@ $(function() { }); }); - +{% if config['REDIS_EVENTSOURCE'] %} var source = null; if (!!window.EventSource) { source = new EventSource('{{ @@ -496,6 +496,7 @@ source.addEventListener('message', function(e) { } }, false); +{% endif %} diff --git a/pagure/ui/issues.py b/pagure/ui/issues.py index d9fb3bd..3ffc688 100644 --- a/pagure/ui/issues.py +++ b/pagure/ui/issues.py @@ -506,40 +506,41 @@ def view_issue(repo, issueid, username=None): ) -@APP.route('//issue//stream') -@APP.route('/fork///issue//stream') -def stream_issue(repo, issueid, username=None): - """ Streams the changes made to an issue live - """ +if APP.config['REDIS_EVENTSOURCE']: + @APP.route('//issue//stream') + @APP.route('/fork///issue//stream') + def stream_issue(repo, issueid, username=None): + """ Streams the changes made to an issue live + """ - repo = pagure.lib.get_project(SESSION, repo, user=username) + repo = pagure.lib.get_project(SESSION, repo, user=username) - if repo is None: - flask.abort(404, 'Project not found') + if repo is None: + flask.abort(404, 'Project not found') - if not repo.settings.get('issue_tracker', True): - flask.abort(404, 'No issue tracker found for this project') + if not repo.settings.get('issue_tracker', True): + flask.abort(404, 'No issue tracker found for this project') - issue = pagure.lib.search_issues(SESSION, repo, issueid=issueid) + issue = pagure.lib.search_issues(SESSION, repo, issueid=issueid) - if issue is None or issue.project != repo: - flask.abort(404, 'Issue not found') + if issue is None or issue.project != repo: + flask.abort(404, 'Issue not found') - if issue.private and not is_repo_admin(repo) \ - and (not authenticated() or - not issue.user.user == flask.g.fas_user.username): - flask.abort( - 403, 'This issue is private and you are not allowed to view it') + if issue.private and not is_repo_admin(repo) \ + and (not authenticated() or + not issue.user.user == flask.g.fas_user.username): + flask.abort( + 403, 'This issue is private and you are not allowed to view it') - pubsub = REDIS.pubsub() - pubsub.subscribe(issue.uid) - def event_stream(pubsub): - for message in pubsub.listen(): - yield 'data: %s\n\n' % message['data'] + pubsub = REDIS.pubsub() + pubsub.subscribe(issue.uid) + def event_stream(pubsub): + for message in pubsub.listen(): + yield 'data: %s\n\n' % message['data'] - return flask.Response( - event_stream(pubsub), - mimetype="text/event-stream") + return flask.Response( + event_stream(pubsub), + mimetype="text/event-stream") @APP.route('//issue//drop', methods=['POST']) From 70166726e1c270024fa390975fbfeb171c54ec97 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Jun 18 2015 11:04:27 +0000 Subject: [PATCH 26/58] Use a connection pool to connect to redis --- diff --git a/pagure/__init__.py b/pagure/__init__.py index 2174776..8c9d825 100644 --- a/pagure/__init__.py +++ b/pagure/__init__.py @@ -54,10 +54,11 @@ if 'PAGURE_CONFIG' in os.environ: FAS = FAS(APP) SESSION = pagure.lib.create_session(APP.config['DB_URL']) -REDIS = redis.StrictRedis( +POOL = redis.ConnectionPool( host=APP.config['REDIS_HOST'], port=APP.config['REDIS_PORT'], db=APP.config['REDIS_DB']) +REDIS = redis.StrictRedis(connection_pool=POOL) if not APP.debug: APP.logger.addHandler(pagure.mail_logging.get_mail_handler( From 1dd40f076a3c1d69c4f893d10f28d7d7ebd70c1b Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Jun 18 2015 11:04:27 +0000 Subject: [PATCH 27/58] Add the pagure streaming server This is a trollius (asyncio) application listening for messages sent to redis and streaming the change from redis to the sockets it's connected to. This is thus our eventsource server allowing to push changes to the UI without having to reload the whole page --- diff --git a/pagure-stream-server.py b/pagure-stream-server.py new file mode 100644 index 0000000..d513988 --- /dev/null +++ b/pagure-stream-server.py @@ -0,0 +1,171 @@ +#!/usr/bin/env python + +""" + (c) 2014-2015 - Copyright Red Hat Inc + + Authors: + Pierre-Yves Chibon + + +Streaming server for pagure's eventsource feature +This server takes messages sent to redis and publish them at the specified +endpoint + +To test, run this script and in another terminal +nc localhost 8080 + HELLO + + GET /test/issue/26?foo=bar HTTP/1.1 + +""" + +import datetime +import logging +import os +import urlparse + +import trollius +import trollius_redis + +log = logging.getLogger(__name__) + + +if 'PAGURE_CONFIG' not in os.environ \ + and os.path.exists('/etc/pagure/pagure.cfg'): + print 'Using configuration file `/etc/pagure/pagure.cfg`' + os.environ['PAGURE_CONFIG'] = '/etc/pagure/pagure.cfg' + + +import pagure +import pagure.lib + + +clients = {} + + +@trollius.coroutine +def handle_client(client_reader, client_writer): + # give client a chance to respond, timeout after 10 seconds + data = yield trollius.From(trollius.wait_for( + client_reader.readline(), + timeout=10.0)) + + if data is None: + log.warning("Expected ticket uid, received None") + return + + data = data.decode().rstrip().split() + log.info("Received %s", data) + if not data: + log.warning("No URL provided: %s" % data) + return + + if not '/' in data[1]: + log.warning("Invalid URL provided: %s" % data[1]) + return + + url = urlparse.urlsplit(data[1]) + + client_writer.write(( + "HTTP/1.0 200 OK\n" + "Content-Type: text/event-stream\n" + "Cache: nocache\n" + "Connection: keep-alive\n" + "Access-Control-Allow-Origin: *\n\n" + ).encode()) + + username = None + if url.path.startswith('/fork'): + username, repo, issue, issueid = url.path.split('/')[2:6] + else: + repo, issue, issueid = url.path.split('/')[1:4] + + repo = pagure.lib.get_project(pagure.SESSION, repo, user=username) + + if repo is None: + log.warning("Project '%s' not found" % repo) + return + + if not repo.settings.get('issue_tracker', True): + log.warning("No issue tracker found for this project") + return + + issue = pagure.lib.search_issues(pagure.SESSION, repo, issueid=issueid) + + if issue is None or issue.project != repo: + log.warning("Issue '%s' not found" % issueid) + return + + if issue.private: + # TODO: find a way to do auth + log.warning( + "This issue is private and you are not allowed to view it") + return + + try: + connection = yield trollius.From(trollius_redis.Connection.create( + host=pagure.APP.config['REDIS_HOST'], + port=pagure.APP.config['REDIS_PORT'], + db=pagure.APP.config['REDIS_DB'])) + + # Create subscriber. + subscriber = yield trollius.From(connection.start_subscribe()) + + # Subscribe to channel. + yield trollius.From(subscriber.subscribe([issue.uid])) + + # Inside a while loop, wait for incoming events. + while True: + reply = yield trollius.From(subscriber.next_published()) + #print(u'Received: ', repr(reply.value), u'on channel', reply.channel) + log.info(reply) + log.info("Sending %s", reply.value) + client_writer.write(('data: %s\n\n' % reply.value).encode()) + yield trollius.From(client_writer.drain()) + + except trollius.ConnectionResetError: + pass + finally: + # Wathever happens, close the connection. + connection.close() + client_writer.close() + + +def main(): + + try: + loop = trollius.get_event_loop() + coro = trollius.start_server( + handle_client, host=None, port=8080, loop=loop) + server = loop.run_until_complete(coro) + print('Serving on {}'.format(server.sockets[0].getsockname())) + loop.run_forever() + except KeyboardInterrupt: + pass + except trollius.ConnectionResetError: + pass + + # Close the server + server.close() + log.info("End Connection") + loop.run_until_complete(server.wait_closed()) + loop.close() + log.info("End") + + +if __name__ == '__main__': + log = logging.getLogger("") + formatter = logging.Formatter( + "%(asctime)s %(levelname)s [%(module)s:%(lineno)d] %(message)s") + + # setup console logging + log.setLevel(logging.DEBUG) + ch = logging.StreamHandler() + ch.setLevel(logging.DEBUG) + + aslog = logging.getLogger("asyncio") + aslog.setLevel(logging.DEBUG) + + ch.setFormatter(formatter) + log.addHandler(ch) + main() From 0e2fa4392ae1f89b065338a6202ff1f8da9a0a5d Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Jun 18 2015 11:04:27 +0000 Subject: [PATCH 28/58] Drop the stream_issue endpoint from the issue controller Since we now stream changes via the pagure-stream-server server, no need to keep this code in. --- diff --git a/pagure/ui/issues.py b/pagure/ui/issues.py index 3ffc688..fb060b9 100644 --- a/pagure/ui/issues.py +++ b/pagure/ui/issues.py @@ -506,43 +506,6 @@ def view_issue(repo, issueid, username=None): ) -if APP.config['REDIS_EVENTSOURCE']: - @APP.route('//issue//stream') - @APP.route('/fork///issue//stream') - def stream_issue(repo, issueid, username=None): - """ Streams the changes made to an issue live - """ - - repo = pagure.lib.get_project(SESSION, repo, user=username) - - if repo is None: - flask.abort(404, 'Project not found') - - if not repo.settings.get('issue_tracker', True): - flask.abort(404, 'No issue tracker found for this project') - - issue = pagure.lib.search_issues(SESSION, repo, issueid=issueid) - - if issue is None or issue.project != repo: - flask.abort(404, 'Issue not found') - - if issue.private and not is_repo_admin(repo) \ - and (not authenticated() or - not issue.user.user == flask.g.fas_user.username): - flask.abort( - 403, 'This issue is private and you are not allowed to view it') - - pubsub = REDIS.pubsub() - pubsub.subscribe(issue.uid) - def event_stream(pubsub): - for message in pubsub.listen(): - yield 'data: %s\n\n' % message['data'] - - return flask.Response( - event_stream(pubsub), - mimetype="text/event-stream") - - @APP.route('//issue//drop', methods=['POST']) @APP.route('/fork///issue//drop', methods=['POST']) From d5eb2de732c5c9f800f621d5025e7da07212542d Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Jun 18 2015 11:04:27 +0000 Subject: [PATCH 29/58] Replace the REDIS_HOST default location from localhost to 0.0.0.0 Don't ask me why but this pleases pagure-stream-server (ie trollius_redis) while localhost confuses it completely. --- diff --git a/pagure/default_config.py b/pagure/default_config.py index 220ce52..d89a6c5 100644 --- a/pagure/default_config.py +++ b/pagure/default_config.py @@ -49,7 +49,7 @@ IP_ALLOWED_INTERNAL = ['127.0.0.1', 'localhost', '::1'] # Redis configuration REDIS_EVENTSOURCE = True -REDIS_HOST = 'localhost' +REDIS_HOST = '0.0.0.0' REDIS_PORT = 6379 REDIS_DB = 0 From 8b17aa21534612ab9dfa955b87d02c4842bd1baa Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Jun 18 2015 11:04:27 +0000 Subject: [PATCH 30/58] Replace the REDIS_EVENTSOURCE boolean by a EVENTSOURCE_SOURCE defaulting to None This configuration key aims at providing the location where the streaming server is running and accessible to pagure (for example: https://pagure.io:8080). --- diff --git a/pagure/default_config.py b/pagure/default_config.py index d89a6c5..d426f68 100644 --- a/pagure/default_config.py +++ b/pagure/default_config.py @@ -48,7 +48,7 @@ MAX_CONTENT_LENGTH = 4 * 1024 * 1024 # 4 megabytes IP_ALLOWED_INTERNAL = ['127.0.0.1', 'localhost', '::1'] # Redis configuration -REDIS_EVENTSOURCE = True +EVENTSOURCE_SOURCE = None REDIS_HOST = '0.0.0.0' REDIS_PORT = 6379 REDIS_DB = 0 From 0e56d13de8cdaacac6d43c65de81c7aa7b27000b Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Jun 18 2015 11:04:27 +0000 Subject: [PATCH 31/58] Replace the REDIS_EVENTSOURCE by a EVENTSOURCE_SOURCE and use it as source EVENTSOURCE_SOURCE being the location where the streaming server is available --- diff --git a/pagure/templates/issue.html b/pagure/templates/issue.html index a674602..5b1588b 100644 --- a/pagure/templates/issue.html +++ b/pagure/templates/issue.html @@ -328,12 +328,10 @@ $(function() { }); }); -{% if config['REDIS_EVENTSOURCE'] %} +{% if config['EVENTSOURCE_SOURCE'] %} var source = null; if (!!window.EventSource) { - source = new EventSource('{{ - url_for("stream_issue", username=username, - repo=repo.name, issueid=issueid) }}'); + source = new EventSource('{{ config["EVENTSOURCE_SOURCE"] }}'); } window.onbeforeunload = function() { From c6552d0cb5b4a02d30da9ea8d2a026739f80b8fc Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Jun 18 2015 11:04:27 +0000 Subject: [PATCH 32/58] Move pagure-stream-server.py into an ev-server folder This ev-server (for eventsource server) will contain all the files related to the eventsource server --- diff --git a/ev-server/pagure-stream-server.py b/ev-server/pagure-stream-server.py new file mode 100644 index 0000000..d513988 --- /dev/null +++ b/ev-server/pagure-stream-server.py @@ -0,0 +1,171 @@ +#!/usr/bin/env python + +""" + (c) 2014-2015 - Copyright Red Hat Inc + + Authors: + Pierre-Yves Chibon + + +Streaming server for pagure's eventsource feature +This server takes messages sent to redis and publish them at the specified +endpoint + +To test, run this script and in another terminal +nc localhost 8080 + HELLO + + GET /test/issue/26?foo=bar HTTP/1.1 + +""" + +import datetime +import logging +import os +import urlparse + +import trollius +import trollius_redis + +log = logging.getLogger(__name__) + + +if 'PAGURE_CONFIG' not in os.environ \ + and os.path.exists('/etc/pagure/pagure.cfg'): + print 'Using configuration file `/etc/pagure/pagure.cfg`' + os.environ['PAGURE_CONFIG'] = '/etc/pagure/pagure.cfg' + + +import pagure +import pagure.lib + + +clients = {} + + +@trollius.coroutine +def handle_client(client_reader, client_writer): + # give client a chance to respond, timeout after 10 seconds + data = yield trollius.From(trollius.wait_for( + client_reader.readline(), + timeout=10.0)) + + if data is None: + log.warning("Expected ticket uid, received None") + return + + data = data.decode().rstrip().split() + log.info("Received %s", data) + if not data: + log.warning("No URL provided: %s" % data) + return + + if not '/' in data[1]: + log.warning("Invalid URL provided: %s" % data[1]) + return + + url = urlparse.urlsplit(data[1]) + + client_writer.write(( + "HTTP/1.0 200 OK\n" + "Content-Type: text/event-stream\n" + "Cache: nocache\n" + "Connection: keep-alive\n" + "Access-Control-Allow-Origin: *\n\n" + ).encode()) + + username = None + if url.path.startswith('/fork'): + username, repo, issue, issueid = url.path.split('/')[2:6] + else: + repo, issue, issueid = url.path.split('/')[1:4] + + repo = pagure.lib.get_project(pagure.SESSION, repo, user=username) + + if repo is None: + log.warning("Project '%s' not found" % repo) + return + + if not repo.settings.get('issue_tracker', True): + log.warning("No issue tracker found for this project") + return + + issue = pagure.lib.search_issues(pagure.SESSION, repo, issueid=issueid) + + if issue is None or issue.project != repo: + log.warning("Issue '%s' not found" % issueid) + return + + if issue.private: + # TODO: find a way to do auth + log.warning( + "This issue is private and you are not allowed to view it") + return + + try: + connection = yield trollius.From(trollius_redis.Connection.create( + host=pagure.APP.config['REDIS_HOST'], + port=pagure.APP.config['REDIS_PORT'], + db=pagure.APP.config['REDIS_DB'])) + + # Create subscriber. + subscriber = yield trollius.From(connection.start_subscribe()) + + # Subscribe to channel. + yield trollius.From(subscriber.subscribe([issue.uid])) + + # Inside a while loop, wait for incoming events. + while True: + reply = yield trollius.From(subscriber.next_published()) + #print(u'Received: ', repr(reply.value), u'on channel', reply.channel) + log.info(reply) + log.info("Sending %s", reply.value) + client_writer.write(('data: %s\n\n' % reply.value).encode()) + yield trollius.From(client_writer.drain()) + + except trollius.ConnectionResetError: + pass + finally: + # Wathever happens, close the connection. + connection.close() + client_writer.close() + + +def main(): + + try: + loop = trollius.get_event_loop() + coro = trollius.start_server( + handle_client, host=None, port=8080, loop=loop) + server = loop.run_until_complete(coro) + print('Serving on {}'.format(server.sockets[0].getsockname())) + loop.run_forever() + except KeyboardInterrupt: + pass + except trollius.ConnectionResetError: + pass + + # Close the server + server.close() + log.info("End Connection") + loop.run_until_complete(server.wait_closed()) + loop.close() + log.info("End") + + +if __name__ == '__main__': + log = logging.getLogger("") + formatter = logging.Formatter( + "%(asctime)s %(levelname)s [%(module)s:%(lineno)d] %(message)s") + + # setup console logging + log.setLevel(logging.DEBUG) + ch = logging.StreamHandler() + ch.setLevel(logging.DEBUG) + + aslog = logging.getLogger("asyncio") + aslog.setLevel(logging.DEBUG) + + ch.setFormatter(formatter) + log.addHandler(ch) + main() diff --git a/pagure-stream-server.py b/pagure-stream-server.py deleted file mode 100644 index d513988..0000000 --- a/pagure-stream-server.py +++ /dev/null @@ -1,171 +0,0 @@ -#!/usr/bin/env python - -""" - (c) 2014-2015 - Copyright Red Hat Inc - - Authors: - Pierre-Yves Chibon - - -Streaming server for pagure's eventsource feature -This server takes messages sent to redis and publish them at the specified -endpoint - -To test, run this script and in another terminal -nc localhost 8080 - HELLO - - GET /test/issue/26?foo=bar HTTP/1.1 - -""" - -import datetime -import logging -import os -import urlparse - -import trollius -import trollius_redis - -log = logging.getLogger(__name__) - - -if 'PAGURE_CONFIG' not in os.environ \ - and os.path.exists('/etc/pagure/pagure.cfg'): - print 'Using configuration file `/etc/pagure/pagure.cfg`' - os.environ['PAGURE_CONFIG'] = '/etc/pagure/pagure.cfg' - - -import pagure -import pagure.lib - - -clients = {} - - -@trollius.coroutine -def handle_client(client_reader, client_writer): - # give client a chance to respond, timeout after 10 seconds - data = yield trollius.From(trollius.wait_for( - client_reader.readline(), - timeout=10.0)) - - if data is None: - log.warning("Expected ticket uid, received None") - return - - data = data.decode().rstrip().split() - log.info("Received %s", data) - if not data: - log.warning("No URL provided: %s" % data) - return - - if not '/' in data[1]: - log.warning("Invalid URL provided: %s" % data[1]) - return - - url = urlparse.urlsplit(data[1]) - - client_writer.write(( - "HTTP/1.0 200 OK\n" - "Content-Type: text/event-stream\n" - "Cache: nocache\n" - "Connection: keep-alive\n" - "Access-Control-Allow-Origin: *\n\n" - ).encode()) - - username = None - if url.path.startswith('/fork'): - username, repo, issue, issueid = url.path.split('/')[2:6] - else: - repo, issue, issueid = url.path.split('/')[1:4] - - repo = pagure.lib.get_project(pagure.SESSION, repo, user=username) - - if repo is None: - log.warning("Project '%s' not found" % repo) - return - - if not repo.settings.get('issue_tracker', True): - log.warning("No issue tracker found for this project") - return - - issue = pagure.lib.search_issues(pagure.SESSION, repo, issueid=issueid) - - if issue is None or issue.project != repo: - log.warning("Issue '%s' not found" % issueid) - return - - if issue.private: - # TODO: find a way to do auth - log.warning( - "This issue is private and you are not allowed to view it") - return - - try: - connection = yield trollius.From(trollius_redis.Connection.create( - host=pagure.APP.config['REDIS_HOST'], - port=pagure.APP.config['REDIS_PORT'], - db=pagure.APP.config['REDIS_DB'])) - - # Create subscriber. - subscriber = yield trollius.From(connection.start_subscribe()) - - # Subscribe to channel. - yield trollius.From(subscriber.subscribe([issue.uid])) - - # Inside a while loop, wait for incoming events. - while True: - reply = yield trollius.From(subscriber.next_published()) - #print(u'Received: ', repr(reply.value), u'on channel', reply.channel) - log.info(reply) - log.info("Sending %s", reply.value) - client_writer.write(('data: %s\n\n' % reply.value).encode()) - yield trollius.From(client_writer.drain()) - - except trollius.ConnectionResetError: - pass - finally: - # Wathever happens, close the connection. - connection.close() - client_writer.close() - - -def main(): - - try: - loop = trollius.get_event_loop() - coro = trollius.start_server( - handle_client, host=None, port=8080, loop=loop) - server = loop.run_until_complete(coro) - print('Serving on {}'.format(server.sockets[0].getsockname())) - loop.run_forever() - except KeyboardInterrupt: - pass - except trollius.ConnectionResetError: - pass - - # Close the server - server.close() - log.info("End Connection") - loop.run_until_complete(server.wait_closed()) - loop.close() - log.info("End") - - -if __name__ == '__main__': - log = logging.getLogger("") - formatter = logging.Formatter( - "%(asctime)s %(levelname)s [%(module)s:%(lineno)d] %(message)s") - - # setup console logging - log.setLevel(logging.DEBUG) - ch = logging.StreamHandler() - ch.setLevel(logging.DEBUG) - - aslog = logging.getLogger("asyncio") - aslog.setLevel(logging.DEBUG) - - ch.setFormatter(formatter) - log.addHandler(ch) - main() From cb5ecf28c231763255bad19095907806da1b8cf2 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Jun 18 2015 11:04:27 +0000 Subject: [PATCH 33/58] Add a systemd init file for the pagure-ev service --- diff --git a/ev-server/pagure_ev.service b/ev-server/pagure_ev.service new file mode 100644 index 0000000..fc98702 --- /dev/null +++ b/ev-server/pagure_ev.service @@ -0,0 +1,15 @@ +[Unit] +Description=Pagure EventSource server (Allowing live refresh of the pages +supporting it) +After=network.target +Documentation=https://pagure.io/pagure + +[Service] +ExecStart=/usr/bin/python2 /usr/share/pagure/pagure-stream-server.py +Type=simple +User=git +Group=git +Restart=on-failure + +[Install] +WantedBy=multi-user.target From 71973610866310bde93815cb6db78e6b4a27628d Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Jun 18 2015 11:04:27 +0000 Subject: [PATCH 34/58] Fix the location of the pagure-stream-server file --- diff --git a/ev-server/pagure_ev.service b/ev-server/pagure_ev.service index fc98702..fff3081 100644 --- a/ev-server/pagure_ev.service +++ b/ev-server/pagure_ev.service @@ -5,7 +5,7 @@ After=network.target Documentation=https://pagure.io/pagure [Service] -ExecStart=/usr/bin/python2 /usr/share/pagure/pagure-stream-server.py +ExecStart=/usr/bin/python2 /usr/share/pagure-ev/pagure-stream-server.py Type=simple User=git Group=git From 70c695f3a7c9a29ca96202cf3451099005a4d848 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Jun 18 2015 11:04:27 +0000 Subject: [PATCH 35/58] Add a new pagure subpackage: pagure-ev package for the eventsource server --- diff --git a/files/pagure.spec b/files/pagure.spec index 23469d2..678c7b8 100644 --- a/files/pagure.spec +++ b/files/pagure.spec @@ -71,7 +71,6 @@ Requires: mod_wsgi # No dependency of the app per se, but required to make it working. Requires: gitolite3 - %description Pagure is a light-weight git-centered forge based on pygit2. @@ -98,6 +97,21 @@ Milters (Mail filters) allowing the integration of pagure and emails. This is useful for example to allow commenting on a ticket by email. +%package ev +Summary: EventSource server for pagure +BuildArch: noarch + +Requires: python-redis +Requires: python-trollius +Requires: python-trollius-redis +Requires(post): systemd +Requires(preun): systemd +Requires(postun): systemd +%description ev +Pagure comes with an eventsource server allowing live update of the pages +supporting it. This packages provides it. + + %prep %setup -q @@ -142,15 +156,28 @@ install -m 644 milters/pagure_milter.service \ install -m 644 milters/comment_email_milter.py \ $RPM_BUILD_ROOT/%{_datadir}/pagure/comment_email_milter.py +# Install the eventsource +mkdir -p $RPM_BUILD_ROOT/%{_datadir}/pagure-ev +install -m 644 ev-server/pagure-stream-server.py \ + $RPM_BUILD_ROOT/%{_datadir}/pagure-ev/pagure-stream-server.py +install -m 644 ev-server/pagure_ev.service \ + $RPM_BUILD_ROOT/%{_unitdir}/pagure_ev.service + %post milters %systemd_post pagure_milter.service +%post ev +%systemd_post pagure_milter.service %preun milters %systemd_preun pagure_milter.service +%preun ev +%systemd_preun pagure_milter.service %postun milters %systemd_postun_with_restart pagure_milter.service +%postun ev +%systemd_postun_with_restart pagure_milter.service %files @@ -176,6 +203,12 @@ install -m 644 milters/comment_email_milter.py \ %{_datadir}/pagure/comment_email_milter.py* +%files +%license LICENSE +%{_datadir}/pagure-ev/ +%{_unitdir}/pagure_milter.service + + %changelog * Tue Jun 16 2015 Pierre-Yves Chibon - 0.1.15-1 - Update 0.1.15 From ad454d35e28f0c4b197757673d7470cdfda6252f Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Jun 18 2015 11:04:27 +0000 Subject: [PATCH 36/58] Add python-trollius-redis to the list of requirements (for the eventsource server) --- diff --git a/requirements.txt b/requirements.txt index b0801d2..f69f5a8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,6 +21,7 @@ python-openid python-openid-cla python-openid-teams python-redis +python-trollius-redis six sqlalchemy >= 0.8 straight.plugin==1.4.0-post-1 From 54093822b5c7955be543438d97dae35dab73f97e Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Jun 18 2015 11:04:27 +0000 Subject: [PATCH 37/58] Adjust the name of the python library redis-py and trollius-redis --- diff --git a/requirements.txt b/requirements.txt index f69f5a8..e7503a0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,9 +20,9 @@ python-fedora python-openid python-openid-cla python-openid-teams -python-redis -python-trollius-redis +redis-py six sqlalchemy >= 0.8 straight.plugin==1.4.0-post-1 +trollius-redis wtforms From be6d5878843d089ccb486b5270ee8724007d2a82 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Jun 18 2015 11:04:27 +0000 Subject: [PATCH 38/58] Turns out it's just redis --- diff --git a/requirements.txt b/requirements.txt index e7503a0..82074eb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,7 +20,7 @@ python-fedora python-openid python-openid-cla python-openid-teams -redis-py +redis six sqlalchemy >= 0.8 straight.plugin==1.4.0-post-1 From e93b76306aa05bfd78e88828400934ad438d9a2e Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Jun 18 2015 11:04:27 +0000 Subject: [PATCH 39/58] Only instantiate the connection with redis if there is an eventsource at the end --- diff --git a/pagure/__init__.py b/pagure/__init__.py index 8c9d825..16ed98c 100644 --- a/pagure/__init__.py +++ b/pagure/__init__.py @@ -54,11 +54,13 @@ if 'PAGURE_CONFIG' in os.environ: FAS = FAS(APP) SESSION = pagure.lib.create_session(APP.config['DB_URL']) -POOL = redis.ConnectionPool( - host=APP.config['REDIS_HOST'], - port=APP.config['REDIS_PORT'], - db=APP.config['REDIS_DB']) -REDIS = redis.StrictRedis(connection_pool=POOL) +REDIS=None +if APP.config['EVENTSOURCE_SOURCE']: + POOL = redis.ConnectionPool( + host=APP.config['REDIS_HOST'], + port=APP.config['REDIS_PORT'], + db=APP.config['REDIS_DB']) + REDIS = redis.StrictRedis(connection_pool=POOL) if not APP.debug: APP.logger.addHandler(pagure.mail_logging.get_mail_handler( From f35cc13512fd6244943afdad1041ac6065f7f2e3 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Jun 18 2015 11:04:27 +0000 Subject: [PATCH 40/58] Adjust the unit-tests for the change in the UI --- diff --git a/tests/test_progit_flask_ui_issues.py b/tests/test_progit_flask_ui_issues.py index a34c61c..ccd94a4 100644 --- a/tests/test_progit_flask_ui_issues.py +++ b/tests/test_progit_flask_ui_issues.py @@ -1122,8 +1122,11 @@ class PagureFlaskIssuestests(tests.Modeltests): '/test/issue/1/edit', data=data, follow_redirects=True) self.assertEqual(output.status_code, 200) self.assertTrue( - '#1 Test issue #1' + '
  • Edited successfully issue #1
    • ' in output.data) + self.assertTrue( + '#1 ' + 'Test issue #1' in output.data) self.assertEqual(output.data.count( ''), 1) self.assertEqual(output.data.count( From 1f61289c95c9eb7e184210f88eb1c3c067285ee3 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Jun 18 2015 11:04:27 +0000 Subject: [PATCH 41/58] Refactor the JS to move most of the logic into its own file This also splits the logic over multiple functions to make the code easier to read --- diff --git a/pagure/static/issue_ev.js b/pagure/static/issue_ev.js new file mode 100644 index 0000000..756d7b0 --- /dev/null +++ b/pagure/static/issue_ev.js @@ -0,0 +1,179 @@ +clean_entry= function(text, element) { + var _out = [] + var _data = $.trim(text).split(','); + var y=0; + for (var j=0; i<_data.length; i++){ + if ($.trim(_data[j]) == element) { + continue; + } + _out[y] = $.trim(_data[j]); + y+=1; + } + return _out; +} + +add_tags = function(data, _issues_url) { + console.log('adding ' + data.added_tags); + var field = $('#taglist'); + var field2 = $('#tag'); + var _data = field.html(); + var _curval = field2.val(); + + for (i=0; i' + tag + '    '; + + if (_curval) { + _curval += ','; + } + _curval += tag; + } + + field.html(_data); + field2.val(_curval); +} + +remove_tags = function(data, _issues_url) { + console.log('removing ' + data.removed_tags); + var field = $('#taglist'); + var field2 = $('#tag'); + var _data = field.html(); + var _data2 = field2.val(); + for (var i=0; i' + tag + ''; + _data = clean_entry(_data, _turl).join(); + _data2 = clean_entry(_data2, tag).join(); + } + field.html(_data); + field2.val(_data2); +} + +assigne_issue = function(data, _issues_url) { + console.log('assigning ' + data.assigned); + var field = $('#assigneduser'); + var _url = _issues_url + '?assignee=' + data.assigned.name + '">' + data.assigned.name + ''; + field.html(_url); + field = $('#assignee'); + field.val(data.assigned.name); +} + +unassigne_issue = function(data) { + console.log('un-assigning '); + var field = $('#assigneduser'); + field.html(' '); + field = $('#assignee'); + field.val(''); +} + +add_deps = function(data, issue_uid, _issue_url) { + console.log('adding ' + data.added_dependency); + if (data.issue_uid == issue_uid){ + if (data.type == "children"){ + var field = $('#blockers'); + var field2 = $('#blocks'); + } else { + var field = $('#dependencies'); + var field2 = $('#depends'); + } + } + var dep = data.added_dependency; + var _data = $.trim(field.html()); + var _url = _issue_url.replace('/-1', '/' + dep) + dep + ''; + _data += ',' + _url; + field.html(_data); + var _curval = field2.val(); + if (_curval && _curval != ',') { + _curval += ','; + } + field2.val(_curval + dep); +} + +remove_deps = function(data, issue_uid, _issue_url) { + console.log('Removing ' + data.removed_dependency); + if (data.issue_uid == issue_uid){ + if (data.type == "children"){ + var field = $('#dependencies'); + var field2 = $('#depends'); + } else { + var field = $('#blockers'); + var field2 = $('#blocks'); + } + } + var dep = data.removed_dependency; + // Set links + var _data = $.trim(field.html()).split(','); + var _url = _issue_url.replace('/-1', '/' + dep) + dep + ''; + field.html(clean_entry(field.html(), _url).join()); + // Set the value in the input field + field2.val(clean_entry(field2.val(), dep).join()); +} + +add_comment = function(data) { + console.log('Adding comment ' + data.comment_added); + var field = $('#comments'); + var _data = '
    \ +
    \ + \ + \ + ' + data.comment_user + '\ + - seconds ago \ + \ + \ +
    \ +
    \ +

    ' + data.comment_added + '

    \ +
    \ +
    '; + field.html(field.html() + _data); +} + +update_issue = function(data) { + console.log('Adjusting issue ' + data.fields); + for (i=0; i' + data.issue.content + '

    '); + } + } +} + +process_event = function(data, issue_uid, _issue_url, _issues_url){ + console.log(data); + if (data.added_tags){ + add_tags(data, _issues_url); + } + else if (data.removed_tags){ + remove_tags(data, _issues_url); + } + else if (data.assigned){ + assigne_issue(data, _issues_url); + } + else if (data.unassigned){ + unassigne_issue(data); + } + else if (data.added_dependency){ + add_deps(data, issue_uid, _issue_url); + } + else if (data.removed_dependency){ + remove_deps(data, issue_uid, _issue_url); + } + else if (data.comment_added){ + add_comment(data); + } + else if (data.fields){ + update_issue(data); + } +} diff --git a/pagure/templates/issue.html b/pagure/templates/issue.html index 5b1588b..9a94959 100644 --- a/pagure/templates/issue.html +++ b/pagure/templates/issue.html @@ -328,175 +328,35 @@ $(function() { }); }); + + {% if config['EVENTSOURCE_SOURCE'] %} + + + +{% endif %} {% endblock %} From 0fd6e06823b3bff676f7ee05310042320427553e Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Jun 18 2015 11:04:27 +0000 Subject: [PATCH 42/58] Fix installing the pagure_ev systemd unit in the -ev subpackage --- diff --git a/files/pagure.spec b/files/pagure.spec index 678c7b8..fc32b74 100644 --- a/files/pagure.spec +++ b/files/pagure.spec @@ -167,17 +167,17 @@ install -m 644 ev-server/pagure_ev.service \ %post milters %systemd_post pagure_milter.service %post ev -%systemd_post pagure_milter.service +%systemd_post pagure_ev.service %preun milters %systemd_preun pagure_milter.service %preun ev -%systemd_preun pagure_milter.service +%systemd_preun pagure_ev.service %postun milters %systemd_postun_with_restart pagure_milter.service %postun ev -%systemd_postun_with_restart pagure_milter.service +%systemd_postun_with_restart pagure_ev.service %files @@ -206,7 +206,7 @@ install -m 644 ev-server/pagure_ev.service \ %files %license LICENSE %{_datadir}/pagure-ev/ -%{_unitdir}/pagure_milter.service +%{_unitdir}/pagure_ev.service %changelog From 897587df352645df4b95adadfa8aeecf1c61cb1f Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Jun 18 2015 11:04:27 +0000 Subject: [PATCH 43/58] Adjust the update_issue endpoint to return ok if we updated via javascript If the POST url contains a ?js=1 and all goes fine, the method will return a simple 'Ok', otherwise it will return some html which means something went wrong. On the JS side, we can then check if the update did go through or not. --- diff --git a/pagure/ui/issues.py b/pagure/ui/issues.py index fb060b9..6030cf7 100644 --- a/pagure/ui/issues.py +++ b/pagure/ui/issues.py @@ -38,6 +38,8 @@ from pagure import (APP, SESSION, REDIS, LOG, __get_file_in_tree, @cla_required def update_issue(repo, issueid, username=None): ''' Add a comment to an issue. ''' + is_js = flask.request.args.get('js', False) + repo = pagure.lib.get_project(SESSION, repo, user=username) if flask.request.method == 'GET': @@ -89,6 +91,7 @@ def update_issue(repo, issueid, username=None): SESSION.commit() flask.flash('Comment removed') except SQLAlchemyError, err: # pragma: no cover + is_js = False SESSION.rollback() LOG.error(err) flask.flash( @@ -193,15 +196,20 @@ def update_issue(repo, issueid, username=None): flask.flash(message) except pagure.exceptions.PagureException, err: + is_js = False SESSION.rollback() flask.flash(err.message, 'error') except SQLAlchemyError, err: # pragma: no cover + is_js = False SESSION.rollback() APP.logger.exception(err) flask.flash(str(err), 'error') - return flask.redirect(flask.url_for( - 'view_issue', username=username, repo=repo.name, issueid=issueid)) + if is_js: + return 'ok' + else: + return flask.redirect(flask.url_for( + 'view_issue', username=username, repo=repo.name, issueid=issueid)) @APP.route('//tag//edit/', methods=('GET', 'POST')) From 476f21b4684e5eeefb0c4b31cfc160c75261d451 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Jun 18 2015 11:04:27 +0000 Subject: [PATCH 44/58] If the user is logged in and there is an eventsource server, do async update This basically allows to make changes (updates) on the ticket without having to reload the whole page letting the eventsource server do the UI refresh. --- diff --git a/pagure/templates/issue.html b/pagure/templates/issue.html index 9a94959..9f0e55c 100644 --- a/pagure/templates/issue.html +++ b/pagure/templates/issue.html @@ -30,6 +30,7 @@ {% if authenticated and form %}
    {{ form.csrf_token }} {% endif %} @@ -205,8 +206,6 @@ $(document).ready(function() { $("#file-picker").on("change", function() { doUpload("{{ form.csrf_token.current_token }}", this.files); }); - - }); {% endif %} @@ -355,6 +354,29 @@ source.addEventListener('message', function(e) { process_event(data, "{{ issue.uid }}", _issue_url, _issues_url); }, false); + +{% if authenticated and form %} +function try_async_comment(form) { + $.post( form.action + "?js=1", $(form).serialize() ) + .done(function(data) { + if(data == 'ok') { + {# The event-source server will automatically refresh the UI #} + $('#comment').val(''); + } else { + // Make the browser submit the form sync + $(form).off('submit'); + form.submit(); + } + }) + .fail(function() { + // Make the browser submit the form sync + $(form).off('submit'); + form.submit(); + }) + return false; +}; +{% endif %} + {% endif %} From 4df770a04ef9d91471279a41d9ba4407f278627a Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Jun 18 2015 11:04:28 +0000 Subject: [PATCH 45/58] Do not flash messages if the update was made via javascript --- diff --git a/pagure/ui/issues.py b/pagure/ui/issues.py index 6030cf7..29de606 100644 --- a/pagure/ui/issues.py +++ b/pagure/ui/issues.py @@ -43,7 +43,8 @@ def update_issue(repo, issueid, username=None): repo = pagure.lib.get_project(SESSION, repo, user=username) if flask.request.method == 'GET': - flask.flash('Invalid method: GET', 'error') + if not is_js: + flask.flash('Invalid method: GET', 'error') return flask.redirect(flask.url_for( 'view_issue', username=username, repo=repo.name, issueid=issueid)) @@ -89,13 +90,16 @@ def update_issue(repo, issueid, username=None): SESSION.delete(comment) try: SESSION.commit() - flask.flash('Comment removed') + if not is_js: + flask.flash('Comment removed') except SQLAlchemyError, err: # pragma: no cover is_js = False SESSION.rollback() LOG.error(err) - flask.flash( - 'Could not remove the comment: %s' % commentid, 'error') + if not is_js: + flask.flash( + 'Could not remove the comment: %s' % commentid, + 'error') comment = form.comment.data depends = [] @@ -134,7 +138,7 @@ def update_issue(repo, issueid, username=None): redis=REDIS, ) SESSION.commit() - if message: + if message and not is_js: flask.flash(message) if repo_admin: @@ -144,8 +148,9 @@ def update_issue(repo, issueid, username=None): username=flask.g.fas_user.username, ticketfolder=APP.config['TICKETS_FOLDER'], redis=REDIS) - for message in messages: - flask.flash(message) + if not is_js: + for message in messages: + flask.flash(message) # Assign or update assignee of the ticket message = pagure.lib.add_issue_assignee( @@ -156,7 +161,7 @@ def update_issue(repo, issueid, username=None): ticketfolder=APP.config['TICKETS_FOLDER'], redis=REDIS, ) - if message: + if message and not is_js: SESSION.commit() flask.flash(message) @@ -182,8 +187,9 @@ def update_issue(repo, issueid, username=None): ticketfolder=APP.config['TICKETS_FOLDER'], redis=REDIS, ) - for message in messages: - flask.flash(message) + if not is_js: + for message in messages: + flask.flash(message) # Update ticket(s) depending on this one messages = pagure.lib.update_blocked_issue( @@ -192,18 +198,21 @@ def update_issue(repo, issueid, username=None): ticketfolder=APP.config['TICKETS_FOLDER'], redis=REDIS, ) - for message in messages: - flask.flash(message) + if not is_js: + for message in messages: + flask.flash(message) except pagure.exceptions.PagureException, err: is_js = False SESSION.rollback() - flask.flash(err.message, 'error') + if not is_js: + flask.flash(err.message, 'error') except SQLAlchemyError, err: # pragma: no cover is_js = False SESSION.rollback() APP.logger.exception(err) - flask.flash(str(err), 'error') + if not is_js: + flask.flash(str(err), 'error') if is_js: return 'ok' From 24c556683c1ee7871289cded0adcc02f8950c9cc Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Jun 18 2015 11:04:28 +0000 Subject: [PATCH 46/58] Fix the JS logic in the issue template to allow deleting a comment --- diff --git a/pagure/templates/issue.html b/pagure/templates/issue.html index 9f0e55c..831ef15 100644 --- a/pagure/templates/issue.html +++ b/pagure/templates/issue.html @@ -357,7 +357,14 @@ source.addEventListener('message', function(e) { {% if authenticated and form %} function try_async_comment(form) { - $.post( form.action + "?js=1", $(form).serialize() ) + var _url = form.action + "?js=1"; + var _data = $(form).serialize(); + var btn = $(document.activeElement); + if (btn[0].value){ + _data += '&drop_comment=' + btn[0].value; + return true; + } + $.post( _url, _data ) .done(function(data) { if(data == 'ok') { {# The event-source server will automatically refresh the UI #} From 0d7c618bbf694cce1a5ffd6483a10d100db6f3d5 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Jun 18 2015 11:07:09 +0000 Subject: [PATCH 47/58] This file did not exist in 2014 --- diff --git a/ev-server/pagure-stream-server.py b/ev-server/pagure-stream-server.py index d513988..490cd55 100644 --- a/ev-server/pagure-stream-server.py +++ b/ev-server/pagure-stream-server.py @@ -1,7 +1,7 @@ #!/usr/bin/env python """ - (c) 2014-2015 - Copyright Red Hat Inc + (c) 2015 - Copyright Red Hat Inc Authors: Pierre-Yves Chibon From 509340a8f7582f023ca9accf9fba3c2da30b465e Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Jun 18 2015 11:34:20 +0000 Subject: [PATCH 48/58] Adjust the example configuration file to include the eventsource/redis options --- diff --git a/files/pagure.cfg.sample b/files/pagure.cfg.sample index d610db1..e61f2fb 100644 --- a/files/pagure.cfg.sample +++ b/files/pagure.cfg.sample @@ -117,6 +117,16 @@ BLACKLISTED_PROJECTS = ['static', 'pv'] ### the IP filter IP_ALLOWED_INTERNAL = ['127.0.0.1', 'localhost', '::1'] +### EventSource/Redis configuration +# The eventsource integration is what allows pagure to refresh the content +# on your page when someone else comments on the ticket (and this without +# asking you to reload the page. +# By default it is off, ie: EVENTSOURCE_SOURCE is None +EVENTSOURCE_SOURCE = None +REDIS_HOST = '0.0.0.0' +REDIS_PORT = 6379 +REDIS_DB = 0 + # Authentication related configuration option From 792267d416a3dfcaa2ee576fc315c930ec131e9f Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Jun 18 2015 11:40:03 +0000 Subject: [PATCH 49/58] Start the pagure_ev service after redis started --- diff --git a/ev-server/pagure_ev.service b/ev-server/pagure_ev.service index fff3081..c507457 100644 --- a/ev-server/pagure_ev.service +++ b/ev-server/pagure_ev.service @@ -1,7 +1,7 @@ [Unit] Description=Pagure EventSource server (Allowing live refresh of the pages supporting it) -After=network.target +After=redis.target Documentation=https://pagure.io/pagure [Service] From 05f4542a95f359daf3b7ad060c53d0d66d4ca107 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Jun 18 2015 11:46:02 +0000 Subject: [PATCH 50/58] Move pagure-stream-server into /usr/libexect/pagure-ev and make it executable --- diff --git a/ev-server/pagure_ev.service b/ev-server/pagure_ev.service index c507457..b202a0e 100644 --- a/ev-server/pagure_ev.service +++ b/ev-server/pagure_ev.service @@ -5,7 +5,7 @@ After=redis.target Documentation=https://pagure.io/pagure [Service] -ExecStart=/usr/bin/python2 /usr/share/pagure-ev/pagure-stream-server.py +ExecStart=/usr/libexec/pagure-ev/pagure-stream-server.py Type=simple User=git Group=git diff --git a/files/pagure.spec b/files/pagure.spec index fc32b74..b118060 100644 --- a/files/pagure.spec +++ b/files/pagure.spec @@ -157,9 +157,9 @@ install -m 644 milters/comment_email_milter.py \ $RPM_BUILD_ROOT/%{_datadir}/pagure/comment_email_milter.py # Install the eventsource -mkdir -p $RPM_BUILD_ROOT/%{_datadir}/pagure-ev -install -m 644 ev-server/pagure-stream-server.py \ - $RPM_BUILD_ROOT/%{_datadir}/pagure-ev/pagure-stream-server.py +mkdir -p $RPM_BUILD_ROOT/%{_libexecdir}/pagure-ev +install -m 755 ev-server/pagure-stream-server.py \ + $RPM_BUILD_ROOT/%{_libexecdir}/pagure-ev/pagure-stream-server.py install -m 644 ev-server/pagure_ev.service \ $RPM_BUILD_ROOT/%{_unitdir}/pagure_ev.service @@ -205,7 +205,7 @@ install -m 644 ev-server/pagure_ev.service \ %files %license LICENSE -%{_datadir}/pagure-ev/ +%{_libexecdir}/pagure-ev/ %{_unitdir}/pagure_ev.service From 3916b5dc36f799671bba9c502dbcccbaa6e214af Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Jun 18 2015 11:49:49 +0000 Subject: [PATCH 51/58] Move cleaning a text via bleach in the backend --- diff --git a/pagure/lib/__init__.py b/pagure/lib/__init__.py index fb2428a..fbe26e6 100644 --- a/pagure/lib/__init__.py +++ b/pagure/lib/__init__.py @@ -20,6 +20,7 @@ import shutil import tempfile import uuid +import bleach import sqlalchemy import sqlalchemy.schema from datetime import timedelta @@ -2261,3 +2262,18 @@ def text2markdown(text): return markdown.markdown('\n'.join(ntext)) return '' + +def clean_input(text): + """ For a given html text, escape everything we do not want to support + to avoid potential security breach. + """ + attrs = bleach.ALLOWED_ATTRIBUTES + attrs['img'] = filter_img_src + return bleach.clean( + text, + tags=bleach.ALLOWED_TAGS + [ + 'p', 'br', 'div', 'h1', 'h2', 'h3', 'table', 'td', 'tr', 'th', + 'col', 'tbody', 'pre', 'img', 'hr', + ], + attributes=attrs, + ) diff --git a/pagure/ui/filters.py b/pagure/ui/filters.py index 703dc46..53c9024 100644 --- a/pagure/ui/filters.py +++ b/pagure/ui/filters.py @@ -13,7 +13,6 @@ import textwrap import urlparse import arrow -import bleach import flask from pygments import highlight @@ -304,16 +303,7 @@ def no_js(content): """ Template filter replacing by From f1e2a92a52dc96ce0cf4253605cc60c577311e2a Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Jun 18 2015 11:59:01 +0000 Subject: [PATCH 55/58] Move around required import --- diff --git a/pagure/lib/__init__.py b/pagure/lib/__init__.py index aab1bd7..e0c2871 100644 --- a/pagure/lib/__init__.py +++ b/pagure/lib/__init__.py @@ -18,6 +18,7 @@ import markdown import os import shutil import tempfile +import urlparse import uuid import bleach diff --git a/pagure/ui/filters.py b/pagure/ui/filters.py index 9b096d5..1c256d6 100644 --- a/pagure/ui/filters.py +++ b/pagure/ui/filters.py @@ -10,7 +10,6 @@ import datetime import textwrap -import urlparse import arrow import flask From dbb6f869040053632b50033bc1673f7ed0600c1a Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Jun 18 2015 11:59:09 +0000 Subject: [PATCH 56/58] Another unit-test needing a fix --- diff --git a/tests/test_progit_lib.py b/tests/test_progit_lib.py index 17c8438..5242aed 100644 --- a/tests/test_progit_lib.py +++ b/tests/test_progit_lib.py @@ -253,7 +253,7 @@ class PagureLibtests(tests.Modeltests): private=True ) self.session.commit() - self.assertEqual(msg, 'Edited successfully issue #2') + self.assertEqual(msg, 'Successfully edited issue #2') @patch('pagure.lib.git.update_git') @patch('pagure.lib.notify.send_email') From 4fb0af1b71bc3c51bbed9763b8159e81c6611dbe Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Jun 18 2015 12:11:32 +0000 Subject: [PATCH 57/58] Specify what EVENTSOURCE_SOURCE could be --- diff --git a/files/pagure.cfg.sample b/files/pagure.cfg.sample index e61f2fb..161159d 100644 --- a/files/pagure.cfg.sample +++ b/files/pagure.cfg.sample @@ -121,7 +121,10 @@ IP_ALLOWED_INTERNAL = ['127.0.0.1', 'localhost', '::1'] # The eventsource integration is what allows pagure to refresh the content # on your page when someone else comments on the ticket (and this without # asking you to reload the page. -# By default it is off, ie: EVENTSOURCE_SOURCE is None +# By default it is off, ie: EVENTSOURCE_SOURCE is None, to turn it on, specify +# here what the URL of the eventsource server is, for example: +# https://ev.pagure.io or https://pagure.io:8080 or whatever you are using +# (Note: the urls send to it start with a '/' so no need to add one yourself) EVENTSOURCE_SOURCE = None REDIS_HOST = '0.0.0.0' REDIS_PORT = 6379 From 4c012f086669c3dcda4594e146f36baa6667b37b Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Jun 18 2015 12:12:58 +0000 Subject: [PATCH 58/58] URLs are sent --- diff --git a/files/pagure.cfg.sample b/files/pagure.cfg.sample index 161159d..ac2045a 100644 --- a/files/pagure.cfg.sample +++ b/files/pagure.cfg.sample @@ -124,7 +124,7 @@ IP_ALLOWED_INTERNAL = ['127.0.0.1', 'localhost', '::1'] # By default it is off, ie: EVENTSOURCE_SOURCE is None, to turn it on, specify # here what the URL of the eventsource server is, for example: # https://ev.pagure.io or https://pagure.io:8080 or whatever you are using -# (Note: the urls send to it start with a '/' so no need to add one yourself) +# (Note: the urls sent to it start with a '/' so no need to add one yourself) EVENTSOURCE_SOURCE = None REDIS_HOST = '0.0.0.0' REDIS_PORT = 6379