From 4448b52f686cf04ed87fc064a28cbff4be55d0aa Mon Sep 17 00:00:00 2001 From: Eric Barbour Date: Jun 13 2016 21:06:57 +0000 Subject: [PATCH 1/2] Front end for live update for feed widget --- diff --git a/ev-server/hubs-stream-server.py b/ev-server/hubs-stream-server.py new file mode 100644 index 0000000..7945f7d --- /dev/null +++ b/ev-server/hubs-stream-server.py @@ -0,0 +1,172 @@ +import logging +import urlparse +import trollius +import trollius_redis +from trollius import From +import urllib2 +import json + +log = logging.getLogger(__name__) +SERVER = None +REDIS_HOST = '0.0.0.0' +REDIS_PORT = 6379 +REDIS_DB = 0 +EVENTSOURCE_PORT = 9090 + +''' +you need to + +pip install trollius trollius_redis + +dnf install redis httpie +systemctl start redis + + +usage: http get 0.0.0.0:8080/ + +''' + +def get_recent_posts(): + delta = "delta=86400" # one day worth of data + rows_per_page = "rows_per_page=50" + url = "https://apps.fedoraproject.org/datagrepper/raw" + "?" + delta + "&" + rows_per_page + request = urllib2.Request(url) + contents = urllib2.urlopen(request).read() + json_response = json.loads(contents) + return json_response['raw_messages'] + +@trollius.coroutine +def handle_client(client_reader, client_writer): + data = None + while True: + # give client a chance to respond, timeout after 10 seconds + line = yield trollius.From(trollius.wait_for( + client_reader.readline(), + timeout=10.0)) + if not line.decode().strip(): + break + line = line.decode().rstrip() + if data is None: + data = line + + if data is None: + log.warning("Expected ticket uid, received None") + return + + data = data.decode().rstrip().split() + log.info("Received %s", data) + if not data: + log.warning("No URL provided: %s" % data) + return + + if not '/' in data[1]: + log.warning("Invalid URL provided: %s" % data[1]) + return + if '/inform' in data[1]: + log.warning("booting off the post /inform ") + return + + url = urlparse.urlsplit(data[1]) + + origin = '*' # pagure.APP.config.get('APP_URL') + if origin.endswith('/'): + origin = origin[:-1] + + client_writer.write(( + "HTTP/1.0 200 OK\n" + "Content-Type: text/event-stream\n" + "Cache: nocache\n" + "Connection: keep-alive\n" + "Access-Control-Allow-Origin: %s\n\n" % origin + ).encode()) + + connection = yield trollius.From(trollius_redis.Connection.create( + host=REDIS_HOST, + port=REDIS_PORT, + db=REDIS_DB)) + + try: + + # Create subscriber. + subscriber = yield trollius.From(connection.start_subscribe()) + + # Subscribe to channel. + #yield trollius.From(subscriber.subscribe(['hubs.%s' % obj['uid']])) + + posts = get_recent_posts() + + # send 50 latest posts + num_post_sent = 0 + while num_post_sent < 50: + #reply = yield trollius.From(subscriber.next_published()) # wait for reply from user + reply = posts[num_post_sent] + reply = json.dumps(reply) + #print(u'Received: ', repr(reply.value), u'on channel', reply.channel) + log.info(reply) + log.info("Sending post %s %s", str(num_post_sent), reply) + client_writer.write(('data: %s\n\n' % reply).encode()) + yield trollius.From(client_writer.drain()) + num_post_sent += 1 + yield From(trollius.sleep(2)) + + except trollius.ConnectionResetError as err: + log.exception("ERROR: ConnectionResetError in handle_client") + except Exception as err: + log.exception("ERROR: Exception in handle_client") + finally: + # Wathever happens, close the connection. + connection.close() + client_writer.close() + +@trollius.coroutine +def greet_every_two_seconds(): + while True: + print('Hello World') + yield From(trollius.sleep(2)) + +# loop = trollius.get_event_loop() +# loop.run_until_complete(greet_every_two_seconds()) + +def main(): + global SERVER + + try: + loop = trollius.get_event_loop() + coro = trollius.start_server( + handle_client, + host=None, + port=EVENTSOURCE_PORT, + loop=loop) + SERVER = loop.run_until_complete(coro) + log.info('Serving server at {}'.format(SERVER.sockets[0].getsockname())) + loop.run_forever() + except KeyboardInterrupt: + pass + except trollius.ConnectionResetError as err: + log.exception("ERROR: ConnectionResetError in main") + except Exception as err: + log.exception("ERROR: Exception in main") + finally: + # Close the server + SERVER.close() + log.info("End Connection") + loop.run_until_complete(SERVER.wait_closed()) + loop.close() + log.info("End") + +if __name__ == '__main__': + log = logging.getLogger("") + formatter = logging.Formatter( + "%(asctime)s %(levelname)s [%(module)s:%(lineno)d] %(message)s") + + # setup console logging + log.setLevel(logging.DEBUG) + ch = logging.StreamHandler() + ch.setLevel(logging.DEBUG) + + aslog = logging.getLogger("asyncio") + aslog.setLevel(logging.DEBUG) + + ch.setFormatter(formatter) + log.addHandler(ch) + main() diff --git a/hubs/app.py b/hubs/app.py index fe661ab..d412a7f 100755 --- a/hubs/app.py +++ b/hubs/app.py @@ -1,7 +1,11 @@ +import os +import json +import uuid +import logging import datetime import functools -import logging -import os +import pygments.formatters + import flask import flask.json @@ -10,12 +14,17 @@ import six from flask.ext.openid import OpenID +import fmn.lib import hubs.models import hubs.widgets import datanommer.models from hubs.utils import username2avatar +from hubs.widgets.feed import ( + apply_markup, rehydrate_preference, + get_remote_preference +) app = flask.Flask(__name__) @@ -628,3 +637,53 @@ def hub_leave(hub): return flask.abort(400) session.commit() return flask.redirect(flask.url_for('hub', name=hub.name)) + +@app.route('/api/fedmsg/markup', methods=['GET']) +def markup_fedmsg(): + try: + data = flask.request.args.get('message') + plugin = flask.request.args.get('plugin') + except KeyError: + return flask.abort(400) + + widget = hubs.models.Widget.by_plugin(session, plugin) + context = widget.config.get('fmn_context') + + messages = [] + message = json.loads(data) + + preference = get_remote_preference(flask.g.auth.nickname, context) + if preference: + try: + preference = rehydrate_preference(preference) + except ImportError: + pass + + paths = fmn.lib.load_rules(root='fmn.rules') + recipients = fmn.lib.recipients( + [preference], message, paths, fedmsg_config) + + if recipients: + messages.append(message) + + matches = fedmsg.meta.conglomerate(messages, lexers=True, **fedmsg_config) + + for match in matches: + match['markup'] = apply_markup(match) + for _, constituent in match['msg_ids'].items(): + constituent['markup'] = apply_markup(constituent) + if constituent['long_form'] != constituent['subtitle']: + if constituent.get('lexer'): + constituent['long_form'] = pygments.highlight( + constituent['long_form'], + constituent['lexer'], + pygments.formatters.HtmlFormatter(style='monokai'), + ) + else: + markup = u"
{long_form}
".format(**constituent) + constituent['long_form'] = markup + + # And tack on a unique identifier for each top level entry. + match['dom_id'] = six.text_type(uuid.uuid4()) + + return flask.jsonify(matches) diff --git a/hubs/models.py b/hubs/models.py index 1731076..32580a6 100755 --- a/hubs/models.py +++ b/hubs/models.py @@ -303,6 +303,10 @@ class Widget(BASE): def by_idx(cls, session, idx): return session.query(cls).filter_by(idx=idx).first() + @classmethod + def by_plugin(cls, session, plugin): + return session.query(cls).filter_by(plugin=plugin).first() + get = by_idx @property diff --git a/hubs/static/js/request_ev.js b/hubs/static/js/request_ev.js new file mode 100644 index 0000000..c0f9d09 --- /dev/null +++ b/hubs/static/js/request_ev.js @@ -0,0 +1,105 @@ +var addNewMessage = function (matches) { + if (matches.length === 0) { + return null; + } + + var match = matches[0]; // server only sends one at a time + var elem = '
\ +
\ +
\ +
\ + \ + \ + \ +
\ +
\ +

' + match['markup'] +'

'+ match['human_time']; + + if(match['msg_ids'].length === 1 && Object.values(match['msg_ids'])[0]['long_form'] !== Object.values(match['msg_ids'])[0]['subtitle'] ) { + var constituent = Object.values(match['msg_ids'])[0]; + elem += '
\ + \ + \ + \ + collapse\ + \ +
\ +
'; + + if (constituent['icon'] != constituent['__icon__']) { + elem += '
\ + \ + ' + constituent['subtitle'] + '\ + \ +
'; + } + elem += '
' + constituent['long_form']; + + if (constituent['link'] && constituent['long_form'].indexOf(constituent['link']) === -1 ){ + elem += '\ + Read more '; + } + elem += '
'; + } + + + if( match['msg_ids'].length > 1) { + elem += '
\ + \ + \ + expand\ + \ + \ +
\ + '; + } + elem += '
\ +
\ +
\ +
'; + return elem; + }; +process_event = function (data) { + var category = null; + var elem; + var originalTitle = document.title; + if (data.match_added) { + elem = add_match(data); + category = 'New feed event'; + } + else { + console.log('Unknown data'); + } + return elem; + + if (category && !document.hasFocus()) { + var int = setInterval(function(){ + var title = document.title; + document.title = (title === originalTitle) ? category : originalTitle; + }, 750); + + $(window).focus(function () { + clearInterval(int); + document.title = originalTitle; + }); + } +}; diff --git a/hubs/tests/__init__.py b/hubs/tests/__init__.py index e69de29..073cc9a 100644 --- a/hubs/tests/__init__.py +++ b/hubs/tests/__init__.py @@ -0,0 +1,136 @@ +from contextlib import contextmanager +import json +from datetime import datetime, timedelta +from os.path import dirname +import unittest + +import flask +import requests +import vcr + +import hubs.models + +cassette_dir = dirname(dirname(__file__)) + '/vcr-request-data/' + + +class APPTest(unittest.TestCase): + def setUp(self): + filename = cassette_dir + self.id() + self.vcr = vcr.use_cassette(filename, record_mode='new_episodes') + self.vcr.__enter__() + + import hubs.app + hubs.app.fedmsg_config = { + 'hubs.sqlalchemy.uri': 'sqlite://', # in memory + } + hubs.app.session = hubs.models.init( + hubs.app.fedmsg_config['hubs.sqlalchemy.uri'], + create=True, + ) + self.app = hubs.app.app.test_client() + self.app.testing = True + self.session = hubs.app.session + self.populate() + + def tearDown(self): + self.vcr.__exit__() + self.session.remove() + + def populate(self): + for user in ['devyani7', 'dhrish', 'shalini', 'ralph', 'decause']: + openid = '%s.id.fedoraproject.org' % user + fullname = user.title() + hubs.models.User.get_or_create( + hubs.app.session, openid=openid, fullname=fullname) + + hubs.app.session.flush() + + hub = hubs.models.Hub.by_name(hubs.app.session, 'ralph') + widget = hubs.models.Widget( + plugin='about', + index=500, + _config=json.dumps({"text": "Testing."}), + ) + hub.widgets.append(widget) + + for team in ['i18n', 'infra', 'old']: + hub = hubs.models.Hub(name=team, summary="the %s team" % team) + if hub.name == 'old': + hub.last_refreshed = datetime.utcnow() - timedelta(days=100) + hub.last_updated = datetime.utcnow() - timedelta(days=200) + + hubs.app.session.add(hub) + widget = hubs.models.Widget(plugin='meetings', index=11, + _config=json.dumps({'calendar': team})) + hub.widgets.append(widget) + + # TODO - test that it is in the registry + # TODO - test that it has all the things it needs + + def widget_instance(self, hubname, plugin): + hub = hubs.models.Hub.by_name(self.session, hubname) + if not hub: + raise ValueError("No such hub %r" % hubname) + for widget in hub.widgets: + if widget.plugin == plugin: + return widget + raise KeyError("No such widget found for %s/%s" % (hubname, plugin)) + + @staticmethod + def get_fedmsg(idx): + url = 'https://apps.fedoraproject.org/datagrepper/id' + response = requests.get(url, params=dict(id=idx)) + if not bool(response): + raise IOError("Failed to talk to %r %r" % (response.url, response)) + return response.json() + + +@contextmanager +def auth_set(APP, auth): + """ Set the provided user as fas_user in the provided application.""" + + # Hack used to remove the before_request function set by + # flask.ext.fas_openid.FAS which otherwise kills our effort to set a + # flask.g.fas_user. + from flask import appcontext_pushed, g + APP.before_request_funcs[None] = [] + + def handler(sender, **kwargs): + g.auth = auth + flask.session['openid'] = auth.openid + + with appcontext_pushed.connected_to(handler, APP): + yield + + +class FakeUser(object): + """ Fake user used to test the fedocallib library. """ + def __init__(self, username='username'): + """ Constructor. + :arg groups: list of the groups in which this fake user is + supposed to be. + """ + self.username = username + self.openid = username + 'id.fedoraproject.org' + self.booksmarks = [] + + def __getitem__(self, key): + return self.dic[key] + + +class FakeAuthorization(object): + """ Fake user used to test the fedocallib library. """ + def __init__(self, username='username'): + """ Constructor. + :arg groups: list of the groups in which this fake user is + supposed to be. + """ + self.logged_in = True + self.fullname = 'fullname: ' + username + self.email = 'email: ' + username + self.openid = username + 'id.fedoraproject.org' + self.user = FakeUser(username) + self.avatar = 'avatar_src_url' + + def __getitem__(self, key): + return self.dic[key] diff --git a/hubs/tests/test_widgets/__init__.py b/hubs/tests/test_widgets/__init__.py index 38d77af..df0f707 100644 --- a/hubs/tests/test_widgets/__init__.py +++ b/hubs/tests/test_widgets/__init__.py @@ -3,14 +3,32 @@ from datetime import datetime, timedelta from os.path import dirname import unittest -import requests import vcr +import requests +from contextlib import contextmanager import hubs.models cassette_dir = dirname(dirname(__file__)) + '/vcr-request-data/' +@contextmanager +def user_set(APP, user): + """ Set the provided user as fas_user in the provided application.""" + + # Hack used to remove the before_request function set by + # flask.ext.fas_openid.FAS which otherwise kills our effort to set a + # flask.g.fas_user. + from flask import appcontext_pushed, g + + def handler(sender, **kwargs): + g.fas_user = user + g.fas_session_id = b'123' + + with appcontext_pushed.connected_to(handler, APP): + yield + + class WidgetTest(unittest.TestCase): def setUp(self): filename = cassette_dir + self.id() diff --git a/hubs/tests/test_widgets/test_feed.py b/hubs/tests/test_widgets/test_feed.py new file mode 100644 index 0000000..2de0b46 --- /dev/null +++ b/hubs/tests/test_widgets/test_feed.py @@ -0,0 +1,290 @@ +import json + +import hubs.tests +import hubs.models + + +class TestFeed(hubs.tests.APPTest): + # TODO this test relies on a specific filter for the atelic user + # It would be better to fake preferences on FAS + user = hubs.tests.FakeAuthorization('atelic') + message_involved = { + 'timestamp': 1465567413.0, + 'topic': 'io.pagure.prod.pagure.issue.new', + 'msg_id': '2016-0490127f-473f-4d01-ASDF-a864a56dc302', + 'msg': { + 'action': 'created', + 'comment': { + 'body': 'hm, there is something odd, the stackage page is being inconsistent, one time the version number for LTS is linked, one time it isn\'t.\r\n\r\nIt\'s as if there are 2 servers running 2 different versions of the application', + 'created_at': '2016-06-10T14:03:31Z', + 'html_url': 'https://github.com/fedora-infra/anitya/issues/283#issuecomment-225190438', + 'id': 225190438, + 'updated_at': '2016-06-10T14:03:31Z', + 'url': 'https://api.github.com/repos/fedora-infra/anitya/issues/comments/225190438', + 'user': { + 'gravatar_id': '', + 'html_url': 'https://github.com/pypingou', + 'id': 1240038, + 'login': 'pypingou', + 'site_admin': False, + 'type': 'User', + 'url': 'https://api.github.com/users/pypingou' + } + }, + 'fas_usernames': { + 'fedora-infra': 'github_org_fedora-infra', + 'pypingou': 'pingou' + }, + 'issue': { + 'assignee': None, + 'body': '@juhp it seems that the stackage backend is broken: https://release-monitoring.org/projects/updates/failed?name=&log=stackage want to look at it?', + 'closed_at': '2016-04-29T14:53:10Z', + 'comments': 8, + 'created_at': '2016-04-15T16:15:57Z', + 'html_url': 'https://github.com/fedora-infra/anitya/issues/283', + 'id': 148703615, + 'labels': [], + 'locked': False, + 'milestone': None, + 'number': 283, + 'state': 'closed', + 'title': 'stackage backend broken', + 'updated_at': '2016-06-10T14:03:31Z', + 'url': 'https://api.github.com/repos/fedora-infra/anitya/issues/283', + 'user': { + 'gravatar_id': '', + 'html_url': 'https://github.com/pypingou', + 'id': 1240038, + 'login': 'pypingou', + 'site_admin': False, + 'type': 'User', + 'url': 'https://api.github.com/users/pypingou' + } + }, + 'organization': { + 'description': 'Fedora Infrastructure Team', + 'id': 3316637, + 'login': 'fedora-infra', + 'url': 'https://api.github.com/orgs/fedora-infra' + }, + 'repository': { + 'created_at': '2013-11-29T10:17:26Z', + 'default_branch': 'master', + 'description': 'A cross-distribution upstream release project', + 'fork': False, + 'forks': 33, + 'forks_count': 33, + 'full_name': 'fedora-infra/anitya', + 'has_downloads': True, + 'has_issues': True, + 'has_pages': False, + 'has_wiki': False, + 'homepage': 'https://release-monitoring.org', + 'html_url': 'https://github.com/fedora-infra/anitya', + 'id': 14798348, + 'language': 'Python', + 'name': 'anitya', + 'open_issues': 25, + 'open_issues_count': 25, + 'owner': { + 'gravatar_id': '', + 'html_url': 'https://github.com/fedora-infra', + 'id': 3316637, + 'login': 'fedora-infra', + 'site_admin': False, + 'type': 'Organization', + 'url': 'https://api.github.com/users/fedora-infra' + }, + 'private': False, + 'pushed_at': '2016-06-05T18:16:15Z', + 'size': 4364, + 'stargazers_count': 56, + 'updated_at': '2016-06-10T09:43:58Z', + 'url': 'https://api.github.com/repos/fedora-infra/anitya', + 'watchers': 56, + 'watchers_count': 56 + }, + 'sender': { + 'gravatar_id': '', + 'html_url': 'https://github.com/pypingou', + 'id': 1240038, + 'login': 'pypingou', + 'site_admin': False, + 'type': 'User', + 'url': 'https://api.github.com/users/pypingou' + } + }, + 'arguments': { + 'categories': [ + 'pagure' + ], + 'contains': [], + 'delta': 172800.0, + 'end': 1465839365.0, + 'grouped': False, + 'meta': [], + 'not_categories': [], + 'not_packages': [], + 'not_topics': [], + 'not_users': [], + 'order': 'desc', + 'packages': [], + 'page': 1, + 'rows_per_page': 1, + 'start': 1465666565.0, + 'topics': [], + 'users': [] + }, + 'count': 1, + 'pages': 225, + 'total': 225 + } + message_not_involved = { + 'topic': 'org.fedoraproject.prod.github.issue.comment', + 'i': 1, + 'msg': { + 'action': 'created', + 'comment': { + 'body': 'hm, there is something odd, the stackage page is being inconsistent, one time the version number for LTS is linked, one time it isn\'t.\r\n\r\nIt\'s as if there are 2 servers running 2 different versions of the application', + 'created_at': '2016-06-10T14:03:31Z', + 'html_url': 'https://github.com/fedora-infra/anitya/issues/283#issuecomment-225190438', + 'id': 225190438, + 'updated_at': '2016-06-10T14:03:31Z', + 'url': 'https://api.github.com/repos/fedora-infra/anitya/issues/comments/225190438', + 'user': { + 'gravatar_id': '', + 'html_url': 'https://github.com/pypingou', + 'id': 1240038, + 'login': 'pypingou', + 'site_admin': False, + 'type': 'User', + 'url': 'https://api.github.com/users/pypingou' + } + }, + 'fas_usernames': { + 'fedora-infra': 'github_org_fedora-infra', + 'pypingou': 'pingou' + }, + 'issue': { + 'assignee': None, + 'body': '@juhp it seems that the stackage backend is broken: https://release-monitoring.org/projects/updates/failed?name=&log=stackage want to look at it?', + 'closed_at': '2016-04-29T14:53:10Z', + 'comments': 8, + 'created_at': '2016-04-15T16:15:57Z', + 'html_url': 'https://github.com/fedora-infra/anitya/issues/283', + 'id': 148703615, + 'labels': [], + 'locked': False, + 'milestone': None, + 'number': 283, + 'state': 'closed', + 'title': 'stackage backend broken', + 'updated_at': '2016-06-10T14:03:31Z', + 'url': 'https://api.github.com/repos/fedora-infra/anitya/issues/283', + 'user': { + 'gravatar_id': '', + 'html_url': 'https://github.com/pypingou', + 'id': 1240038, + 'login': 'pypingou', + 'site_admin': False, + 'type': 'User', + 'url': 'https://api.github.com/users/pypingou' + } + }, + 'organization': { + 'description': 'Fedora Infrastructure Team', + 'id': 3316637, + 'login': 'fedora-infra', + 'url': 'https://api.github.com/orgs/fedora-infra' + }, + 'repository': { + 'created_at': '2013-11-29T10:17:26Z', + 'default_branch': 'master', + 'description': 'A cross-distribution upstream release monitoring project', + 'fork': False, + 'forks': 33, + 'forks_count': 33, + 'full_name': 'fedora-infra/anitya', + 'has_downloads': True, + 'has_issues': True, + 'has_pages': False, + 'has_wiki': False, + 'homepage': 'https://release-monitoring.org', + 'html_url': 'https://github.com/fedora-infra/anitya', + 'id': 14798348, + 'language': 'Python', + 'name': 'anitya', + 'open_issues': 25, + 'open_issues_count': 25, + 'owner': { + 'gravatar_id': '', + 'html_url': 'https://github.com/fedora-infra', + 'id': 3316637, + 'login': 'fedora-infra', + 'site_admin': False, + 'type': 'Organization', + 'url': 'https://api.github.com/users/fedora-infra' + }, + 'private': False, + 'pushed_at': '2016-06-05T18:16:15Z', + 'size': 4364, + 'stargazers_count': 56, + 'updated_at': '2016-06-10T09:43:58Z', + 'url': 'https://api.github.com/repos/fedora-infra/anitya', + 'watchers': 56, + 'watchers_count': 56 + }, + 'sender': { + 'gravatar_id': '', + 'html_url': 'https://github.com/pypingou', + 'id': 1240038, + 'login': 'pypingou', + 'site_admin': False, + 'type': 'User', + 'url': 'https://api.github.com/users/pypingou' + } + }, + 'msg_id': '2016-0490127f-473f-4d01-8e3a-a864a56dc302', + 'signature': 'MNpAVdrPW78dcSe94FOS+/6QFBYo/LkrV5ee9tQedkhurHY1BGpaFHlU5l8cNVEpB7rr/zhHgQ+V\nmE3zYPmW/aREgvUoQgz74TR5F6/nK4Q1cD97VkFgHCadxZC6kjXd4H3BahEYnD+erj1iaArcdZaI\nXyKLZUBJLYL4JiXlmec=\n', + 'source_name': 'datanommer', + 'source_version': '0.6.5', + 'timestamp': 1465567413.0, + 'topic': 'org.fedoraproject.prod.github.issue.comment' + } + + plugin = 'feed' + + def test_returns_match_if_involved(self): + payload = { + 'message': json.dumps(self.message_involved), + 'plugin': 'feed' + } + with self.app.session_transaction() as sess: + sess['openid'] = 'atelic@fedoraproject.org' + sess['nickname'] = 'atelic' + + with hubs.tests.test_widgets.user_set(self.app, self.user): + response = self.app.get('/api/fedmsg/markup', + query_string=payload) + + assert response.status_code == 200, response.status_code + data = json.loads(response.data) + self.assertTrue(data) + assert isinstance(data[0], dict) + + def test_returns_no_match_if_not_involved(self): + payload = { + 'message': json.dumps(self.message_not_involved), + 'plugin': 'feed' + } + with self.app.session_transaction() as sess: + sess['openid'] = 'atelic@fedoraproject.org' + sess['nickname'] = 'atelic' + + with hubs.tests.test_widgets.user_set(self.app, self.user): + response = self.app.get('/api/fedmsg/markup', + query_string=payload) + + assert response.status_code == 200, response.status_code + data = json.loads(response.data) + self.assertFalse(data) diff --git a/hubs/widgets/templates/feed.html b/hubs/widgets/templates/feed.html index 534a51a..cf7fd05 100644 --- a/hubs/widgets/templates/feed.html +++ b/hubs/widgets/templates/feed.html @@ -72,3 +72,33 @@ {% endfor %} + + + + From 571dad8837c604ad97881c4b5f94aed687cdb5a2 Mon Sep 17 00:00:00 2001 From: Eric Barbour Date: Jun 15 2016 16:22:46 +0000 Subject: [PATCH 2/2] Add tests for feed update and improve speed --- diff --git a/ev-server/hubs-stream-server.py b/ev-server/hubs-stream-server.py index 7945f7d..78fa3ad 100644 --- a/ev-server/hubs-stream-server.py +++ b/ev-server/hubs-stream-server.py @@ -1,10 +1,15 @@ +''' +This file is for development purposes only. + +The end goal is for the streaming server to be implemented on the FMN side +''' +import json import logging -import urlparse import trollius import trollius_redis -from trollius import From import urllib2 -import json +import urlparse +from trollius import From log = logging.getLogger(__name__) SERVER = None @@ -59,15 +64,13 @@ def handle_client(client_reader, client_writer): log.warning("No URL provided: %s" % data) return - if not '/' in data[1]: + if '/' not in data[1]: log.warning("Invalid URL provided: %s" % data[1]) return if '/inform' in data[1]: log.warning("booting off the post /inform ") return - url = urlparse.urlsplit(data[1]) - origin = '*' # pagure.APP.config.get('APP_URL') if origin.endswith('/'): origin = origin[:-1] @@ -88,20 +91,18 @@ def handle_client(client_reader, client_writer): try: # Create subscriber. - subscriber = yield trollius.From(connection.start_subscribe()) + # subscriber = yield trollius.From(connection.start_subscribe()) # Subscribe to channel. - #yield trollius.From(subscriber.subscribe(['hubs.%s' % obj['uid']])) + # yield trollius.From(subscriber.subscribe(['hubs.%s' % obj['uid']])) posts = get_recent_posts() # send 50 latest posts num_post_sent = 0 while num_post_sent < 50: - #reply = yield trollius.From(subscriber.next_published()) # wait for reply from user reply = posts[num_post_sent] reply = json.dumps(reply) - #print(u'Received: ', repr(reply.value), u'on channel', reply.channel) log.info(reply) log.info("Sending post %s %s", str(num_post_sent), reply) client_writer.write(('data: %s\n\n' % reply).encode()) @@ -109,15 +110,16 @@ def handle_client(client_reader, client_writer): num_post_sent += 1 yield From(trollius.sleep(2)) - except trollius.ConnectionResetError as err: + except trollius.ConnectionResetError: log.exception("ERROR: ConnectionResetError in handle_client") - except Exception as err: + except Exception: log.exception("ERROR: Exception in handle_client") finally: # Wathever happens, close the connection. connection.close() client_writer.close() + @trollius.coroutine def greet_every_two_seconds(): while True: @@ -127,6 +129,7 @@ def greet_every_two_seconds(): # loop = trollius.get_event_loop() # loop.run_until_complete(greet_every_two_seconds()) + def main(): global SERVER diff --git a/hubs/app.py b/hubs/app.py index d412a7f..6c23aa8 100755 --- a/hubs/app.py +++ b/hubs/app.py @@ -1,15 +1,14 @@ -import os -import json -import uuid -import logging import datetime import functools -import pygments.formatters - +import json +import logging +import os +import uuid import flask import flask.json import munch +import pygments.formatters import six from flask.ext.openid import OpenID @@ -52,6 +51,7 @@ import fedmsg.config import fedmsg.meta fedmsg_config = fedmsg.config.load_config() fedmsg.meta.make_processors(**fedmsg_config) +PATHS = fmn.lib.load_rules(root='fmn.rules') session = hubs.models.init(fedmsg_config['hubs.sqlalchemy.uri']) datanommer.models.init(fedmsg_config['datanommer.sqlalchemy.uri']) @@ -641,27 +641,34 @@ def hub_leave(hub): @app.route('/api/fedmsg/markup', methods=['GET']) def markup_fedmsg(): try: - data = flask.request.args.get('message') - plugin = flask.request.args.get('plugin') + data = flask.request.args['message'] + plugin = flask.request.args['plugin'] except KeyError: return flask.abort(400) widget = hubs.models.Widget.by_plugin(session, plugin) + if not widget: + return flask.abort(400) + context = widget.config.get('fmn_context') messages = [] message = json.loads(data) - preference = get_remote_preference(flask.g.auth.nickname, context) + try: + nickname = flask.g.auth.nickname + except AttributeError: # Not logged in + return flask.abort(403) + + preference = get_remote_preference(nickname, context) if preference: try: preference = rehydrate_preference(preference) except ImportError: pass - paths = fmn.lib.load_rules(root='fmn.rules') recipients = fmn.lib.recipients( - [preference], message, paths, fedmsg_config) + [preference], message, PATHS, fedmsg_config) if recipients: messages.append(message) diff --git a/hubs/tests/__init__.py b/hubs/tests/__init__.py index 073cc9a..e69de29 100644 --- a/hubs/tests/__init__.py +++ b/hubs/tests/__init__.py @@ -1,136 +0,0 @@ -from contextlib import contextmanager -import json -from datetime import datetime, timedelta -from os.path import dirname -import unittest - -import flask -import requests -import vcr - -import hubs.models - -cassette_dir = dirname(dirname(__file__)) + '/vcr-request-data/' - - -class APPTest(unittest.TestCase): - def setUp(self): - filename = cassette_dir + self.id() - self.vcr = vcr.use_cassette(filename, record_mode='new_episodes') - self.vcr.__enter__() - - import hubs.app - hubs.app.fedmsg_config = { - 'hubs.sqlalchemy.uri': 'sqlite://', # in memory - } - hubs.app.session = hubs.models.init( - hubs.app.fedmsg_config['hubs.sqlalchemy.uri'], - create=True, - ) - self.app = hubs.app.app.test_client() - self.app.testing = True - self.session = hubs.app.session - self.populate() - - def tearDown(self): - self.vcr.__exit__() - self.session.remove() - - def populate(self): - for user in ['devyani7', 'dhrish', 'shalini', 'ralph', 'decause']: - openid = '%s.id.fedoraproject.org' % user - fullname = user.title() - hubs.models.User.get_or_create( - hubs.app.session, openid=openid, fullname=fullname) - - hubs.app.session.flush() - - hub = hubs.models.Hub.by_name(hubs.app.session, 'ralph') - widget = hubs.models.Widget( - plugin='about', - index=500, - _config=json.dumps({"text": "Testing."}), - ) - hub.widgets.append(widget) - - for team in ['i18n', 'infra', 'old']: - hub = hubs.models.Hub(name=team, summary="the %s team" % team) - if hub.name == 'old': - hub.last_refreshed = datetime.utcnow() - timedelta(days=100) - hub.last_updated = datetime.utcnow() - timedelta(days=200) - - hubs.app.session.add(hub) - widget = hubs.models.Widget(plugin='meetings', index=11, - _config=json.dumps({'calendar': team})) - hub.widgets.append(widget) - - # TODO - test that it is in the registry - # TODO - test that it has all the things it needs - - def widget_instance(self, hubname, plugin): - hub = hubs.models.Hub.by_name(self.session, hubname) - if not hub: - raise ValueError("No such hub %r" % hubname) - for widget in hub.widgets: - if widget.plugin == plugin: - return widget - raise KeyError("No such widget found for %s/%s" % (hubname, plugin)) - - @staticmethod - def get_fedmsg(idx): - url = 'https://apps.fedoraproject.org/datagrepper/id' - response = requests.get(url, params=dict(id=idx)) - if not bool(response): - raise IOError("Failed to talk to %r %r" % (response.url, response)) - return response.json() - - -@contextmanager -def auth_set(APP, auth): - """ Set the provided user as fas_user in the provided application.""" - - # Hack used to remove the before_request function set by - # flask.ext.fas_openid.FAS which otherwise kills our effort to set a - # flask.g.fas_user. - from flask import appcontext_pushed, g - APP.before_request_funcs[None] = [] - - def handler(sender, **kwargs): - g.auth = auth - flask.session['openid'] = auth.openid - - with appcontext_pushed.connected_to(handler, APP): - yield - - -class FakeUser(object): - """ Fake user used to test the fedocallib library. """ - def __init__(self, username='username'): - """ Constructor. - :arg groups: list of the groups in which this fake user is - supposed to be. - """ - self.username = username - self.openid = username + 'id.fedoraproject.org' - self.booksmarks = [] - - def __getitem__(self, key): - return self.dic[key] - - -class FakeAuthorization(object): - """ Fake user used to test the fedocallib library. """ - def __init__(self, username='username'): - """ Constructor. - :arg groups: list of the groups in which this fake user is - supposed to be. - """ - self.logged_in = True - self.fullname = 'fullname: ' + username - self.email = 'email: ' + username - self.openid = username + 'id.fedoraproject.org' - self.user = FakeUser(username) - self.avatar = 'avatar_src_url' - - def __getitem__(self, key): - return self.dic[key] diff --git a/hubs/tests/test_widgets/__init__.py b/hubs/tests/test_widgets/__init__.py index df0f707..38d77af 100644 --- a/hubs/tests/test_widgets/__init__.py +++ b/hubs/tests/test_widgets/__init__.py @@ -3,32 +3,14 @@ from datetime import datetime, timedelta from os.path import dirname import unittest -import vcr import requests -from contextlib import contextmanager +import vcr import hubs.models cassette_dir = dirname(dirname(__file__)) + '/vcr-request-data/' -@contextmanager -def user_set(APP, user): - """ Set the provided user as fas_user in the provided application.""" - - # Hack used to remove the before_request function set by - # flask.ext.fas_openid.FAS which otherwise kills our effort to set a - # flask.g.fas_user. - from flask import appcontext_pushed, g - - def handler(sender, **kwargs): - g.fas_user = user - g.fas_session_id = b'123' - - with appcontext_pushed.connected_to(handler, APP): - yield - - class WidgetTest(unittest.TestCase): def setUp(self): filename = cassette_dir + self.id() diff --git a/hubs/tests/test_widgets/test_feed.py b/hubs/tests/test_widgets/test_feed.py deleted file mode 100644 index 2de0b46..0000000 --- a/hubs/tests/test_widgets/test_feed.py +++ /dev/null @@ -1,290 +0,0 @@ -import json - -import hubs.tests -import hubs.models - - -class TestFeed(hubs.tests.APPTest): - # TODO this test relies on a specific filter for the atelic user - # It would be better to fake preferences on FAS - user = hubs.tests.FakeAuthorization('atelic') - message_involved = { - 'timestamp': 1465567413.0, - 'topic': 'io.pagure.prod.pagure.issue.new', - 'msg_id': '2016-0490127f-473f-4d01-ASDF-a864a56dc302', - 'msg': { - 'action': 'created', - 'comment': { - 'body': 'hm, there is something odd, the stackage page is being inconsistent, one time the version number for LTS is linked, one time it isn\'t.\r\n\r\nIt\'s as if there are 2 servers running 2 different versions of the application', - 'created_at': '2016-06-10T14:03:31Z', - 'html_url': 'https://github.com/fedora-infra/anitya/issues/283#issuecomment-225190438', - 'id': 225190438, - 'updated_at': '2016-06-10T14:03:31Z', - 'url': 'https://api.github.com/repos/fedora-infra/anitya/issues/comments/225190438', - 'user': { - 'gravatar_id': '', - 'html_url': 'https://github.com/pypingou', - 'id': 1240038, - 'login': 'pypingou', - 'site_admin': False, - 'type': 'User', - 'url': 'https://api.github.com/users/pypingou' - } - }, - 'fas_usernames': { - 'fedora-infra': 'github_org_fedora-infra', - 'pypingou': 'pingou' - }, - 'issue': { - 'assignee': None, - 'body': '@juhp it seems that the stackage backend is broken: https://release-monitoring.org/projects/updates/failed?name=&log=stackage want to look at it?', - 'closed_at': '2016-04-29T14:53:10Z', - 'comments': 8, - 'created_at': '2016-04-15T16:15:57Z', - 'html_url': 'https://github.com/fedora-infra/anitya/issues/283', - 'id': 148703615, - 'labels': [], - 'locked': False, - 'milestone': None, - 'number': 283, - 'state': 'closed', - 'title': 'stackage backend broken', - 'updated_at': '2016-06-10T14:03:31Z', - 'url': 'https://api.github.com/repos/fedora-infra/anitya/issues/283', - 'user': { - 'gravatar_id': '', - 'html_url': 'https://github.com/pypingou', - 'id': 1240038, - 'login': 'pypingou', - 'site_admin': False, - 'type': 'User', - 'url': 'https://api.github.com/users/pypingou' - } - }, - 'organization': { - 'description': 'Fedora Infrastructure Team', - 'id': 3316637, - 'login': 'fedora-infra', - 'url': 'https://api.github.com/orgs/fedora-infra' - }, - 'repository': { - 'created_at': '2013-11-29T10:17:26Z', - 'default_branch': 'master', - 'description': 'A cross-distribution upstream release project', - 'fork': False, - 'forks': 33, - 'forks_count': 33, - 'full_name': 'fedora-infra/anitya', - 'has_downloads': True, - 'has_issues': True, - 'has_pages': False, - 'has_wiki': False, - 'homepage': 'https://release-monitoring.org', - 'html_url': 'https://github.com/fedora-infra/anitya', - 'id': 14798348, - 'language': 'Python', - 'name': 'anitya', - 'open_issues': 25, - 'open_issues_count': 25, - 'owner': { - 'gravatar_id': '', - 'html_url': 'https://github.com/fedora-infra', - 'id': 3316637, - 'login': 'fedora-infra', - 'site_admin': False, - 'type': 'Organization', - 'url': 'https://api.github.com/users/fedora-infra' - }, - 'private': False, - 'pushed_at': '2016-06-05T18:16:15Z', - 'size': 4364, - 'stargazers_count': 56, - 'updated_at': '2016-06-10T09:43:58Z', - 'url': 'https://api.github.com/repos/fedora-infra/anitya', - 'watchers': 56, - 'watchers_count': 56 - }, - 'sender': { - 'gravatar_id': '', - 'html_url': 'https://github.com/pypingou', - 'id': 1240038, - 'login': 'pypingou', - 'site_admin': False, - 'type': 'User', - 'url': 'https://api.github.com/users/pypingou' - } - }, - 'arguments': { - 'categories': [ - 'pagure' - ], - 'contains': [], - 'delta': 172800.0, - 'end': 1465839365.0, - 'grouped': False, - 'meta': [], - 'not_categories': [], - 'not_packages': [], - 'not_topics': [], - 'not_users': [], - 'order': 'desc', - 'packages': [], - 'page': 1, - 'rows_per_page': 1, - 'start': 1465666565.0, - 'topics': [], - 'users': [] - }, - 'count': 1, - 'pages': 225, - 'total': 225 - } - message_not_involved = { - 'topic': 'org.fedoraproject.prod.github.issue.comment', - 'i': 1, - 'msg': { - 'action': 'created', - 'comment': { - 'body': 'hm, there is something odd, the stackage page is being inconsistent, one time the version number for LTS is linked, one time it isn\'t.\r\n\r\nIt\'s as if there are 2 servers running 2 different versions of the application', - 'created_at': '2016-06-10T14:03:31Z', - 'html_url': 'https://github.com/fedora-infra/anitya/issues/283#issuecomment-225190438', - 'id': 225190438, - 'updated_at': '2016-06-10T14:03:31Z', - 'url': 'https://api.github.com/repos/fedora-infra/anitya/issues/comments/225190438', - 'user': { - 'gravatar_id': '', - 'html_url': 'https://github.com/pypingou', - 'id': 1240038, - 'login': 'pypingou', - 'site_admin': False, - 'type': 'User', - 'url': 'https://api.github.com/users/pypingou' - } - }, - 'fas_usernames': { - 'fedora-infra': 'github_org_fedora-infra', - 'pypingou': 'pingou' - }, - 'issue': { - 'assignee': None, - 'body': '@juhp it seems that the stackage backend is broken: https://release-monitoring.org/projects/updates/failed?name=&log=stackage want to look at it?', - 'closed_at': '2016-04-29T14:53:10Z', - 'comments': 8, - 'created_at': '2016-04-15T16:15:57Z', - 'html_url': 'https://github.com/fedora-infra/anitya/issues/283', - 'id': 148703615, - 'labels': [], - 'locked': False, - 'milestone': None, - 'number': 283, - 'state': 'closed', - 'title': 'stackage backend broken', - 'updated_at': '2016-06-10T14:03:31Z', - 'url': 'https://api.github.com/repos/fedora-infra/anitya/issues/283', - 'user': { - 'gravatar_id': '', - 'html_url': 'https://github.com/pypingou', - 'id': 1240038, - 'login': 'pypingou', - 'site_admin': False, - 'type': 'User', - 'url': 'https://api.github.com/users/pypingou' - } - }, - 'organization': { - 'description': 'Fedora Infrastructure Team', - 'id': 3316637, - 'login': 'fedora-infra', - 'url': 'https://api.github.com/orgs/fedora-infra' - }, - 'repository': { - 'created_at': '2013-11-29T10:17:26Z', - 'default_branch': 'master', - 'description': 'A cross-distribution upstream release monitoring project', - 'fork': False, - 'forks': 33, - 'forks_count': 33, - 'full_name': 'fedora-infra/anitya', - 'has_downloads': True, - 'has_issues': True, - 'has_pages': False, - 'has_wiki': False, - 'homepage': 'https://release-monitoring.org', - 'html_url': 'https://github.com/fedora-infra/anitya', - 'id': 14798348, - 'language': 'Python', - 'name': 'anitya', - 'open_issues': 25, - 'open_issues_count': 25, - 'owner': { - 'gravatar_id': '', - 'html_url': 'https://github.com/fedora-infra', - 'id': 3316637, - 'login': 'fedora-infra', - 'site_admin': False, - 'type': 'Organization', - 'url': 'https://api.github.com/users/fedora-infra' - }, - 'private': False, - 'pushed_at': '2016-06-05T18:16:15Z', - 'size': 4364, - 'stargazers_count': 56, - 'updated_at': '2016-06-10T09:43:58Z', - 'url': 'https://api.github.com/repos/fedora-infra/anitya', - 'watchers': 56, - 'watchers_count': 56 - }, - 'sender': { - 'gravatar_id': '', - 'html_url': 'https://github.com/pypingou', - 'id': 1240038, - 'login': 'pypingou', - 'site_admin': False, - 'type': 'User', - 'url': 'https://api.github.com/users/pypingou' - } - }, - 'msg_id': '2016-0490127f-473f-4d01-8e3a-a864a56dc302', - 'signature': 'MNpAVdrPW78dcSe94FOS+/6QFBYo/LkrV5ee9tQedkhurHY1BGpaFHlU5l8cNVEpB7rr/zhHgQ+V\nmE3zYPmW/aREgvUoQgz74TR5F6/nK4Q1cD97VkFgHCadxZC6kjXd4H3BahEYnD+erj1iaArcdZaI\nXyKLZUBJLYL4JiXlmec=\n', - 'source_name': 'datanommer', - 'source_version': '0.6.5', - 'timestamp': 1465567413.0, - 'topic': 'org.fedoraproject.prod.github.issue.comment' - } - - plugin = 'feed' - - def test_returns_match_if_involved(self): - payload = { - 'message': json.dumps(self.message_involved), - 'plugin': 'feed' - } - with self.app.session_transaction() as sess: - sess['openid'] = 'atelic@fedoraproject.org' - sess['nickname'] = 'atelic' - - with hubs.tests.test_widgets.user_set(self.app, self.user): - response = self.app.get('/api/fedmsg/markup', - query_string=payload) - - assert response.status_code == 200, response.status_code - data = json.loads(response.data) - self.assertTrue(data) - assert isinstance(data[0], dict) - - def test_returns_no_match_if_not_involved(self): - payload = { - 'message': json.dumps(self.message_not_involved), - 'plugin': 'feed' - } - with self.app.session_transaction() as sess: - sess['openid'] = 'atelic@fedoraproject.org' - sess['nickname'] = 'atelic' - - with hubs.tests.test_widgets.user_set(self.app, self.user): - response = self.app.get('/api/fedmsg/markup', - query_string=payload) - - assert response.status_code == 200, response.status_code - data = json.loads(response.data) - self.assertFalse(data)