From 21a8046ee47c79d34f08bbcb21d9ba8b0302da05 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Dec 22 2016 10:52:08 +0000 Subject: [PATCH 1/8] Create the pagure-logcom service and add a README presenting it --- diff --git a/pagure-logcom/README.rst b/pagure-logcom/README.rst new file mode 100644 index 0000000..b60fb19 --- /dev/null +++ b/pagure-logcom/README.rst @@ -0,0 +1,12 @@ +Pagure LogCom +============= + +This is the service logging in the user's commits to be displayed in the +database. +This service is triggered by a git hook, sending a notification that a push +happened. This service receive the notification and goes over all the commit +that got pushed and logs the activity corresponding to that user. + + * Run:: + + PAGURE_CONFIG=/path/to/config PYTHONPATH=. python pagure-logcom/pagure_logcom_server.py From ae0a96604a57649e4f9ef8b0b8b4ac0d8e2574d9 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Dec 22 2016 10:52:08 +0000 Subject: [PATCH 2/8] Include the pagure-logcom service in the released tarball --- diff --git a/MANIFEST.in b/MANIFEST.in index d2ddd0d..35104f1 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -2,6 +2,7 @@ include LICENSE README.rst requirements.txt UPGRADING.rst include createdb.py recursive-include pagure * recursive-include pagure-ci * +recursive-include pagure-logcom * recursive-include files * recursive-include milters * recursive-include tests * From 3e603f0591f729235fa58798025f2929f7bbfa09 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Dec 22 2016 10:52:08 +0000 Subject: [PATCH 3/8] Add the service file for pagure_logcom --- diff --git a/pagure-logcom/pagure_logcom.service b/pagure-logcom/pagure_logcom.service new file mode 100644 index 0000000..577c775 --- /dev/null +++ b/pagure-logcom/pagure_logcom.service @@ -0,0 +1,14 @@ +[Unit] +Description=Pagure Logging Commit service +After=redis.target +Documentation=https://pagure.io/pagure + +[Service] +ExecStart=/usr/libexec/pagure-logcom/pagure_logcom_server.py +Type=simple +User=git +Group=git +Restart=on-failure + +[Install] +WantedBy=multi-user.target From 623e665dad6c419879def37ab53b572b070ffd5e Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Dec 22 2016 10:52:08 +0000 Subject: [PATCH 4/8] Add the logcom service This service aims at logging the commits to the pagure_logs table outside of the push process (ie outside of git push). This will save time as otherwise we were logging all the commits in the git push, so if you were pushing, for example, the linux kernel tree, it was taking up to 45 minutes on a local push (from within the same machine). With this change, it will still take that long to log all the commits and save them to the DB, but at least, we will end the git push sooner. --- diff --git a/pagure-logcom/pagure_logcom_server.py b/pagure-logcom/pagure_logcom_server.py new file mode 100644 index 0000000..a3ec61d --- /dev/null +++ b/pagure-logcom/pagure_logcom_server.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +""" + (c) 2016 - Copyright Red Hat Inc + + Authors: + Pierre-Yves Chibon + + +This server listens to message sent via redis post commits and log the +user's activity in the database. + +Using this mechanism, we no longer need to block the git push until all the +activity has been logged (which is you push the kernel tree for the first +time can be really time-consuming). + +""" + +import json +import logging +import os +import requests + +import trollius +import trollius_redis + + +LOG = logging.getLogger(__name__) + +if 'PAGURE_CONFIG' not in os.environ \ + and os.path.exists('/etc/pagure/pagure.cfg'): + print 'Using configuration file `/etc/pagure/pagure.cfg`' + os.environ['PAGURE_CONFIG'] = '/etc/pagure/pagure.cfg' + + +import pagure +import pagure.lib + + +@trollius.coroutine +def handle_messages(): + ''' Handles connecting to redis and acting upon messages received. + In this case, it means triggering a build on jenkins based on the + information provided. + ''' + + host = pagure.APP.config.get('REDIS_HOST', '0.0.0.0') + port = pagure.APP.config.get('REDIS_PORT', 6379) + dbname = pagure.APP.config.get('REDIS_DB', 0) + connection = yield trollius.From(trollius_redis.Connection.create( + host=host, port=port, db=dbname)) + + # Create subscriber. + subscriber = yield trollius.From(connection.start_subscribe()) + + # Subscribe to channel. + yield trollius.From(subscriber.subscribe(['pagure.logcom'])) + + # Inside a while loop, wait for incoming events. + while True: + reply = yield trollius.From(subscriber.next_published()) + LOG.info( + 'Received: %s on channel: %s', + repr(reply.value), reply.channel) + data = json.loads(reply.value) + + commits = data['commits'] + abspath = data['abspath'] + repo = data['project']['name'] + username = data['project']['username']['name'] \ + if data['project']['parent'] else None + namespace = data['project']['namespace'] + + session = pagure.lib.create_session(pagure.APP.config['DB_URL']) + print(session.bind.engine.url) + + LOG.info('Looking for project: %s%s of %s', + '%s/' % namespacerepo if namespace else '', + repo, username) + project = pagure.lib.get_project( + pagure.SESSION, repo, user=username, namespace=namespace) + + if not project: + LOG.info('No project found') + continue + + LOG.info('Found project: %s', project.fullname) + + LOG.info('Processing %s commits in %s', len(commits), abspath) + + pagure.lib.git.log_commits_to_db( + pagure.SESSION, project, commits, abspath) + + session.close() + LOG.info('Ready for another') + + +def main(): + ''' Start the main async loop. ''' + + try: + loop = trollius.get_event_loop() + tasks = [ + trollius.async(handle_messages()), + ] + loop.run_until_complete(trollius.wait(tasks)) + loop.run_forever() + except KeyboardInterrupt: + pass + except trollius.ConnectionResetError: + pass + + LOG.info("End Connection") + loop.close() + LOG.info("End") + + +if __name__ == '__main__': + formatter = logging.Formatter( + "%(asctime)s %(levelname)s [%(module)s:%(lineno)d] %(message)s") + + logging.basicConfig(level=logging.DEBUG) + + # setup console logging + LOG.setLevel(logging.DEBUG) + shellhandler = logging.StreamHandler() + shellhandler.setLevel(logging.DEBUG) + + aslog = logging.getLogger("asyncio") + aslog.setLevel(logging.DEBUG) + aslog = logging.getLogger("trollius") + aslog.setLevel(logging.DEBUG) + + shellhandler.setFormatter(formatter) + LOG.addHandler(shellhandler) + main() From 4c310393d02d2c45b2788fecf5133acd52a45fbc Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Dec 22 2016 10:52:08 +0000 Subject: [PATCH 5/8] Move logging the commits in the DB outside the default hook --- diff --git a/pagure/hooks/files/default_hook.py b/pagure/hooks/files/default_hook.py index f0bb3ae..33f3c8b 100755 --- a/pagure/hooks/files/default_hook.py +++ b/pagure/hooks/files/default_hook.py @@ -5,6 +5,7 @@ """ from __future__ import print_function +import json import os import sys @@ -21,6 +22,8 @@ import pagure import pagure.exceptions import pagure.lib.link +from pagure.lib import REDIS + abspath = os.path.abspath(os.environ['GIT_DIR']) @@ -69,8 +72,16 @@ def run_as_post_receive_hook(): commits = pagure.lib.git.get_revs_between( oldrev, newrev, abspath, refname) - pagure.lib.git.log_commits_to_db( - pagure.SESSION, project, commits, abspath) + + if REDIS: + print('Sending to redis to log activity') + REDIS.publish('pagure.logcom', + json.dumps({ + 'project': project.to_json(public=True), + 'abspath': abspath, + 'commits': commits, + } + )) try: # Reset the merge_status of all opened PR to refresh their cache From 79b62ca87c0fb64ef17cde8c3ced5e9d78a3d657 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Dec 22 2016 10:52:08 +0000 Subject: [PATCH 6/8] Let's not forget to commit at the end of the transaction, it helps saving it --- diff --git a/pagure-logcom/pagure_logcom_server.py b/pagure-logcom/pagure_logcom_server.py index a3ec61d..c9ce468 100644 --- a/pagure-logcom/pagure_logcom_server.py +++ b/pagure-logcom/pagure_logcom_server.py @@ -73,7 +73,6 @@ def handle_messages(): namespace = data['project']['namespace'] session = pagure.lib.create_session(pagure.APP.config['DB_URL']) - print(session.bind.engine.url) LOG.info('Looking for project: %s%s of %s', '%s/' % namespacerepo if namespace else '', @@ -90,9 +89,14 @@ def handle_messages(): LOG.info('Processing %s commits in %s', len(commits), abspath) pagure.lib.git.log_commits_to_db( - pagure.SESSION, project, commits, abspath) - - session.close() + session, project, commits, abspath) + + try: + session.commit() + except SQLAlchemyError as err: # pragma: no cover + session.rollback() + finally: + session.close() LOG.info('Ready for another') From 76d5e093fd4ab8bca26bbf40f0f7ad33f2ccf656 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Dec 22 2016 10:52:08 +0000 Subject: [PATCH 7/8] Adjust pagure_logcom server based on feedback from the review - The import requests was in the wrong import block - We adjust the docstring of the handle_message method and included in it a small documentation regarding the expected format of message --- diff --git a/pagure-logcom/pagure_logcom_server.py b/pagure-logcom/pagure_logcom_server.py index c9ce468..6827027 100644 --- a/pagure-logcom/pagure_logcom_server.py +++ b/pagure-logcom/pagure_logcom_server.py @@ -20,8 +20,8 @@ time can be really time-consuming). import json import logging import os -import requests +import requests import trollius import trollius_redis @@ -41,8 +41,30 @@ import pagure.lib @trollius.coroutine def handle_messages(): ''' Handles connecting to redis and acting upon messages received. - In this case, it means triggering a build on jenkins based on the - information provided. + In this case, it means logging into the DB the commits specified in the + message for the specified repo. + + The currently accepted message format looks like: + + :: + + { + "project": { + "name": "foo", + "namespace": null, + "parent": null, + "username": { + "name": "user" + } + }, + "abspath": "/srv/git/repositories/pagure.git", + "commits": [ + "b7b4059c44d692d7df3227ce58ce01191e5407bd", + "f8d0899bb6654590ffdef66b539fd3b8cf873b35", + "9b6fdc48d3edab82d3de28953271ea52b0a96117" + ] + } + ''' host = pagure.APP.config.get('REDIS_HOST', '0.0.0.0') From e2027b47724f359ec4f24884f4ad9bdb0b41461b Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Dec 22 2016 10:52:08 +0000 Subject: [PATCH 8/8] Rename the LOG variable to _log, it's a private variable to that module --- diff --git a/pagure-logcom/pagure_logcom_server.py b/pagure-logcom/pagure_logcom_server.py index 6827027..0861610 100644 --- a/pagure-logcom/pagure_logcom_server.py +++ b/pagure-logcom/pagure_logcom_server.py @@ -26,7 +26,7 @@ import trollius import trollius_redis -LOG = logging.getLogger(__name__) +_log = logging.getLogger(__name__) if 'PAGURE_CONFIG' not in os.environ \ and os.path.exists('/etc/pagure/pagure.cfg'): @@ -82,7 +82,7 @@ def handle_messages(): # Inside a while loop, wait for incoming events. while True: reply = yield trollius.From(subscriber.next_published()) - LOG.info( + _log.info( 'Received: %s on channel: %s', repr(reply.value), reply.channel) data = json.loads(reply.value) @@ -96,19 +96,19 @@ def handle_messages(): session = pagure.lib.create_session(pagure.APP.config['DB_URL']) - LOG.info('Looking for project: %s%s of %s', + _log.info('Looking for project: %s%s of %s', '%s/' % namespacerepo if namespace else '', repo, username) project = pagure.lib.get_project( pagure.SESSION, repo, user=username, namespace=namespace) if not project: - LOG.info('No project found') + _log.info('No project found') continue - LOG.info('Found project: %s', project.fullname) + _log.info('Found project: %s', project.fullname) - LOG.info('Processing %s commits in %s', len(commits), abspath) + _log.info('Processing %s commits in %s', len(commits), abspath) pagure.lib.git.log_commits_to_db( session, project, commits, abspath) @@ -119,7 +119,7 @@ def handle_messages(): session.rollback() finally: session.close() - LOG.info('Ready for another') + _log.info('Ready for another') def main(): @@ -137,9 +137,9 @@ def main(): except trollius.ConnectionResetError: pass - LOG.info("End Connection") + _log.info("End Connection") loop.close() - LOG.info("End") + _log.info("End") if __name__ == '__main__': @@ -149,7 +149,7 @@ if __name__ == '__main__': logging.basicConfig(level=logging.DEBUG) # setup console logging - LOG.setLevel(logging.DEBUG) + _log.setLevel(logging.DEBUG) shellhandler = logging.StreamHandler() shellhandler.setLevel(logging.DEBUG) @@ -159,5 +159,5 @@ if __name__ == '__main__': aslog.setLevel(logging.DEBUG) shellhandler.setFormatter(formatter) - LOG.addHandler(shellhandler) + _log.addHandler(shellhandler) main()