From 74ab5f252a98f029160625681649d60b0abaada5 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Feb 15 2017 09:05:26 +0000 Subject: [PATCH 1/7] Ensure there are message(s) returned before storing it Otherwise we were ending up with an error when trying to format the list. --- diff --git a/pagure/lib/git.py b/pagure/lib/git.py index f47aa34..d81d2ea 100644 --- a/pagure/lib/git.py +++ b/pagure/lib/git.py @@ -540,7 +540,7 @@ def update_ticket_from_git( else: # Edit existing issue - messages.extend(pagure.lib.edit_issue( + msgs = pagure.lib.edit_issue( session, issue=issue, ticketfolder=None, @@ -551,7 +551,9 @@ def update_ticket_from_git( status=json_data.get('status'), close_status=json_data.get('close_status'), private=json_data.get('private'), - )) + ) + if msgs: + messages.extend(msgs) session.commit() @@ -610,30 +612,38 @@ def update_ticket_from_git( # Update tags tags = json_data.get('tags', []) - messages.extend(pagure.lib.update_tags( - session, issue, tags, username=user.user, ticketfolder=None)) + msgs = pagure.lib.update_tags( + session, issue, tags, username=user.user, ticketfolder=None) + if msgs: + messages.extend(msgs) # Update assignee assignee = get_user_from_json(session, json_data, key='assignee') if assignee: - messages.append(pagure.lib.add_issue_assignee( + msg = pagure.lib.add_issue_assignee( session, issue, assignee.username, - user=user.user, ticketfolder=None, notify=False)) + user=user.user, ticketfolder=None, notify=False) + if msg: + messages.append(msg) # Update depends depends = json_data.get('depends', []) - messages.extend(pagure.lib.update_dependency_issue( + msgs = pagure.lib.update_dependency_issue( session, issue.project, issue, depends, - username=user.user, ticketfolder=None)) + username=user.user, ticketfolder=None) + if msgs: + messages.extend(msgs) # Update blocks blocks = json_data.get('blocks', []) - messages.extend(pagure.lib.update_blocked_issue( + msgs = pagure.lib.update_blocked_issue( session, issue.project, issue, blocks, - username=user.user, ticketfolder=None)) + username=user.user, ticketfolder=None) + if msgs: + messages.extend(msgs) for comment in json_data['comments']: - user = get_user_from_json(session, comment) + usercomment = get_user_from_json(session, comment) commentobj = pagure.lib.get_issue_comment( session, issue_uid, comment['id']) if not commentobj: @@ -641,12 +651,13 @@ def update_ticket_from_git( session, issue=issue, comment=comment['comment'], - user=user.username, + user=usercomment.username, ticketfolder=None, notify=False, date_created=datetime.datetime.utcfromtimestamp( float(comment['date_created'])), ) + if messages: pagure.lib.add_metadata_update_notif( session=session, From c82b7188ec7c99d687a3add36e610ec02ae2d734 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Feb 15 2017 09:05:26 +0000 Subject: [PATCH 2/7] Introduce the pagure-loadjson service This service takes the task of updating the database from the information present in the git repo off the git hook and into its own process. It will make migrating large repo to pagure an easier process. It also offers an easier monitoring of the progress. --- diff --git a/pagure-loadjson/README.rst b/pagure-loadjson/README.rst new file mode 100644 index 0000000..d2223d5 --- /dev/null +++ b/pagure-loadjson/README.rst @@ -0,0 +1,13 @@ +Pagure loadjson +=============== + +This is the service loads into the database the JSON files representing +issues or pull-requests. + +This service is triggered by a git hook, sending a notification that a push +happened. This service receive the notification and find the list of file +that changed and load them into the database. + + * Run:: + + PAGURE_CONFIG=/path/to/config PYTHONPATH=. python pagure-loadjson/pagure_loadjson_server.py diff --git a/pagure-loadjson/pagure_loadjson.service b/pagure-loadjson/pagure_loadjson.service new file mode 100644 index 0000000..2c963a6 --- /dev/null +++ b/pagure-loadjson/pagure_loadjson.service @@ -0,0 +1,14 @@ +[Unit] +Description=Pagure service loading JSON files into the DB +After=redis.target +Documentation=https://pagure.io/pagure + +[Service] +ExecStart=/usr/libexec/pagure-loadjson/pagure_loadjson_server.py +Type=simple +User=git +Group=git +Restart=on-failure + +[Install] +WantedBy=multi-user.target diff --git a/pagure-loadjson/pagure_loadjson_server.py b/pagure-loadjson/pagure_loadjson_server.py new file mode 100644 index 0000000..8848b08 --- /dev/null +++ b/pagure-loadjson/pagure_loadjson_server.py @@ -0,0 +1,230 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +""" + (c) 2017 - Copyright Red Hat Inc + + Authors: + Pierre-Yves Chibon + + +This server listens to message sent to redis via post commits hook and find +the list of files modified by the commits listed in the message and sync +them into the database. + +Using this mechanism, we no longer need to block the git push until all the +files have been uploaded (which when migrating some large projects over to +pagure can be really time-consuming). + +""" + +import json +import logging +import os + +import requests +import trollius +import trollius_redis + + +_log = logging.getLogger(__name__) + +if 'PAGURE_CONFIG' not in os.environ \ + and os.path.exists('/etc/pagure/pagure.cfg'): + print 'Using configuration file `/etc/pagure/pagure.cfg`' + os.environ['PAGURE_CONFIG'] = '/etc/pagure/pagure.cfg' + + +import pagure +import pagure.lib + + +def get_files_to_load(title, new_commits_list, abspath): + + _log.info('%s: Retrieve the list of files changed' % title) + file_list = [] + new_commits_list.reverse() + n = len(new_commits_list) + for idx, commit in enumerate(new_commits_list): + if (idx % 100) == 0: + _log.info( + 'Loading files change in commits for %s: %s/%s', + title, idx, n) + if commit == new_commits_list[0]: + filenames = pagure.lib.git.read_git_lines( + ['diff-tree', '--no-commit-id', '--name-only', '-r', '--root', + commit], abspath) + else: + filenames = pagure.lib.git.read_git_lines( + ['diff-tree', '--no-commit-id', '--name-only', '-r', commit], + abspath) + for line in filenames: + if line.strip(): + file_list.append(line.strip()) + + return file_list + + +@trollius.coroutine +def handle_messages(): + ''' Handles connecting to redis and acting upon messages received. + In this case, it means logging into the DB the commits specified in the + message for the specified repo. + + The currently accepted message format looks like: + + :: + + { + "project": { + "name": "foo", + "namespace": null, + "parent": null, + "username": { + "name": "user" + } + }, + "abspath": "/srv/git/repositories/pagure.git", + "commits": [ + "b7b4059c44d692d7df3227ce58ce01191e5407bd", + "f8d0899bb6654590ffdef66b539fd3b8cf873b35", + "9b6fdc48d3edab82d3de28953271ea52b0a96117" + ] + } + + ''' + + host = pagure.APP.config.get('REDIS_HOST', '0.0.0.0') + port = pagure.APP.config.get('REDIS_PORT', 6379) + dbname = pagure.APP.config.get('REDIS_DB', 0) + connection = yield trollius.From(trollius_redis.Connection.create( + host=host, port=port, db=dbname)) + + # Create subscriber. + subscriber = yield trollius.From(connection.start_subscribe()) + + # Subscribe to channel. + yield trollius.From(subscriber.subscribe(['pagure.loadjson'])) + + # Inside a while loop, wait for incoming events. + while True: + reply = yield trollius.From(subscriber.next_published()) + _log.info( + 'Received: %s on channel: %s', + repr(reply.value), reply.channel) + data = json.loads(reply.value) + + commits = data['commits'] + abspath = data['abspath'] + repo = data['project']['name'] + username = data['project']['username']['name'] \ + if data['project']['parent'] else None + namespace = data['project']['namespace'] + data_type = data['data_type'] + + if data_type not in ['ticket', 'pull-request']: + _log.info('Invalid data_type retrieved: %s', data_type) + continue + + session = pagure.lib.create_session(pagure.APP.config['DB_URL']) + + _log.info('Looking for project: %s%s of user: %s', + '%s/' % namespacerepo if namespace else '', + repo, username) + project = pagure.lib.get_project( + session, repo, user=username, namespace=namespace) + + if not project: + _log.info('No project found') + continue + + _log.info('Found project: %s', project.fullname) + + _log.info( + '%s: Processing %s commits in %s', project.fullname, + len(commits), abspath) + + file_list = set(get_files_to_load(project.fullname, commits, abspath)) + n = len(file_list) + _log.info('%s files to process' % n) + + for idx, filename in enumerate(file_list): + _log.info('Loading: %s -- %s/%s', filename, idx, n) + json_data = None + data = ''.join( + pagure.lib.git.read_git_lines( + ['show', 'HEAD:%s' % filename], abspath)) + if data and not filename.startswith('files/'): + try: + json_data = json.loads(data) + except: + pass + if json_data: + try: + if data_type == 'ticket': + pagure.lib.git.update_ticket_from_git( + session, + reponame=repo, + namespace=namespace, + username=username, + issue_uid=filename, + json_data=json_data + ) + except Exception as err: + _log.info('data: %s', json_data) + session.rollback() + _log.exception(err) + break + + try: + session.commit() + except SQLAlchemyError as err: # pragma: no cover + session.rollback() + finally: + session.close() + _log.info('Ready for another') + + +def main(): + ''' Start the main async loop. ''' + + try: + loop = trollius.get_event_loop() + tasks = [ + trollius.async(handle_messages()), + ] + loop.run_until_complete(trollius.wait(tasks)) + loop.run_forever() + except KeyboardInterrupt: + pass + except trollius.ConnectionResetError: + pass + + _log.info("End Connection") + loop.close() + _log.info("End") + + +if __name__ == '__main__': + formatter = logging.Formatter( + "%(asctime)s %(levelname)s [%(module)s:%(lineno)d] %(message)s") + + logging.basicConfig(level=logging.DEBUG) + + # setup console logging + _log.setLevel(logging.DEBUG) + shellhandler = logging.StreamHandler() + shellhandler.setLevel(logging.DEBUG) + + aslog = logging.getLogger("asyncio") + aslog.setLevel(logging.DEBUG) + aslog = logging.getLogger("trollius") + aslog.setLevel(logging.DEBUG) + + # Turn down the logs coming from python-markdown + mklog = logging.getLogger("MARKDOWN") + mklog.setLevel(logging.WARN) + + shellhandler.setFormatter(formatter) + _log.addHandler(shellhandler) + main() diff --git a/pagure/hooks/files/pagure_hook_tickets.py b/pagure/hooks/files/pagure_hook_tickets.py old mode 100755 new mode 100644 index 18847db..f684ce3 --- a/pagure/hooks/files/pagure_hook_tickets.py +++ b/pagure/hooks/files/pagure_hook_tickets.py @@ -4,6 +4,7 @@ """Pagure specific hook to update tickets stored in the database based on the information pushed in the tickets git repository. """ +from __future__ import print_function import json import os @@ -20,81 +21,57 @@ if 'PAGURE_CONFIG' not in os.environ \ import pagure import pagure.lib.git - -abspath = os.path.abspath(os.environ['GIT_DIR']) +from pagure.lib import REDIS -def get_files_to_load(new_commits_list): +abspath = os.path.abspath(os.environ['GIT_DIR']) - print 'Files changed by new commits:\n' - file_list = [] - new_commits_list.reverse() - for commit in new_commits_list: - if commit == new_commits_list[0]: - filenames = pagure.lib.git.read_git_lines( - ['diff-tree', '--no-commit-id', '--name-only', '-r', '--root', - commit], abspath) - else: - filenames = pagure.lib.git.read_git_lines( - ['diff-tree', '--no-commit-id', '--name-only', '-r', commit], - abspath) - for line in filenames: - if line.strip(): - file_list.append(line.strip()) - return file_list +def run_as_post_receive_hook(): + repo = pagure.lib.git.get_repo_name(abspath) + username = pagure.lib.git.get_username(abspath) + namespace = pagure.lib.git.get_repo_namespace( + abspath, gitfolder=pagure.APP.config['TICKETS_FOLDER']) + if pagure.APP.config.get('HOOK_DEBUG', False): + print('repo:', repo) + print('user:', username) + print('namespace:', namespace) -def run_as_post_receive_hook(): + project = pagure.lib.get_project( + pagure.SESSION, repo, user=username, namespace=namespace) - file_list = set() for line in sys.stdin: if pagure.APP.config.get('HOOK_DEBUG', False): - print line + print(line) (oldrev, newrev, refname) = line.strip().split(' ', 2) if pagure.APP.config.get('HOOK_DEBUG', False): - print ' -- Old rev' - print oldrev - print ' -- New rev' - print newrev - print ' -- Ref name' - print refname + print(' -- Old rev') + print(oldrev) + print(' -- New rev') + print(newrev) + print(' -- Ref name') + print(refname) if set(newrev) == set(['0']): - print "Deleting a reference/branch, so we won't run the "\ - "pagure hook" + print("Deleting a reference/branch, so we won't run the " + "pagure hook") return - tmp = set(get_files_to_load( - pagure.lib.git.get_revs_between(oldrev, newrev, abspath, refname))) - file_list = file_list.union(tmp) - - reponame = pagure.lib.git.get_repo_name(abspath) - username = pagure.lib.git.get_username(abspath) - namespace = pagure.lib.git.get_repo_namespace( - abspath, gitfolder=pagure.APP.config['TICKETS_FOLDER']) - print 'repo:', reponame, username, namespace - - for filename in file_list: - print 'To load: %s' % filename - json_data = None - data = ''.join( - pagure.lib.git.read_git_lines( - ['show', 'HEAD:%s' % filename], abspath)) - if data and 'files' not in filename: - try: - json_data = json.loads(data) - except: - pass - if json_data: - pagure.lib.git.update_ticket_from_git( - pagure.SESSION, - reponame=reponame, - namespace=namespace, - username=username, - issue_uid=filename, - json_data=json_data) + commits = pagure.lib.git.get_revs_between( + oldrev, newrev, abspath, refname) + + if REDIS: + print('Sending to redis to log activity') + REDIS.publish('pagure.loadjson', + json.dumps({ + 'project': project.to_json(public=True), + 'abspath': abspath, + 'commits': commits, + 'data_type': 'ticket', + } + )) def main(args): From 69437809a79d28829a5d7798a5502875e94bb4a6 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Feb 15 2017 09:05:26 +0000 Subject: [PATCH 3/7] Email the user about the output of the upload to the DB With this change we will inform the user by email about the outcome of the load. If an error has occurred, it will be included in the email, so that an admin can then look further into it. --- diff --git a/pagure-loadjson/pagure_loadjson_server.py b/pagure-loadjson/pagure_loadjson_server.py index 8848b08..8ab6f91 100644 --- a/pagure-loadjson/pagure_loadjson_server.py +++ b/pagure-loadjson/pagure_loadjson_server.py @@ -21,11 +21,13 @@ pagure can be really time-consuming). import json import logging import os +import traceback import requests import trollius import trollius_redis +from sqlalchemy.exc import SQLAlchemyError _log = logging.getLogger(__name__) @@ -36,9 +38,29 @@ if 'PAGURE_CONFIG' not in os.environ \ import pagure +import pagure.exceptions import pagure.lib +import pagure.lib.notify +def format_callstack(): + """ Format the callstack to find out the stack trace. """ + ind = 0 + for ind, frame in enumerate(f[0] for f in inspect.stack()): + if '__name__' not in frame.f_globals: + continue + modname = frame.f_globals['__name__'].split('.')[0] + if modname != "logging": + break + + def _format_frame(frame): + """ Format the frame. """ + return ' File "%s", line %i in %s\n %s' % (frame) + + stack = traceback.extract_stack() + stack = stack[:-ind] + return "\n".join([_format_frame(frame) for frame in stack]) + def get_files_to_load(title, new_commits_list, abspath): _log.info('%s: Retrieve the list of files changed' % title) @@ -89,7 +111,9 @@ def handle_messages(): "b7b4059c44d692d7df3227ce58ce01191e5407bd", "f8d0899bb6654590ffdef66b539fd3b8cf873b35", "9b6fdc48d3edab82d3de28953271ea52b0a96117" - ] + ], + "data_type": "ticket", + "agent": "pingou", } ''' @@ -121,6 +145,7 @@ def handle_messages(): if data['project']['parent'] else None namespace = data['project']['namespace'] data_type = data['data_type'] + agent = data['agent'] if data_type not in ['ticket', 'pull-request']: _log.info('Invalid data_type retrieved: %s', data_type) @@ -147,9 +172,11 @@ def handle_messages(): file_list = set(get_files_to_load(project.fullname, commits, abspath)) n = len(file_list) _log.info('%s files to process' % n) + mail_body = [] for idx, filename in enumerate(file_list): - _log.info('Loading: %s -- %s/%s', filename, idx, n) + _log.info('Loading: %s -- %s/%s', filename, idx+1, n) + tmp = 'Loading: %s -- %s/%s' % (filename, idx+1, n) json_data = None data = ''.join( pagure.lib.git.read_git_lines( @@ -170,14 +197,32 @@ def handle_messages(): issue_uid=filename, json_data=json_data ) + tmp += ' ... ... Done' except Exception as err: _log.info('data: %s', json_data) session.rollback() _log.exception(err) + tmp += ' ... ... FAILED\n' + tmp += format_callstack() break + finally: + mail_body.append(tmp) try: session.commit() + _log.info( + 'Emailing results for %s to %s', project.fullname, agent) + try: + if not agent: + raise pagure.exceptions.PagureException( + 'No agent found: %s' % agent) + user_obj = pagure.lib.get_user(session, agent) + pagure.lib.notify.send_email( + '\n'.join(mail_body), + 'Issue import report', + user_obj.default_email) + except pagure.exceptions.PagureException as err: + _log.exception('Could not find user %s' % agent) except SQLAlchemyError as err: # pragma: no cover session.rollback() finally: diff --git a/pagure/hooks/files/pagure_hook_tickets.py b/pagure/hooks/files/pagure_hook_tickets.py index f684ce3..84b1150 100644 --- a/pagure/hooks/files/pagure_hook_tickets.py +++ b/pagure/hooks/files/pagure_hook_tickets.py @@ -63,15 +63,21 @@ def run_as_post_receive_hook(): oldrev, newrev, abspath, refname) if REDIS: - print('Sending to redis to log activity') + print('Sending to redis to load the data') REDIS.publish('pagure.loadjson', json.dumps({ 'project': project.to_json(public=True), 'abspath': abspath, 'commits': commits, 'data_type': 'ticket', + 'agent': os.environ.get('GL_USER'), } )) + print( + 'A report will be emailed to you once the load is finished') + else: + print('Hook not configured to connect to pagure-loadjson') + print('/!\ Your data will not be loaded into the database!') def main(args): From bb9ead388abe935ebdafaf550175b86df8135dd3 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Feb 15 2017 09:05:26 +0000 Subject: [PATCH 4/7] Indicate also the skipped files This way we'll have all the steps and not go from (for example) file 3 to file 5. --- diff --git a/pagure-loadjson/pagure_loadjson_server.py b/pagure-loadjson/pagure_loadjson_server.py index 8ab6f91..0b4337a 100644 --- a/pagure-loadjson/pagure_loadjson_server.py +++ b/pagure-loadjson/pagure_loadjson_server.py @@ -207,6 +207,9 @@ def handle_messages(): break finally: mail_body.append(tmp) + else: + tmp += ' ... ... SKIPPED - No JSON data' + mail_body.append(tmp) try: session.commit() From fa9a6c203bdd654ab04cb8151909f1aaade1eb15 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Feb 15 2017 09:05:26 +0000 Subject: [PATCH 5/7] Ship pagure-loadjson in the tarball and install it in the spec file --- diff --git a/MANIFEST.in b/MANIFEST.in index 35104f1..c47d045 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -10,3 +10,4 @@ recursive-include doc * recursive-include alembic * recursive-include ev-server * recursive-include webhook-server * +recursive-include pagure-loadjson * diff --git a/files/pagure.spec b/files/pagure.spec index c8c3ee1..6945ed2 100644 --- a/files/pagure.spec +++ b/files/pagure.spec @@ -175,6 +175,23 @@ pagure-logcom contains the service that logs commits into the database so that the activity calendar heatmap is filled. +%package loadjson +Summary: The loadjson service for pagure +BuildArch: noarch + +BuildRequires: systemd-devel +Requires: python-redis +Requires: python-trollius +Requires: python-trollius-redis +Requires(post): systemd +Requires(preun): systemd +Requires(postun): systemd +%description loadjson +pagure-loadjson is the service allowing to update the database with the +information provided in the JSON blobs that are stored in the tickets and +pull-requests git repo. + + %prep %setup -q @@ -251,6 +268,13 @@ install -m 755 pagure-logcom/pagure_logcom_server.py \ install -m 644 pagure-logcom/pagure_logcom.service \ $RPM_BUILD_ROOT/%{_unitdir}/pagure_logcom.service +# Install the loadjson service +mkdir -p $RPM_BUILD_ROOT/%{_libexecdir}/pagure-loadjson +install -m 755 pagure-loadjson/pagure_loadjson_server.py \ + $RPM_BUILD_ROOT/%{_libexecdir}/pagure-loadjson/pagure_loadjson_server.py +install -m 644 pagure-loadjson/pagure_loadjson.service \ + $RPM_BUILD_ROOT/%{_unitdir}/pagure_loadjson.service + %post milters %systemd_post pagure_milter.service @@ -262,6 +286,8 @@ install -m 644 pagure-logcom/pagure_logcom.service \ %systemd_post pagure_ci.service %post logcom %systemd_post pagure_logcom.service +%post loadjson +%systemd_post pagure_loadjson.service %preun milters %systemd_preun pagure_milter.service @@ -273,6 +299,8 @@ install -m 644 pagure-logcom/pagure_logcom.service \ %systemd_preun pagure_ci.service %preun logcom %systemd_preun pagure_logcom.service +%preun loadjson +%systemd_preun pagure_loadjson.service %postun milters %systemd_postun_with_restart pagure_milter.service @@ -284,6 +312,8 @@ install -m 644 pagure-logcom/pagure_logcom.service \ %systemd_postun_with_restart pagure_ci.service %postun logcom %systemd_postun_with_restart pagure_logcom.service +%postun loadjson +%systemd_postun_with_restart pagure_loadjson.service %files @@ -334,6 +364,12 @@ install -m 644 pagure-logcom/pagure_logcom.service \ %{_unitdir}/pagure_logcom.service +%files loadjson +%license LICENSE +%{_libexecdir}/pagure-loadjson/ +%{_unitdir}/pagure_loadjson.service + + %changelog * Mon Feb 13 2017 Pierre-Yves Chibon - 2.12.1-1 - Update to 2.12.1 From 196048f6a417c68bd112ffecdd7453b986afae38 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Feb 15 2017 09:05:26 +0000 Subject: [PATCH 6/7] Be honest about which data type loadjson supports atm --- diff --git a/files/pagure.spec b/files/pagure.spec index 6945ed2..2fd39b5 100644 --- a/files/pagure.spec +++ b/files/pagure.spec @@ -188,8 +188,8 @@ Requires(preun): systemd Requires(postun): systemd %description loadjson pagure-loadjson is the service allowing to update the database with the -information provided in the JSON blobs that are stored in the tickets and -pull-requests git repo. +information provided in the JSON blobs that are stored in the tickets (and +in the future pull-requests) git repo. %prep diff --git a/pagure-loadjson/README.rst b/pagure-loadjson/README.rst index d2223d5..10c1827 100644 --- a/pagure-loadjson/README.rst +++ b/pagure-loadjson/README.rst @@ -2,7 +2,7 @@ Pagure loadjson =============== This is the service loads into the database the JSON files representing -issues or pull-requests. +issues (and in the future also the pull-requests). This service is triggered by a git hook, sending a notification that a push happened. This service receive the notification and find the list of file From 1a6c11e1af3e642b6a4b46723d36cd273846c90c Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Feb 15 2017 09:05:26 +0000 Subject: [PATCH 7/7] Document how to install the loadjson service --- diff --git a/doc/index.rst b/doc/index.rst index d5ea43f..bf330f1 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -36,6 +36,7 @@ Contents: install_evs install_webhooks install_pagure_ci + install_pagure_loadjson configuration development contributing diff --git a/doc/install_pagure_loadjson.rst b/doc/install_pagure_loadjson.rst new file mode 100644 index 0000000..1f743aa --- /dev/null +++ b/doc/install_pagure_loadjson.rst @@ -0,0 +1,47 @@ +Installing pagure-loadjson +========================== + +pagure-loadjson is the service that updates the database based on the content +of the JSON blob pushed into the ticket git repository (and in the future +for pull-requests as well). + + +Configure your system +--------------------- + +* Install the required dependencies + +:: + + python-redis + python-trollius-redis + python-trollius + +.. note:: We ship a systemd unit file for pagure_loadjson but we welcome patches + for scripts for other init systems. + + +* Install the files of pagure-loadjon as follow: + ++--------------------------------------------------+----------------------------------------------------+ +| Source | Destination | ++==================================================+====================================================+ +| ``pagure-loadjson/pagure_loadjson_server.py`` | ``/usr/libexec/pagure-loadjson/pagure_loadjson.py``| ++--------------------------------------------------+----------------------------------------------------+ +| ``pagure-loadjson/pagure_loadjson.service`` | ``/etc/systemd/system/pagure_loadjson.service`` | ++--------------------------------------------------+----------------------------------------------------+ + +The first file is the pagure-loadjson service itself, triggered by the git +hook (shipped with pagure itself) and loading the JSON files into the database. + +The second file is the systemd service file. + + +* Activate the service and ensure it's started upon boot: + +:: + + systemctl enable redis + systemctl start redis + systemctl enable pagure_loadjson + systemctl start pagure_loadjson