From 23eab50d5b00e964c0a9e8abc4f382d1324c8b9f Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Jun 13 2017 21:01:30 +0000 Subject: [PATCH 1/7] Rename webhook-server into pagure-webhook This will make this service easier to identify and more consistent with the other services we ship. --- diff --git a/doc/install_webhooks.rst b/doc/install_webhooks.rst index 8e5e0b0..42531e9 100644 --- a/doc/install_webhooks.rst +++ b/doc/install_webhooks.rst @@ -29,9 +29,9 @@ Configure your system +----------------------------------------------+----------------------------------------------------------+ | Source | Destination | +==============================================+==========================================================+ -| ``webhook-server/pagure-webhook-server.py`` | ``/usr/libexec/pagure-webhook/pagure-webhook-server.py`` | +| ``pagure-webhook/pagure-webhook-server.py`` | ``/usr/libexec/pagure-webhook/pagure-webhook-server.py`` | +----------------------------------------------+----------------------------------------------------------+ -| ``webhook-server/pagure_webhook.service`` | ``/etc/systemd/system/pagure_webhook.service`` | +| ``pagure-webhook/pagure_webhook.service`` | ``/etc/systemd/system/pagure_webhook.service`` | +----------------------------------------------+----------------------------------------------------------+ The first file is the script of the web-hook server itself. diff --git a/files/pagure.spec b/files/pagure.spec index 0cf7e9c..766c80f 100644 --- a/files/pagure.spec +++ b/files/pagure.spec @@ -254,9 +254,9 @@ install -m 644 ev-server/pagure_ev.service \ # Install the web-hook mkdir -p $RPM_BUILD_ROOT/%{_libexecdir}/pagure-webhook -install -m 755 webhook-server/pagure-webhook-server.py \ +install -m 755 pagure-webhook/pagure-webhook-server.py \ $RPM_BUILD_ROOT/%{_libexecdir}/pagure-webhook/pagure-webhook-server.py -install -m 644 webhook-server/pagure_webhook.service \ +install -m 644 pagure-webhook/pagure_webhook.service \ $RPM_BUILD_ROOT/%{_unitdir}/pagure_webhook.service # Install the ci service diff --git a/pagure-webhook/pagure-webhook-server.py b/pagure-webhook/pagure-webhook-server.py new file mode 100644 index 0000000..cbcb1dc --- /dev/null +++ b/pagure-webhook/pagure-webhook-server.py @@ -0,0 +1,191 @@ +#!/usr/bin/env python + +""" + (c) 2015 - Copyright Red Hat Inc + + Authors: + Pierre-Yves Chibon + + +This server listens to message sent via redis and send the corresponding +web-hook request. + +Using this mechanism, we no longer block the main application if the +receiving end is offline or so. + +""" + +import datetime +import hashlib +import hmac +import json +import logging +import os +import requests +import time +import uuid + +import six +import trollius +import trollius_redis + +from kitchen.text.converters import to_bytes + + +log = logging.getLogger(__name__) + + +if 'PAGURE_CONFIG' not in os.environ \ + and os.path.exists('/etc/pagure/pagure.cfg'): + print 'Using configuration file `/etc/pagure/pagure.cfg`' + os.environ['PAGURE_CONFIG'] = '/etc/pagure/pagure.cfg' + + +import pagure +import pagure.lib +from pagure.exceptions import PagureEvException + +_i = 0 + + +def call_web_hooks(project, topic, msg, urls): + ''' Sends the web-hook notification. ''' + log.info( + "Processing project: %s - topic: %s", project.fullname, topic) + log.debug('msg: %s', msg) + + # Send web-hooks notification + global _i + _i += 1 + year = datetime.datetime.now().year + if isinstance(topic, six.text_type): + topic = to_bytes(topic, encoding='utf8', nonstring="passthru") + msg['pagure_instance'] = pagure.APP.config['APP_URL'] + msg['project_fullname'] = project.fullname + msg = dict( + topic=topic.decode('utf-8'), + msg=msg, + timestamp=int(time.time()), + msg_id=str(year) + '-' + str(uuid.uuid4()), + i=_i, + ) + + content = json.dumps(msg) + hashhex = hmac.new( + str(project.hook_token), content, hashlib.sha1).hexdigest() + hashhex256 = hmac.new( + str(project.hook_token), content, hashlib.sha256).hexdigest() + headers = { + 'X-Pagure': pagure.APP.config['APP_URL'], + 'X-Pagure-project': project.fullname, + 'X-Pagure-Signature': hashhex, + 'X-Pagure-Signature-256': hashhex256, + 'X-Pagure-Topic': topic, + 'Content-Type': 'application/json', + } + for url in urls: + url = url.strip() + log.info('Calling url %s' % url) + try: + req = requests.post( + url, + headers=headers, + data=content, + timeout=60, + ) + if not req: + log.info( + 'An error occured while querying: %s - ' + 'Error code: %s' % (url, req.status_code)) + except (requests.exceptions.RequestException, Exception) as err: + log.info( + 'An error occured while querying: %s - Error: %s' % ( + url, err)) + + +@trollius.coroutine +def handle_messages(): + host = pagure.APP.config.get('REDIS_HOST', '0.0.0.0') + port = pagure.APP.config.get('REDIS_PORT', 6379) + dbname = pagure.APP.config.get('REDIS_DB', 0) + connection = yield trollius.From(trollius_redis.Connection.create( + host=host, port=port, db=dbname)) + + # Create subscriber. + subscriber = yield trollius.From(connection.start_subscribe()) + + # Subscribe to channel. + yield trollius.From(subscriber.subscribe(['pagure.hook'])) + + # Inside a while loop, wait for incoming events. + while True: + reply = yield trollius.From(subscriber.next_published()) + log.info( + 'Received: %s on channel: %s', + repr(reply.value), reply.channel) + data = json.loads(reply.value) + username = None + if data['project'].startswith('forks'): + username, projectname = data['project'].split('/', 2)[1:] + else: + projectname = data['project'] + + namespace = None + if '/' in projectname: + namespace, projectname = projectname.split('/', 1) + + log.info( + 'Searching %s/%s/%s' % (username, namespace, projectname)) + session = pagure.lib.create_session(pagure.APP.config['DB_URL']) + project = pagure.lib._get_project( + session=session, name=projectname, user=username, + namespace=namespace) + if not project: + log.info('No project found with these criteria') + session.close() + continue + urls = project.settings.get('Web-hooks') + session.close() + if not urls: + log.info('No URLs set: %s' % urls) + continue + urls = urls.split('\n') + log.info('Got the project, going to the webhooks') + call_web_hooks(project, data['topic'], data['msg'], urls) + + +def main(): + server = None + try: + loop = trollius.get_event_loop() + tasks = [ + trollius.async(handle_messages()), + ] + loop.run_until_complete(trollius.wait(tasks)) + loop.run_forever() + except KeyboardInterrupt: + pass + except trollius.ConnectionResetError: + pass + + log.info("End Connection") + loop.close() + log.info("End") + + +if __name__ == '__main__': + log = logging.getLogger("") + formatter = logging.Formatter( + "%(asctime)s %(levelname)s [%(module)s:%(lineno)d] %(message)s") + + # setup console logging + log.setLevel(logging.DEBUG) + ch = logging.StreamHandler() + ch.setLevel(logging.DEBUG) + + aslog = logging.getLogger("asyncio") + aslog.setLevel(logging.DEBUG) + + ch.setFormatter(formatter) + log.addHandler(ch) + main() diff --git a/pagure-webhook/pagure_webhook.service b/pagure-webhook/pagure_webhook.service new file mode 100644 index 0000000..e9d0512 --- /dev/null +++ b/pagure-webhook/pagure_webhook.service @@ -0,0 +1,14 @@ +[Unit] +Description=Pagure WebHook server (Allowing web-hook notifications) +After=redis.target +Documentation=https://pagure.io/pagure + +[Service] +ExecStart=/usr/libexec/pagure-webhook/pagure-webhook-server.py +Type=simple +User=git +Group=git +Restart=on-failure + +[Install] +WantedBy=multi-user.target diff --git a/webhook-server/pagure-webhook-server.py b/webhook-server/pagure-webhook-server.py deleted file mode 100644 index cbcb1dc..0000000 --- a/webhook-server/pagure-webhook-server.py +++ /dev/null @@ -1,191 +0,0 @@ -#!/usr/bin/env python - -""" - (c) 2015 - Copyright Red Hat Inc - - Authors: - Pierre-Yves Chibon - - -This server listens to message sent via redis and send the corresponding -web-hook request. - -Using this mechanism, we no longer block the main application if the -receiving end is offline or so. - -""" - -import datetime -import hashlib -import hmac -import json -import logging -import os -import requests -import time -import uuid - -import six -import trollius -import trollius_redis - -from kitchen.text.converters import to_bytes - - -log = logging.getLogger(__name__) - - -if 'PAGURE_CONFIG' not in os.environ \ - and os.path.exists('/etc/pagure/pagure.cfg'): - print 'Using configuration file `/etc/pagure/pagure.cfg`' - os.environ['PAGURE_CONFIG'] = '/etc/pagure/pagure.cfg' - - -import pagure -import pagure.lib -from pagure.exceptions import PagureEvException - -_i = 0 - - -def call_web_hooks(project, topic, msg, urls): - ''' Sends the web-hook notification. ''' - log.info( - "Processing project: %s - topic: %s", project.fullname, topic) - log.debug('msg: %s', msg) - - # Send web-hooks notification - global _i - _i += 1 - year = datetime.datetime.now().year - if isinstance(topic, six.text_type): - topic = to_bytes(topic, encoding='utf8', nonstring="passthru") - msg['pagure_instance'] = pagure.APP.config['APP_URL'] - msg['project_fullname'] = project.fullname - msg = dict( - topic=topic.decode('utf-8'), - msg=msg, - timestamp=int(time.time()), - msg_id=str(year) + '-' + str(uuid.uuid4()), - i=_i, - ) - - content = json.dumps(msg) - hashhex = hmac.new( - str(project.hook_token), content, hashlib.sha1).hexdigest() - hashhex256 = hmac.new( - str(project.hook_token), content, hashlib.sha256).hexdigest() - headers = { - 'X-Pagure': pagure.APP.config['APP_URL'], - 'X-Pagure-project': project.fullname, - 'X-Pagure-Signature': hashhex, - 'X-Pagure-Signature-256': hashhex256, - 'X-Pagure-Topic': topic, - 'Content-Type': 'application/json', - } - for url in urls: - url = url.strip() - log.info('Calling url %s' % url) - try: - req = requests.post( - url, - headers=headers, - data=content, - timeout=60, - ) - if not req: - log.info( - 'An error occured while querying: %s - ' - 'Error code: %s' % (url, req.status_code)) - except (requests.exceptions.RequestException, Exception) as err: - log.info( - 'An error occured while querying: %s - Error: %s' % ( - url, err)) - - -@trollius.coroutine -def handle_messages(): - host = pagure.APP.config.get('REDIS_HOST', '0.0.0.0') - port = pagure.APP.config.get('REDIS_PORT', 6379) - dbname = pagure.APP.config.get('REDIS_DB', 0) - connection = yield trollius.From(trollius_redis.Connection.create( - host=host, port=port, db=dbname)) - - # Create subscriber. - subscriber = yield trollius.From(connection.start_subscribe()) - - # Subscribe to channel. - yield trollius.From(subscriber.subscribe(['pagure.hook'])) - - # Inside a while loop, wait for incoming events. - while True: - reply = yield trollius.From(subscriber.next_published()) - log.info( - 'Received: %s on channel: %s', - repr(reply.value), reply.channel) - data = json.loads(reply.value) - username = None - if data['project'].startswith('forks'): - username, projectname = data['project'].split('/', 2)[1:] - else: - projectname = data['project'] - - namespace = None - if '/' in projectname: - namespace, projectname = projectname.split('/', 1) - - log.info( - 'Searching %s/%s/%s' % (username, namespace, projectname)) - session = pagure.lib.create_session(pagure.APP.config['DB_URL']) - project = pagure.lib._get_project( - session=session, name=projectname, user=username, - namespace=namespace) - if not project: - log.info('No project found with these criteria') - session.close() - continue - urls = project.settings.get('Web-hooks') - session.close() - if not urls: - log.info('No URLs set: %s' % urls) - continue - urls = urls.split('\n') - log.info('Got the project, going to the webhooks') - call_web_hooks(project, data['topic'], data['msg'], urls) - - -def main(): - server = None - try: - loop = trollius.get_event_loop() - tasks = [ - trollius.async(handle_messages()), - ] - loop.run_until_complete(trollius.wait(tasks)) - loop.run_forever() - except KeyboardInterrupt: - pass - except trollius.ConnectionResetError: - pass - - log.info("End Connection") - loop.close() - log.info("End") - - -if __name__ == '__main__': - log = logging.getLogger("") - formatter = logging.Formatter( - "%(asctime)s %(levelname)s [%(module)s:%(lineno)d] %(message)s") - - # setup console logging - log.setLevel(logging.DEBUG) - ch = logging.StreamHandler() - ch.setLevel(logging.DEBUG) - - aslog = logging.getLogger("asyncio") - aslog.setLevel(logging.DEBUG) - - ch.setFormatter(formatter) - log.addHandler(ch) - main() diff --git a/webhook-server/pagure_webhook.service b/webhook-server/pagure_webhook.service deleted file mode 100644 index e9d0512..0000000 --- a/webhook-server/pagure_webhook.service +++ /dev/null @@ -1,14 +0,0 @@ -[Unit] -Description=Pagure WebHook server (Allowing web-hook notifications) -After=redis.target -Documentation=https://pagure.io/pagure - -[Service] -ExecStart=/usr/libexec/pagure-webhook/pagure-webhook-server.py -Type=simple -User=git -Group=git -Restart=on-failure - -[Install] -WantedBy=multi-user.target From 132515af96592a4b6173767dc9d7c0616aad8a3b Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Jun 13 2017 21:01:30 +0000 Subject: [PATCH 2/7] Rename ev-server into pagure-ev This will make this service easier to identify and more consistent with the other services we ship. --- diff --git a/MANIFEST.in b/MANIFEST.in index c47d045..a3a2897 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -8,6 +8,6 @@ recursive-include milters * recursive-include tests * recursive-include doc * recursive-include alembic * -recursive-include ev-server * -recursive-include webhook-server * +recursive-include pagure-ev * +recursive-include pagure-webhook * recursive-include pagure-loadjson * diff --git a/ansible/roles/pagure-dev/files/pagure_ev.service b/ansible/roles/pagure-dev/files/pagure_ev.service index 573f99d..9b9a821 100644 --- a/ansible/roles/pagure-dev/files/pagure_ev.service +++ b/ansible/roles/pagure-dev/files/pagure_ev.service @@ -6,7 +6,7 @@ Documentation=https://pagure.io/pagure [Service] Environment="PAGURE_CONFIG=/home/vagrant/pagure.cfg" ExecStart=/home/vagrant/.virtualenvs/python2-pagure/bin/python \ - /home/vagrant/devel/ev-server/pagure_stream_server.py + /home/vagrant/devel/pagure-ev/pagure_stream_server.py Type=simple [Install] diff --git a/doc/install_evs.rst b/doc/install_evs.rst index 0ba7edc..13c95fb 100644 --- a/doc/install_evs.rst +++ b/doc/install_evs.rst @@ -28,9 +28,9 @@ The eventsource server is easy to set-up. +----------------------------------------+-----------------------------------------------------+ | Source | Destination | +========================================+=====================================================+ -| ``ev-server/pagure_stream_server.py`` | ``/usr/libexec/pagure-ev/pagure_stream_server.py`` | +| ``pagure-ev/pagure_stream_server.py`` | ``/usr/libexec/pagure-ev/pagure_stream_server.py`` | +----------------------------------------+-----------------------------------------------------+ -| ``ev-server/pagure_ev.service`` | ``/etc/systemd/system/pagure_ev.service`` | +| ``pagure-ev/pagure_ev.service`` | ``/etc/systemd/system/pagure_ev.service`` | +----------------------------------------+-----------------------------------------------------+ The first file is the script of the SSE server itself. diff --git a/docker/ev b/docker/ev index 0e225d5..9b1753f 100644 --- a/docker/ev +++ b/docker/ev @@ -17,7 +17,7 @@ RUN dnf install -y python2-devel python-setuptools python-nose py-bcrypt python- RUN dnf install -y python2-celery WORKDIR /code -ENTRYPOINT ["/usr/bin/python", "/code/ev-server/pagure_stream_server.py"] +ENTRYPOINT ["/usr/bin/python", "/code/pagure-ev/pagure_stream_server.py"] # Code injection is last to make optimal use of caches VOLUME ["/code"] diff --git a/ev-server/pagure_ev.service b/ev-server/pagure_ev.service deleted file mode 100644 index 27e864b..0000000 --- a/ev-server/pagure_ev.service +++ /dev/null @@ -1,14 +0,0 @@ -[Unit] -Description=Pagure EventSource server (Allowing live refresh of the pages supporting it) -After=redis.target -Documentation=https://pagure.io/pagure - -[Service] -ExecStart=/usr/libexec/pagure-ev/pagure_stream_server.py -Type=simple -User=git -Group=git -Restart=on-failure - -[Install] -WantedBy=multi-user.target diff --git a/ev-server/pagure_stream_server.py b/ev-server/pagure_stream_server.py deleted file mode 100644 index c7c0a36..0000000 --- a/ev-server/pagure_stream_server.py +++ /dev/null @@ -1,327 +0,0 @@ -#!/usr/bin/env python - -""" - (c) 2015-2017 - Copyright Red Hat Inc - - Authors: - Pierre-Yves Chibon - - -Streaming server for pagure's eventsource feature -This server takes messages sent to redis and publish them at the specified -endpoint - -To test, run this script and in another terminal -nc localhost 8080 - HELLO - - GET /test/issue/26?foo=bar HTTP/1.1 - -""" - -import logging -import os -import urlparse - -import redis -import trollius - -log = logging.getLogger(__name__) - - -if 'PAGURE_CONFIG' not in os.environ \ - and os.path.exists('/etc/pagure/pagure.cfg'): - print 'Using configuration file `/etc/pagure/pagure.cfg`' - os.environ['PAGURE_CONFIG'] = '/etc/pagure/pagure.cfg' - - -import pagure # noqa: E402 -import pagure.lib # noqa: E402 -from pagure.exceptions import PagureEvException # noqa: E402 - -SERVER = None -POOL = redis.ConnectionPool( - host=pagure.APP.config['REDIS_HOST'], - port=pagure.APP.config['REDIS_PORT'], - db=pagure.APP.config['REDIS_DB']) - - -def _get_issue(repo, objid): - """Get a Ticket (issue) instance for a given repo (Project) and - objid (issue number). - """ - issue = None - if not repo.settings.get('issue_tracker', True): - raise PagureEvException("No issue tracker found for this project") - - issue = pagure.lib.search_issues( - pagure.SESSION, repo, issueid=objid) - - if issue is None or issue.project != repo: - raise PagureEvException("Issue '%s' not found" % objid) - - if issue.private: - # TODO: find a way to do auth - raise PagureEvException( - "This issue is private and you are not allowed to view it") - - return issue - - -def _get_pull_request(repo, objid): - """Get a PullRequest instance for a given repo (Project) and objid - (request number). - """ - if not repo.settings.get('pull_requests', True): - raise PagureEvException( - "No pull-request tracker found for this project") - - request = pagure.lib.search_pull_requests( - pagure.SESSION, project_id=repo.id, requestid=objid) - - if request is None or request.project != repo: - raise PagureEvException("Pull-Request '%s' not found" % objid) - - return request - - -# Dict representing known object types that we handle requests for, -# and the bound functions for getting an object instance from the -# parsed path data. Has to come after the functions it binds -OBJECTS = { - 'issue': _get_issue, - 'pull-request': _get_pull_request -} - - -def _parse_path(path): - """Get the repo name, object type, object ID, and (if present) - username and/or namespace from a URL path component. Will only - handle the known object types from the OBJECTS dict. Assumes: - * Project name comes immediately before object type - * Object ID comes immediately after object type - * If a fork, path starts with /fork/(username) - * Namespace, if present, comes after fork username (if present) or at start - * No other components come before the project name - * None of the parsed items can contain a / - """ - username = None - namespace = None - # path always starts with / so split and throw away first item - items = path.split('/')[1:] - # find the *last* match for any object type - try: - objtype = [item for item in items if item in OBJECTS][-1] - except IndexError: - raise PagureEvException( - "No known object type found in path: %s" % path) - try: - # objid is the item after objtype, we need all items up to it - items = items[:items.index(objtype) + 2] - # now strip the repo, objtype and objid off the end - (repo, objtype, objid) = items[-3:] - items = items[:-3] - except (IndexError, ValueError): - raise PagureEvException( - "No project or object ID found in path: %s" % path) - # now check for a fork - if items and items[0] == 'fork': - try: - # get the username and strip it and 'fork' - username = items[1] - items = items[2:] - except IndexError: - raise PagureEvException( - "Path starts with /fork but no user found! Path: %s" % path) - # if we still have an item left, it must be the namespace - if items: - namespace = items.pop(0) - # if we have any items left at this point, we've no idea - if items: - raise PagureEvException( - "More path components than expected! Path: %s" % path) - - return username, namespace, repo, objtype, objid - - -def get_obj_from_path(path): - """ Return the Ticket or Request object based on the path provided. - """ - (username, namespace, reponame, objtype, objid) = _parse_path(path) - repo = pagure.get_authorized_project( - pagure.SESSION, reponame, user=username, namespace=namespace) - - if repo is None: - raise PagureEvException("Project '%s' not found" % reponame) - - # find the appropriate object getter function from OBJECTS - try: - getfunc = OBJECTS[objtype] - except KeyError: - raise PagureEvException("Invalid object provided: '%s'" % objtype) - - return getfunc(repo, objid) - - -@trollius.coroutine -def handle_client(client_reader, client_writer): - data = None - while True: - # give client a chance to respond, timeout after 10 seconds - line = yield trollius.From(trollius.wait_for( - client_reader.readline(), - timeout=10.0)) - if not line.decode().strip(): - break - line = line.decode().rstrip() - if data is None: - data = line - - if data is None: - log.warning("Expected ticket uid, received None") - return - - data = data.decode().rstrip().split() - log.info("Received %s", data) - if not data: - log.warning("No URL provided: %s" % data) - return - - if '/' not in data[1]: - log.warning("Invalid URL provided: %s" % data[1]) - return - - url = urlparse.urlsplit(data[1]) - - try: - obj = get_obj_from_path(url.path) - except PagureEvException as err: - log.warning(err.message) - return - - origin = pagure.APP.config.get('APP_URL') - if origin.endswith('/'): - origin = origin[:-1] - - client_writer.write(( - "HTTP/1.0 200 OK\n" - "Content-Type: text/event-stream\n" - "Cache: nocache\n" - "Connection: keep-alive\n" - "Access-Control-Allow-Origin: %s\n\n" % origin - ).encode()) - - - conn = redis.Redis(connection_pool=POOL) - subscriber = conn.pubsub(ignore_subscribe_messages=True) - - try: - subscriber.subscribe('pagure.%s' % obj.uid) - - # Inside a while loop, wait for incoming events. - oncall = 0 - while True: - msg = subscriber.get_message() - if msg is None: - # Send a ping to see if the client is still alive - if oncall >= 5: - # Only send a ping once every 5 seconds - client_writer.write(('event: ping\n\n').encode()) - oncall = 0 - oncall += 1 - yield trollius.From(client_writer.drain()) - yield trollius.From(trollius.sleep(1)) - else: - log.info("Sending %s", msg['data']) - client_writer.write(('data: %s\n\n' % msg['data']).encode()) - yield trollius.From(client_writer.drain()) - - except OSError: - log.info("Client closed connection") - except trollius.ConnectionResetError as err: - log.exception("ERROR: ConnectionResetError in handle_client") - except Exception as err: - log.exception("ERROR: Exception in handle_client") - log.info(type(err)) - finally: - # Wathever happens, close the connection. - log.info("Client left. Goodbye!") - subscriber.close() - client_writer.close() - - -@trollius.coroutine -def stats(client_reader, client_writer): - - try: - log.info('Clients: %s', SERVER.active_count) - client_writer.write(( - "HTTP/1.0 200 OK\n" - "Cache: nocache\n\n" - ).encode()) - client_writer.write(('data: %s\n\n' % SERVER.active_count).encode()) - yield trollius.From(client_writer.drain()) - - except trollius.ConnectionResetError as err: - log.info(err) - finally: - client_writer.close() - return - - -def main(): - global SERVER - - try: - loop = trollius.get_event_loop() - coro = trollius.start_server( - handle_client, - host=None, - port=pagure.APP.config['EVENTSOURCE_PORT'], - loop=loop) - SERVER = loop.run_until_complete(coro) - log.info( - 'Serving server at {}'.format(SERVER.sockets[0].getsockname())) - if pagure.APP.config.get('EV_STATS_PORT'): - stats_coro = trollius.start_server( - stats, - host=None, - port=pagure.APP.config.get('EV_STATS_PORT'), - loop=loop) - stats_server = loop.run_until_complete(stats_coro) - log.info('Serving stats at {}'.format( - stats_server.sockets[0].getsockname())) - loop.run_forever() - except KeyboardInterrupt: - pass - except trollius.ConnectionResetError as err: - log.exception("ERROR: ConnectionResetError in main") - except Exception: - log.exception("ERROR: Exception in main") - finally: - # Close the server - SERVER.close() - if pagure.APP.config.get('EV_STATS_PORT'): - stats_server.close() - log.info("End Connection") - loop.run_until_complete(SERVER.wait_closed()) - loop.close() - log.info("End") - - -if __name__ == '__main__': - log = logging.getLogger("") - formatter = logging.Formatter( - "%(asctime)s %(levelname)s [%(module)s:%(lineno)d] %(message)s") - - # setup console logging - log.setLevel(logging.DEBUG) - ch = logging.StreamHandler() - ch.setLevel(logging.DEBUG) - - aslog = logging.getLogger("asyncio") - aslog.setLevel(logging.DEBUG) - - ch.setFormatter(formatter) - log.addHandler(ch) - main() diff --git a/files/pagure.spec b/files/pagure.spec index 766c80f..e373923 100644 --- a/files/pagure.spec +++ b/files/pagure.spec @@ -247,9 +247,9 @@ install -m 644 milters/comment_email_milter.py \ # Install the eventsource mkdir -p $RPM_BUILD_ROOT/%{_libexecdir}/pagure-ev -install -m 755 ev-server/pagure_stream_server.py \ +install -m 755 pagure-ev/pagure_stream_server.py \ $RPM_BUILD_ROOT/%{_libexecdir}/pagure-ev/pagure_stream_server.py -install -m 644 ev-server/pagure_ev.service \ +install -m 644 pagure-ev/pagure_ev.service \ $RPM_BUILD_ROOT/%{_unitdir}/pagure_ev.service # Install the web-hook diff --git a/pagure-ev/pagure_ev.service b/pagure-ev/pagure_ev.service new file mode 100644 index 0000000..27e864b --- /dev/null +++ b/pagure-ev/pagure_ev.service @@ -0,0 +1,14 @@ +[Unit] +Description=Pagure EventSource server (Allowing live refresh of the pages supporting it) +After=redis.target +Documentation=https://pagure.io/pagure + +[Service] +ExecStart=/usr/libexec/pagure-ev/pagure_stream_server.py +Type=simple +User=git +Group=git +Restart=on-failure + +[Install] +WantedBy=multi-user.target diff --git a/pagure-ev/pagure_stream_server.py b/pagure-ev/pagure_stream_server.py new file mode 100644 index 0000000..c7c0a36 --- /dev/null +++ b/pagure-ev/pagure_stream_server.py @@ -0,0 +1,327 @@ +#!/usr/bin/env python + +""" + (c) 2015-2017 - Copyright Red Hat Inc + + Authors: + Pierre-Yves Chibon + + +Streaming server for pagure's eventsource feature +This server takes messages sent to redis and publish them at the specified +endpoint + +To test, run this script and in another terminal +nc localhost 8080 + HELLO + + GET /test/issue/26?foo=bar HTTP/1.1 + +""" + +import logging +import os +import urlparse + +import redis +import trollius + +log = logging.getLogger(__name__) + + +if 'PAGURE_CONFIG' not in os.environ \ + and os.path.exists('/etc/pagure/pagure.cfg'): + print 'Using configuration file `/etc/pagure/pagure.cfg`' + os.environ['PAGURE_CONFIG'] = '/etc/pagure/pagure.cfg' + + +import pagure # noqa: E402 +import pagure.lib # noqa: E402 +from pagure.exceptions import PagureEvException # noqa: E402 + +SERVER = None +POOL = redis.ConnectionPool( + host=pagure.APP.config['REDIS_HOST'], + port=pagure.APP.config['REDIS_PORT'], + db=pagure.APP.config['REDIS_DB']) + + +def _get_issue(repo, objid): + """Get a Ticket (issue) instance for a given repo (Project) and + objid (issue number). + """ + issue = None + if not repo.settings.get('issue_tracker', True): + raise PagureEvException("No issue tracker found for this project") + + issue = pagure.lib.search_issues( + pagure.SESSION, repo, issueid=objid) + + if issue is None or issue.project != repo: + raise PagureEvException("Issue '%s' not found" % objid) + + if issue.private: + # TODO: find a way to do auth + raise PagureEvException( + "This issue is private and you are not allowed to view it") + + return issue + + +def _get_pull_request(repo, objid): + """Get a PullRequest instance for a given repo (Project) and objid + (request number). + """ + if not repo.settings.get('pull_requests', True): + raise PagureEvException( + "No pull-request tracker found for this project") + + request = pagure.lib.search_pull_requests( + pagure.SESSION, project_id=repo.id, requestid=objid) + + if request is None or request.project != repo: + raise PagureEvException("Pull-Request '%s' not found" % objid) + + return request + + +# Dict representing known object types that we handle requests for, +# and the bound functions for getting an object instance from the +# parsed path data. Has to come after the functions it binds +OBJECTS = { + 'issue': _get_issue, + 'pull-request': _get_pull_request +} + + +def _parse_path(path): + """Get the repo name, object type, object ID, and (if present) + username and/or namespace from a URL path component. Will only + handle the known object types from the OBJECTS dict. Assumes: + * Project name comes immediately before object type + * Object ID comes immediately after object type + * If a fork, path starts with /fork/(username) + * Namespace, if present, comes after fork username (if present) or at start + * No other components come before the project name + * None of the parsed items can contain a / + """ + username = None + namespace = None + # path always starts with / so split and throw away first item + items = path.split('/')[1:] + # find the *last* match for any object type + try: + objtype = [item for item in items if item in OBJECTS][-1] + except IndexError: + raise PagureEvException( + "No known object type found in path: %s" % path) + try: + # objid is the item after objtype, we need all items up to it + items = items[:items.index(objtype) + 2] + # now strip the repo, objtype and objid off the end + (repo, objtype, objid) = items[-3:] + items = items[:-3] + except (IndexError, ValueError): + raise PagureEvException( + "No project or object ID found in path: %s" % path) + # now check for a fork + if items and items[0] == 'fork': + try: + # get the username and strip it and 'fork' + username = items[1] + items = items[2:] + except IndexError: + raise PagureEvException( + "Path starts with /fork but no user found! Path: %s" % path) + # if we still have an item left, it must be the namespace + if items: + namespace = items.pop(0) + # if we have any items left at this point, we've no idea + if items: + raise PagureEvException( + "More path components than expected! Path: %s" % path) + + return username, namespace, repo, objtype, objid + + +def get_obj_from_path(path): + """ Return the Ticket or Request object based on the path provided. + """ + (username, namespace, reponame, objtype, objid) = _parse_path(path) + repo = pagure.get_authorized_project( + pagure.SESSION, reponame, user=username, namespace=namespace) + + if repo is None: + raise PagureEvException("Project '%s' not found" % reponame) + + # find the appropriate object getter function from OBJECTS + try: + getfunc = OBJECTS[objtype] + except KeyError: + raise PagureEvException("Invalid object provided: '%s'" % objtype) + + return getfunc(repo, objid) + + +@trollius.coroutine +def handle_client(client_reader, client_writer): + data = None + while True: + # give client a chance to respond, timeout after 10 seconds + line = yield trollius.From(trollius.wait_for( + client_reader.readline(), + timeout=10.0)) + if not line.decode().strip(): + break + line = line.decode().rstrip() + if data is None: + data = line + + if data is None: + log.warning("Expected ticket uid, received None") + return + + data = data.decode().rstrip().split() + log.info("Received %s", data) + if not data: + log.warning("No URL provided: %s" % data) + return + + if '/' not in data[1]: + log.warning("Invalid URL provided: %s" % data[1]) + return + + url = urlparse.urlsplit(data[1]) + + try: + obj = get_obj_from_path(url.path) + except PagureEvException as err: + log.warning(err.message) + return + + origin = pagure.APP.config.get('APP_URL') + if origin.endswith('/'): + origin = origin[:-1] + + client_writer.write(( + "HTTP/1.0 200 OK\n" + "Content-Type: text/event-stream\n" + "Cache: nocache\n" + "Connection: keep-alive\n" + "Access-Control-Allow-Origin: %s\n\n" % origin + ).encode()) + + + conn = redis.Redis(connection_pool=POOL) + subscriber = conn.pubsub(ignore_subscribe_messages=True) + + try: + subscriber.subscribe('pagure.%s' % obj.uid) + + # Inside a while loop, wait for incoming events. + oncall = 0 + while True: + msg = subscriber.get_message() + if msg is None: + # Send a ping to see if the client is still alive + if oncall >= 5: + # Only send a ping once every 5 seconds + client_writer.write(('event: ping\n\n').encode()) + oncall = 0 + oncall += 1 + yield trollius.From(client_writer.drain()) + yield trollius.From(trollius.sleep(1)) + else: + log.info("Sending %s", msg['data']) + client_writer.write(('data: %s\n\n' % msg['data']).encode()) + yield trollius.From(client_writer.drain()) + + except OSError: + log.info("Client closed connection") + except trollius.ConnectionResetError as err: + log.exception("ERROR: ConnectionResetError in handle_client") + except Exception as err: + log.exception("ERROR: Exception in handle_client") + log.info(type(err)) + finally: + # Wathever happens, close the connection. + log.info("Client left. Goodbye!") + subscriber.close() + client_writer.close() + + +@trollius.coroutine +def stats(client_reader, client_writer): + + try: + log.info('Clients: %s', SERVER.active_count) + client_writer.write(( + "HTTP/1.0 200 OK\n" + "Cache: nocache\n\n" + ).encode()) + client_writer.write(('data: %s\n\n' % SERVER.active_count).encode()) + yield trollius.From(client_writer.drain()) + + except trollius.ConnectionResetError as err: + log.info(err) + finally: + client_writer.close() + return + + +def main(): + global SERVER + + try: + loop = trollius.get_event_loop() + coro = trollius.start_server( + handle_client, + host=None, + port=pagure.APP.config['EVENTSOURCE_PORT'], + loop=loop) + SERVER = loop.run_until_complete(coro) + log.info( + 'Serving server at {}'.format(SERVER.sockets[0].getsockname())) + if pagure.APP.config.get('EV_STATS_PORT'): + stats_coro = trollius.start_server( + stats, + host=None, + port=pagure.APP.config.get('EV_STATS_PORT'), + loop=loop) + stats_server = loop.run_until_complete(stats_coro) + log.info('Serving stats at {}'.format( + stats_server.sockets[0].getsockname())) + loop.run_forever() + except KeyboardInterrupt: + pass + except trollius.ConnectionResetError as err: + log.exception("ERROR: ConnectionResetError in main") + except Exception: + log.exception("ERROR: Exception in main") + finally: + # Close the server + SERVER.close() + if pagure.APP.config.get('EV_STATS_PORT'): + stats_server.close() + log.info("End Connection") + loop.run_until_complete(SERVER.wait_closed()) + loop.close() + log.info("End") + + +if __name__ == '__main__': + log = logging.getLogger("") + formatter = logging.Formatter( + "%(asctime)s %(levelname)s [%(module)s:%(lineno)d] %(message)s") + + # setup console logging + log.setLevel(logging.DEBUG) + ch = logging.StreamHandler() + ch.setLevel(logging.DEBUG) + + aslog = logging.getLogger("asyncio") + aslog.setLevel(logging.DEBUG) + + ch.setFormatter(formatter) + log.addHandler(ch) + main() diff --git a/tests/test_stream_server.py b/tests/test_stream_server.py index 3541cc1..ab35799 100644 --- a/tests/test_stream_server.py +++ b/tests/test_stream_server.py @@ -24,7 +24,7 @@ import mock sys.path.insert(0, os.path.join(os.path.dirname( os.path.abspath(__file__)), '..')) sys.path.insert(0, os.path.join(os.path.dirname( - os.path.abspath(__file__)), '../ev-server')) + os.path.abspath(__file__)), '../pagure-ev')) import pagure # pylint: disable=wrong-import-position from pagure.exceptions import PagureEvException # pylint: disable=wrong-import-position From a7d88a08c7719ae3bc3013c3c28eedcf3942540b Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Jun 13 2017 21:01:30 +0000 Subject: [PATCH 3/7] Rename the milter service to pagure-milter This makes it easier to identify and is more consistent with how we named our other services. --- diff --git a/MANIFEST.in b/MANIFEST.in index a3a2897..3d60809 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -4,7 +4,7 @@ recursive-include pagure * recursive-include pagure-ci * recursive-include pagure-logcom * recursive-include files * -recursive-include milters * +recursive-include pagure-milters * recursive-include tests * recursive-include doc * recursive-include alembic * diff --git a/doc/install_milter.rst b/doc/install_milter.rst index dc1524f..f1025c9 100644 --- a/doc/install_milter.rst +++ b/doc/install_milter.rst @@ -53,15 +53,15 @@ In postfix this is done via: * Install the files of the milter as follow: -+--------------------------------------+---------------------------------------------------+ -| Source | Destination | -+======================================+===================================================+ -| ``milters/comment_email_milter.py`` | ``/usr/share//pagure/comment_email_milter.py`` | -+--------------------------------------+---------------------------------------------------+ -| ``milters/milter_tempfile.conf`` | ``/usr/lib/tmpfiles.d/pagure-milter.conf`` | -+--------------------------------------+---------------------------------------------------+ -| ``milters/pagure_milter.service`` | ``/etc/systemd/system/pagure_milter.service`` | -+--------------------------------------+---------------------------------------------------+ ++---------------------------------------------+---------------------------------------------------+ +| Source | Destination | ++=============================================+===================================================+ +| ``pagure-milters/comment_email_milter.py`` | ``/usr/share/pagure/comment_email_milter.py`` | ++---------------------------------------------+---------------------------------------------------+ +| ``pagure-milters/milter_tempfile.conf`` | ``/usr/lib/tmpfiles.d/pagure-milter.conf`` | ++---------------------------------------------+---------------------------------------------------+ +| ``pagure-milters/pagure_milter.service`` | ``/etc/systemd/system/pagure_milter.service`` | ++---------------------------------------------+---------------------------------------------------+ The first file is the script of the milter itself. diff --git a/doc/milter.rst b/doc/milter.rst index 5ab8ec2..73cfdba 100644 --- a/doc/milter.rst +++ b/doc/milter.rst @@ -11,8 +11,8 @@ Pagure's milter is designed to be run on the same machine as the mail server (postfix by default). Postfix connecting to the milter via a unix socket. The milter itself is a service managed by systemd. -You can find all the relevant files for the milter under the ``milters`` folder -in the sources. +You can find all the relevant files for the milter under the +``pagure-milters`` folder in the sources. Install the milter @@ -58,5 +58,5 @@ is two lines in the ``main.cf`` file of postfix: These two lines are pointing to the unix socket used by postfix to communicate with the milter. This socket is defined in the milter file itself, in the -sources: ``milters/comment_email_milter.py``. +sources: ``pagure-milters/comment_email_milter.py``. diff --git a/files/pagure.spec b/files/pagure.spec index e373923..888fd0d 100644 --- a/files/pagure.spec +++ b/files/pagure.spec @@ -238,11 +238,11 @@ install -m 644 files/pagure_worker.service \ mkdir -p $RPM_BUILD_ROOT/%{_localstatedir}/run/pagure mkdir -p $RPM_BUILD_ROOT/%{_tmpfilesdir} mkdir -p $RPM_BUILD_ROOT/%{_unitdir} -install -m 0644 milters/milter_tempfile.conf \ +install -m 0644 pagure-milters/milter_tempfile.conf \ $RPM_BUILD_ROOT/%{_tmpfilesdir}/%{name}-milter.conf -install -m 644 milters/pagure_milter.service \ +install -m 644 pagure-milters/pagure_milter.service \ $RPM_BUILD_ROOT/%{_unitdir}/pagure_milter.service -install -m 644 milters/comment_email_milter.py \ +install -m 644 pagure-milters/comment_email_milter.py \ $RPM_BUILD_ROOT/%{_datadir}/pagure/comment_email_milter.py # Install the eventsource diff --git a/milters/comment_email_milter.py b/milters/comment_email_milter.py deleted file mode 100644 index 477cdf4..0000000 --- a/milters/comment_email_milter.py +++ /dev/null @@ -1,247 +0,0 @@ -#!/usr/bin/env python2 -# -*- coding: utf-8 -*- - -# Milter calls methods of your class at milter events. -# Return REJECT,TEMPFAIL,ACCEPT to short circuit processing for a message. -# You can also add/del recipients, replacebody, add/del headers, etc. - -import base64 -import email -import hashlib -import os -import urlparse -import StringIO -import sys -import time -from socket import AF_INET, AF_INET6 -from multiprocessing import Process as Thread, Queue - -import Milter -import requests - -from Milter.utils import parse_addr - -logq = Queue(maxsize=4) - - -if 'PAGURE_CONFIG' not in os.environ \ - and os.path.exists('/etc/pagure/pagure.cfg'): - os.environ['PAGURE_CONFIG'] = '/etc/pagure/pagure.cfg' - - -import pagure - - -def get_email_body(emailobj): - ''' Return the body of the email, preferably in text. - ''' - body = None - if emailobj.is_multipart(): - for payload in emailobj.get_payload(): - body = payload.get_payload() - if payload.get_content_type() == 'text/plain': - break - else: - body = emailobj.get_payload() - - enc = emailobj['Content-Transfer-Encoding'] - if enc == 'base64': - body = base64.decodestring(body) - - return body - - -def clean_item(item): - ''' For an item provided as return the content, if there are no - <> then return the string. - ''' - if '<' in item: - item = item.split('<')[1] - if '>' in item: - item = item.split('>')[0] - - return item - - -class PagureMilter(Milter.Base): - - def __init__(self): # A new instance with each new connection. - self.id = Milter.uniqueID() # Integer incremented with each call. - self.fp = None - - def log(self, message): - print(message) - sys.stdout.flush() - - def envfrom(self, mailfrom, *str): - self.log("mail from: %s - %s" % (mailfrom, str)) - self.fromparms = Milter.dictfromlist(str) - # NOTE: self.fp is only an *internal* copy of message data. You - # must use addheader, chgheader, replacebody to change the message - # on the MTA. - self.fp = StringIO.StringIO() - self.canon_from = '@'.join(parse_addr(mailfrom)) - self.fp.write('From %s %s\n' % (self.canon_from, time.ctime())) - return Milter.CONTINUE - - @Milter.noreply - def header(self, name, hval): - ''' Headers ''' - # add header to buffer - self.fp.write("%s: %s\n" % (name, hval)) - return Milter.CONTINUE - - @Milter.noreply - def eoh(self): - ''' End of Headers ''' - self.fp.write("\n") - return Milter.CONTINUE - - @Milter.noreply - def body(self, chunk): - ''' Body ''' - self.fp.write(chunk) - return Milter.CONTINUE - - @Milter.noreply - def envrcpt(self, to, *str): - rcptinfo = to, Milter.dictfromlist(str) - print rcptinfo - - return Milter.CONTINUE - - def eom(self): - ''' End of Message ''' - self.fp.seek(0) - msg = email.message_from_file(self.fp) - - msg_id = msg.get('In-Reply-To', None) - if msg_id is None: - self.log('No In-Reply-To, keep going') - return Milter.CONTINUE - - # Ensure we don't get extra lines in the message-id - msg_id = msg_id.split('\n')[0].strip() - - self.log('msg-ig %s' % msg_id) - self.log('To %s' % msg['to']) - self.log('Cc %s' % msg.get('cc')) - self.log('From %s' % msg['From']) - - # Ensure the user replied to his/her own notification, not that - # they are trying to forge their ID into someone else's - salt = pagure.APP.config.get('SALT_EMAIL') - m = hashlib.sha512('%s%s%s' % (msg_id, salt, clean_item(msg['From']))) - email_address = msg['to'] - if 'reply+' in msg.get('cc', ''): - email_address = msg['cc'] - if not 'reply+' in email_address: - self.log( - 'No valid recipient email found in To/Cc: %s' - % email_address) - tohash = email_address.split('@')[0].split('+')[-1] - if m.hexdigest() != tohash: - self.log('hash: %s' % m.hexdigest()) - self.log('tohash: %s' % tohash) - self.log('Hash does not correspond to the destination') - return Milter.CONTINUE - - if msg['From'] and msg['From'] == pagure.APP.config.get('FROM_EMAIL'): - self.log("Let's not process the email we send") - return Milter.CONTINUE - - msg_id = clean_item(msg_id) - - if msg_id and '-ticket-' in msg_id: - self.log('Processing issue') - return self.handle_ticket_email(msg, msg_id) - elif msg_id and '-pull-request-' in msg_id: - self.log('Processing pull-request') - return self.handle_request_email(msg, msg_id) - else: - self.log('Not a pagure ticket or pull-request email, let it go') - return Milter.CONTINUE - - - def handle_ticket_email(self, emailobj, msg_id): - ''' Add the email as a comment on a ticket. ''' - uid = msg_id.split('-ticket-')[-1].split('@')[0] - parent_id = None - if '-' in uid: - uid, parent_id = uid.rsplit('-', 1) - if '/' in uid: - uid = uid.split('/')[0] - self.log('uid %s' % uid) - self.log('parent_id %s' % parent_id) - - data = { - 'objid': uid, - 'comment': get_email_body(emailobj), - 'useremail': clean_item(emailobj['From']), - } - url = pagure.APP.config.get('APP_URL') - - if url.endswith('/'): - url = url[:-1] - url = '%s/pv/ticket/comment/' % url - req = requests.put(url, data=data) - if req.status_code == 200: - self.log('Comment added') - return Milter.ACCEPT - self.log('Could not add the comment to pagure') - return Milter.CONTINUE - - def handle_request_email(self, emailobj, msg_id): - ''' Add the email as a comment on a request. ''' - uid = msg_id.split('-pull-request-')[-1].split('@')[0] - parent_id = None - if '-' in uid: - uid, parent_id = uid.rsplit('-', 1) - if '/' in uid: - uid = uid.split('/')[0] - self.log('uid %s' % uid) - self.log('parent_id %s' % parent_id) - - data = { - 'objid': uid, - 'comment': get_email_body(emailobj), - 'useremail': clean_item(emailobj['From']), - } - url = pagure.APP.config.get('APP_URL') - - if url.endswith('/'): - url = url[:-1] - url = '%s/pv/pull-request/comment/' % url - req = requests.put(url, data=data) - - return Milter.ACCEPT - - -def background(): - while True: - t = logq.get() - if not t: break - msg,id,ts = t - print "%s [%d]" % (time.strftime('%Y%b%d %H:%M:%S',time.localtime(ts)),id), - # 2005Oct13 02:34:11 [1] msg1 msg2 msg3 ... - for i in msg: print i, - print - - -def main(): - bt = Thread(target=background) - bt.start() - socketname = "/var/run/pagure/paguresock" - timeout = 600 - # Register to have the Milter factory create instances of your class: - Milter.factory = PagureMilter - print "%s pagure milter startup" % time.strftime('%Y%b%d %H:%M:%S') - sys.stdout.flush() - Milter.runmilter("paguremilter", socketname, timeout) - logq.put(None) - bt.join() - print "%s pagure milter shutdown" % time.strftime('%Y%b%d %H:%M:%S') - - -if __name__ == "__main__": - main() diff --git a/milters/milter_tempfile.conf b/milters/milter_tempfile.conf deleted file mode 100644 index 3e92e09..0000000 --- a/milters/milter_tempfile.conf +++ /dev/null @@ -1 +0,0 @@ -d /var/run/pagure 0755 postfix postfix diff --git a/milters/pagure_milter.service b/milters/pagure_milter.service deleted file mode 100644 index 7cc6b01..0000000 --- a/milters/pagure_milter.service +++ /dev/null @@ -1,14 +0,0 @@ -[Unit] -Description=Pagure SMTP filter (Milter) Daemon (talk to postfix over a socket) -After=postfix.target -Documentation=https://github.com/pypingou/pagure - -[Service] -ExecStart=/usr/bin/python2 /usr/share/pagure/comment_email_milter.py -Type=simple -User=postfix -Group=postfix -Restart=on-failure - -[Install] -WantedBy=multi-user.target diff --git a/pagure-milters/comment_email_milter.py b/pagure-milters/comment_email_milter.py new file mode 100644 index 0000000..477cdf4 --- /dev/null +++ b/pagure-milters/comment_email_milter.py @@ -0,0 +1,247 @@ +#!/usr/bin/env python2 +# -*- coding: utf-8 -*- + +# Milter calls methods of your class at milter events. +# Return REJECT,TEMPFAIL,ACCEPT to short circuit processing for a message. +# You can also add/del recipients, replacebody, add/del headers, etc. + +import base64 +import email +import hashlib +import os +import urlparse +import StringIO +import sys +import time +from socket import AF_INET, AF_INET6 +from multiprocessing import Process as Thread, Queue + +import Milter +import requests + +from Milter.utils import parse_addr + +logq = Queue(maxsize=4) + + +if 'PAGURE_CONFIG' not in os.environ \ + and os.path.exists('/etc/pagure/pagure.cfg'): + os.environ['PAGURE_CONFIG'] = '/etc/pagure/pagure.cfg' + + +import pagure + + +def get_email_body(emailobj): + ''' Return the body of the email, preferably in text. + ''' + body = None + if emailobj.is_multipart(): + for payload in emailobj.get_payload(): + body = payload.get_payload() + if payload.get_content_type() == 'text/plain': + break + else: + body = emailobj.get_payload() + + enc = emailobj['Content-Transfer-Encoding'] + if enc == 'base64': + body = base64.decodestring(body) + + return body + + +def clean_item(item): + ''' For an item provided as return the content, if there are no + <> then return the string. + ''' + if '<' in item: + item = item.split('<')[1] + if '>' in item: + item = item.split('>')[0] + + return item + + +class PagureMilter(Milter.Base): + + def __init__(self): # A new instance with each new connection. + self.id = Milter.uniqueID() # Integer incremented with each call. + self.fp = None + + def log(self, message): + print(message) + sys.stdout.flush() + + def envfrom(self, mailfrom, *str): + self.log("mail from: %s - %s" % (mailfrom, str)) + self.fromparms = Milter.dictfromlist(str) + # NOTE: self.fp is only an *internal* copy of message data. You + # must use addheader, chgheader, replacebody to change the message + # on the MTA. + self.fp = StringIO.StringIO() + self.canon_from = '@'.join(parse_addr(mailfrom)) + self.fp.write('From %s %s\n' % (self.canon_from, time.ctime())) + return Milter.CONTINUE + + @Milter.noreply + def header(self, name, hval): + ''' Headers ''' + # add header to buffer + self.fp.write("%s: %s\n" % (name, hval)) + return Milter.CONTINUE + + @Milter.noreply + def eoh(self): + ''' End of Headers ''' + self.fp.write("\n") + return Milter.CONTINUE + + @Milter.noreply + def body(self, chunk): + ''' Body ''' + self.fp.write(chunk) + return Milter.CONTINUE + + @Milter.noreply + def envrcpt(self, to, *str): + rcptinfo = to, Milter.dictfromlist(str) + print rcptinfo + + return Milter.CONTINUE + + def eom(self): + ''' End of Message ''' + self.fp.seek(0) + msg = email.message_from_file(self.fp) + + msg_id = msg.get('In-Reply-To', None) + if msg_id is None: + self.log('No In-Reply-To, keep going') + return Milter.CONTINUE + + # Ensure we don't get extra lines in the message-id + msg_id = msg_id.split('\n')[0].strip() + + self.log('msg-ig %s' % msg_id) + self.log('To %s' % msg['to']) + self.log('Cc %s' % msg.get('cc')) + self.log('From %s' % msg['From']) + + # Ensure the user replied to his/her own notification, not that + # they are trying to forge their ID into someone else's + salt = pagure.APP.config.get('SALT_EMAIL') + m = hashlib.sha512('%s%s%s' % (msg_id, salt, clean_item(msg['From']))) + email_address = msg['to'] + if 'reply+' in msg.get('cc', ''): + email_address = msg['cc'] + if not 'reply+' in email_address: + self.log( + 'No valid recipient email found in To/Cc: %s' + % email_address) + tohash = email_address.split('@')[0].split('+')[-1] + if m.hexdigest() != tohash: + self.log('hash: %s' % m.hexdigest()) + self.log('tohash: %s' % tohash) + self.log('Hash does not correspond to the destination') + return Milter.CONTINUE + + if msg['From'] and msg['From'] == pagure.APP.config.get('FROM_EMAIL'): + self.log("Let's not process the email we send") + return Milter.CONTINUE + + msg_id = clean_item(msg_id) + + if msg_id and '-ticket-' in msg_id: + self.log('Processing issue') + return self.handle_ticket_email(msg, msg_id) + elif msg_id and '-pull-request-' in msg_id: + self.log('Processing pull-request') + return self.handle_request_email(msg, msg_id) + else: + self.log('Not a pagure ticket or pull-request email, let it go') + return Milter.CONTINUE + + + def handle_ticket_email(self, emailobj, msg_id): + ''' Add the email as a comment on a ticket. ''' + uid = msg_id.split('-ticket-')[-1].split('@')[0] + parent_id = None + if '-' in uid: + uid, parent_id = uid.rsplit('-', 1) + if '/' in uid: + uid = uid.split('/')[0] + self.log('uid %s' % uid) + self.log('parent_id %s' % parent_id) + + data = { + 'objid': uid, + 'comment': get_email_body(emailobj), + 'useremail': clean_item(emailobj['From']), + } + url = pagure.APP.config.get('APP_URL') + + if url.endswith('/'): + url = url[:-1] + url = '%s/pv/ticket/comment/' % url + req = requests.put(url, data=data) + if req.status_code == 200: + self.log('Comment added') + return Milter.ACCEPT + self.log('Could not add the comment to pagure') + return Milter.CONTINUE + + def handle_request_email(self, emailobj, msg_id): + ''' Add the email as a comment on a request. ''' + uid = msg_id.split('-pull-request-')[-1].split('@')[0] + parent_id = None + if '-' in uid: + uid, parent_id = uid.rsplit('-', 1) + if '/' in uid: + uid = uid.split('/')[0] + self.log('uid %s' % uid) + self.log('parent_id %s' % parent_id) + + data = { + 'objid': uid, + 'comment': get_email_body(emailobj), + 'useremail': clean_item(emailobj['From']), + } + url = pagure.APP.config.get('APP_URL') + + if url.endswith('/'): + url = url[:-1] + url = '%s/pv/pull-request/comment/' % url + req = requests.put(url, data=data) + + return Milter.ACCEPT + + +def background(): + while True: + t = logq.get() + if not t: break + msg,id,ts = t + print "%s [%d]" % (time.strftime('%Y%b%d %H:%M:%S',time.localtime(ts)),id), + # 2005Oct13 02:34:11 [1] msg1 msg2 msg3 ... + for i in msg: print i, + print + + +def main(): + bt = Thread(target=background) + bt.start() + socketname = "/var/run/pagure/paguresock" + timeout = 600 + # Register to have the Milter factory create instances of your class: + Milter.factory = PagureMilter + print "%s pagure milter startup" % time.strftime('%Y%b%d %H:%M:%S') + sys.stdout.flush() + Milter.runmilter("paguremilter", socketname, timeout) + logq.put(None) + bt.join() + print "%s pagure milter shutdown" % time.strftime('%Y%b%d %H:%M:%S') + + +if __name__ == "__main__": + main() diff --git a/pagure-milters/milter_tempfile.conf b/pagure-milters/milter_tempfile.conf new file mode 100644 index 0000000..3e92e09 --- /dev/null +++ b/pagure-milters/milter_tempfile.conf @@ -0,0 +1 @@ +d /var/run/pagure 0755 postfix postfix diff --git a/pagure-milters/pagure_milter.service b/pagure-milters/pagure_milter.service new file mode 100644 index 0000000..7cc6b01 --- /dev/null +++ b/pagure-milters/pagure_milter.service @@ -0,0 +1,14 @@ +[Unit] +Description=Pagure SMTP filter (Milter) Daemon (talk to postfix over a socket) +After=postfix.target +Documentation=https://github.com/pypingou/pagure + +[Service] +ExecStart=/usr/bin/python2 /usr/share/pagure/comment_email_milter.py +Type=simple +User=postfix +Group=postfix +Restart=on-failure + +[Install] +WantedBy=multi-user.target From 7b7b8396f48ada5fa39b510c34cff06062c11189 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Jun 13 2017 21:01:30 +0000 Subject: [PATCH 4/7] Move all the default folders into a lcl (as in local) folder --- diff --git a/.gitignore b/.gitignore index 4934a79..699b160 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ docs/ requests/ releases/ remotes/ +lcl/ pagure_env/ # Other python/editor specific files to ignore dist/ diff --git a/README.rst b/README.rst index 048c3bd..c643c5c 100644 --- a/README.rst +++ b/README.rst @@ -98,7 +98,7 @@ Manually * Create the folder that will receive the projects, forks, docs, requests and tickets' git repo:: - mkdir repos docs forks tickets requests + mkdir -p lcl/{repos,docs,forks,tickets,requests,remotes,attachments} * Create the inital database scheme:: diff --git a/pagure/default_config.py b/pagure/default_config.py index 4d9e3b5..a808189 100644 --- a/pagure/default_config.py +++ b/pagure/default_config.py @@ -98,6 +98,7 @@ EVENTSOURCE_PORT = 8080 GIT_FOLDER = os.path.join( os.path.abspath(os.path.dirname(__file__)), '..', + 'lcl', 'repos' ) @@ -105,6 +106,7 @@ GIT_FOLDER = os.path.join( DOCS_FOLDER = os.path.join( os.path.abspath(os.path.dirname(__file__)), '..', + 'lcl', 'docs' ) @@ -112,6 +114,7 @@ DOCS_FOLDER = os.path.join( TICKETS_FOLDER = os.path.join( os.path.abspath(os.path.dirname(__file__)), '..', + 'lcl', 'tickets' ) @@ -119,6 +122,7 @@ TICKETS_FOLDER = os.path.join( REQUESTS_FOLDER = os.path.join( os.path.abspath(os.path.dirname(__file__)), '..', + 'lcl', 'requests' ) @@ -126,6 +130,7 @@ REQUESTS_FOLDER = os.path.join( REMOTE_GIT_FOLDER = os.path.join( os.path.abspath(os.path.dirname(__file__)), '..', + 'lcl', 'remotes' ) @@ -133,6 +138,7 @@ REMOTE_GIT_FOLDER = os.path.join( ATTACHMENTS_FOLDER = os.path.join( os.path.abspath(os.path.dirname(__file__)), '..', + 'lcl', 'attachments' ) @@ -143,12 +149,19 @@ VIRUS_SCAN_ATTACHMENTS = False GITOLITE_CONFIG = os.path.join( os.path.abspath(os.path.dirname(__file__)), '..', + 'lcl', 'gitolite.conf' ) # Configuration keys to specify where the upload folder is and what is its # name -UPLOAD_FOLDER_PATH = './releases' +UPLOAD_FOLDER_PATH = os.path.join( + os.path.abspath(os.path.dirname(__file__)), + '..', + 'lcl', + 'releases' +) + # Home folder of the gitolite user -- Folder where to run gl-compile-conf from GITOLITE_HOME = None From dedefcfcd70aa804ee7d7139f13f38098ab20c47 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Jun 13 2017 21:01:30 +0000 Subject: [PATCH 5/7] Move all the development related files into a dev/ folder --- diff --git a/README.rst b/README.rst index c643c5c..1757857 100644 --- a/README.rst +++ b/README.rst @@ -37,7 +37,7 @@ https://fedoraproject.org/wiki/Vagrant. An example Vagrantfile is provided as ``Vagrantfile.example``. To use it, just copy it and install Vagrant:: - $ cp Vagrantfile.example Vagrantfile + $ cp dev/Vagrantfile.example Vagrantfile $ sudo dnf install ansible libvirt vagrant-libvirt vagrant-sshfs vagrant-hostmanager $ vagrant up diff --git a/Vagrantfile.example b/Vagrantfile.example deleted file mode 100644 index 8a181f5..0000000 --- a/Vagrantfile.example +++ /dev/null @@ -1,71 +0,0 @@ -# -*- mode: ruby -*- -# vi: set ft=ruby : - -VAGRANTFILE_API_VERSION = "2" - -Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| - config.vm.box_url = "https://download.fedoraproject.org/pub/fedora/linux/releases/25/CloudImages/x86_64/images/Fedora-Cloud-Base-Vagrant-25-1.3.x86_64.vagrant-libvirt.box" - config.vm.box = "f25-cloud-libvirt" - - - # Forward traffic on the host to the development server on the guest - config.vm.network "forwarded_port", guest: 5000, host: 5000 - # Forward traffic on the host to Redis on the guest - config.vm.network "forwarded_port", guest: 6379, host: 6379 - # Forward traffic on the host to the SSE server on the guest - config.vm.network "forwarded_port", guest: 8080, host: 8080 - - if Vagrant.has_plugin?("vagrant-hostmanager") - config.hostmanager.enabled = true - config.hostmanager.manage_host = true - end - - # Vagrant can share the source directory using rsync, NFS, or SSHFS (with the vagrant-sshfs - # plugin). By default it rsyncs the current working directory to /vagrant. - # - # If you would prefer to use NFS to share the directory uncomment this and configure NFS - # config.vm.synced_folder ".", "/vagrant", type: "nfs", nfs_version: 4, nfs_udp: false - config.vm.synced_folder ".", "/vagrant", disabled: true - config.vm.synced_folder ".", "/home/vagrant/devel", - type: "sshfs", - sshfs_opts_append: "-o nonempty" - - # To cache update packages (which is helpful if frequently doing `vagrant destroy && vagrant up`) - # you can create a local directory and share it to the guest's DNF cache. The directory needs to - # exist, so create it before you uncomment the line below. - #Dir.mkdir('.dnf-cache') unless File.exists?('.dnf-cache') - #config.vm.synced_folder ".dnf-cache", "/var/cache/dnf", - # type: "sshfs", - # sshfs_opts_append: "-o nonempty" - - # Comment this line if you would like to disable the automatic update during provisioning - config.vm.provision "shell", inline: "sudo dnf upgrade -y" - - # bootstrap and run with ansible - config.vm.provision "shell", inline: "sudo dnf -y install python2-dnf libselinux-python" - config.vm.provision "ansible" do |ansible| - ansible.playbook = "ansible/vagrant-playbook.yml" - end - - - # Create the "pagure" box - config.vm.define "pagure" do |pagure| - pagure.vm.host_name = "pagure-dev.example.com" - - pagure.vm.provider :libvirt do |domain| - # Season to taste - domain.cpus = 4 - domain.graphics_type = "spice" - domain.memory = 2048 - domain.video_type = "qxl" - - # Uncomment the following line if you would like to enable libvirt's unsafe cache - # mode. It is called unsafe for a reason, as it causes the virtual host to ignore all - # fsync() calls from the guest. Only do this if you are comfortable with the possibility of - # your development guest becoming corrupted (in which case you should only need to do a - # vagrant destroy and vagrant up to get a new one). - # - # domain.volume_cache = "unsafe" - end - end -end diff --git a/ansible/roles/pagure-dev/files/bashrc b/ansible/roles/pagure-dev/files/bashrc deleted file mode 100644 index 4b7ac84..0000000 --- a/ansible/roles/pagure-dev/files/bashrc +++ /dev/null @@ -1,37 +0,0 @@ -# .bashrc - -# Source global definitions -if [ -f /etc/bashrc ]; then - . /etc/bashrc -fi - -# Uncomment the following line if you don't like systemctl's auto-paging feature: -# export SYSTEMD_PAGER= - -# User specific aliases and functions -# If adding new functions to this file, note that you can add help text to the function -# by defining a variable with name __help containing the help text - -export PAGURE_CONFIG=~/pagure.cfg - -pstart (){ - systemctl --user start pagure.service pagure-docs.service pagure_ci.service\ - pagure_ev.service pagure_webhook.service - echo 'The application is running on http://localhost:5000/' -} - -pstop (){ - systemctl --user stop pagure.service pagure-docs.service pagure_ci.service\ - pagure_ev.service pagure_webhook.service -} - -prestart (){ - systemctl --user restart pagure.service pagure-docs.service pagure_ci.service\ - pagure_ev.service pagure_webhook.service - echo 'The application is running on http://localhost:5000/' -} - -pstatus (){ - systemctl --user status pagure.service pagure-docs.service pagure_ci.service\ - pagure_ev.service pagure_webhook.service -} diff --git a/ansible/roles/pagure-dev/files/clamd.conf b/ansible/roles/pagure-dev/files/clamd.conf deleted file mode 100644 index 7e9cfa5..0000000 --- a/ansible/roles/pagure-dev/files/clamd.conf +++ /dev/null @@ -1,684 +0,0 @@ -## -## Example config file for the Clam AV daemon -## Please read the clamd.conf(5) manual before editing this file. -## - - -# Comment or remove the line below. -# Example - -# Uncomment this option to enable logging. -# LogFile must be writable for the user running daemon. -# A full path is required. -# Default: disabled -#LogFile /var/log/clamd. - -# By default the log file is locked for writing - the lock protects against -# running clamd multiple times (if want to run another clamd, please -# copy the configuration file, change the LogFile variable, and run -# the daemon with --config-file option). -# This option disables log file locking. -# Default: no -#LogFileUnlock yes - -# Maximum size of the log file. -# Value of 0 disables the limit. -# You may use 'M' or 'm' for megabytes (1M = 1m = 1048576 bytes) -# and 'K' or 'k' for kilobytes (1K = 1k = 1024 bytes). To specify the size -# in bytes just don't use modifiers. If LogFileMaxSize is enabled, log -# rotation (the LogRotate option) will always be enabled. -# Default: 1M -#LogFileMaxSize 2M - -# Log time with each message. -# Default: no -#LogTime yes - -# Also log clean files. Useful in debugging but drastically increases the -# log size. -# Default: no -#LogClean yes - -# Use system logger (can work together with LogFile). -# Default: no -LogSyslog yes - -# Specify the type of syslog messages - please refer to 'man syslog' -# for facility names. -# Default: LOG_LOCAL6 -#LogFacility LOG_MAIL - -# Enable verbose logging. -# Default: no -#LogVerbose yes - -# Enable log rotation. Always enabled when LogFileMaxSize is enabled. -# Default: no -#LogRotate yes - -# Log additional information about the infected file, such as its -# size and hash, together with the virus name. -#ExtendedDetectionInfo yes - -# This option allows you to save a process identifier of the listening -# daemon (main thread). -# Default: disabled -#PidFile /var/run/clamd./clamd.pid - -# Optional path to the global temporary directory. -# Default: system specific (usually /tmp or /var/tmp). -#TemporaryDirectory /var/tmp - -# Path to the database directory. -# Default: hardcoded (depends on installation options) -#DatabaseDirectory /var/lib/clamav - -# Only load the official signatures published by the ClamAV project. -# Default: no -#OfficialDatabaseOnly no - -# The daemon can work in local mode, network mode or both. -# Due to security reasons we recommend the local mode. - -# Path to a local socket file the daemon will listen on. -# Default: disabled (must be specified by a user) -LocalSocket /var/lib/clamav/clamd.sock - -# Sets the group ownership on the unix socket. -# Default: disabled (the primary group of the user running clamd) -LocalSocketGroup clamupdate - -# Sets the permissions on the unix socket to the specified mode. -# Default: disabled (socket is world accessible) -#LocalSocketMode 660 - -# Remove stale socket after unclean shutdown. -# Default: yes -#FixStaleSocket yes - -# TCP port address. -# Default: no -#TCPSocket 3310 - -# TCP address. -# By default we bind to INADDR_ANY, probably not wise. -# Enable the following to provide some degree of protection -# from the outside world. This option can be specified multiple -# times if you want to listen on multiple IPs. IPv6 is now supported. -# Default: no -#TCPAddr 127.0.0.1 - -# Maximum length the queue of pending connections may grow to. -# Default: 200 -#MaxConnectionQueueLength 30 - -# Clamd uses FTP-like protocol to receive data from remote clients. -# If you are using clamav-milter to balance load between remote clamd daemons -# on firewall servers you may need to tune the options below. - -# Close the connection when the data size limit is exceeded. -# The value should match your MTA's limit for a maximum attachment size. -# Default: 25M -#StreamMaxLength 10M - -# Limit port range. -# Default: 1024 -#StreamMinPort 30000 -# Default: 2048 -#StreamMaxPort 32000 - -# Maximum number of threads running at the same time. -# Default: 10 -#MaxThreads 20 - -# Waiting for data from a client socket will timeout after this time (seconds). -# Default: 120 -#ReadTimeout 300 - -# This option specifies the time (in seconds) after which clamd should -# timeout if a client doesn't provide any initial command after connecting. -# Default: 5 -#CommandReadTimeout 5 - -# This option specifies how long to wait (in miliseconds) if the send buffer is full. -# Keep this value low to prevent clamd hanging -# -# Default: 500 -#SendBufTimeout 200 - -# Maximum number of queued items (including those being processed by MaxThreads threads) -# It is recommended to have this value at least twice MaxThreads if possible. -# WARNING: you shouldn't increase this too much to avoid running out of file descriptors, -# the following condition should hold: -# MaxThreads*MaxRecursion + (MaxQueue - MaxThreads) + 6< RLIMIT_NOFILE (usual max is 1024) -# -# Default: 100 -#MaxQueue 200 - -# Waiting for a new job will timeout after this time (seconds). -# Default: 30 -#IdleTimeout 60 - -# Don't scan files and directories matching regex -# This directive can be used multiple times -# Default: scan all -#ExcludePath ^/proc/ -#ExcludePath ^/sys/ - -# Maximum depth directories are scanned at. -# Default: 15 -#MaxDirectoryRecursion 20 - -# Follow directory symlinks. -# Default: no -#FollowDirectorySymlinks yes - -# Follow regular file symlinks. -# Default: no -#FollowFileSymlinks yes - -# Scan files and directories on other filesystems. -# Default: yes -#CrossFilesystems yes - -# Perform a database check. -# Default: 600 (10 min) -#SelfCheck 600 - -# Execute a command when virus is found. In the command string %v will -# be replaced with the virus name. -# Default: no -#VirusEvent /usr/local/bin/send_sms 123456789 "VIRUS ALERT: %v" - -# Run as another user (clamd must be started by root for this option to work) -# Default: don't drop privileges -User clamupdate - -# Initialize supplementary group access (clamd must be started by root). -# Default: no -AllowSupplementaryGroups yes - -# Stop daemon when libclamav reports out of memory condition. -#ExitOnOOM yes - -# Don't fork into background. -# Default: no -#Foreground yes - -# Enable debug messages in libclamav. -# Default: no -#Debug yes - -# Do not remove temporary files (for debug purposes). -# Default: no -#LeaveTemporaryFiles yes - -# Permit use of the ALLMATCHSCAN command. If set to no, clamd will reject -# any ALLMATCHSCAN command as invalid. -# Default: yes -#AllowAllMatchScan no - -# Detect Possibly Unwanted Applications. -# Default: no -#DetectPUA yes - -# Exclude a specific PUA category. This directive can be used multiple times. -# See https://github.com/vrtadmin/clamav-faq/blob/master/faq/faq-pua.md for -# the complete list of PUA categories. -# Default: Load all categories (if DetectPUA is activated) -#ExcludePUA NetTool -#ExcludePUA PWTool - -# Only include a specific PUA category. This directive can be used multiple -# times. -# Default: Load all categories (if DetectPUA is activated) -#IncludePUA Spy -#IncludePUA Scanner -#IncludePUA RAT - -# In some cases (eg. complex malware, exploits in graphic files, and others), -# ClamAV uses special algorithms to provide accurate detection. This option -# controls the algorithmic detection. -# Default: yes -#AlgorithmicDetection yes - -# This option causes memory or nested map scans to dump the content to disk. -# If you turn on this option, more data is written to disk and is available -# when the LeaveTemporaryFiles option is enabled. -#ForceToDisk yes - -# This option allows you to disable the caching feature of the engine. By -# default, the engine will store an MD5 in a cache of any files that are -# not flagged as virus or that hit limits checks. Disabling the cache will -# have a negative performance impact on large scans. -# Default: no -#DisableCache yes - -## -## Executable files -## - -# PE stands for Portable Executable - it's an executable file format used -# in all 32 and 64-bit versions of Windows operating systems. This option allows -# ClamAV to perform a deeper analysis of executable files and it's also -# required for decompression of popular executable packers such as UPX, FSG, -# and Petite. If you turn off this option, the original files will still be -# scanned, but without additional processing. -# Default: yes -#ScanPE yes - -# Certain PE files contain an authenticode signature. By default, we check -# the signature chain in the PE file against a database of trusted and -# revoked certificates if the file being scanned is marked as a virus. -# If any certificate in the chain validates against any trusted root, but -# does not match any revoked certificate, the file is marked as whitelisted. -# If the file does match a revoked certificate, the file is marked as virus. -# The following setting completely turns off authenticode verification. -# Default: no -#DisableCertCheck yes - -# Executable and Linking Format is a standard format for UN*X executables. -# This option allows you to control the scanning of ELF files. -# If you turn off this option, the original files will still be scanned, but -# without additional processing. -# Default: yes -#ScanELF yes - -# With this option clamav will try to detect broken executables (both PE and -# ELF) and mark them as Broken.Executable. -# Default: no -#DetectBrokenExecutables yes - - -## -## Documents -## - -# This option enables scanning of OLE2 files, such as Microsoft Office -# documents and .msi files. -# If you turn off this option, the original files will still be scanned, but -# without additional processing. -# Default: yes -#ScanOLE2 yes - -# With this option enabled OLE2 files with VBA macros, which were not -# detected by signatures will be marked as "Heuristics.OLE2.ContainsMacros". -# Default: no -#OLE2BlockMacros no - -# This option enables scanning within PDF files. -# If you turn off this option, the original files will still be scanned, but -# without decoding and additional processing. -# Default: yes -#ScanPDF yes - -# This option enables scanning within SWF files. -# If you turn off this option, the original files will still be scanned, but -# without decoding and additional processing. -# Default: yes -#ScanSWF yes - -# This option enables scanning xml-based document files supported by libclamav. -# If you turn off this option, the original files will still be scanned, but -# without additional processing. -# Default: yes -#ScanXMLDOCS yes - -# This option enables scanning of HWP3 files. -# If you turn off this option, the original files will still be scanned, but -# without additional processing. -# Default: yes -#ScanHWP3 yes - - -## -## Mail files -## - -# Enable internal e-mail scanner. -# If you turn off this option, the original files will still be scanned, but -# without parsing individual messages/attachments. -# Default: yes -#ScanMail yes - -# Scan RFC1341 messages split over many emails. -# You will need to periodically clean up $TemporaryDirectory/clamav-partial directory. -# WARNING: This option may open your system to a DoS attack. -# Never use it on loaded servers. -# Default: no -#ScanPartialMessages yes - -# With this option enabled ClamAV will try to detect phishing attempts by using -# signatures. -# Default: yes -#PhishingSignatures yes - -# Scan URLs found in mails for phishing attempts using heuristics. -# Default: yes -#PhishingScanURLs yes - -# Always block SSL mismatches in URLs, even if the URL isn't in the database. -# This can lead to false positives. -# -# Default: no -#PhishingAlwaysBlockSSLMismatch no - -# Always block cloaked URLs, even if URL isn't in database. -# This can lead to false positives. -# -# Default: no -#PhishingAlwaysBlockCloak no - -# Detect partition intersections in raw disk images using heuristics. -# Default: no -#PartitionIntersection no - -# Allow heuristic match to take precedence. -# When enabled, if a heuristic scan (such as phishingScan) detects -# a possible virus/phish it will stop scan immediately. Recommended, saves CPU -# scan-time. -# When disabled, virus/phish detected by heuristic scans will be reported only at -# the end of a scan. If an archive contains both a heuristically detected -# virus/phish, and a real malware, the real malware will be reported -# -# Keep this disabled if you intend to handle "*.Heuristics.*" viruses -# differently from "real" malware. -# If a non-heuristically-detected virus (signature-based) is found first, -# the scan is interrupted immediately, regardless of this config option. -# -# Default: no -#HeuristicScanPrecedence yes - - -## -## Data Loss Prevention (DLP) -## - -# Enable the DLP module -# Default: No -#StructuredDataDetection yes - -# This option sets the lowest number of Credit Card numbers found in a file -# to generate a detect. -# Default: 3 -#StructuredMinCreditCardCount 5 - -# This option sets the lowest number of Social Security Numbers found -# in a file to generate a detect. -# Default: 3 -#StructuredMinSSNCount 5 - -# With this option enabled the DLP module will search for valid -# SSNs formatted as xxx-yy-zzzz -# Default: yes -#StructuredSSNFormatNormal yes - -# With this option enabled the DLP module will search for valid -# SSNs formatted as xxxyyzzzz -# Default: no -#StructuredSSNFormatStripped yes - - -## -## HTML -## - -# Perform HTML normalisation and decryption of MS Script Encoder code. -# Default: yes -# If you turn off this option, the original files will still be scanned, but -# without additional processing. -#ScanHTML yes - - -## -## Archives -## - -# ClamAV can scan within archives and compressed files. -# If you turn off this option, the original files will still be scanned, but -# without unpacking and additional processing. -# Default: yes -#ScanArchive yes - -# Mark encrypted archives as viruses (Encrypted.Zip, Encrypted.RAR). -# Default: no -#ArchiveBlockEncrypted no - - -## -## Limits -## - -# The options below protect your system against Denial of Service attacks -# using archive bombs. - -# This option sets the maximum amount of data to be scanned for each input file. -# Archives and other containers are recursively extracted and scanned up to this -# value. -# Value of 0 disables the limit -# Note: disabling this limit or setting it too high may result in severe damage -# to the system. -# Default: 100M -#MaxScanSize 150M - -# Files larger than this limit won't be scanned. Affects the input file itself -# as well as files contained inside it (when the input file is an archive, a -# document or some other kind of container). -# Value of 0 disables the limit. -# Note: disabling this limit or setting it too high may result in severe damage -# to the system. -# Default: 25M -#MaxFileSize 30M - -# Nested archives are scanned recursively, e.g. if a Zip archive contains a RAR -# file, all files within it will also be scanned. This options specifies how -# deeply the process should be continued. -# Note: setting this limit too high may result in severe damage to the system. -# Default: 16 -#MaxRecursion 10 - -# Number of files to be scanned within an archive, a document, or any other -# container file. -# Value of 0 disables the limit. -# Note: disabling this limit or setting it too high may result in severe damage -# to the system. -# Default: 10000 -#MaxFiles 15000 - -# Maximum size of a file to check for embedded PE. Files larger than this value -# will skip the additional analysis step. -# Note: disabling this limit or setting it too high may result in severe damage -# to the system. -# Default: 10M -#MaxEmbeddedPE 10M - -# Maximum size of a HTML file to normalize. HTML files larger than this value -# will not be normalized or scanned. -# Note: disabling this limit or setting it too high may result in severe damage -# to the system. -# Default: 10M -#MaxHTMLNormalize 10M - -# Maximum size of a normalized HTML file to scan. HTML files larger than this -# value after normalization will not be scanned. -# Note: disabling this limit or setting it too high may result in severe damage -# to the system. -# Default: 2M -#MaxHTMLNoTags 2M - -# Maximum size of a script file to normalize. Script content larger than this -# value will not be normalized or scanned. -# Note: disabling this limit or setting it too high may result in severe damage -# to the system. -# Default: 5M -#MaxScriptNormalize 5M - -# Maximum size of a ZIP file to reanalyze type recognition. ZIP files larger -# than this value will skip the step to potentially reanalyze as PE. -# Note: disabling this limit or setting it too high may result in severe damage -# to the system. -# Default: 1M -#MaxZipTypeRcg 1M - -# This option sets the maximum number of partitions of a raw disk image to be scanned. -# Raw disk images with more partitions than this value will have up to the value number -# partitions scanned. Negative values are not allowed. -# Note: setting this limit too high may result in severe damage or impact performance. -# Default: 50 -#MaxPartitions 128 - -# This option sets the maximum number of icons within a PE to be scanned. -# PE files with more icons than this value will have up to the value number icons scanned. -# Negative values are not allowed. -# WARNING: setting this limit too high may result in severe damage or impact performance. -# Default: 100 -#MaxIconsPE 200 - -# This option sets the maximum recursive calls for HWP3 parsing during scanning. -# HWP3 files using more than this limit will be terminated and alert the user. -# Scans will be unable to scan any HWP3 attachments if the recursive limit is reached. -# Negative values are not allowed. -# WARNING: setting this limit too high may result in severe damage or impact performance. -# Default: 16 -#MaxRecHWP3 16 - -# This option sets the maximum calls to the PCRE match function during an instance of regex matching. -# Instances using more than this limit will be terminated and alert the user but the scan will continue. -# For more information on match_limit, see the PCRE documentation. -# Negative values are not allowed. -# WARNING: setting this limit too high may severely impact performance. -# Default: 10000 -#PCREMatchLimit 20000 - -# This option sets the maximum recursive calls to the PCRE match function during an instance of regex matching. -# Instances using more than this limit will be terminated and alert the user but the scan will continue. -# For more information on match_limit_recursion, see the PCRE documentation. -# Negative values are not allowed and values > PCREMatchLimit are superfluous. -# WARNING: setting this limit too high may severely impact performance. -# Default: 5000 -#PCRERecMatchLimit 10000 - -# This option sets the maximum filesize for which PCRE subsigs will be executed. -# Files exceeding this limit will not have PCRE subsigs executed unless a subsig is encompassed to a smaller buffer. -# Negative values are not allowed. -# Setting this value to zero disables the limit. -# WARNING: setting this limit too high or disabling it may severely impact performance. -# Default: 25M -#PCREMaxFileSize 100M - - -## -## On-access Scan Settings -## - -# Enable on-access scanning. Currently, this is supported via fanotify. -# Clamuko/Dazuko support has been deprecated. -# Default: no -#ScanOnAccess yes - -# Set the mount point to be scanned. The mount point specified, or the mount point -# containing the specified directory will be watched. If any directories are specified, -# this option will preempt the DDD system. This will notify only. It can be used multiple times. -# (On-access scan only) -# Default: disabled -#OnAccessMountPath / -#OnAccessMountPath /home/user - -# Don't scan files larger than OnAccessMaxFileSize -# Value of 0 disables the limit. -# Default: 5M -#OnAccessMaxFileSize 10M - -# Set the include paths (all files inside them will be scanned). You can have -# multiple OnAccessIncludePath directives but each directory must be added -# in a separate line. (On-access scan only) -# Default: disabled -#OnAccessIncludePath /home -#OnAccessIncludePath /students - -# Set the exclude paths. All subdirectories are also excluded. -# (On-access scan only) -# Default: disabled -#OnAccessExcludePath /home/bofh - -# With this option you can whitelist specific UIDs. Processes with these UIDs -# will be able to access all files. -# This option can be used multiple times (one per line). -# Default: disabled -#OnAccessExcludeUID 0 - -# Toggles dynamic directory determination. Allows for recursively watching include paths. -# (On-access scan only) -# Default: no -#OnAccessDisableDDD yes - -# Modifies fanotify blocking behaviour when handling permission events. -# If off, fanotify will only notify if the file scanned is a virus, -# and not perform any blocking. -# (On-access scan only) -# Default: no -#OnAccessPrevention yes - -# Toggles extra scanning and notifications when a file or directory is created or moved. -# Requires the DDD system to kick-off extra scans. -# (On-access scan only) -# Default: no -#OnAccessExtraScanning yes - -## -## Bytecode -## - -# With this option enabled ClamAV will load bytecode from the database. -# It is highly recommended you keep this option on, otherwise you'll miss detections for many new viruses. -# Default: yes -#Bytecode yes - -# Bytecode mode -# -# This option has been set to 'ForceInterpreter' in Fedora due to -# security concerns by default. You might need to enable the -# 'antivirus_use_jit' SELinux boolean after setting this option to -# the more efficient 'ForceJIT' value. -# -# Default: ForceInterpreter -#ByteCodeMode ForceInterpreter - -# Set bytecode security level. -# Possible values: -# None - no security at all, meant for debugging. DO NOT USE THIS ON PRODUCTION SYSTEMS -# This value is only available if clamav was built with --enable-debug! -# TrustSigned - trust bytecode loaded from signed .c[lv]d files, -# insert runtime safety checks for bytecode loaded from other sources -# Paranoid - don't trust any bytecode, insert runtime checks for all -# Recommended: TrustSigned, because bytecode in .cvd files already has these checks -# Note that by default only signed bytecode is loaded, currently you can only -# load unsigned bytecode in --enable-debug mode. -# -# Default: TrustSigned -#BytecodeSecurity TrustSigned - -# Set bytecode timeout in miliseconds. -# -# Default: 5000 -# BytecodeTimeout 1000 - -## -## Statistics gathering and submitting -## - -# Enable statistical reporting. -# Default: no -#StatsEnabled yes - -# Disable submission of individual PE sections for files flagged as malware. -# Default: no -#StatsPEDisabled yes - -# HostID in the form of an UUID to use when submitting statistical information. -# Default: auto -#StatsHostID auto - -# Time in seconds to wait for the stats server to come back with a response -# Default: 10 -#StatsTimeout 10 diff --git a/ansible/roles/pagure-dev/files/gitolite3.rc b/ansible/roles/pagure-dev/files/gitolite3.rc deleted file mode 100644 index 1a20d42..0000000 --- a/ansible/roles/pagure-dev/files/gitolite3.rc +++ /dev/null @@ -1,195 +0,0 @@ -# configuration variables for gitolite - -# This file is in perl syntax. But you do NOT need to know perl to edit it -- -# just mind the commas, use single quotes unless you know what you're doing, -# and make sure the brackets and braces stay matched up! - -# (Tip: perl allows a comma after the last item in a list also!) - -# HELP for commands can be had by running the command with "-h". - -# HELP for all the other FEATURES can be found in the documentation (look for -# "list of non-core programs shipped with gitolite" in the master index) or -# directly in the corresponding source file. - -%RC = ( - - # ------------------------------------------------------------------ - - # default umask gives you perms of '0700'; see the rc file docs for - # how/why you might change this - UMASK => 0077, - - # look for "git-config" in the documentation - GIT_CONFIG_KEYS => '', - - # comment out if you don't need all the extra detail in the logfile - LOG_EXTRA => 1, - # syslog options - # 1. leave this section as is for normal gitolite logging - # 2. uncomment this line to log only to syslog: - # LOG_DEST => 'syslog', - # 3. uncomment this line to log to syslog and the normal gitolite log: - # LOG_DEST => 'syslog,normal', - - # roles. add more roles (like MANAGER, TESTER, ...) here. - # WARNING: if you make changes to this hash, you MUST run 'gitolite - # compile' afterward, and possibly also 'gitolite trigger POST_COMPILE' - ROLES => { - READERS => 1, - WRITERS => 1, - }, - - # enable caching (currently only Redis). PLEASE RTFM BEFORE USING!!! - # CACHE => 'Redis', - - # ------------------------------------------------------------------ - - # rc variables used by various features - - # the 'info' command prints this as additional info, if it is set - # SITE_INFO => 'Please see http://blahblah/gitolite for more help', - - # the CpuTime feature uses these - # display user, system, and elapsed times to user after each git operation - # DISPLAY_CPU_TIME => 1, - # display a warning if total CPU times (u, s, cu, cs) crosses this limit - # CPU_TIME_WARN_LIMIT => 0.1, - - # the Mirroring feature needs this - # HOSTNAME => "foo", - - # TTL for redis cache; PLEASE SEE DOCUMENTATION BEFORE UNCOMMENTING! - # CACHE_TTL => 600, - - # ------------------------------------------------------------------ - - # suggested locations for site-local gitolite code (see cust.html) - - # this one is managed directly on the server - # LOCAL_CODE => "$ENV{HOME}/local", - - # or you can use this, which lets you put everything in a subdirectory - # called "local" in your gitolite-admin repo. For a SECURITY WARNING - # on this, see http://gitolite.com/gitolite/non-core.html#pushcode - # LOCAL_CODE => "$rc{GL_ADMIN_BASE}/local", - - # ------------------------------------------------------------------ - - # List of commands and features to enable - - ENABLE => [ - - # COMMANDS - - # These are the commands enabled by default - 'help', - 'desc', - 'info', - 'perms', - 'writable', - - # Uncomment or add new commands here. - # 'create', - # 'fork', - # 'mirror', - # 'readme', - # 'sskm', - # 'D', - - # These FEATURES are enabled by default. - - # essential (unless you're using smart-http mode) - 'ssh-authkeys', - - # creates git-config enties from gitolite.conf file entries like 'config foo.bar = baz' - 'git-config', - - # creates git-daemon-export-ok files; if you don't use git-daemon, comment this out - 'daemon', - - # creates projects.list file; if you don't use gitweb, comment this out - #'gitweb', - - # These FEATURES are disabled by default; uncomment to enable. If you - # need to add new ones, ask on the mailing list :-) - - # user-visible behaviour - - # prevent wild repos auto-create on fetch/clone - # 'no-create-on-read', - # no auto-create at all (don't forget to enable the 'create' command!) - # 'no-auto-create', - - # access a repo by another (possibly legacy) name - # 'Alias', - - # give some users direct shell access. See documentation in - # sts.html for details on the following two choices. - # "Shell $ENV{HOME}/.gitolite.shell-users", - # 'Shell alice bob', - - # set default roles from lines like 'option default.roles-1 = ...', etc. - # 'set-default-roles', - - # show more detailed messages on deny - # 'expand-deny-messages', - - # show a message of the day - # 'Motd', - - # system admin stuff - - # enable mirroring (don't forget to set the HOSTNAME too!) - # 'Mirroring', - - # allow people to submit pub files with more than one key in them - # 'ssh-authkeys-split', - - # selective read control hack - # 'partial-copy', - - # manage local, gitolite-controlled, copies of read-only upstream repos - # 'upstream', - - # updates 'description' file instead of 'gitweb.description' config item - # 'cgit', - - # allow repo-specific hooks to be added - # 'repo-specific-hooks', - - # performance, logging, monitoring... - - # be nice - # 'renice 10', - - # log CPU times (user, system, cumulative user, cumulative system) - # 'CpuTime', - - # syntactic_sugar for gitolite.conf and included files - - # allow backslash-escaped continuation lines in gitolite.conf - # 'continuation-lines', - - # create implicit user groups from directory names in keydir/ - # 'keysubdirs-as-groups', - - # allow simple line-oriented macros - # 'macros', - - # Kindergarten mode - - # disallow various things that sensible people shouldn't be doing anyway - # 'Kindergarten', - ], - -); - -# ------------------------------------------------------------------------------ -# per perl rules, this should be the last line in such a file: -1; - -# Local variables: -# mode: perl -# End: -# vim: set syn=perl: diff --git a/ansible/roles/pagure-dev/files/motd b/ansible/roles/pagure-dev/files/motd deleted file mode 100644 index bca778c..0000000 --- a/ansible/roles/pagure-dev/files/motd +++ /dev/null @@ -1,22 +0,0 @@ - -Welcome to the Pagure development environment! - -Here are some tips: - -* Pagure is installed in a Python virtualenv. Use `workon python2-pagure` to - enter the virtualenv. - -* The code for Pagure is located at ~/devel/ - -* You can populate the database with the `dev-data.py` script in the repository - -* Run `pstart` to start the development server and `pstop` to stop it. - -* Logs for the server are available with `journalctl`; the services are run - as systemd user units in ~/.config/systemd/user/ - -Once you start the server you can navigate to http://localhost:5000/ -in your browser on the host to access your Pagure development environment. - -Happy hacking! - diff --git a/ansible/roles/pagure-dev/files/pagure-docs.service b/ansible/roles/pagure-dev/files/pagure-docs.service deleted file mode 100644 index beeeca3..0000000 --- a/ansible/roles/pagure-dev/files/pagure-docs.service +++ /dev/null @@ -1,12 +0,0 @@ -[Unit] -Description=Runs the Pagure documentation server -After=network.target - -[Service] -Environment="PAGURE_CONFIG=/home/vagrant/pagure.cfg" -ExecStart=/home/vagrant/.virtualenvs/python2-pagure/bin/python \ - /home/vagrant/devel/rundocserver.py --host 0.0.0.0 -Type=simple - -[Install] -WantedBy=multi-user.target diff --git a/ansible/roles/pagure-dev/files/pagure.cfg b/ansible/roles/pagure-dev/files/pagure.cfg deleted file mode 100644 index a813f09..0000000 --- a/ansible/roles/pagure-dev/files/pagure.cfg +++ /dev/null @@ -1,173 +0,0 @@ -import os -from datetime import timedelta - -### Set the time after which the admin session expires -# There are two sessions on pagure, login that holds for 31 days and -# the session defined here after which an user has to re-login. -# This session is used when accessing all administrative parts of pagure -# (ie: changing a project's or a user's settings) -ADMIN_SESSION_LIFETIME = timedelta(minutes=20000000) - -### Secret key for the Flask application -SECRET_KEY='' - -### url to the database server: -#DB_URL=mysql://user:pass@host/db_name -#DB_URL=postgres://user:pass@host/db_name -DB_URL = 'sqlite:////home/vagrant/pagure_data/pagure_dev.sqlite' - -### The FAS group in which the admin of pagure are -ADMIN_GROUP = ['sysadmin-main'] - -### Hard-coded list of global admins -PAGURE_ADMIN_USERS = [] - -### The URL at which the project is available. -APP_URL = '*' -### The URL at which the documentation of projects will be available -## This should be in a different domain to avoid XSS issues since we want -## to allow raw html to be displayed (different domain, ie not a sub-domain). -DOC_APP_URL = '*' - -# Avoid sending emails while developing by default -EMAIL_SEND = False -EMAIL_ERROR = 'vagrant@localhost' - -### The URL to use to clone git repositories. -GIT_URL_SSH = 'ssh://vagrant@pagure-dev.example.com/' -GIT_URL_GIT = 'http://pagure-dev.example.com:5000/' - -### Folder containing to the git repos -STORAGE_ROOT = '/home/vagrant/pagure_data/' - -GIT_FOLDER = os.path.join(STORAGE_ROOT, 'repos') - -### Folder containing the docs repos -DOCS_FOLDER = os.path.join(STORAGE_ROOT, 'docs') - -### Folder containing the tickets repos -TICKETS_FOLDER = os.path.join(STORAGE_ROOT, 'tickets') - -### Folder containing the pull-requests repos -REQUESTS_FOLDER = os.path.join(STORAGE_ROOT, 'requests') - -### Folder containing the clones for the remote pull-requests -REMOTE_GIT_FOLDER = os.path.join(STORAGE_ROOT, 'remotes') - -### Whether to enable scanning for viruses in attachments -VIRUS_SCAN_ATTACHMENTS = False - -### Home folder of the gitolite user -### Folder where to run gl-compile-conf from -GITOLITE_HOME = '/home/vagrant/' - -### Configuration file for gitolite -GITOLITE_CONFIG = os.path.join(GITOLITE_HOME, '.gitolite/conf/gitolite.conf') - -### Version of gitolite used: 2 or 3? -GITOLITE_VERSION = 3 - -### Folder containing all the public ssh keys for gitolite -GITOLITE_KEYDIR = os.path.join(GITOLITE_HOME, '.gitolite/keydir/') - -### Path to the gitolite.rc file -GL_RC = '/home/vagrant/.gitolite.rc' - -### Path to the /bin directory where the gitolite tools can be found -GL_BINDIR = '/usr/bin/' - - -# SSH Information - -### The ssh certificates of the git server to be provided to the user -### /!\ format is important -# SSH_KEYS = {'RSA': {'fingerprint': '', 'pubkey': ''}} - - - -# Optional configuration - -### Number of items displayed per page -# Used when listing items -ITEM_PER_PAGE = 50 - -### Maximum size of the uploaded content -# Used to limit the size of file attached to a ticket for example -MAX_CONTENT_LENGTH = 4 * 1024 * 1024 # 4 megabytes - -### Lenght for short commits ids or file hex -SHORT_LENGTH = 6 - -### List of blacklisted project names that can conflicts for pagure's URLs -### or other -BLACKLISTED_PROJECTS = [ - 'static', 'pv', 'releases', 'new', 'api', 'settings', - 'logout', 'login', 'users', 'groups', 'projects'] - -### IP addresses allowed to access the internal endpoints -### These endpoints are used by the milter and are security sensitive, thus -### the IP filter -IP_ALLOWED_INTERNAL = ['127.0.0.1', 'localhost', '::1',] - -### EventSource/Web-Hook/Redis configuration -# The eventsource integration is what allows pagure to refresh the content -# on your page when someone else comments on the ticket (and this without -# asking you to reload the page. -# By default it is off, ie: EVENTSOURCE_SOURCE is None, to turn it on, specify -# here what the URL of the eventsource server is, for example: -# https://ev.pagure.io or https://pagure.io:8080 or whatever you are using -# (Note: the urls sent to it start with a '/' so no need to add one yourself) -EVENTSOURCE_SOURCE = 'http://localhost:8080' -# Port where the event source server is running (maybe be the same port -# as the one specified in EVENTSOURCE_SOURCE or a different one if you -# have something running in front of the server such as apache or stunnel). -EVENTSOURCE_PORT = 8080 -# If this port is specified, the event source server will run another server -# at this port and will provide information about the number of active -# connections running on the first (main) event source server -#EV_STATS_PORT = 8888 -# Web-hook can be turned on or off allowing using them for notifications, or -# not. -WEBHOOK = True - -### Redis configuration -# A redis server is required for both the Event-Source server or the web-hook -# server. -REDIS_HOST = '127.0.0.1' -REDIS_PORT = 6379 -REDIS_DB = 0 - -# Authentication related configuration option - -### Switch the authentication method -# Specify which authentication method to use, defaults to `fas` can be or -# `local` -# Default: ``fas``. -PAGURE_AUTH = 'fas' - -# When this is set to True, the session cookie will only be returned to the -# server via ssl (https). If you connect to the server via plain http, the -# cookie will not be sent. This prevents sniffing of the cookie contents. -# This may be set to False when testing your application but should always -# be set to True in production. -# Default: ``True``. -SESSION_COOKIE_SECURE = False - -# The name of the cookie used to store the session id. -# Default: ``.pagure``. -SESSION_COOKIE_NAME = 'pagure' - -# Boolean specifying whether to check the user's IP address when retrieving -# its session. This make things more secure (thus is on by default) but -# under certain setup it might not work (for example is there are proxies -# in front of the application). -CHECK_SESSION_IP = True - -# Used by SESSION_COOKIE_PATH -APPLICATION_ROOT = '/' - -# Allow the backward compatiblity endpoints for the old URLs schema to -# see the commits of a repo. This is only interesting if you pagure instance -# was running since before version 1.3 and if you care about backward -# compatibility in your URLs. -OLD_VIEW_COMMIT_ENABLED = False diff --git a/ansible/roles/pagure-dev/files/pagure.service b/ansible/roles/pagure-dev/files/pagure.service deleted file mode 100644 index 7999bc9..0000000 --- a/ansible/roles/pagure-dev/files/pagure.service +++ /dev/null @@ -1,11 +0,0 @@ -[Unit] -Description=The Pagure web service -After=network.target - -[Service] -Environment="PAGURE_CONFIG=/home/vagrant/pagure.cfg" -ExecStart=/home/vagrant/.virtualenvs/python2-pagure/bin/python %h/devel/runserver.py --host 0.0.0.0 -Type=simple - -[Install] -WantedBy=multi-user.target diff --git a/ansible/roles/pagure-dev/files/pagure_ci.service b/ansible/roles/pagure-dev/files/pagure_ci.service deleted file mode 100644 index b9e427a..0000000 --- a/ansible/roles/pagure-dev/files/pagure_ci.service +++ /dev/null @@ -1,13 +0,0 @@ -[Unit] -Description=Pagure Continuous Integration service -After=redis.target -Documentation=https://pagure.io/pagure - -[Service] -Environment="PAGURE_CONFIG=/home/vagrant/pagure.cfg" -ExecStart=/home/vagrant/.virtualenvs/python2-pagure/bin/python \ - /home/vagrant/devel/pagure-ci/pagure_ci_server.py -Type=simple - -[Install] -WantedBy=multi-user.target diff --git a/ansible/roles/pagure-dev/files/pagure_ev.service b/ansible/roles/pagure-dev/files/pagure_ev.service deleted file mode 100644 index 9b9a821..0000000 --- a/ansible/roles/pagure-dev/files/pagure_ev.service +++ /dev/null @@ -1,13 +0,0 @@ -[Unit] -Description=Pagure EventSource server (Allowing live refresh of the pages supporting it) -After=redis.target -Documentation=https://pagure.io/pagure - -[Service] -Environment="PAGURE_CONFIG=/home/vagrant/pagure.cfg" -ExecStart=/home/vagrant/.virtualenvs/python2-pagure/bin/python \ - /home/vagrant/devel/pagure-ev/pagure_stream_server.py -Type=simple - -[Install] -WantedBy=multi-user.target diff --git a/ansible/roles/pagure-dev/files/pagure_webhook.service b/ansible/roles/pagure-dev/files/pagure_webhook.service deleted file mode 100644 index 601e296..0000000 --- a/ansible/roles/pagure-dev/files/pagure_webhook.service +++ /dev/null @@ -1,13 +0,0 @@ -[Unit] -Description=Pagure WebHook server (Allowing web-hook notifications) -After=redis.target -Documentation=https://pagure.io/pagure - -[Service] -Environment="PAGURE_CONFIG=/home/vagrant/pagure.cfg" -ExecStart=/home/vagrant/.virtualenvs/python2-pagure/bin/python \ - /home/vagrant/devel/webhook-server/pagure-webhook-server.py -Type=simple - -[Install] -WantedBy=multi-user.target diff --git a/ansible/roles/pagure-dev/tasks/clamav.yml b/ansible/roles/pagure-dev/tasks/clamav.yml deleted file mode 100644 index 537d95a..0000000 --- a/ansible/roles/pagure-dev/tasks/clamav.yml +++ /dev/null @@ -1,30 +0,0 @@ ---- - -- name: Install ClamAV packages - dnf: name={{ item }} state=present - with_items: - - clamav-data-empty - - clamav-server - - clamav-server-systemd - - clamav-update - -- name: Configure freshclam - replace: - dest: /etc/freshclam.conf - regexp: "Example*" - replace: "" - -- name: Install Pagure's ClamAV configuration - copy: - src: clamd.conf - dest: /etc/clamd.d/pagure.conf - -# pyclamd expects /etc/clamd.conf -- name: Link /etc/clamd.conf to our pagure config - file: src=/etc/clamd.d/pagure.conf dest=/etc/clamd.conf state=link - -- name: Download latest ClamAV database - command: freshclam - -- name: Start ClamAV - service: name=clamd@pagure state=started enabled=yes diff --git a/ansible/roles/pagure-dev/tasks/eventsource.yml b/ansible/roles/pagure-dev/tasks/eventsource.yml deleted file mode 100644 index 98ee627..0000000 --- a/ansible/roles/pagure-dev/tasks/eventsource.yml +++ /dev/null @@ -1,13 +0,0 @@ ---- - -- name: Install Redis - dnf: name={{ item }} state=present - with_items: - - python-redis - - python-trollius - - python-trollius-redis - - redis - - -- name: Start Redis - service: name=redis state=started enabled=yes diff --git a/ansible/roles/pagure-dev/tasks/gitolite.yml b/ansible/roles/pagure-dev/tasks/gitolite.yml deleted file mode 100644 index 61080bd..0000000 --- a/ansible/roles/pagure-dev/tasks/gitolite.yml +++ /dev/null @@ -1,25 +0,0 @@ ---- - -- name: Install gitolite3 - dnf: name={{ item }} state=present - with_items: - - gitolite3 - -- name: Install gitolite.rc to ~/.gitolite.rc - become_user: "{{ ansible_env.SUDO_USER }}" - copy: - src: gitolite3.rc - dest: /home/{{ ansible_env.SUDO_USER }}/.gitolite.rc - -- name: Create a key for gitolite - become_user: "{{ ansible_env.SUDO_USER }}" - command: ssh-keygen -f gitolite_rsa -t rsa -N '' - args: - chdir: /home/{{ ansible_env.SUDO_USER }} - creates: /home/{{ ansible_env.SUDO_USER }}/gitolite_rsa.pub - -- name: Setup gitolite - become_user: "{{ ansible_env.SUDO_USER }}" - command: gitolite setup -pk gitolite_rsa.pub - args: - chdir: /home/{{ ansible_env.SUDO_USER }} diff --git a/ansible/roles/pagure-dev/tasks/main.yml b/ansible/roles/pagure-dev/tasks/main.yml deleted file mode 100644 index 64799fb..0000000 --- a/ansible/roles/pagure-dev/tasks/main.yml +++ /dev/null @@ -1,176 +0,0 @@ ---- - -- include: clamav.yml -- include: eventsource.yml -- include: gitolite.yml -- include: milter.yml -- include: postgres.yml - -- name: Install helpful development packages - dnf: name={{ item }} state=present - with_items: - - git - - ngrep - - nmap-ncat - - python-rpdb - - tmux - - tree - - vim-enhanced - -- name: Install Pagure development packages - dnf: name={{ item }} state=present - with_items: - - gcc - - libgit2-devel - - libffi-devel - - libjpeg-devel - - make - - python-alembic - - python-arrow - - python-binaryornot - - python-bleach - - python-blinker - - python-chardet - - python-cryptography - - python-docutils - - python-enum34 - - python2-eventlet - - python-fedora-flask - - python-flask - - python-flask-wtf - - python-flask-multistatic - - python2-jinja2 - - python-markdown - - python-munch - - python-openid-cla - - python-openid-teams - - python-pip - - python-psutil - - python-pygit2 - - python-pygments - - python-redis - - python-sqlalchemy - - python-straight-plugin - - python-virtualenvwrapper - - python-wtforms - - python-devel - - python3-devel - - redhat-rpm-config - -- name: register the libgit2 version installed - shell: rpm -q libgit2|cut -d \- -f 2| cut -d \. -f 1,2 - register: libgit2_version - -# Add various helpful configuration files -- name: Install a custom bashrc - become_user: "{{ ansible_env.SUDO_USER }}" - copy: src=bashrc dest=/home/{{ ansible_env.SUDO_USER }}/.bashrc - -- name: Install the message of the day - copy: src=motd dest=/etc/motd - - -# Install Pagure inside a virtualenv and configure it -- name: Install pygit2 in the virtualenv - become_user: "{{ ansible_env.SUDO_USER }}" - pip: - name: "{{ item }}" - virtualenv: /home/{{ ansible_env.SUDO_USER }}/.virtualenvs/python2-pagure/ - virtualenv_python: python2 - with_items: - - "pygit2=={{ libgit2_version.stdout_lines[0] }}.*" - -- name: Install Pagure Python dependencies into a virtualenv - become_user: "{{ ansible_env.SUDO_USER }}" - pip: - requirements: /home/{{ ansible_env.SUDO_USER }}/devel/{{ item }} - virtualenv: /home/{{ ansible_env.SUDO_USER }}/.virtualenvs/python2-pagure/ - virtualenv_python: python2 - with_items: - - "requirements.txt" - - "tests_requirements.txt" - -- name: Install Pagure package into a virtualenv - become_user: "{{ ansible_env.SUDO_USER }}" - pip: - name: /home/{{ ansible_env.SUDO_USER }}/devel/ - extra_args: '-e' - virtualenv: /home/{{ ansible_env.SUDO_USER }}/.virtualenvs/python2-pagure/ - -- name: Install Pagure package into /usr/lib - pip: - name: /home/{{ ansible_env.SUDO_USER }}/devel/ - extra_args: '-e' - -- name: Install the pagure configuration - become_user: "{{ ansible_env.SUDO_USER }}" - copy: src=pagure.cfg dest=/home/{{ ansible_env.SUDO_USER }}/pagure.cfg - -- name: Creates pagure data directories - become_user: "{{ ansible_env.SUDO_USER }}" - file: path=/home/{{ ansible_env.SUDO_USER }}/pagure_data/{{ item }} state=directory - with_items: - - forks - - docs - - tickets - - requests - - remotes - -- name: Link the pagure repos directory to gitolite - become_user: "{{ ansible_env.SUDO_USER }}" - file: - path: /home/{{ ansible_env.SUDO_USER }}/pagure_data/repos - src: /home/{{ ansible_env.SUDO_USER }}/repositories - state: link - -- name: Add a working copy of alembic.ini - become_user: "{{ ansible_env.SUDO_USER }}" - copy: - src: /home/{{ ansible_env.SUDO_USER }}/devel/files/alembic.ini - dest: /home/{{ ansible_env.SUDO_USER }}/alembic.ini - remote_src: True - -- name: Configure alembic to use our development database - become_user: "{{ ansible_env.SUDO_USER }}" - replace: - dest: /home/{{ ansible_env.SUDO_USER }}/alembic.ini - regexp: "sqlalchemy.url = sqlite:////var/tmp/pagure_dev.sqlite" - replace: "sqlalchemy.url = sqlite:////home/{{ ansible_env.SUDO_USER }}/pagure_data/pagure_dev.sqlite" - -- name: Configure alembic to point to the pagure migration folder - become_user: "{{ ansible_env.SUDO_USER }}" - replace: - dest: /home/{{ ansible_env.SUDO_USER }}/alembic.ini - regexp: "script_location = /usr/share/pagure/alembic" - replace: "script_location = /home/vagrant/devel/alembic/" - -- name: Create the Pagure database - become_user: "{{ ansible_env.SUDO_USER }}" - command: .virtualenvs/python2-pagure/bin/python devel/createdb.py - args: - creates: /home/{{ ansible_env.SUDO_USER }}/pagure_data/pagure_dev.sqlite - chdir: "/home/{{ ansible_env.SUDO_USER }}/" - -- name: Stamp the database with its current migration - become_user: "{{ ansible_env.SUDO_USER }}" - shell: alembic stamp $(alembic heads | awk '{ print $1 }') - args: - chdir: "/home/{{ ansible_env.SUDO_USER }}/" - -- name: Create systemd user unit directory - become_user: "{{ ansible_env.SUDO_USER }}" - file: - path: /home/{{ ansible_env.SUDO_USER }}/.config/systemd/user/ - state: directory - -- name: Install the Pagure service files for systemd - become_user: "{{ ansible_env.SUDO_USER }}" - copy: - src: "{{ item }}" - dest: /home/{{ ansible_env.SUDO_USER }}/.config/systemd/user/{{ item }} - with_items: - - pagure.service - - pagure-docs.service - - pagure_ci.service - - pagure_ev.service - - pagure_webhook.service diff --git a/ansible/roles/pagure-dev/tasks/milter.yml b/ansible/roles/pagure-dev/tasks/milter.yml deleted file mode 100644 index b18e4dc..0000000 --- a/ansible/roles/pagure-dev/tasks/milter.yml +++ /dev/null @@ -1,10 +0,0 @@ ---- - -- name: Install Pagure milter packages - dnf: name={{ item }} state=present - with_items: - - postfix - - python-pymilter - -- name: Start Postfix - service: name=postfix state=started enabled=yes diff --git a/ansible/roles/pagure-dev/tasks/postgres.yml b/ansible/roles/pagure-dev/tasks/postgres.yml deleted file mode 100644 index c839074..0000000 --- a/ansible/roles/pagure-dev/tasks/postgres.yml +++ /dev/null @@ -1,38 +0,0 @@ ---- - -- name: Install postgresql packages - dnf: name={{ item }} state=present - with_items: - - postgresql - - postgresql-server - - postgresql-devel # Allows pip installing psycopg2 is desired - - python-psycopg2 - -- name: Initialize PostgreSQL - command: postgresql-setup initdb - args: - creates: /var/lib/pgsql/data/pg_hba.conf - -- replace: - dest: /var/lib/pgsql/data/pg_hba.conf - regexp: "local all all peer" - replace: "local all all trust" - -- replace: - dest: /var/lib/pgsql/data/pg_hba.conf - regexp: "host all all 127.0.0.1/32 ident" - replace: "host all all 127.0.0.1/32 trust" - -- replace: - dest: /var/lib/pgsql/data/pg_hba.conf - regexp: "host all all ::1/128 ident" - replace: "host all all ::1/128 trust" - -- name: Start postgresql - service: name=postgresql state=restarted enabled=yes - -- name: Add a pagure postgres user - postgresql_user: name=pagure role_attr_flags=SUPERUSER,LOGIN - -- name: Create a database for pagure - postgresql_db: name=pagure owner=pagure diff --git a/ansible/vagrant-playbook.yml b/ansible/vagrant-playbook.yml deleted file mode 100644 index e67cf8a..0000000 --- a/ansible/vagrant-playbook.yml +++ /dev/null @@ -1,7 +0,0 @@ ---- -- hosts: all - become: true - become_method: sudo - vars: - roles: - - pagure-dev diff --git a/dev/Vagrantfile.example b/dev/Vagrantfile.example new file mode 100644 index 0000000..dbcc925 --- /dev/null +++ b/dev/Vagrantfile.example @@ -0,0 +1,71 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +VAGRANTFILE_API_VERSION = "2" + +Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| + config.vm.box_url = "https://download.fedoraproject.org/pub/fedora/linux/releases/25/CloudImages/x86_64/images/Fedora-Cloud-Base-Vagrant-25-1.3.x86_64.vagrant-libvirt.box" + config.vm.box = "f25-cloud-libvirt" + + + # Forward traffic on the host to the development server on the guest + config.vm.network "forwarded_port", guest: 5000, host: 5000 + # Forward traffic on the host to Redis on the guest + config.vm.network "forwarded_port", guest: 6379, host: 6379 + # Forward traffic on the host to the SSE server on the guest + config.vm.network "forwarded_port", guest: 8080, host: 8080 + + if Vagrant.has_plugin?("vagrant-hostmanager") + config.hostmanager.enabled = true + config.hostmanager.manage_host = true + end + + # Vagrant can share the source directory using rsync, NFS, or SSHFS (with the vagrant-sshfs + # plugin). By default it rsyncs the current working directory to /vagrant. + # + # If you would prefer to use NFS to share the directory uncomment this and configure NFS + # config.vm.synced_folder ".", "/vagrant", type: "nfs", nfs_version: 4, nfs_udp: false + config.vm.synced_folder ".", "/vagrant", disabled: true + config.vm.synced_folder ".", "/home/vagrant/devel", + type: "sshfs", + sshfs_opts_append: "-o nonempty" + + # To cache update packages (which is helpful if frequently doing `vagrant destroy && vagrant up`) + # you can create a local directory and share it to the guest's DNF cache. The directory needs to + # exist, so create it before you uncomment the line below. + #Dir.mkdir('.dnf-cache') unless File.exists?('.dnf-cache') + #config.vm.synced_folder ".dnf-cache", "/var/cache/dnf", + # type: "sshfs", + # sshfs_opts_append: "-o nonempty" + + # Comment this line if you would like to disable the automatic update during provisioning + config.vm.provision "shell", inline: "sudo dnf upgrade -y" + + # bootstrap and run with ansible + config.vm.provision "shell", inline: "sudo dnf -y install python2-dnf libselinux-python" + config.vm.provision "ansible" do |ansible| + ansible.playbook = "dev/ansible/vagrant-playbook.yml" + end + + + # Create the "pagure" box + config.vm.define "pagure" do |pagure| + pagure.vm.host_name = "pagure-dev.example.com" + + pagure.vm.provider :libvirt do |domain| + # Season to taste + domain.cpus = 4 + domain.graphics_type = "spice" + domain.memory = 2048 + domain.video_type = "qxl" + + # Uncomment the following line if you would like to enable libvirt's unsafe cache + # mode. It is called unsafe for a reason, as it causes the virtual host to ignore all + # fsync() calls from the guest. Only do this if you are comfortable with the possibility of + # your development guest becoming corrupted (in which case you should only need to do a + # vagrant destroy and vagrant up to get a new one). + # + # domain.volume_cache = "unsafe" + end + end +end diff --git a/dev/ansible/roles/pagure-dev/files/bashrc b/dev/ansible/roles/pagure-dev/files/bashrc new file mode 100644 index 0000000..4b7ac84 --- /dev/null +++ b/dev/ansible/roles/pagure-dev/files/bashrc @@ -0,0 +1,37 @@ +# .bashrc + +# Source global definitions +if [ -f /etc/bashrc ]; then + . /etc/bashrc +fi + +# Uncomment the following line if you don't like systemctl's auto-paging feature: +# export SYSTEMD_PAGER= + +# User specific aliases and functions +# If adding new functions to this file, note that you can add help text to the function +# by defining a variable with name __help containing the help text + +export PAGURE_CONFIG=~/pagure.cfg + +pstart (){ + systemctl --user start pagure.service pagure-docs.service pagure_ci.service\ + pagure_ev.service pagure_webhook.service + echo 'The application is running on http://localhost:5000/' +} + +pstop (){ + systemctl --user stop pagure.service pagure-docs.service pagure_ci.service\ + pagure_ev.service pagure_webhook.service +} + +prestart (){ + systemctl --user restart pagure.service pagure-docs.service pagure_ci.service\ + pagure_ev.service pagure_webhook.service + echo 'The application is running on http://localhost:5000/' +} + +pstatus (){ + systemctl --user status pagure.service pagure-docs.service pagure_ci.service\ + pagure_ev.service pagure_webhook.service +} diff --git a/dev/ansible/roles/pagure-dev/files/clamd.conf b/dev/ansible/roles/pagure-dev/files/clamd.conf new file mode 100644 index 0000000..7fcb1d7 --- /dev/null +++ b/dev/ansible/roles/pagure-dev/files/clamd.conf @@ -0,0 +1,684 @@ +## +## Example config file for the Clam AV daemon +## Please read the clamd.conf(5) manual before editing this file. +## + + +# Comment or remove the line below. +# Example + +# Uncomment this option to enable logging. +# LogFile must be writable for the user running daemon. +# A full path is required. +# Default: disabled +#LogFile /var/log/clamd. + +# By default the log file is locked for writing - the lock protects against +# running clamd multiple times (if want to run another clamd, please +# copy the configuration file, change the LogFile variable, and run +# the daemon with --config-file option). +# This option disables log file locking. +# Default: no +#LogFileUnlock yes + +# Maximum size of the log file. +# Value of 0 disables the limit. +# You may use 'M' or 'm' for megabytes (1M = 1m = 1048576 bytes) +# and 'K' or 'k' for kilobytes (1K = 1k = 1024 bytes). To specify the size +# in bytes just don't use modifiers. If LogFileMaxSize is enabled, log +# rotation (the LogRotate option) will always be enabled. +# Default: 1M +#LogFileMaxSize 2M + +# Log time with each message. +# Default: no +#LogTime yes + +# Also log clean files. Useful in debugging but drastically increases the +# log size. +# Default: no +#LogClean yes + +# Use system logger (can work together with LogFile). +# Default: no +LogSyslog yes + +# Specify the type of syslog messages - please refer to 'man syslog' +# for facility names. +# Default: LOG_LOCAL6 +#LogFacility LOG_MAIL + +# Enable verbose logging. +# Default: no +#LogVerbose yes + +# Enable log rotation. Always enabled when LogFileMaxSize is enabled. +# Default: no +#LogRotate yes + +# Log additional information about the infected file, such as its +# size and hash, together with the virus name. +#ExtendedDetectionInfo yes + +# This option allows you to save a process identifier of the listening +# daemon (main thread). +# Default: disabled +#PidFile /var/run/clamd./clamd.pid + +# Optional path to the global temporary directory. +# Default: system specific (usually /tmp or /var/tmp). +#TemporaryDirectory /var/tmp + +# Path to the database directory. +# Default: hardcoded (depends on installation options) +#DatabaseDirectory /var/lib/clamav + +# Only load the official signatures published by the ClamAV project. +# Default: no +#OfficialDatabaseOnly no + +# The daemon can work in local mode, network mode or both. +# Due to security reasons we recommend the local mode. + +# Path to a local socket file the daemon will listen on. +# Default: disabled (must be specified by a user) +LocalSocket /var/lib/clamav/clamd.sock + +# Sets the group ownership on the unix socket. +# Default: disabled (the primary group of the user running clamd) +LocalSocketGroup clamupdate + +# Sets the permissions on the unix socket to the specified mode. +# Default: disabled (socket is world accessible) +#LocalSocketMode 660 + +# Remove stale socket after unclean shutdown. +# Default: yes +#FixStaleSocket yes + +# TCP port address. +# Default: no +#TCPSocket 3310 + +# TCP address. +# By default we bind to INADDR_ANY, probably not wise. +# Enable the following to provide some degree of protection +# from the outside world. This option can be specified multiple +# times if you want to listen on multiple IPs. IPv6 is now supported. +# Default: no +#TCPAddr 127.0.0.1 + +# Maximum length the queue of pending connections may grow to. +# Default: 200 +#MaxConnectionQueueLength 30 + +# Clamd uses FTP-like protocol to receive data from remote clients. +# If you are using clamav-milter to balance load between remote clamd daemons +# on firewall servers you may need to tune the options below. + +# Close the connection when the data size limit is exceeded. +# The value should match your MTA's limit for a maximum attachment size. +# Default: 25M +#StreamMaxLength 10M + +# Limit port range. +# Default: 1024 +#StreamMinPort 30000 +# Default: 2048 +#StreamMaxPort 32000 + +# Maximum number of threads running at the same time. +# Default: 10 +#MaxThreads 20 + +# Waiting for data from a client socket will timeout after this time (seconds). +# Default: 120 +#ReadTimeout 300 + +# This option specifies the time (in seconds) after which clamd should +# timeout if a client doesn't provide any initial command after connecting. +# Default: 5 +#CommandReadTimeout 5 + +# This option specifies how long to wait (in miliseconds) if the send buffer is full. +# Keep this value low to prevent clamd hanging +# +# Default: 500 +#SendBufTimeout 200 + +# Maximum number of queued items (including those being processed by MaxThreads threads) +# It is recommended to have this value at least twice MaxThreads if possible. +# WARNING: you shouldn't increase this too much to avoid running out of file descriptors, +# the following condition should hold: +# MaxThreads*MaxRecursion + (MaxQueue - MaxThreads) + 6< RLIMIT_NOFILE (usual max is 1024) +# +# Default: 100 +#MaxQueue 200 + +# Waiting for a new job will timeout after this time (seconds). +# Default: 30 +#IdleTimeout 60 + +# Don't scan files and directories matching regex +# This directive can be used multiple times +# Default: scan all +#ExcludePath ^/proc/ +#ExcludePath ^/sys/ + +# Maximum depth directories are scanned at. +# Default: 15 +#MaxDirectoryRecursion 20 + +# Follow directory symlinks. +# Default: no +#FollowDirectorySymlinks yes + +# Follow regular file symlinks. +# Default: no +#FollowFileSymlinks yes + +# Scan files and directories on other filesystems. +# Default: yes +#CrossFilesystems yes + +# Perform a database check. +# Default: 600 (10 min) +#SelfCheck 600 + +# Execute a command when virus is found. In the command string %v will +# be replaced with the virus name. +# Default: no +#VirusEvent /usr/local/bin/send_sms 123456789 "VIRUS ALERT: %v" + +# Run as another user (clamd must be started by root for this option to work) +# Default: don't drop privileges +User clamupdate + +# Initialize supplementary group access (clamd must be started by root). +# Default: no +AllowSupplementaryGroups yes + +# Stop daemon when libclamav reports out of memory condition. +#ExitOnOOM yes + +# Don't fork into background. +# Default: no +#Foreground yes + +# Enable debug messages in libclamav. +# Default: no +#Debug yes + +# Do not remove temporary files (for debug purposes). +# Default: no +#LeaveTemporaryFiles yes + +# Permit use of the ALLMATCHSCAN command. If set to no, clamd will reject +# any ALLMATCHSCAN command as invalid. +# Default: yes +#AllowAllMatchScan no + +# Detect Possibly Unwanted Applications. +# Default: no +#DetectPUA yes + +# Exclude a specific PUA category. This directive can be used multiple times. +# See https://github.com/vrtadmin/clamav-faq/blob/master/faq/faq-pua.md for +# the complete list of PUA categories. +# Default: Load all categories (if DetectPUA is activated) +#ExcludePUA NetTool +#ExcludePUA PWTool + +# Only include a specific PUA category. This directive can be used multiple +# times. +# Default: Load all categories (if DetectPUA is activated) +#IncludePUA Spy +#IncludePUA Scanner +#IncludePUA RAT + +# In some cases (eg. complex malware, exploits in graphic files, and others), +# ClamAV uses special algorithms to provide accurate detection. This option +# controls the algorithmic detection. +# Default: yes +#AlgorithmicDetection yes + +# This option causes memory or nested map scans to dump the content to disk. +# If you turn on this option, more data is written to disk and is available +# when the LeaveTemporaryFiles option is enabled. +#ForceToDisk yes + +# This option allows you to disable the caching feature of the engine. By +# default, the engine will store an MD5 in a cache of any files that are +# not flagged as virus or that hit limits checks. Disabling the cache will +# have a negative performance impact on large scans. +# Default: no +#DisableCache yes + +## +## Executable files +## + +# PE stands for Portable Executable - it's an executable file format used +# in all 32 and 64-bit versions of Windows operating systems. This option allows +# ClamAV to perform a deeper analysis of executable files and it's also +# required for decompression of popular executable packers such as UPX, FSG, +# and Petite. If you turn off this option, the original files will still be +# scanned, but without additional processing. +# Default: yes +#ScanPE yes + +# Certain PE files contain an authenticode signature. By default, we check +# the signature chain in the PE file against a database of trusted and +# revoked certificates if the file being scanned is marked as a virus. +# If any certificate in the chain validates against any trusted root, but +# does not match any revoked certificate, the file is marked as whitelisted. +# If the file does match a revoked certificate, the file is marked as virus. +# The following setting completely turns off authenticode verification. +# Default: no +#DisableCertCheck yes + +# Executable and Linking Format is a standard format for UN*X executables. +# This option allows you to control the scanning of ELF files. +# If you turn off this option, the original files will still be scanned, but +# without additional processing. +# Default: yes +#ScanELF yes + +# With this option clamav will try to detect broken executables (both PE and +# ELF) and mark them as Broken.Executable. +# Default: no +#DetectBrokenExecutables yes + + +## +## Documents +## + +# This option enables scanning of OLE2 files, such as Microsoft Office +# documents and .msi files. +# If you turn off this option, the original files will still be scanned, but +# without additional processing. +# Default: yes +#ScanOLE2 yes + +# With this option enabled OLE2 files with VBA macros, which were not +# detected by signatures will be marked as "Heuristics.OLE2.ContainsMacros". +# Default: no +#OLE2BlockMacros no + +# This option enables scanning within PDF files. +# If you turn off this option, the original files will still be scanned, but +# without decoding and additional processing. +# Default: yes +#ScanPDF yes + +# This option enables scanning within SWF files. +# If you turn off this option, the original files will still be scanned, but +# without decoding and additional processing. +# Default: yes +#ScanSWF yes + +# This option enables scanning xml-based document files supported by libclamav. +# If you turn off this option, the original files will still be scanned, but +# without additional processing. +# Default: yes +#ScanXMLDOCS yes + +# This option enables scanning of HWP3 files. +# If you turn off this option, the original files will still be scanned, but +# without additional processing. +# Default: yes +#ScanHWP3 yes + + +## +## Mail files +## + +# Enable internal e-mail scanner. +# If you turn off this option, the original files will still be scanned, but +# without parsing individual messages/attachments. +# Default: yes +#ScanMail yes + +# Scan RFC1341 messages split over many emails. +# You will need to periodically clean up $TemporaryDirectory/clamav-partial directory. +# WARNING: This option may open your system to a DoS attack. +# Never use it on loaded servers. +# Default: no +#ScanPartialMessages yes + +# With this option enabled ClamAV will try to detect phishing attempts by using +# signatures. +# Default: yes +#PhishingSignatures yes + +# Scan URLs found in mails for phishing attempts using heuristics. +# Default: yes +#PhishingScanURLs yes + +# Always block SSL mismatches in URLs, even if the URL isn't in the database. +# This can lead to false positives. +# +# Default: no +#PhishingAlwaysBlockSSLMismatch no + +# Always block cloaked URLs, even if URL isn't in database. +# This can lead to false positives. +# +# Default: no +#PhishingAlwaysBlockCloak no + +# Detect partition intersections in raw disk images using heuristics. +# Default: no +#PartitionIntersection no + +# Allow heuristic match to take precedence. +# When enabled, if a heuristic scan (such as phishingScan) detects +# a possible virus/phish it will stop scan immediately. Recommended, saves CPU +# scan-time. +# When disabled, virus/phish detected by heuristic scans will be reported only at +# the end of a scan. If an archive contains both a heuristically detected +# virus/phish, and a real malware, the real malware will be reported +# +# Keep this disabled if you intend to handle "*.Heuristics.*" viruses +# differently from "real" malware. +# If a non-heuristically-detected virus (signature-based) is found first, +# the scan is interrupted immediately, regardless of this config option. +# +# Default: no +#HeuristicScanPrecedence yes + + +## +## Data Loss Prevention (DLP) +## + +# Enable the DLP module +# Default: No +#StructuredDataDetection yes + +# This option sets the lowest number of Credit Card numbers found in a file +# to generate a detect. +# Default: 3 +#StructuredMinCreditCardCount 5 + +# This option sets the lowest number of Social Security Numbers found +# in a file to generate a detect. +# Default: 3 +#StructuredMinSSNCount 5 + +# With this option enabled the DLP module will search for valid +# SSNs formatted as xxx-yy-zzzz +# Default: yes +#StructuredSSNFormatNormal yes + +# With this option enabled the DLP module will search for valid +# SSNs formatted as xxxyyzzzz +# Default: no +#StructuredSSNFormatStripped yes + + +## +## HTML +## + +# Perform HTML normalisation and decryption of MS Script Encoder code. +# Default: yes +# If you turn off this option, the original files will still be scanned, but +# without additional processing. +#ScanHTML yes + + +## +## Archives +## + +# ClamAV can scan within archives and compressed files. +# If you turn off this option, the original files will still be scanned, but +# without unpacking and additional processing. +# Default: yes +#ScanArchive yes + +# Mark encrypted archives as viruses (Encrypted.Zip, Encrypted.RAR). +# Default: no +#ArchiveBlockEncrypted no + + +## +## Limits +## + +# The options below protect your system against Denial of Service attacks +# using archive bombs. + +# This option sets the maximum amount of data to be scanned for each input file. +# Archives and other containers are recursively extracted and scanned up to this +# value. +# Value of 0 disables the limit +# Note: disabling this limit or setting it too high may result in severe damage +# to the system. +# Default: 100M +#MaxScanSize 150M + +# Files larger than this limit won't be scanned. Affects the input file itself +# as well as files contained inside it (when the input file is an archive, a +# document or some other kind of container). +# Value of 0 disables the limit. +# Note: disabling this limit or setting it too high may result in severe damage +# to the system. +# Default: 25M +#MaxFileSize 30M + +# Nested archives are scanned recursively, e.g. if a Zip archive contains a RAR +# file, all files within it will also be scanned. This options specifies how +# deeply the process should be continued. +# Note: setting this limit too high may result in severe damage to the system. +# Default: 16 +#MaxRecursion 10 + +# Number of files to be scanned within an archive, a document, or any other +# container file. +# Value of 0 disables the limit. +# Note: disabling this limit or setting it too high may result in severe damage +# to the system. +# Default: 10000 +#MaxFiles 15000 + +# Maximum size of a file to check for embedded PE. Files larger than this value +# will skip the additional analysis step. +# Note: disabling this limit or setting it too high may result in severe damage +# to the system. +# Default: 10M +#MaxEmbeddedPE 10M + +# Maximum size of a HTML file to normalize. HTML files larger than this value +# will not be normalized or scanned. +# Note: disabling this limit or setting it too high may result in severe damage +# to the system. +# Default: 10M +#MaxHTMLNormalize 10M + +# Maximum size of a normalized HTML file to scan. HTML files larger than this +# value after normalization will not be scanned. +# Note: disabling this limit or setting it too high may result in severe damage +# to the system. +# Default: 2M +#MaxHTMLNoTags 2M + +# Maximum size of a script file to normalize. Script content larger than this +# value will not be normalized or scanned. +# Note: disabling this limit or setting it too high may result in severe damage +# to the system. +# Default: 5M +#MaxScriptNormalize 5M + +# Maximum size of a ZIP file to reanalyze type recognition. ZIP files larger +# than this value will skip the step to potentially reanalyze as PE. +# Note: disabling this limit or setting it too high may result in severe damage +# to the system. +# Default: 1M +#MaxZipTypeRcg 1M + +# This option sets the maximum number of partitions of a raw disk image to be scanned. +# Raw disk images with more partitions than this value will have up to the value number +# partitions scanned. Negative values are not allowed. +# Note: setting this limit too high may result in severe damage or impact performance. +# Default: 50 +#MaxPartitions 128 + +# This option sets the maximum number of icons within a PE to be scanned. +# PE files with more icons than this value will have up to the value number icons scanned. +# Negative values are not allowed. +# WARNING: setting this limit too high may result in severe damage or impact performance. +# Default: 100 +#MaxIconsPE 200 + +# This option sets the maximum recursive calls for HWP3 parsing during scanning. +# HWP3 files using more than this limit will be terminated and alert the user. +# Scans will be unable to scan any HWP3 attachments if the recursive limit is reached. +# Negative values are not allowed. +# WARNING: setting this limit too high may result in severe damage or impact performance. +# Default: 16 +#MaxRecHWP3 16 + +# This option sets the maximum calls to the PCRE match function during an instance of regex matching. +# Instances using more than this limit will be terminated and alert the user but the scan will continue. +# For more information on match_limit, see the PCRE documentation. +# Negative values are not allowed. +# WARNING: setting this limit too high may severely impact performance. +# Default: 10000 +#PCREMatchLimit 20000 + +# This option sets the maximum recursive calls to the PCRE match function during an instance of regex matching. +# Instances using more than this limit will be terminated and alert the user but the scan will continue. +# For more information on match_limit_recursion, see the PCRE documentation. +# Negative values are not allowed and values > PCREMatchLimit are superfluous. +# WARNING: setting this limit too high may severely impact performance. +# Default: 5000 +#PCRERecMatchLimit 10000 + +# This option sets the maximum filesize for which PCRE subsigs will be executed. +# Files exceeding this limit will not have PCRE subsigs executed unless a subsig is encompassed to a smaller buffer. +# Negative values are not allowed. +# Setting this value to zero disables the limit. +# WARNING: setting this limit too high or disabling it may severely impact performance. +# Default: 25M +#PCREMaxFileSize 100M + + +## +## On-access Scan Settings +## + +# Enable on-access scanning. Currently, this is supported via fanotify. +# Clamuko/Dazuko support has been deprecated. +# Default: no +#ScanOnAccess yes + +# Set the mount point to be scanned. The mount point specified, or the mount point +# containing the specified directory will be watched. If any directories are specified, +# this option will preempt the DDD system. This will notify only. It can be used multiple times. +# (On-access scan only) +# Default: disabled +#OnAccessMountPath / +#OnAccessMountPath /home/user + +# Don't scan files larger than OnAccessMaxFileSize +# Value of 0 disables the limit. +# Default: 5M +#OnAccessMaxFileSize 10M + +# Set the include paths (all files inside them will be scanned). You can have +# multiple OnAccessIncludePath directives but each directory must be added +# in a separate line. (On-access scan only) +# Default: disabled +#OnAccessIncludePath /home +#OnAccessIncludePath /students + +# Set the exclude paths. All subdirectories are also excluded. +# (On-access scan only) +# Default: disabled +#OnAccessExcludePath /home/bofh + +# With this option you can whitelist specific UIDs. Processes with these UIDs +# will be able to access all files. +# This option can be used multiple times (one per line). +# Default: disabled +#OnAccessExcludeUID 0 + +# Toggles dynamic directory determination. Allows for recursively watching include paths. +# (On-access scan only) +# Default: no +#OnAccessDisableDDD yes + +# Modifies fanotify blocking behaviour when handling permission events. +# If off, fanotify will only notify if the file scanned is a virus, +# and not perform any blocking. +# (On-access scan only) +# Default: no +#OnAccessPrevention yes + +# Toggles extra scanning and notifications when a file or directory is created or moved. +# Requires the DDD system to kick-off extra scans. +# (On-access scan only) +# Default: no +#OnAccessExtraScanning yes + +## +## Bytecode +## + +# With this option enabled ClamAV will load bytecode from the database. +# It is highly recommended you keep this option on, otherwise you'll miss detections for many new viruses. +# Default: yes +#Bytecode yes + +# Bytecode mode +# +# This option has been set to 'ForceInterpreter' in Fedora due to +# security concerns by default. You might need to enable the +# 'antivirus_use_jit' SELinux boolean after setting this option to +# the more efficient 'ForceJIT' value. +# +# Default: ForceInterpreter +#ByteCodeMode ForceInterpreter + +# Set bytecode security level. +# Possible values: +# None - no security at all, meant for debugging. DO NOT USE THIS ON PRODUCTION SYSTEMS +# This value is only available if clamav was built with --enable-debug! +# TrustSigned - trust bytecode loaded from signed .c[lv]d files, +# insert runtime safety checks for bytecode loaded from other sources +# Paranoid - don't trust any bytecode, insert runtime checks for all +# Recommended: TrustSigned, because bytecode in .cvd files already has these checks +# Note that by default only signed bytecode is loaded, currently you can only +# load unsigned bytecode in --enable-debug mode. +# +# Default: TrustSigned +#BytecodeSecurity TrustSigned + +# Set bytecode timeout in miliseconds. +# +# Default: 5000 +# BytecodeTimeout 1000 + +## +## Statistics gathering and submitting +## + +# Enable statistical reporting. +# Default: no +#StatsEnabled yes + +# Disable submission of individual PE sections for files flagged as malware. +# Default: no +#StatsPEDisabled yes + +# HostID in the form of an UUID to use when submitting statistical information. +# Default: auto +#StatsHostID auto + +# Time in seconds to wait for the stats server to come back with a response +# Default: 10 +#StatsTimeout 10 diff --git a/dev/ansible/roles/pagure-dev/files/gitolite3.rc b/dev/ansible/roles/pagure-dev/files/gitolite3.rc new file mode 100644 index 0000000..1a20d42 --- /dev/null +++ b/dev/ansible/roles/pagure-dev/files/gitolite3.rc @@ -0,0 +1,195 @@ +# configuration variables for gitolite + +# This file is in perl syntax. But you do NOT need to know perl to edit it -- +# just mind the commas, use single quotes unless you know what you're doing, +# and make sure the brackets and braces stay matched up! + +# (Tip: perl allows a comma after the last item in a list also!) + +# HELP for commands can be had by running the command with "-h". + +# HELP for all the other FEATURES can be found in the documentation (look for +# "list of non-core programs shipped with gitolite" in the master index) or +# directly in the corresponding source file. + +%RC = ( + + # ------------------------------------------------------------------ + + # default umask gives you perms of '0700'; see the rc file docs for + # how/why you might change this + UMASK => 0077, + + # look for "git-config" in the documentation + GIT_CONFIG_KEYS => '', + + # comment out if you don't need all the extra detail in the logfile + LOG_EXTRA => 1, + # syslog options + # 1. leave this section as is for normal gitolite logging + # 2. uncomment this line to log only to syslog: + # LOG_DEST => 'syslog', + # 3. uncomment this line to log to syslog and the normal gitolite log: + # LOG_DEST => 'syslog,normal', + + # roles. add more roles (like MANAGER, TESTER, ...) here. + # WARNING: if you make changes to this hash, you MUST run 'gitolite + # compile' afterward, and possibly also 'gitolite trigger POST_COMPILE' + ROLES => { + READERS => 1, + WRITERS => 1, + }, + + # enable caching (currently only Redis). PLEASE RTFM BEFORE USING!!! + # CACHE => 'Redis', + + # ------------------------------------------------------------------ + + # rc variables used by various features + + # the 'info' command prints this as additional info, if it is set + # SITE_INFO => 'Please see http://blahblah/gitolite for more help', + + # the CpuTime feature uses these + # display user, system, and elapsed times to user after each git operation + # DISPLAY_CPU_TIME => 1, + # display a warning if total CPU times (u, s, cu, cs) crosses this limit + # CPU_TIME_WARN_LIMIT => 0.1, + + # the Mirroring feature needs this + # HOSTNAME => "foo", + + # TTL for redis cache; PLEASE SEE DOCUMENTATION BEFORE UNCOMMENTING! + # CACHE_TTL => 600, + + # ------------------------------------------------------------------ + + # suggested locations for site-local gitolite code (see cust.html) + + # this one is managed directly on the server + # LOCAL_CODE => "$ENV{HOME}/local", + + # or you can use this, which lets you put everything in a subdirectory + # called "local" in your gitolite-admin repo. For a SECURITY WARNING + # on this, see http://gitolite.com/gitolite/non-core.html#pushcode + # LOCAL_CODE => "$rc{GL_ADMIN_BASE}/local", + + # ------------------------------------------------------------------ + + # List of commands and features to enable + + ENABLE => [ + + # COMMANDS + + # These are the commands enabled by default + 'help', + 'desc', + 'info', + 'perms', + 'writable', + + # Uncomment or add new commands here. + # 'create', + # 'fork', + # 'mirror', + # 'readme', + # 'sskm', + # 'D', + + # These FEATURES are enabled by default. + + # essential (unless you're using smart-http mode) + 'ssh-authkeys', + + # creates git-config enties from gitolite.conf file entries like 'config foo.bar = baz' + 'git-config', + + # creates git-daemon-export-ok files; if you don't use git-daemon, comment this out + 'daemon', + + # creates projects.list file; if you don't use gitweb, comment this out + #'gitweb', + + # These FEATURES are disabled by default; uncomment to enable. If you + # need to add new ones, ask on the mailing list :-) + + # user-visible behaviour + + # prevent wild repos auto-create on fetch/clone + # 'no-create-on-read', + # no auto-create at all (don't forget to enable the 'create' command!) + # 'no-auto-create', + + # access a repo by another (possibly legacy) name + # 'Alias', + + # give some users direct shell access. See documentation in + # sts.html for details on the following two choices. + # "Shell $ENV{HOME}/.gitolite.shell-users", + # 'Shell alice bob', + + # set default roles from lines like 'option default.roles-1 = ...', etc. + # 'set-default-roles', + + # show more detailed messages on deny + # 'expand-deny-messages', + + # show a message of the day + # 'Motd', + + # system admin stuff + + # enable mirroring (don't forget to set the HOSTNAME too!) + # 'Mirroring', + + # allow people to submit pub files with more than one key in them + # 'ssh-authkeys-split', + + # selective read control hack + # 'partial-copy', + + # manage local, gitolite-controlled, copies of read-only upstream repos + # 'upstream', + + # updates 'description' file instead of 'gitweb.description' config item + # 'cgit', + + # allow repo-specific hooks to be added + # 'repo-specific-hooks', + + # performance, logging, monitoring... + + # be nice + # 'renice 10', + + # log CPU times (user, system, cumulative user, cumulative system) + # 'CpuTime', + + # syntactic_sugar for gitolite.conf and included files + + # allow backslash-escaped continuation lines in gitolite.conf + # 'continuation-lines', + + # create implicit user groups from directory names in keydir/ + # 'keysubdirs-as-groups', + + # allow simple line-oriented macros + # 'macros', + + # Kindergarten mode + + # disallow various things that sensible people shouldn't be doing anyway + # 'Kindergarten', + ], + +); + +# ------------------------------------------------------------------------------ +# per perl rules, this should be the last line in such a file: +1; + +# Local variables: +# mode: perl +# End: +# vim: set syn=perl: diff --git a/dev/ansible/roles/pagure-dev/files/motd b/dev/ansible/roles/pagure-dev/files/motd new file mode 100644 index 0000000..9a2a76c --- /dev/null +++ b/dev/ansible/roles/pagure-dev/files/motd @@ -0,0 +1,21 @@ + +Welcome to the Pagure development environment! + +Here are some tips: + +* Pagure is installed in a Python virtualenv. Use `workon python2-pagure` to + enter the virtualenv. + +* The code for Pagure is located at ~/devel/ + +* You can populate the database with the `dev-data.py` script in the repository + +* Run `pstart` to start the development server and `pstop` to stop it. + +* Logs for the server are available with `journalctl`; the services are run + as systemd user units in ~/.config/systemd/user/ + +Once you start the server you can navigate to http://localhost:5000/ +in your browser on the host to access your Pagure development environment. + +Happy hacking! diff --git a/dev/ansible/roles/pagure-dev/files/pagure-docs.service b/dev/ansible/roles/pagure-dev/files/pagure-docs.service new file mode 100644 index 0000000..beeeca3 --- /dev/null +++ b/dev/ansible/roles/pagure-dev/files/pagure-docs.service @@ -0,0 +1,12 @@ +[Unit] +Description=Runs the Pagure documentation server +After=network.target + +[Service] +Environment="PAGURE_CONFIG=/home/vagrant/pagure.cfg" +ExecStart=/home/vagrant/.virtualenvs/python2-pagure/bin/python \ + /home/vagrant/devel/rundocserver.py --host 0.0.0.0 +Type=simple + +[Install] +WantedBy=multi-user.target diff --git a/dev/ansible/roles/pagure-dev/files/pagure.cfg b/dev/ansible/roles/pagure-dev/files/pagure.cfg new file mode 100644 index 0000000..a813f09 --- /dev/null +++ b/dev/ansible/roles/pagure-dev/files/pagure.cfg @@ -0,0 +1,173 @@ +import os +from datetime import timedelta + +### Set the time after which the admin session expires +# There are two sessions on pagure, login that holds for 31 days and +# the session defined here after which an user has to re-login. +# This session is used when accessing all administrative parts of pagure +# (ie: changing a project's or a user's settings) +ADMIN_SESSION_LIFETIME = timedelta(minutes=20000000) + +### Secret key for the Flask application +SECRET_KEY='' + +### url to the database server: +#DB_URL=mysql://user:pass@host/db_name +#DB_URL=postgres://user:pass@host/db_name +DB_URL = 'sqlite:////home/vagrant/pagure_data/pagure_dev.sqlite' + +### The FAS group in which the admin of pagure are +ADMIN_GROUP = ['sysadmin-main'] + +### Hard-coded list of global admins +PAGURE_ADMIN_USERS = [] + +### The URL at which the project is available. +APP_URL = '*' +### The URL at which the documentation of projects will be available +## This should be in a different domain to avoid XSS issues since we want +## to allow raw html to be displayed (different domain, ie not a sub-domain). +DOC_APP_URL = '*' + +# Avoid sending emails while developing by default +EMAIL_SEND = False +EMAIL_ERROR = 'vagrant@localhost' + +### The URL to use to clone git repositories. +GIT_URL_SSH = 'ssh://vagrant@pagure-dev.example.com/' +GIT_URL_GIT = 'http://pagure-dev.example.com:5000/' + +### Folder containing to the git repos +STORAGE_ROOT = '/home/vagrant/pagure_data/' + +GIT_FOLDER = os.path.join(STORAGE_ROOT, 'repos') + +### Folder containing the docs repos +DOCS_FOLDER = os.path.join(STORAGE_ROOT, 'docs') + +### Folder containing the tickets repos +TICKETS_FOLDER = os.path.join(STORAGE_ROOT, 'tickets') + +### Folder containing the pull-requests repos +REQUESTS_FOLDER = os.path.join(STORAGE_ROOT, 'requests') + +### Folder containing the clones for the remote pull-requests +REMOTE_GIT_FOLDER = os.path.join(STORAGE_ROOT, 'remotes') + +### Whether to enable scanning for viruses in attachments +VIRUS_SCAN_ATTACHMENTS = False + +### Home folder of the gitolite user +### Folder where to run gl-compile-conf from +GITOLITE_HOME = '/home/vagrant/' + +### Configuration file for gitolite +GITOLITE_CONFIG = os.path.join(GITOLITE_HOME, '.gitolite/conf/gitolite.conf') + +### Version of gitolite used: 2 or 3? +GITOLITE_VERSION = 3 + +### Folder containing all the public ssh keys for gitolite +GITOLITE_KEYDIR = os.path.join(GITOLITE_HOME, '.gitolite/keydir/') + +### Path to the gitolite.rc file +GL_RC = '/home/vagrant/.gitolite.rc' + +### Path to the /bin directory where the gitolite tools can be found +GL_BINDIR = '/usr/bin/' + + +# SSH Information + +### The ssh certificates of the git server to be provided to the user +### /!\ format is important +# SSH_KEYS = {'RSA': {'fingerprint': '', 'pubkey': ''}} + + + +# Optional configuration + +### Number of items displayed per page +# Used when listing items +ITEM_PER_PAGE = 50 + +### Maximum size of the uploaded content +# Used to limit the size of file attached to a ticket for example +MAX_CONTENT_LENGTH = 4 * 1024 * 1024 # 4 megabytes + +### Lenght for short commits ids or file hex +SHORT_LENGTH = 6 + +### List of blacklisted project names that can conflicts for pagure's URLs +### or other +BLACKLISTED_PROJECTS = [ + 'static', 'pv', 'releases', 'new', 'api', 'settings', + 'logout', 'login', 'users', 'groups', 'projects'] + +### IP addresses allowed to access the internal endpoints +### These endpoints are used by the milter and are security sensitive, thus +### the IP filter +IP_ALLOWED_INTERNAL = ['127.0.0.1', 'localhost', '::1',] + +### EventSource/Web-Hook/Redis configuration +# The eventsource integration is what allows pagure to refresh the content +# on your page when someone else comments on the ticket (and this without +# asking you to reload the page. +# By default it is off, ie: EVENTSOURCE_SOURCE is None, to turn it on, specify +# here what the URL of the eventsource server is, for example: +# https://ev.pagure.io or https://pagure.io:8080 or whatever you are using +# (Note: the urls sent to it start with a '/' so no need to add one yourself) +EVENTSOURCE_SOURCE = 'http://localhost:8080' +# Port where the event source server is running (maybe be the same port +# as the one specified in EVENTSOURCE_SOURCE or a different one if you +# have something running in front of the server such as apache or stunnel). +EVENTSOURCE_PORT = 8080 +# If this port is specified, the event source server will run another server +# at this port and will provide information about the number of active +# connections running on the first (main) event source server +#EV_STATS_PORT = 8888 +# Web-hook can be turned on or off allowing using them for notifications, or +# not. +WEBHOOK = True + +### Redis configuration +# A redis server is required for both the Event-Source server or the web-hook +# server. +REDIS_HOST = '127.0.0.1' +REDIS_PORT = 6379 +REDIS_DB = 0 + +# Authentication related configuration option + +### Switch the authentication method +# Specify which authentication method to use, defaults to `fas` can be or +# `local` +# Default: ``fas``. +PAGURE_AUTH = 'fas' + +# When this is set to True, the session cookie will only be returned to the +# server via ssl (https). If you connect to the server via plain http, the +# cookie will not be sent. This prevents sniffing of the cookie contents. +# This may be set to False when testing your application but should always +# be set to True in production. +# Default: ``True``. +SESSION_COOKIE_SECURE = False + +# The name of the cookie used to store the session id. +# Default: ``.pagure``. +SESSION_COOKIE_NAME = 'pagure' + +# Boolean specifying whether to check the user's IP address when retrieving +# its session. This make things more secure (thus is on by default) but +# under certain setup it might not work (for example is there are proxies +# in front of the application). +CHECK_SESSION_IP = True + +# Used by SESSION_COOKIE_PATH +APPLICATION_ROOT = '/' + +# Allow the backward compatiblity endpoints for the old URLs schema to +# see the commits of a repo. This is only interesting if you pagure instance +# was running since before version 1.3 and if you care about backward +# compatibility in your URLs. +OLD_VIEW_COMMIT_ENABLED = False diff --git a/dev/ansible/roles/pagure-dev/files/pagure.service b/dev/ansible/roles/pagure-dev/files/pagure.service new file mode 100644 index 0000000..7999bc9 --- /dev/null +++ b/dev/ansible/roles/pagure-dev/files/pagure.service @@ -0,0 +1,11 @@ +[Unit] +Description=The Pagure web service +After=network.target + +[Service] +Environment="PAGURE_CONFIG=/home/vagrant/pagure.cfg" +ExecStart=/home/vagrant/.virtualenvs/python2-pagure/bin/python %h/devel/runserver.py --host 0.0.0.0 +Type=simple + +[Install] +WantedBy=multi-user.target diff --git a/dev/ansible/roles/pagure-dev/files/pagure_ci.service b/dev/ansible/roles/pagure-dev/files/pagure_ci.service new file mode 100644 index 0000000..b9e427a --- /dev/null +++ b/dev/ansible/roles/pagure-dev/files/pagure_ci.service @@ -0,0 +1,13 @@ +[Unit] +Description=Pagure Continuous Integration service +After=redis.target +Documentation=https://pagure.io/pagure + +[Service] +Environment="PAGURE_CONFIG=/home/vagrant/pagure.cfg" +ExecStart=/home/vagrant/.virtualenvs/python2-pagure/bin/python \ + /home/vagrant/devel/pagure-ci/pagure_ci_server.py +Type=simple + +[Install] +WantedBy=multi-user.target diff --git a/dev/ansible/roles/pagure-dev/files/pagure_ev.service b/dev/ansible/roles/pagure-dev/files/pagure_ev.service new file mode 100644 index 0000000..9b9a821 --- /dev/null +++ b/dev/ansible/roles/pagure-dev/files/pagure_ev.service @@ -0,0 +1,13 @@ +[Unit] +Description=Pagure EventSource server (Allowing live refresh of the pages supporting it) +After=redis.target +Documentation=https://pagure.io/pagure + +[Service] +Environment="PAGURE_CONFIG=/home/vagrant/pagure.cfg" +ExecStart=/home/vagrant/.virtualenvs/python2-pagure/bin/python \ + /home/vagrant/devel/pagure-ev/pagure_stream_server.py +Type=simple + +[Install] +WantedBy=multi-user.target diff --git a/dev/ansible/roles/pagure-dev/files/pagure_webhook.service b/dev/ansible/roles/pagure-dev/files/pagure_webhook.service new file mode 100644 index 0000000..601e296 --- /dev/null +++ b/dev/ansible/roles/pagure-dev/files/pagure_webhook.service @@ -0,0 +1,13 @@ +[Unit] +Description=Pagure WebHook server (Allowing web-hook notifications) +After=redis.target +Documentation=https://pagure.io/pagure + +[Service] +Environment="PAGURE_CONFIG=/home/vagrant/pagure.cfg" +ExecStart=/home/vagrant/.virtualenvs/python2-pagure/bin/python \ + /home/vagrant/devel/webhook-server/pagure-webhook-server.py +Type=simple + +[Install] +WantedBy=multi-user.target diff --git a/dev/ansible/roles/pagure-dev/tasks/clamav.yml b/dev/ansible/roles/pagure-dev/tasks/clamav.yml new file mode 100644 index 0000000..537d95a --- /dev/null +++ b/dev/ansible/roles/pagure-dev/tasks/clamav.yml @@ -0,0 +1,30 @@ +--- + +- name: Install ClamAV packages + dnf: name={{ item }} state=present + with_items: + - clamav-data-empty + - clamav-server + - clamav-server-systemd + - clamav-update + +- name: Configure freshclam + replace: + dest: /etc/freshclam.conf + regexp: "Example*" + replace: "" + +- name: Install Pagure's ClamAV configuration + copy: + src: clamd.conf + dest: /etc/clamd.d/pagure.conf + +# pyclamd expects /etc/clamd.conf +- name: Link /etc/clamd.conf to our pagure config + file: src=/etc/clamd.d/pagure.conf dest=/etc/clamd.conf state=link + +- name: Download latest ClamAV database + command: freshclam + +- name: Start ClamAV + service: name=clamd@pagure state=started enabled=yes diff --git a/dev/ansible/roles/pagure-dev/tasks/eventsource.yml b/dev/ansible/roles/pagure-dev/tasks/eventsource.yml new file mode 100644 index 0000000..98ee627 --- /dev/null +++ b/dev/ansible/roles/pagure-dev/tasks/eventsource.yml @@ -0,0 +1,13 @@ +--- + +- name: Install Redis + dnf: name={{ item }} state=present + with_items: + - python-redis + - python-trollius + - python-trollius-redis + - redis + + +- name: Start Redis + service: name=redis state=started enabled=yes diff --git a/dev/ansible/roles/pagure-dev/tasks/gitolite.yml b/dev/ansible/roles/pagure-dev/tasks/gitolite.yml new file mode 100644 index 0000000..61080bd --- /dev/null +++ b/dev/ansible/roles/pagure-dev/tasks/gitolite.yml @@ -0,0 +1,25 @@ +--- + +- name: Install gitolite3 + dnf: name={{ item }} state=present + with_items: + - gitolite3 + +- name: Install gitolite.rc to ~/.gitolite.rc + become_user: "{{ ansible_env.SUDO_USER }}" + copy: + src: gitolite3.rc + dest: /home/{{ ansible_env.SUDO_USER }}/.gitolite.rc + +- name: Create a key for gitolite + become_user: "{{ ansible_env.SUDO_USER }}" + command: ssh-keygen -f gitolite_rsa -t rsa -N '' + args: + chdir: /home/{{ ansible_env.SUDO_USER }} + creates: /home/{{ ansible_env.SUDO_USER }}/gitolite_rsa.pub + +- name: Setup gitolite + become_user: "{{ ansible_env.SUDO_USER }}" + command: gitolite setup -pk gitolite_rsa.pub + args: + chdir: /home/{{ ansible_env.SUDO_USER }} diff --git a/dev/ansible/roles/pagure-dev/tasks/main.yml b/dev/ansible/roles/pagure-dev/tasks/main.yml new file mode 100644 index 0000000..64799fb --- /dev/null +++ b/dev/ansible/roles/pagure-dev/tasks/main.yml @@ -0,0 +1,176 @@ +--- + +- include: clamav.yml +- include: eventsource.yml +- include: gitolite.yml +- include: milter.yml +- include: postgres.yml + +- name: Install helpful development packages + dnf: name={{ item }} state=present + with_items: + - git + - ngrep + - nmap-ncat + - python-rpdb + - tmux + - tree + - vim-enhanced + +- name: Install Pagure development packages + dnf: name={{ item }} state=present + with_items: + - gcc + - libgit2-devel + - libffi-devel + - libjpeg-devel + - make + - python-alembic + - python-arrow + - python-binaryornot + - python-bleach + - python-blinker + - python-chardet + - python-cryptography + - python-docutils + - python-enum34 + - python2-eventlet + - python-fedora-flask + - python-flask + - python-flask-wtf + - python-flask-multistatic + - python2-jinja2 + - python-markdown + - python-munch + - python-openid-cla + - python-openid-teams + - python-pip + - python-psutil + - python-pygit2 + - python-pygments + - python-redis + - python-sqlalchemy + - python-straight-plugin + - python-virtualenvwrapper + - python-wtforms + - python-devel + - python3-devel + - redhat-rpm-config + +- name: register the libgit2 version installed + shell: rpm -q libgit2|cut -d \- -f 2| cut -d \. -f 1,2 + register: libgit2_version + +# Add various helpful configuration files +- name: Install a custom bashrc + become_user: "{{ ansible_env.SUDO_USER }}" + copy: src=bashrc dest=/home/{{ ansible_env.SUDO_USER }}/.bashrc + +- name: Install the message of the day + copy: src=motd dest=/etc/motd + + +# Install Pagure inside a virtualenv and configure it +- name: Install pygit2 in the virtualenv + become_user: "{{ ansible_env.SUDO_USER }}" + pip: + name: "{{ item }}" + virtualenv: /home/{{ ansible_env.SUDO_USER }}/.virtualenvs/python2-pagure/ + virtualenv_python: python2 + with_items: + - "pygit2=={{ libgit2_version.stdout_lines[0] }}.*" + +- name: Install Pagure Python dependencies into a virtualenv + become_user: "{{ ansible_env.SUDO_USER }}" + pip: + requirements: /home/{{ ansible_env.SUDO_USER }}/devel/{{ item }} + virtualenv: /home/{{ ansible_env.SUDO_USER }}/.virtualenvs/python2-pagure/ + virtualenv_python: python2 + with_items: + - "requirements.txt" + - "tests_requirements.txt" + +- name: Install Pagure package into a virtualenv + become_user: "{{ ansible_env.SUDO_USER }}" + pip: + name: /home/{{ ansible_env.SUDO_USER }}/devel/ + extra_args: '-e' + virtualenv: /home/{{ ansible_env.SUDO_USER }}/.virtualenvs/python2-pagure/ + +- name: Install Pagure package into /usr/lib + pip: + name: /home/{{ ansible_env.SUDO_USER }}/devel/ + extra_args: '-e' + +- name: Install the pagure configuration + become_user: "{{ ansible_env.SUDO_USER }}" + copy: src=pagure.cfg dest=/home/{{ ansible_env.SUDO_USER }}/pagure.cfg + +- name: Creates pagure data directories + become_user: "{{ ansible_env.SUDO_USER }}" + file: path=/home/{{ ansible_env.SUDO_USER }}/pagure_data/{{ item }} state=directory + with_items: + - forks + - docs + - tickets + - requests + - remotes + +- name: Link the pagure repos directory to gitolite + become_user: "{{ ansible_env.SUDO_USER }}" + file: + path: /home/{{ ansible_env.SUDO_USER }}/pagure_data/repos + src: /home/{{ ansible_env.SUDO_USER }}/repositories + state: link + +- name: Add a working copy of alembic.ini + become_user: "{{ ansible_env.SUDO_USER }}" + copy: + src: /home/{{ ansible_env.SUDO_USER }}/devel/files/alembic.ini + dest: /home/{{ ansible_env.SUDO_USER }}/alembic.ini + remote_src: True + +- name: Configure alembic to use our development database + become_user: "{{ ansible_env.SUDO_USER }}" + replace: + dest: /home/{{ ansible_env.SUDO_USER }}/alembic.ini + regexp: "sqlalchemy.url = sqlite:////var/tmp/pagure_dev.sqlite" + replace: "sqlalchemy.url = sqlite:////home/{{ ansible_env.SUDO_USER }}/pagure_data/pagure_dev.sqlite" + +- name: Configure alembic to point to the pagure migration folder + become_user: "{{ ansible_env.SUDO_USER }}" + replace: + dest: /home/{{ ansible_env.SUDO_USER }}/alembic.ini + regexp: "script_location = /usr/share/pagure/alembic" + replace: "script_location = /home/vagrant/devel/alembic/" + +- name: Create the Pagure database + become_user: "{{ ansible_env.SUDO_USER }}" + command: .virtualenvs/python2-pagure/bin/python devel/createdb.py + args: + creates: /home/{{ ansible_env.SUDO_USER }}/pagure_data/pagure_dev.sqlite + chdir: "/home/{{ ansible_env.SUDO_USER }}/" + +- name: Stamp the database with its current migration + become_user: "{{ ansible_env.SUDO_USER }}" + shell: alembic stamp $(alembic heads | awk '{ print $1 }') + args: + chdir: "/home/{{ ansible_env.SUDO_USER }}/" + +- name: Create systemd user unit directory + become_user: "{{ ansible_env.SUDO_USER }}" + file: + path: /home/{{ ansible_env.SUDO_USER }}/.config/systemd/user/ + state: directory + +- name: Install the Pagure service files for systemd + become_user: "{{ ansible_env.SUDO_USER }}" + copy: + src: "{{ item }}" + dest: /home/{{ ansible_env.SUDO_USER }}/.config/systemd/user/{{ item }} + with_items: + - pagure.service + - pagure-docs.service + - pagure_ci.service + - pagure_ev.service + - pagure_webhook.service diff --git a/dev/ansible/roles/pagure-dev/tasks/milter.yml b/dev/ansible/roles/pagure-dev/tasks/milter.yml new file mode 100644 index 0000000..b18e4dc --- /dev/null +++ b/dev/ansible/roles/pagure-dev/tasks/milter.yml @@ -0,0 +1,10 @@ +--- + +- name: Install Pagure milter packages + dnf: name={{ item }} state=present + with_items: + - postfix + - python-pymilter + +- name: Start Postfix + service: name=postfix state=started enabled=yes diff --git a/dev/ansible/roles/pagure-dev/tasks/postgres.yml b/dev/ansible/roles/pagure-dev/tasks/postgres.yml new file mode 100644 index 0000000..c839074 --- /dev/null +++ b/dev/ansible/roles/pagure-dev/tasks/postgres.yml @@ -0,0 +1,38 @@ +--- + +- name: Install postgresql packages + dnf: name={{ item }} state=present + with_items: + - postgresql + - postgresql-server + - postgresql-devel # Allows pip installing psycopg2 is desired + - python-psycopg2 + +- name: Initialize PostgreSQL + command: postgresql-setup initdb + args: + creates: /var/lib/pgsql/data/pg_hba.conf + +- replace: + dest: /var/lib/pgsql/data/pg_hba.conf + regexp: "local all all peer" + replace: "local all all trust" + +- replace: + dest: /var/lib/pgsql/data/pg_hba.conf + regexp: "host all all 127.0.0.1/32 ident" + replace: "host all all 127.0.0.1/32 trust" + +- replace: + dest: /var/lib/pgsql/data/pg_hba.conf + regexp: "host all all ::1/128 ident" + replace: "host all all ::1/128 trust" + +- name: Start postgresql + service: name=postgresql state=restarted enabled=yes + +- name: Add a pagure postgres user + postgresql_user: name=pagure role_attr_flags=SUPERUSER,LOGIN + +- name: Create a database for pagure + postgresql_db: name=pagure owner=pagure diff --git a/dev/ansible/vagrant-playbook.yml b/dev/ansible/vagrant-playbook.yml new file mode 100644 index 0000000..e67cf8a --- /dev/null +++ b/dev/ansible/vagrant-playbook.yml @@ -0,0 +1,7 @@ +--- +- hosts: all + become: true + become_method: sudo + vars: + roles: + - pagure-dev diff --git a/dev/docker-compose.yml b/dev/docker-compose.yml new file mode 100644 index 0000000..1cd9fac --- /dev/null +++ b/dev/docker-compose.yml @@ -0,0 +1,72 @@ +version: '3.2' +volumes: + repos: + attachments: + postgres: +services: + web: + build: + context: ./docker + dockerfile: web + depends_on: + - redis + - postgresql + image: pagure-web:latest + ports: + - "5000:5000" + volumes: + - type: volume + source: ../lcl/repos + target: /repos + read_only: true + - type: volume + source: ../lcl/attachments + target: /attachments + read_only: false + - ..:/code + worker: + build: + context: ./docker + dockerfile: worker + depends_on: + - redis + - postgresql + image: pagure-worker:latest + volumes: + - type: volume + source: ../lcl/epos + target: /repos + read_only: false + - type: volume + source: ../lcl/attachments + target: /attachments + read_only: true + - ..:/code + environment: + - PYTHONPATH=. + - PAGURE_CONFIG=/code/openshift.cfg + ev: + build: + context: ./docker + dockerfile: ev + depends_on: + - redis + image: pagure-ev:latest + ports: + - "8080:8080" + volumes: + - ..:/code + environment: + - PYTHONPATH=. + - PAGURE_CONFIG=/code/openshift.cfg + redis: + image: redis + postgresql: + image: postgres + environment: + - POSTGRES_USER=pagure + - POSTGRES_PASSWORD=pagure + - POSTGRES_DB=pagure + - PGDATA=/var/lib/postgresql/data/pgdata + volumes: + - postgres:/var/lib/postgresql/data/pgdata diff --git a/dev/docker/ev b/dev/docker/ev new file mode 100644 index 0000000..9b1753f --- /dev/null +++ b/dev/docker/ev @@ -0,0 +1,24 @@ +FROM registry.fedoraproject.org/fedora:25 +MAINTAINER Patrick Uiterwijk + +VOLUME ["/repos"] +RUN mkdir /code + +RUN dnf install -y python2-devel python-setuptools python-nose py-bcrypt python-alembic \ + python-arrow python-binaryornot python-bleach python-blinker \ + python-chardet python-cryptography python-docutils python-flask \ + python-flask-wtf python-flask-multistatic python-markdown python-psutil \ + python-pygit2 python-pygments python-fedora python-openid python-openid-cla \ + python-openid-teams python-straight-plugin python-wtforms python-munch \ + python-enum34 python-redis python-sqlalchemy systemd gitolite3 python-filelock \ + python-fedora-flask python2-pillow python2-psycopg2 python-trollius \ + python-trollius-redis + +RUN dnf install -y python2-celery + +WORKDIR /code +ENTRYPOINT ["/usr/bin/python", "/code/pagure-ev/pagure_stream_server.py"] + +# Code injection is last to make optimal use of caches +VOLUME ["/code"] +# Openshift: COPY / /code diff --git a/dev/docker/web b/dev/docker/web new file mode 100644 index 0000000..e0a14c1 --- /dev/null +++ b/dev/docker/web @@ -0,0 +1,26 @@ +FROM registry.fedoraproject.org/fedora:25 +MAINTAINER Patrick Uiterwijk + +VOLUME ["/repos"] +RUN mkdir /code + +RUN dnf install -y python2-devel python-setuptools python-nose py-bcrypt python-alembic \ + python-arrow python-binaryornot python-bleach python-blinker \ + python-chardet python-cryptography python-docutils python-flask \ + python-flask-wtf python-flask-multistatic python-markdown python-psutil \ + python-pygit2 python-pygments python-fedora python-openid python-openid-cla \ + python-openid-teams python-straight-plugin python-wtforms python-munch \ + python-enum34 python-redis python-sqlalchemy systemd gitolite3 python-filelock \ + python-fedora-flask python2-pillow python2-psycopg2 python2-celery \ + findutils +COPY web-run /run.sh + +WORKDIR /code +# Openshift: --no-debug +ENTRYPOINT ["/usr/bin/bash", "/run.sh"] +EXPOSE 5000 + +# Code injection is last to make optimal use of caches +VOLUME ["/code"] +# Openshift: COPY / /code +VOLUME ["/attachments"] diff --git a/dev/docker/web-run b/dev/docker/web-run new file mode 100644 index 0000000..7ca36a8 --- /dev/null +++ b/dev/docker/web-run @@ -0,0 +1,14 @@ +#!/bin/bash -xe +if [ ! -f /attachments/inited ]; +then + echo "Giving Postgres time to start" + sleep 10 + PAGURE_CONFIG=/code/openshift.cfg python createdb.py + alembic --config /code/openshift_alembic.ini heads | awk '{print $1}' | \ + xargs alembic --config /code/openshift_alembic.ini stamp + touch /attachments/inited +else + alembic --config /code/openshift_alembic.ini upgrade head +fi + +exec /usr/bin/python /code/runserver.py --host 0.0.0.0 --config /code/openshift.cfg diff --git a/dev/docker/worker b/dev/docker/worker new file mode 100644 index 0000000..18072d3 --- /dev/null +++ b/dev/docker/worker @@ -0,0 +1,26 @@ +FROM registry.fedoraproject.org/fedora:25 +MAINTAINER Patrick Uiterwijk + +VOLUME ["/repos"] +RUN mkdir /code + +RUN dnf install -y python2-devel python-setuptools python-nose py-bcrypt python-alembic \ + python-arrow python-binaryornot python-bleach python-blinker \ + python-chardet python-cryptography python-docutils python-flask \ + python-flask-wtf python-flask-multistatic python-markdown python-psutil \ + python-pygit2 python-pygments python-fedora python-openid python-openid-cla \ + python-openid-teams python-straight-plugin python-wtforms python-munch \ + python-enum34 python-redis python-sqlalchemy systemd gitolite3 python-filelock \ + python-fedora-flask python2-pillow python2-psycopg2 + +RUN dnf install -y python2-celery + +WORKDIR /code +ENTRYPOINT ["/usr/bin/celery", "-A", "pagure.lib.tasks", "worker", "--loglevel", "info", "--autoreload"] + +# Code injection is last to make optimal use of caches +VOLUME ["/code"] +# Openshift: COPY / /code +VOLUME ["/attachments"] +# Ideally this would run as non-root, but that needs the /repos owned correctly +ENV C_FORCE_ROOT true diff --git a/dev/openshift.cfg b/dev/openshift.cfg new file mode 100644 index 0000000..420aee2 --- /dev/null +++ b/dev/openshift.cfg @@ -0,0 +1,13 @@ +SECRET_KEY = 'klalkdsaskrhjklh3423423' +DB_URL = 'postgresql://pagure:pagure@postgresql/pagure' +INSTANCE_NAME = 'DEVELOPMENT PAGURE' +EMAIL_ERROR = '' +APP_URL = 'http://localhost:5000/' +REDIS_HOST = 'redis' +GIT_FOLDER = '/repos/repos' +DOCS_FOLDER = '/repos/docs' +TICKETS_FOLDER = '/repos/tickets' +REQUESTS_FOLDER = '/repos/requests' +REMOTE_GIT_FOLDER = '/repos/remote' +ATTACHMENTS_FOLDER = '/attachments' +EVENTSOURCE_SOURCE = 'http://localhost:8080' diff --git a/dev/openshift_alembic.ini b/dev/openshift_alembic.ini new file mode 100644 index 0000000..a1c8d53 --- /dev/null +++ b/dev/openshift_alembic.ini @@ -0,0 +1,60 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts +script_location = /code/alembic + +# template used to generate migration files +# file_template = %%(rev)s_%%(slug)s + +# max length of characters to apply to the +# "slug" field +#truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +# sourceless = false + +#sqlalchemy.url = driver://user:pass@localhost/dbname +sqlalchemy.url = postgresql://pagure:pagure@postgresql/pagure + + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index 4876c68..0000000 --- a/docker-compose.yml +++ /dev/null @@ -1,72 +0,0 @@ -version: '3.2' -volumes: - repos: - attachments: - postgres: -services: - web: - build: - context: ./docker - dockerfile: web - depends_on: - - redis - - postgresql - image: pagure-web:latest - ports: - - "5000:5000" - volumes: - - type: volume - source: repos - target: /repos - read_only: true - - type: volume - source: attachments - target: /attachments - read_only: false - - .:/code - worker: - build: - context: ./docker - dockerfile: worker - depends_on: - - redis - - postgresql - image: pagure-worker:latest - volumes: - - type: volume - source: repos - target: /repos - read_only: false - - type: volume - source: attachments - target: /attachments - read_only: true - - .:/code - environment: - - PYTHONPATH=. - - PAGURE_CONFIG=/code/openshift.cfg - ev: - build: - context: ./docker - dockerfile: ev - depends_on: - - redis - image: pagure-ev:latest - ports: - - "8080:8080" - volumes: - - .:/code - environment: - - PYTHONPATH=. - - PAGURE_CONFIG=/code/openshift.cfg - redis: - image: redis - postgresql: - image: postgres - environment: - - POSTGRES_USER=pagure - - POSTGRES_PASSWORD=pagure - - POSTGRES_DB=pagure - - PGDATA=/var/lib/postgresql/data/pgdata - volumes: - - postgres:/var/lib/postgresql/data/pgdata diff --git a/docker/ev b/docker/ev deleted file mode 100644 index 9b1753f..0000000 --- a/docker/ev +++ /dev/null @@ -1,24 +0,0 @@ -FROM registry.fedoraproject.org/fedora:25 -MAINTAINER Patrick Uiterwijk - -VOLUME ["/repos"] -RUN mkdir /code - -RUN dnf install -y python2-devel python-setuptools python-nose py-bcrypt python-alembic \ - python-arrow python-binaryornot python-bleach python-blinker \ - python-chardet python-cryptography python-docutils python-flask \ - python-flask-wtf python-flask-multistatic python-markdown python-psutil \ - python-pygit2 python-pygments python-fedora python-openid python-openid-cla \ - python-openid-teams python-straight-plugin python-wtforms python-munch \ - python-enum34 python-redis python-sqlalchemy systemd gitolite3 python-filelock \ - python-fedora-flask python2-pillow python2-psycopg2 python-trollius \ - python-trollius-redis - -RUN dnf install -y python2-celery - -WORKDIR /code -ENTRYPOINT ["/usr/bin/python", "/code/pagure-ev/pagure_stream_server.py"] - -# Code injection is last to make optimal use of caches -VOLUME ["/code"] -# Openshift: COPY / /code diff --git a/docker/web b/docker/web deleted file mode 100644 index e0a14c1..0000000 --- a/docker/web +++ /dev/null @@ -1,26 +0,0 @@ -FROM registry.fedoraproject.org/fedora:25 -MAINTAINER Patrick Uiterwijk - -VOLUME ["/repos"] -RUN mkdir /code - -RUN dnf install -y python2-devel python-setuptools python-nose py-bcrypt python-alembic \ - python-arrow python-binaryornot python-bleach python-blinker \ - python-chardet python-cryptography python-docutils python-flask \ - python-flask-wtf python-flask-multistatic python-markdown python-psutil \ - python-pygit2 python-pygments python-fedora python-openid python-openid-cla \ - python-openid-teams python-straight-plugin python-wtforms python-munch \ - python-enum34 python-redis python-sqlalchemy systemd gitolite3 python-filelock \ - python-fedora-flask python2-pillow python2-psycopg2 python2-celery \ - findutils -COPY web-run /run.sh - -WORKDIR /code -# Openshift: --no-debug -ENTRYPOINT ["/usr/bin/bash", "/run.sh"] -EXPOSE 5000 - -# Code injection is last to make optimal use of caches -VOLUME ["/code"] -# Openshift: COPY / /code -VOLUME ["/attachments"] diff --git a/docker/web-run b/docker/web-run deleted file mode 100644 index 7ca36a8..0000000 --- a/docker/web-run +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash -xe -if [ ! -f /attachments/inited ]; -then - echo "Giving Postgres time to start" - sleep 10 - PAGURE_CONFIG=/code/openshift.cfg python createdb.py - alembic --config /code/openshift_alembic.ini heads | awk '{print $1}' | \ - xargs alembic --config /code/openshift_alembic.ini stamp - touch /attachments/inited -else - alembic --config /code/openshift_alembic.ini upgrade head -fi - -exec /usr/bin/python /code/runserver.py --host 0.0.0.0 --config /code/openshift.cfg diff --git a/docker/worker b/docker/worker deleted file mode 100644 index 18072d3..0000000 --- a/docker/worker +++ /dev/null @@ -1,26 +0,0 @@ -FROM registry.fedoraproject.org/fedora:25 -MAINTAINER Patrick Uiterwijk - -VOLUME ["/repos"] -RUN mkdir /code - -RUN dnf install -y python2-devel python-setuptools python-nose py-bcrypt python-alembic \ - python-arrow python-binaryornot python-bleach python-blinker \ - python-chardet python-cryptography python-docutils python-flask \ - python-flask-wtf python-flask-multistatic python-markdown python-psutil \ - python-pygit2 python-pygments python-fedora python-openid python-openid-cla \ - python-openid-teams python-straight-plugin python-wtforms python-munch \ - python-enum34 python-redis python-sqlalchemy systemd gitolite3 python-filelock \ - python-fedora-flask python2-pillow python2-psycopg2 - -RUN dnf install -y python2-celery - -WORKDIR /code -ENTRYPOINT ["/usr/bin/celery", "-A", "pagure.lib.tasks", "worker", "--loglevel", "info", "--autoreload"] - -# Code injection is last to make optimal use of caches -VOLUME ["/code"] -# Openshift: COPY / /code -VOLUME ["/attachments"] -# Ideally this would run as non-root, but that needs the /repos owned correctly -ENV C_FORCE_ROOT true diff --git a/openshift.cfg b/openshift.cfg deleted file mode 100644 index 420aee2..0000000 --- a/openshift.cfg +++ /dev/null @@ -1,13 +0,0 @@ -SECRET_KEY = 'klalkdsaskrhjklh3423423' -DB_URL = 'postgresql://pagure:pagure@postgresql/pagure' -INSTANCE_NAME = 'DEVELOPMENT PAGURE' -EMAIL_ERROR = '' -APP_URL = 'http://localhost:5000/' -REDIS_HOST = 'redis' -GIT_FOLDER = '/repos/repos' -DOCS_FOLDER = '/repos/docs' -TICKETS_FOLDER = '/repos/tickets' -REQUESTS_FOLDER = '/repos/requests' -REMOTE_GIT_FOLDER = '/repos/remote' -ATTACHMENTS_FOLDER = '/attachments' -EVENTSOURCE_SOURCE = 'http://localhost:8080' diff --git a/openshift_alembic.ini b/openshift_alembic.ini deleted file mode 100644 index a1c8d53..0000000 --- a/openshift_alembic.ini +++ /dev/null @@ -1,60 +0,0 @@ -# A generic, single database configuration. - -[alembic] -# path to migration scripts -script_location = /code/alembic - -# template used to generate migration files -# file_template = %%(rev)s_%%(slug)s - -# max length of characters to apply to the -# "slug" field -#truncate_slug_length = 40 - -# set to 'true' to run the environment during -# the 'revision' command, regardless of autogenerate -# revision_environment = false - -# set to 'true' to allow .pyc and .pyo files without -# a source .py file to be detected as revisions in the -# versions/ directory -# sourceless = false - -#sqlalchemy.url = driver://user:pass@localhost/dbname -sqlalchemy.url = postgresql://pagure:pagure@postgresql/pagure - - -# Logging configuration -[loggers] -keys = root,sqlalchemy,alembic - -[handlers] -keys = console - -[formatters] -keys = generic - -[logger_root] -level = WARN -handlers = console -qualname = - -[logger_sqlalchemy] -level = WARN -handlers = -qualname = sqlalchemy.engine - -[logger_alembic] -level = INFO -handlers = -qualname = alembic - -[handler_console] -class = StreamHandler -args = (sys.stderr,) -level = NOTSET -formatter = generic - -[formatter_generic] -format = %(levelname)-5.5s [%(name)s] %(message)s -datefmt = %H:%M:%S From 062b9c8676adcf16d91a88d3386d16b4e65b2fe1 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Jun 13 2017 21:01:30 +0000 Subject: [PATCH 6/7] Drop the milter.rst file duplicating install_pagure_milter.rst and not shown anywhere --- diff --git a/doc/milter.rst b/doc/milter.rst deleted file mode 100644 index 73cfdba..0000000 --- a/doc/milter.rst +++ /dev/null @@ -1,62 +0,0 @@ -Pagure's Milter -=============== - -`Milter `_ are script executed by -postfix upon sending or receiving an email. - -We use this system to allow pagure's users to comment on a ticket (or a -pull-request) by directly replying to the email sent as a notification. - -Pagure's milter is designed to be run on the same machine as the mail server -(postfix by default). Postfix connecting to the milter via a unix socket. - -The milter itself is a service managed by systemd. -You can find all the relevant files for the milter under the -``pagure-milters`` folder in the sources. - - -Install the milter ------------------- - -The first step to enable the milter on a pagure instance is thus to install the -``.service`` file for systemd and place the corresponding script that, by -default, should go to ``/usr/share/pagure/comment_email_milter.py``. - -If you are using the RPM, install ``pagure-milters`` should provide and install -all the files correctly. - - -Activate the milter -------------------- - -Make sure the milter is running and will be automaticall started at boot by -running the commands: - -To start the milter: - -:: - - systemctl start pagure_milter - -To ensure the milter is always started at boot time: - -:: - - systemctl enable pagure_milter - - -Activate the milter in postfix ------------------------------- - -To actually activate the milter in postfix is in fact really easy, all it takes -is two lines in the ``main.cf`` file of postfix: - -:: - - non_smtpd_milters = unix:/var/run/pagure/paguresock - smtpd_milters = unix:/var/run/pagure/paguresock - -These two lines are pointing to the unix socket used by postfix to communicate -with the milter. This socket is defined in the milter file itself, in the -sources: ``pagure-milters/comment_email_milter.py``. - From 45a8938cdf6ee6df3de484305ca40cc92d191313 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Jun 14 2017 07:26:18 +0000 Subject: [PATCH 7/7] Update the documentation for the coming 3.0 release --- diff --git a/doc/configuration.rst b/doc/configuration.rst index 7e2b6c9..a2acff8 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -742,7 +742,7 @@ Defaults to: ``None`` GITOLITE_POST_CONFIG -~~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~~~ This configuration key allows you to include some content at the *end* of the gitolite configuration file (such as some project definition or access), diff --git a/doc/index.rst b/doc/index.rst index bf330f1..d283886 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -32,11 +32,12 @@ Contents: overview usage/index install - install_milter - install_evs - install_webhooks + install_pagure_milter + install_pagure_ev + install_pagure_webhooks install_pagure_ci install_pagure_loadjson + install_pagure_logcom configuration development contributing diff --git a/doc/install_evs.rst b/doc/install_evs.rst deleted file mode 100644 index 13c95fb..0000000 --- a/doc/install_evs.rst +++ /dev/null @@ -1,48 +0,0 @@ -Installing pagure's EventSource server -====================================== - -Eventsource or Server Sent Events are messages sent from a server to a web -browser. It allows one to refresh a page "live", ie, without the need to reload -it entirely. - - -Configure your system ---------------------- - -The eventsource server is easy to set-up. - -* Install the required dependencies - -:: - - python-redis - python-trollius - python-trollius-redis - -.. note:: We ship a systemd unit file for pagure_milter but we welcome patches - for scripts for other init systems. - - -* Install the files of the SSE server as follow: - -+----------------------------------------+-----------------------------------------------------+ -| Source | Destination | -+========================================+=====================================================+ -| ``pagure-ev/pagure_stream_server.py`` | ``/usr/libexec/pagure-ev/pagure_stream_server.py`` | -+----------------------------------------+-----------------------------------------------------+ -| ``pagure-ev/pagure_ev.service`` | ``/etc/systemd/system/pagure_ev.service`` | -+----------------------------------------+-----------------------------------------------------+ - -The first file is the script of the SSE server itself. - -The second file is the systemd service file. - - -* Finally, activate the service and ensure it's started upon boot: - -:: - - systemctl enable redis - systemctl start redis - systemctl enable pagure_ev - systemctl start pagure_ev diff --git a/doc/install_milter.rst b/doc/install_milter.rst deleted file mode 100644 index f1025c9..0000000 --- a/doc/install_milter.rst +++ /dev/null @@ -1,79 +0,0 @@ -Installing pagure's milter -========================== - -A milter is a script that is ran by a Mail Transfer Agent (`MTA -`_) -upon receiving an email via either a network or an unix socket. - -If you want more information feel free to check out the corresponding page -on wikipedia: `https://en.wikipedia.org/wiki/Milter -`_. - -Configure your system ---------------------- - -* Install the required dependencies - -:: - - python-pymilter - -.. note:: We ship a systemd unit file for pagure_milter but we welcome patches - for scripts for other init systems. - -.. note:: It also requires a MTA, we used postfix. - - -* Create an alias ``reply`` - -This can be done in ``/etc/aliases``, for example: -:: - - reply: /dev/null - - -* Activate the ability of your MTA, to split users based on the character ``+``. - This way all the emails sent to ``reply+...@example.com`` will be forwarded - to your alias for ``reply``. - - -In postfix this is done via: -:: - - recipient_delimiter = + - -* Hook the milter in the MTA - -In postfix this is done via: -:: - - non_smtpd_milters = unix:/var/run/pagure/paguresock - smtpd_milters = unix:/var/run/pagure/paguresock - - -* Install the files of the milter as follow: - -+---------------------------------------------+---------------------------------------------------+ -| Source | Destination | -+=============================================+===================================================+ -| ``pagure-milters/comment_email_milter.py`` | ``/usr/share/pagure/comment_email_milter.py`` | -+---------------------------------------------+---------------------------------------------------+ -| ``pagure-milters/milter_tempfile.conf`` | ``/usr/lib/tmpfiles.d/pagure-milter.conf`` | -+---------------------------------------------+---------------------------------------------------+ -| ``pagure-milters/pagure_milter.service`` | ``/etc/systemd/system/pagure_milter.service`` | -+---------------------------------------------+---------------------------------------------------+ - -The first file is the script of the milter itself. - -The second file is a file specific for systemd and ensuring the temporary -folders needed by the milter are re-created if needed at each boot. - -The third file is the systemd service file. - - -* Activate the service and ensure it's started upon boot: - -:: - - systemctl enable pagure_milter - systemctl start pagure_milter diff --git a/doc/install_pagure_ev.rst b/doc/install_pagure_ev.rst new file mode 100644 index 0000000..13c95fb --- /dev/null +++ b/doc/install_pagure_ev.rst @@ -0,0 +1,48 @@ +Installing pagure's EventSource server +====================================== + +Eventsource or Server Sent Events are messages sent from a server to a web +browser. It allows one to refresh a page "live", ie, without the need to reload +it entirely. + + +Configure your system +--------------------- + +The eventsource server is easy to set-up. + +* Install the required dependencies + +:: + + python-redis + python-trollius + python-trollius-redis + +.. note:: We ship a systemd unit file for pagure_milter but we welcome patches + for scripts for other init systems. + + +* Install the files of the SSE server as follow: + ++----------------------------------------+-----------------------------------------------------+ +| Source | Destination | ++========================================+=====================================================+ +| ``pagure-ev/pagure_stream_server.py`` | ``/usr/libexec/pagure-ev/pagure_stream_server.py`` | ++----------------------------------------+-----------------------------------------------------+ +| ``pagure-ev/pagure_ev.service`` | ``/etc/systemd/system/pagure_ev.service`` | ++----------------------------------------+-----------------------------------------------------+ + +The first file is the script of the SSE server itself. + +The second file is the systemd service file. + + +* Finally, activate the service and ensure it's started upon boot: + +:: + + systemctl enable redis + systemctl start redis + systemctl enable pagure_ev + systemctl start pagure_ev diff --git a/doc/install_pagure_logcom.rst b/doc/install_pagure_logcom.rst new file mode 100644 index 0000000..4c4c1de --- /dev/null +++ b/doc/install_pagure_logcom.rst @@ -0,0 +1,47 @@ +Installing pagure-logcom +======================== + +pagure-logcom is the service that updates the log table in the database +for every commit made to the main branch of a repository allowing to build +the calendar heatmap presented on every user's page. + + +Configure your system +--------------------- + +* Install the required dependencies + +:: + + python-redis + python-trollius-redis + python-trollius + +.. note:: We ship a systemd unit file for pagure_logcom but we welcome patches + for scripts for other init systems. + + +* Install the files of pagure-loadjon as follow: + ++-----------------------------------------------+-------------------------------------------------------+ +| Source | Destination | ++===============================================+=======================================================+ +| ``pagure-logcom/pagure_logcom_server.py`` | ``/usr/libexec/pagure-logcom/pagure_logcom_server.py``| ++--------------------------------------------------+----------------------------------------------------+ +| ``pagure-logcom/pagure_logcom.service`` | ``/etc/systemd/system/pagure_logcom.service`` | ++-----------------------------------------------+-------------------------------------------------------+ + +The first file is the pagure-logcom service itself, triggered by the git +hook (shipped with pagure itself) and logging the commits into the database. + +The second file is the systemd service file. + + +* Activate the service and ensure it's started upon boot: + +:: + + systemctl enable redis + systemctl start redis + systemctl enable pagure_logcom + systemctl start pagure_logcom diff --git a/doc/install_pagure_milter.rst b/doc/install_pagure_milter.rst new file mode 100644 index 0000000..f1025c9 --- /dev/null +++ b/doc/install_pagure_milter.rst @@ -0,0 +1,79 @@ +Installing pagure's milter +========================== + +A milter is a script that is ran by a Mail Transfer Agent (`MTA +`_) +upon receiving an email via either a network or an unix socket. + +If you want more information feel free to check out the corresponding page +on wikipedia: `https://en.wikipedia.org/wiki/Milter +`_. + +Configure your system +--------------------- + +* Install the required dependencies + +:: + + python-pymilter + +.. note:: We ship a systemd unit file for pagure_milter but we welcome patches + for scripts for other init systems. + +.. note:: It also requires a MTA, we used postfix. + + +* Create an alias ``reply`` + +This can be done in ``/etc/aliases``, for example: +:: + + reply: /dev/null + + +* Activate the ability of your MTA, to split users based on the character ``+``. + This way all the emails sent to ``reply+...@example.com`` will be forwarded + to your alias for ``reply``. + + +In postfix this is done via: +:: + + recipient_delimiter = + + +* Hook the milter in the MTA + +In postfix this is done via: +:: + + non_smtpd_milters = unix:/var/run/pagure/paguresock + smtpd_milters = unix:/var/run/pagure/paguresock + + +* Install the files of the milter as follow: + ++---------------------------------------------+---------------------------------------------------+ +| Source | Destination | ++=============================================+===================================================+ +| ``pagure-milters/comment_email_milter.py`` | ``/usr/share/pagure/comment_email_milter.py`` | ++---------------------------------------------+---------------------------------------------------+ +| ``pagure-milters/milter_tempfile.conf`` | ``/usr/lib/tmpfiles.d/pagure-milter.conf`` | ++---------------------------------------------+---------------------------------------------------+ +| ``pagure-milters/pagure_milter.service`` | ``/etc/systemd/system/pagure_milter.service`` | ++---------------------------------------------+---------------------------------------------------+ + +The first file is the script of the milter itself. + +The second file is a file specific for systemd and ensuring the temporary +folders needed by the milter are re-created if needed at each boot. + +The third file is the systemd service file. + + +* Activate the service and ensure it's started upon boot: + +:: + + systemctl enable pagure_milter + systemctl start pagure_milter diff --git a/doc/install_pagure_webhooks.rst b/doc/install_pagure_webhooks.rst new file mode 100644 index 0000000..42531e9 --- /dev/null +++ b/doc/install_pagure_webhooks.rst @@ -0,0 +1,49 @@ +Installing pagure's web-hooks notification system +================================================= + +Web-hooks are a notification system upon which a system makes a http POST +request with some data upon doing an action. This allows notifying a system +that an action has occurred. + +If you want more information feel free to check out the corresponding page +on wikipedia: `https://en.wikipedia.org/wiki/Webhook +`_. + +Configure your system +--------------------- + +* Install the required dependencies + +:: + + python-redis + python-trollius + python-trollius-redis + +.. note:: We ship a systemd unit file for pagure_webhook but we welcome patches + for scripts for other init systems. + + +* Install the files of the web-hook server as follow: + ++----------------------------------------------+----------------------------------------------------------+ +| Source | Destination | ++==============================================+==========================================================+ +| ``pagure-webhook/pagure-webhook-server.py`` | ``/usr/libexec/pagure-webhook/pagure-webhook-server.py`` | ++----------------------------------------------+----------------------------------------------------------+ +| ``pagure-webhook/pagure_webhook.service`` | ``/etc/systemd/system/pagure_webhook.service`` | ++----------------------------------------------+----------------------------------------------------------+ + +The first file is the script of the web-hook server itself. + +The second file is the systemd service file. + + +* Activate the service and ensure it's started upon boot: + +:: + + systemctl enable redis + systemctl start redis + systemctl enable pagure_webhook + systemctl start pagure_webhook diff --git a/doc/install_webhooks.rst b/doc/install_webhooks.rst deleted file mode 100644 index 42531e9..0000000 --- a/doc/install_webhooks.rst +++ /dev/null @@ -1,49 +0,0 @@ -Installing pagure's web-hooks notification system -================================================= - -Web-hooks are a notification system upon which a system makes a http POST -request with some data upon doing an action. This allows notifying a system -that an action has occurred. - -If you want more information feel free to check out the corresponding page -on wikipedia: `https://en.wikipedia.org/wiki/Webhook -`_. - -Configure your system ---------------------- - -* Install the required dependencies - -:: - - python-redis - python-trollius - python-trollius-redis - -.. note:: We ship a systemd unit file for pagure_webhook but we welcome patches - for scripts for other init systems. - - -* Install the files of the web-hook server as follow: - -+----------------------------------------------+----------------------------------------------------------+ -| Source | Destination | -+==============================================+==========================================================+ -| ``pagure-webhook/pagure-webhook-server.py`` | ``/usr/libexec/pagure-webhook/pagure-webhook-server.py`` | -+----------------------------------------------+----------------------------------------------------------+ -| ``pagure-webhook/pagure_webhook.service`` | ``/etc/systemd/system/pagure_webhook.service`` | -+----------------------------------------------+----------------------------------------------------------+ - -The first file is the script of the web-hook server itself. - -The second file is the systemd service file. - - -* Activate the service and ensure it's started upon boot: - -:: - - systemctl enable redis - systemctl start redis - systemctl enable pagure_webhook - systemctl start pagure_webhook diff --git a/doc/overview.rst b/doc/overview.rst index 66d2e84..43cb87a 100644 --- a/doc/overview.rst +++ b/doc/overview.rst @@ -2,7 +2,7 @@ Overview ======== Pagure is split over multiple components, each having their purpose and all -but one (the core application) being optional. +but two (the core web application and its workers) being optional. These components are: @@ -31,6 +31,21 @@ provide a web UI to the git repositories as well as tickets and pull-requests. This is the main application for the forge. +Pagure workers +-------------- + +Interacting with git repos can be a long process, it varies depending on the +size of the repository itself but also based on hardware performances or +simply the load on the system. +To make pagure capable of handling more load, since pagure 3.0 the interactions +with the git repositories from the web UI is performed by dedicated workers, +allowing async processing of the different tasks. + +The communication between the core application and its worker is based on +`celery <>`_ and defaults to using `redis `_ but any of +the queueing system supported by `celery <>`_ could be used instead. + + Gitolite -------- @@ -90,7 +105,7 @@ information received. Pagure web-hook Server -------------------------- +---------------------- Sends notifications to third party services using POST http requests. @@ -102,3 +117,21 @@ being slow. The flow is: the main pagure server does an action, sends a message over redis, the web-hook server picks it up, build the query and performs the POST request to the specified URLs. + + +Pagure load JSON service +------------------------ + +The load JSON service is an async service updating the database based on +information pushed to the ticket or pull-request git repositories. +This allows updating the database with information pushed to the git +repositories without keeping the connection open with the client. + + +Pagure log com service +---------------------- + +The log com (for log commit) service is an async service updating the log +table of the database on every pushed made to any repository allowing to +build the data for the calendar heatmap graph displayed on every user's +page. diff --git a/doc/overview_simple.ascii b/doc/overview_simple.ascii index a1a129b..62fc7b0 100644 --- a/doc/overview_simple.ascii +++ b/doc/overview_simple.ascii @@ -5,22 +5,31 @@ User's git actions+--------------------->+ Gitolite +-------------------------->+ Git repos | | | | | +------------+ +-----------+ - ^ - | - +-----------------------------+ - | -User's mail client | - ^ +---------------+ | - | Notifications | | | - +----------------------------------+ Mail server | | - | | | - +---------------+ | - ^ | - | | - | | - +--------------+ Updates +--------------+ - | | & queries |{s} | -User's web browser+---------------------->+ Pagure +------------->| Database | - | web server | | | - | | +--------------+ + ^ ^ + | | + +-----------------------------+ | + | | +User's mail client | | + ^ +---------------+ | | + | Notifications | | | | + +----------------------------------+ Mail server | | | + | | | | + +---------------+ | | + ^ | | + | | | + | | | + +--------------+ Updates +--------------+ | + | | & queries |{s} | | +User's web browser+---------------------->+ Pagure +------------->| Database | | + | web server | | | | + | | +--------------+ | + +------+-------+ | + | | + | sends instructions | Updates + v | + +--------------+ | + | | | + | Pagure +----------------------------------+ + | workers | + | | +--------------+ diff --git a/doc/usage/ticket_templates.rst b/doc/usage/ticket_templates.rst index 625d2ff..07e6e9f 100644 --- a/doc/usage/ticket_templates.rst +++ b/doc/usage/ticket_templates.rst @@ -10,7 +10,7 @@ often requested/needed. The templates are provided in the git repository containing the meta-data for the tickets. They must be placed under a ``templates`` folder in this git repository, -end with ``.md``and as the extension suggests can be formatted as markdown. +end with ``.md`` and as the extension suggests can be formatted as markdown. If you create a template ``templates/default.md``, it will be shown by default when someone ask to create a new ticket.